Skip to content
Merged
68 changes: 60 additions & 8 deletions .github/workflows/build-project.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2180,15 +2180,14 @@ jobs:
# lanes, so COFF-only link defects reached master (`undefined symbol:
# obs_encoder_get_caps`, merge_group 42390201886 green, push 42393067345
# red). This is only the cross lane's link step for Windows x64: no clippy,
# Dylint; selected Tier 2 links also cover Darwin, nothing executed.
# Dylint. Selected Tier 2 links run in the independent alias matrix below.
rust-windows-link:
timeout-minutes: 30
name: Windows Link Check 🔗 (x86_64-pc-windows-msvc)
runs-on: ${{ inputs.mq-heavy-labels != '' && fromJSON(inputs.mq-heavy-labels) || 'ubuntu-26.04' }}
needs: [check-event, ci-mode]
# Every fallback non-docs PR links categories. Tier 2 selections always run
# before the queue: existing client proofs do not attest these CMake links.
if: ${{ (github.event_name != 'pull_request' || needs.ci-mode.outputs.fast_path != 'true' || needs.check-event.outputs.cmocka_link == 'true') && !inputs.flatpak-measurement-only && !inputs.native-cache-measurement-only && needs.check-event.outputs.deferred == 'true' && needs.check-event.outputs.docs_only != 'true' && needs.check-event.outputs.windows_link == 'true' }}
# Category links retain their existing member-admission policy.
if: ${{ (github.event_name != 'pull_request' || needs.ci-mode.outputs.fast_path != 'true') && !inputs.flatpak-measurement-only && !inputs.native-cache-measurement-only && needs.check-event.outputs.deferred == 'true' && needs.check-event.outputs.docs_only != 'true' && needs.check-event.outputs.windows_link == 'true' }}
defaults:
run:
shell: bash
Expand Down Expand Up @@ -2225,14 +2224,67 @@ jobs:
save-cache: ${{ github.ref == 'refs/heads/master' }}
cache-suffix: ${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || '' }}

- name: Set Up Native Link Tools
run: sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build pkg-config

- name: Link Windows Test Binaries 🔗
env:
OBS_CMOCKA_LINK: ${{ needs.check-event.outputs.cmocka_link }}
OBS_CMOCKA_LINK: 'false'
run: uv run --no-project ci/windows_link_check.py

cmocka-link:
timeout-minutes: 30
name: Tier 2 Link Check 🔗 (${{ matrix.alias }})
runs-on: ${{ inputs.mq-heavy-labels != '' && fromJSON(inputs.mq-heavy-labels) || 'ubuntu-26.04' }}
needs: [check-event, ci-mode]
# Both full inventories are required independently, including attested PRs.
# No category or other alias dependency: the three link legs can overlap.
if: ${{ !inputs.flatpak-measurement-only && !inputs.native-cache-measurement-only && needs.check-event.outputs.deferred == 'true' && needs.check-event.outputs.docs_only != 'true' && needs.check-event.outputs.cmocka_link == 'true' }}
strategy:
fail-fast: false
matrix:
alias: [win-x64, mac-arm64]
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false

- name: Set Up soldr 🦀
uses: zackees/setup-soldr@db9aa035e488f4dde7d31d5c8cb4ec20ec99c6c1 # merge of forge-ci/setup-soldr#676: persistent-root cook store and recipes (#675)
with:
# Same scope and layer as rust-cross-lint, so it replays master's cache.
verdict-scope: build-project
version: 0.9.45
build-cache: false
# Cross-release archives and Corrosion use separate target trees (#5224).
prebuild-deps: none
cargo-registry-cache: true
cache-key-suffix: linux-host-v3
save-cache-remote: ${{ github.ref == 'refs/heads/master' }}
# The Windows target is added mid-job (see rust-cross-lint, #969).
solo-toolchain-cache: false
# S3 remote cache: bucket forge-ci on DO Spaces nyc3; the 50G watermark is
# enforced only on the merge_group singleton. No bucket/creds = feature off.
s3-bucket: forge-ci
s3-endpoint: nyc3.digitaloceanspaces.com
s3-region: nyc3
s3-cache-limit: 50G
env:
SOLDR_S3_CACHE_ACCESS_KEY: ${{ secrets.SOLDR_S3_CACHE_ACCESS_KEY }}
SOLDR_S3_CACHE_SECRET_KEY: ${{ secrets.SOLDR_S3_CACHE_SECRET_KEY }}

- name: Set Up uv 🐍
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
with:
save-cache: ${{ github.ref == 'refs/heads/master' }}
cache-suffix: ${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || '' }}

# uv wheels avoid the 165 s apt mirror setup observed in run 44495240642.
# CMake 3.31 retains clang-cl /showIncludes, understood by pinned ccache;
# 4.3 forwards -clang:-M* dependency flags which ccache cannot cache.
- name: Link Tier 2 Fixtures 🔗
run: uv run --no-project --with cmake==3.31.10 --with ninja==1.13.0 ci/cmocka_link_check.py --alias ${{ matrix.alias }}

rust-cross-lint-release:
timeout-minutes: 45
name: Rust Cross Lint 🦀 (release targets)
Expand Down
2 changes: 1 addition & 1 deletion ci/ci_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ class Scope:
"cmocka_link": (
"libobs/**", "rust/libobs-rust/**", "rust/obs-build/**", "test/cmocka/**", "deps/**",
"cmake/cross/**", "cmake/common/soldr.cmake", "ci/cmocka_link_check.py",
"ci/windows_link_check.py", "ci/ci_scope.py", ".github/workflows/build-project.yaml",
"ci/windows_link_check.py", "ci/tier2_front_cache.py", "ci/ci_scope.py", ".github/workflows/build-project.yaml",
"Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "CMakeLists.txt",
),
"tier2": (
Expand Down
52 changes: 43 additions & 9 deletions ci/cmocka_link_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
import shlex
import subprocess
import time
import tempfile
from pathlib import Path

import local_gate
import tier2_front_cache

ROOT = Path(__file__).resolve().parent.parent
TRIPLES = {"win-x64": "x86_64-pc-windows-msvc", "mac-arm64": "aarch64-apple-darwin"}
Expand Down Expand Up @@ -77,13 +79,45 @@ def query(option: str) -> str:
return path


def cache_env(work: Path, tools: Path | None = None) -> dict[str, str]:
"""Fresh per-pass cache: share identical support compiles, never link results."""
# Keep the launcher command stable for kept Ninja trees; only cached objects
# and policy are ephemeral. The installer re-verifies its pinned binary.
binary = tier2_front_cache.install(tools or work)
config = work / "ccache.conf"
config.write_text("")
env = {key: value for key, value in local_gate.soldr_env().items() if not key.startswith("CCACHE_")}
env.update(CCACHE_CONFIGPATH=str(config), CCACHE_DIR=str(work / "objects"),
CCACHE_MAXSIZE="512M", CCACHE_COMPILERCHECK="content",
CMAKE_C_COMPILER_LAUNCHER=str(binary), CMAKE_CXX_COMPILER_LAUNCHER=str(binary))
return env


def build_targets(alias: str, work: Path, jobs: int, env: dict[str, str]) -> None:
targets = discover_targets(work / ".cmake/api/v1/reply")
(work / "link-targets.json").write_text(json.dumps(targets, indent=2) + "\n")
print(f"Tier 2 {alias}: linking hybrid libobs and {len(targets) - 1} Rust fixtures ({jobs} Ninja workers)", flush=True)
subprocess.run(["cmake", "--build", str(work), "--parallel", str(jobs), "--target", *targets,
"--", "-k", "0"], cwd=ROOT, env=env, check=True)


def link(alias: str, work: Path, jobs: int) -> None:
with tempfile.TemporaryDirectory(prefix="cmocka-link-cache-") as directory:
env = cache_env(Path(directory), work / "compiler-cache")
try:
_link(alias, work, jobs, env)
finally:
subprocess.run([env["CMAKE_C_COMPILER_LAUNCHER"], "--show-stats"], env=env, check=False)


def _link(alias: str, work: Path, jobs: int, env: dict[str, str]) -> None:
started = time.monotonic()
target = TRIPLES[alias]
local_gate.add_target(target)
# Invoke Cargo even on a warm tree: the cross project's legacy custom
# command only depends on manifests, which cannot detect a Rust source edit.
local_gate.soldr_cargo(["build", "--release", "--locked", "--target", target, "-p", "libobs-rust"])
print(f"Tier 2 {alias}: Rust archive ready in {time.monotonic() - started:.2f}s", flush=True)
work.mkdir(parents=True, exist_ok=True)
wrappers = work / "soldr-tools"
wrappers.mkdir(exist_ok=True)
Expand All @@ -97,26 +131,26 @@ def link(alias: str, work: Path, jobs: int) -> None:
query.touch()
archiver = subprocess.run(["soldr", "cc", "--target", target, "--print-ar"],
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
env = local_gate.soldr_env({"OBS_CROSS_TARGET": alias, "SOLDR_PORCELAIN": "0",
f"AR_{target.replace('-', '_')}": archiver})
env = dict(env, OBS_CROSS_TARGET=alias, SOLDR_PORCELAIN="0")
env[f"AR_{target.replace('-', '_')}"] = archiver
configured = time.monotonic()
subprocess.run(["cmake", "-S", str(ROOT), "-B", str(work), "-G", "Ninja",
f"-DOBS_CROSS_TARGET={alias}", "-DOBS_CROSS_LINK=ON", "-DOBS_CROSS_TIER2=ON",
"-DCMAKE_BUILD_TYPE=Release", f"-DRust_CARGO={wrappers / 'cargo'}",
f"-DRust_COMPILER={wrappers / 'rustc'}", f"-DRust_CARGO_TARGET={target}",
"-DOBS_TEST_SANITIZE_PER_TARGET=OFF", *toolchain_args], cwd=ROOT, env=env, check=True)
targets = discover_targets(work / ".cmake/api/v1/reply")
(work / "link-targets.json").write_text(json.dumps(targets, indent=2) + "\n")
print(f"Tier 2 {alias}: linking hybrid libobs and {len(targets) - 1} Rust fixtures", flush=True)
subprocess.run(["cmake", "--build", str(work), "--parallel", str(jobs), "--target", *targets,
"--", "-k", "0"], cwd=ROOT, env=env, check=True)
"-DOBS_TEST_SANITIZE_PER_TARGET=OFF",
f"-DCMAKE_C_COMPILER_LAUNCHER={env['CMAKE_C_COMPILER_LAUNCHER']}",
f"-DCMAKE_CXX_COMPILER_LAUNCHER={env['CMAKE_CXX_COMPILER_LAUNCHER']}", *toolchain_args], cwd=ROOT, env=env, check=True)
print(f"Tier 2 {alias}: configured in {time.monotonic() - configured:.2f}s", flush=True)
build_targets(alias, work, jobs, env)
print(f"Tier 2 {alias}: linked in {time.monotonic() - started:.2f}s", flush=True)


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--alias", choices=TRIPLES, default="win-x64")
parser.add_argument("--build", type=Path)
parser.add_argument("--jobs", type=int, default=min(4, os.cpu_count() or 1))
parser.add_argument("--jobs", type=int, default=min(4, os.process_cpu_count() or 1))
args = parser.parse_args()
if args.jobs < 1:
parser.error("--jobs must be positive")
Expand Down
10 changes: 8 additions & 2 deletions ci/gate_jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,12 @@ def data_selected(files) -> bool:
)) for path in files)


def cmocka_jobs(selected: Scope) -> set[str]:
"""Both alias jobs are required whenever the Tier 2 link lane is selected."""
return ({f"Build 🧱 / Tier 2 Link Check 🔗 ({alias})" for alias in ("win-x64", "mac-arm64")}
if getattr(selected, "cmocka_link", False) else set())


def expected_jobs(selected: Scope, files: list[str] | tuple[str, ...] = ()) -> set[str]:
"""The queue execution manifest: exact job names, never proof credit.

Expand All @@ -137,6 +143,7 @@ def expected_jobs(selected: Scope, files: list[str] | tuple[str, ...] = ()) -> s
expected.add("Build 🧱 / Fast Gate ⚡")
if selected.windows_link:
expected.add("Build 🧱 / Windows Link Check 🔗 (x86_64-pc-windows-msvc)")
expected.update(cmocka_jobs(selected))
if selected.cross_lint:
expected.update({
"Build 🧱 / Rust Cross Lint 🦀 (x86_64-pc-windows-msvc)",
Expand Down Expand Up @@ -198,8 +205,7 @@ def pr_expected_jobs(selected: Scope, files: list[str], plan: dict) -> set[str]:
expected -= heavy
# Existing client proofs cover category links, not the cmocka CMake
# pass. The base-owned selector must witness that pass even on fast PRs.
if getattr(selected, "cmocka_link", False):
expected.add("Build 🧱 / Windows Link Check 🔗 (x86_64-pc-windows-msvc)")
expected.update(cmocka_jobs(selected))
for key, job_names in names.items():
status = plan["jobs"].get(key, {})
needed = status.get("required_gates", [])
Expand Down
22 changes: 21 additions & 1 deletion ci/test_ci_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ def test_queue_runner_selection_preserves_hosted_fallbacks(self):

heavy = {
"rust-cross-lint", "rust-cross-lint-release", "rust-tests-macos-build",
"rust-linux-tier2-build", "rust-linux-tier2", "rust-windows-link",
"rust-linux-tier2-build", "rust-linux-tier2", "rust-windows-link", "cmocka-link",
}
light = {
"ci.yml": {
Expand Down Expand Up @@ -435,6 +435,26 @@ def test_queue_runs_merge_result_tests_and_windows_link(self):
self.assertIn('echo "windows_link=${windows_link:-true}"', text)
self.assertIn("(github.event_name == 'merge_group' || needs.ci-mode.outputs.rust-test_hit != 'true')", text)

def test_both_cmocka_aliases_are_independent_selected_jobs(self):
from test_windows_link_check import job_block

block = job_block("cmocka-link")
self.assertIn("alias: [win-x64, mac-arm64]", block)
self.assertIn("fail-fast: false", block)
self.assertIn("needs: [check-event, ci-mode]", block)
condition = next(line for line in block.splitlines() if line.startswith(" if:"))
self.assertIn("outputs.cmocka_link == 'true'", condition)
self.assertNotIn("fast_path", condition)
self.assertNotIn("apt-get", block)
self.assertIn("--with cmake==3.31.10 --with ninja==1.13.0", block)
self.assertIn("ci/cmocka_link_check.py --alias ${{ matrix.alias }}", block)
gate = (ROOT / ".github/workflows/ci.yml").read_text()
self.assertIn("build-project", gate.split(" ci-gate:", 1)[1].split(" steps:", 1)[0])
for event in ("pull_request", "merge_group"):
for path in ("libobs/obs.c.h", "ci/cmocka_link_check.py", "ci/tier2_front_cache.py"):
with self.subTest(event=event, path=path):
self.assertTrue(scope([path], event=event).cmocka_link)

def test_core_file_content_is_not_read(self):
# Core files carry no platform defines (the ratchet lint enforces
# it), so a core file is Linux-only whatever it contains (#897).
Expand Down
50 changes: 50 additions & 0 deletions ci/test_cmocka_link_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,56 @@ def test_windows_uses_managed_clang_cl_and_the_target_sdk(self):
self.assertIn("--target=x86_64-pc-windows-msvc", wrapper)


class BuildTest(unittest.TestCase):
def test_each_alias_builds_the_full_inventory_at_the_cpu_budget(self):
inventory = ["libobs-cross-shared", "test_existing_rust", "test_new_steps_rust"]
for alias in cmocka_link_check.TRIPLES:
with self.subTest(alias=alias), tempfile.TemporaryDirectory() as temporary:
work = Path(temporary)
with mock.patch.object(cmocka_link_check, "discover_targets", return_value=inventory), \
mock.patch.object(cmocka_link_check.subprocess, "run") as run:
cmocka_link_check.build_targets(alias, work, 3, {})
args = run.call_args.args[0]
self.assertEqual(args[args.index("--parallel") + 1], "3")
self.assertEqual(args[args.index("--target") + 1:args.index("--")], inventory)
self.assertEqual(json.loads((work / "link-targets.json").read_text()), inventory)

def test_alias_link_failures_propagate(self):
import subprocess

for alias in cmocka_link_check.TRIPLES:
with self.subTest(alias=alias), tempfile.TemporaryDirectory() as temporary:
work = Path(temporary)
failure = subprocess.CalledProcessError(1, ["cmake", "--build"])
with mock.patch.object(cmocka_link_check, "discover_targets", return_value=["libobs-cross-shared", "test_existing_rust"]), \
mock.patch.object(cmocka_link_check.subprocess, "run", side_effect=failure):
with self.assertRaises(subprocess.CalledProcessError):
cmocka_link_check.build_targets(alias, work, 2, {})

def test_kept_tree_uses_a_stable_verified_launcher_with_a_fresh_cache(self):
with tempfile.TemporaryDirectory() as temporary:
work = Path(temporary)
tools = work / "build" / "compiler-cache"
with mock.patch.object(cmocka_link_check.tier2_front_cache, "install", return_value=tools / "ccache") as install:
env = cmocka_link_check.cache_env(work, tools)
install.assert_called_once_with(tools)
self.assertEqual(env["CMAKE_C_COMPILER_LAUNCHER"], str(tools / "ccache"))
self.assertEqual(env["CCACHE_DIR"], str(work / "objects"))

def test_cache_is_fresh_and_cannot_inherit_relaxed_or_remote_policy(self):
with tempfile.TemporaryDirectory() as temporary:
work = Path(temporary)
with mock.patch.object(cmocka_link_check.tier2_front_cache, "install", return_value=work / "ccache"), \
mock.patch.dict("os.environ", {"CCACHE_SLOPPINESS": "time_macros", "CCACHE_REMOTE_STORAGE": "remote", "CCACHE_PREFIX": "unsafe"}):
env = cmocka_link_check.cache_env(work)
self.assertNotIn("CCACHE_SLOPPINESS", env)
self.assertNotIn("CCACHE_REMOTE_STORAGE", env)
self.assertNotIn("CCACHE_PREFIX", env)
self.assertEqual(Path(env["CCACHE_CONFIGPATH"]).read_text(), "")
self.assertEqual(env["CCACHE_MAXSIZE"], "512M")
self.assertEqual(env["CCACHE_COMPILERCHECK"], "content")


class DiscoveryTest(unittest.TestCase):
def test_uses_generated_target_types_and_names_not_source_regexes(self):
with tempfile.TemporaryDirectory() as temporary:
Expand Down
2 changes: 2 additions & 0 deletions ci/test_cook_profiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ def test_build_consumers_declare_the_profile_policy(self):
"rust-cross-lint": "--profile dev -- --tests",
"rust-cross-lint-release": "--profile dev -- --tests",
"rust-tests-macos-build": "--profile dev -- --tests",
"rust-windows-link": "--profile dev -- --tests",
"cmocka-link": None,
"rust-tests-windows": "--profile dev -- --tests",
# Corrosion-only build, its own target dir: no cook (#3027).
"rust-linux-tier2-build": None,
Expand Down
Loading
Loading