From 55253d756d9b35ce972c256a8d4a6e552701bf35 Mon Sep 17 00:00:00 2001 From: Ryan Curran Date: Thu, 29 Oct 2026 08:51:14 -0400 Subject: [PATCH] builder-upgrade: fall back to the ronin CLT dmg when softwareupdate has none softwareupdate on r8-198 listed no Command Line Tools at all, only the 15.8.2 update, so the CLT step could never finish. Install the 16.4 dmg ronin ships instead, and only if the pkg inside reports "signed Apple Software". --- orchestrator/orchestrator/builder_upgrade.py | 25 +++++++++- orchestrator/tests/test_builder_upgrade.py | 51 ++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/orchestrator/orchestrator/builder_upgrade.py b/orchestrator/orchestrator/builder_upgrade.py index 49d4877..8d89a37 100644 --- a/orchestrator/orchestrator/builder_upgrade.py +++ b/orchestrator/orchestrator/builder_upgrade.py @@ -42,6 +42,9 @@ XCODE_URL = "https://ronin-puppet-package-repo.s3.us-west-2.amazonaws.com/macos/public/15/Xcode_16.4.xip" XCODE_BUILD = "16F6" CLT_LABEL = "Command Line Tools for Xcode-16.4" +CLT_DMG_URL = "https://ronin-puppet-package-repo.s3.us-west-2.amazonaws.com/macos/public/common/Command_Line_Tools_for_Xcode_16.4.dmg" +CLT_DMG = "/private/var/tmp/clt.dmg" +CLT_MOUNT = "/private/var/tmp/clt-mnt" INSTALLER = "/Applications/Install macOS Sequoia.app" XCODE_REMOTE = "/var/root/install-xcode.sh" XCODE_DAEMON = "org.mozilla.relops.xcode-install" @@ -482,6 +485,23 @@ def step_install_xcode(h: BuilderHost, *, poll: int = 30, stall: int = 20 * 60, ui.ok(f"Xcode {XCODE_BUILD} selected") +def _install_clt_dmg(h: BuilderHost) -> None: + """softwareupdate offered no CLT at all on r8-198 (2026-10-29: only the 15.8.2 update was + listed). Install the dmg ronin ships instead, and only if the pkg is Apple-signed -- + Software Update pkgs report "signed Apple Software", not a Developer ID naming Apple Inc.""" + ui.wire(f"CLT not offered by softwareupdate -- installing {CLT_DMG_URL.rsplit('/', 1)[-1]}") + _sh(h, f"sudo curl -fsSL --retry 3 -o {CLT_DMG} {shlex.quote(CLT_DMG_URL)}", timeout=1800) + try: + _sh(h, f"sudo mkdir -p {CLT_MOUNT} && sudo hdiutil attach -quiet -nobrowse -readonly -mountpoint {CLT_MOUNT} {CLT_DMG}") + pkg = _sh(h, f"ls {CLT_MOUNT}/*.pkg | head -1").strip() + sig = _sh(h, f"pkgutil --check-signature {shlex.quote(pkg)}", check=False) + if "Status: signed Apple Software" not in sig: + raise ReprovisionError(f"{escape(pkg)} is not Apple-signed: {escape(sig[:200])}") + _sh(h, f"sudo installer -pkg {shlex.quote(pkg)} -target /", timeout=1800) + finally: + _sh(h, f"sudo hdiutil detach -quiet {CLT_MOUNT}; sudo rm -rf {CLT_DMG} {CLT_MOUNT}", check=False) + + def step_install_clt(h: BuilderHost) -> None: """The upgrade leaves a CommandLineTools stub with no usr/bin/git. Done here because the download outlasts puppet's old 300s exec timeout (ronin #1470 raises it).""" @@ -491,9 +511,12 @@ def step_install_clt(h: BuilderHost) -> None: return _sh(h, f"sudo touch {CLT_TRIGGER}") try: - _sh(h, f"sudo softwareupdate -i {shlex.quote(CLT_LABEL)} --verbose", timeout=3600) + # check=False: "No such update" is the case the dmg below covers. + _sh(h, f"sudo softwareupdate -i {shlex.quote(CLT_LABEL)} --verbose", check=False, timeout=3600) finally: _sh(h, f"sudo rm -f {CLT_TRIGGER}", check=False) + if not clt_ready(h): + _install_clt_dmg(h) if not clt_ready(h): raise ReprovisionError("Command Line Tools still missing usr/bin/git after the install") ui.ok("installed") diff --git a/orchestrator/tests/test_builder_upgrade.py b/orchestrator/tests/test_builder_upgrade.py index 2dea2a3..77e1e59 100644 --- a/orchestrator/tests/test_builder_upgrade.py +++ b/orchestrator/tests/test_builder_upgrade.py @@ -477,3 +477,54 @@ def test_runner_maps_builder_upgrade_action(): assert cmd[1:] == ["builder-upgrade", "macmini-r8-124", "--job", "7", "--unquarantine"] dry = runner._job_cmd("macmini-r8-124", {"id": 8, "action": "builder-upgrade", "params": {"dry_run": True}}, None) assert "--dry-run" in dry and "--unquarantine" not in dry + + +def _clt_fake(*, softwareupdate_installs: bool, sig: str = "Status: signed Apple Software"): + state = {"installed": False} + + def softwareupdate(): + state["installed"] = softwareupdate_installs + return _cp("" if softwareupdate_installs else "Command Line Tools for Xcode-16.4: No such update", rc=0 if softwareupdate_installs else 1) + + def installer(): + state["installed"] = True + return _cp("installer: The install was successful.") + + def ready(): + return _cp("", rc=0 if state["installed"] else 1) + + return FakeSSH({ + "CommandLineTools/usr/bin/git": ready, + "softwareupdate -i": softwareupdate, + "ls /private/var/tmp/clt-mnt": "/private/var/tmp/clt-mnt/Command Line Tools.pkg\n", + "pkgutil --check-signature": sig, + "installer -pkg": installer, + }) + + +def test_install_clt_uses_softwareupdate_when_offered(): + fake = _clt_fake(softwareupdate_installs=True) + with patch("orchestrator.builder_upgrade.ssh.run", fake.run): + bu.step_install_clt(_host()) + assert not any("curl" in c for c in fake.commands()) + + +def test_install_clt_falls_back_to_the_ronin_dmg(): + # r8-198: softwareupdate listed no CLT at all. + fake = _clt_fake(softwareupdate_installs=False) + with patch("orchestrator.builder_upgrade.ssh.run", fake.run): + bu.step_install_clt(_host()) + cmds = fake.commands() + assert any(bu.CLT_DMG_URL in c for c in cmds) + assert any("installer -pkg" in c for c in cmds) + assert "hdiutil detach" in cmds[-2] or any("hdiutil detach" in c for c in cmds[-3:]) + + +def test_install_clt_refuses_a_dmg_pkg_not_signed_by_apple(): + fake = _clt_fake(softwareupdate_installs=False, sig="Status: no signature") + with patch("orchestrator.builder_upgrade.ssh.run", fake.run): + with pytest.raises(ReprovisionError, match="not Apple-signed"): + bu.step_install_clt(_host()) + cmds = fake.commands() + assert not any("installer -pkg" in c for c in cmds) + assert any("hdiutil detach" in c for c in cmds)