builder-upgrade: fall back to the ronin CLT dmg when softwareupdate has none

orchestrator/orchestrator/builder_upgrade.py

@@ -42,6 +42,9 @@
 XCODE_URL = "https://ronin-puppet-package-repo.s3.us-west-2.amazonaws.com/macos/public/15/Xcode_16.4.xip"
 XCODE_BUILD = "16F6"
 CLT_LABEL = "Command Line Tools for Xcode-16.4"
+CLT_DMG_URL = "https://ronin-puppet-package-repo.s3.us-west-2.amazonaws.com/macos/public/common/Command_Line_Tools_for_Xcode_16.4.dmg"
+CLT_DMG = "/private/var/tmp/clt.dmg"
+CLT_MOUNT = "/private/var/tmp/clt-mnt"
 INSTALLER = "/Applications/Install macOS Sequoia.app"
 XCODE_REMOTE = "/var/root/install-xcode.sh"
 XCODE_DAEMON = "org.mozilla.relops.xcode-install"
@@ -482,6 +485,23 @@ def step_install_xcode(h: BuilderHost, *, poll: int = 30, stall: int = 20 * 60,
     ui.ok(f"Xcode {XCODE_BUILD} selected")
 
 
+def _install_clt_dmg(h: BuilderHost) -> None:
+    """softwareupdate offered no CLT at all on r8-198 (2026-10-29: only the 15.8.2 update was
+    listed). Install the dmg ronin ships instead, and only if the pkg is Apple-signed --
+    Software Update pkgs report "signed Apple Software", not a Developer ID naming Apple Inc."""
+    ui.wire(f"CLT not offered by softwareupdate -- installing {CLT_DMG_URL.rsplit('/', 1)[-1]}")
+    _sh(h, f"sudo curl -fsSL --retry 3 -o {CLT_DMG} {shlex.quote(CLT_DMG_URL)}", timeout=1800)
+    try:
+        _sh(h, f"sudo mkdir -p {CLT_MOUNT} && sudo hdiutil attach -quiet -nobrowse -readonly -mountpoint {CLT_MOUNT} {CLT_DMG}")
+        pkg = _sh(h, f"ls {CLT_MOUNT}/*.pkg | head -1").strip()
+        sig = _sh(h, f"pkgutil --check-signature {shlex.quote(pkg)}", check=False)
+        if "Status: signed Apple Software" not in sig:
+            raise ReprovisionError(f"{escape(pkg)} is not Apple-signed: {escape(sig[:200])}")
+        _sh(h, f"sudo installer -pkg {shlex.quote(pkg)} -target /", timeout=1800)
+    finally:
+        _sh(h, f"sudo hdiutil detach -quiet {CLT_MOUNT}; sudo rm -rf {CLT_DMG} {CLT_MOUNT}", check=False)
+
+
 def step_install_clt(h: BuilderHost) -> None:
     """The upgrade leaves a CommandLineTools stub with no usr/bin/git. Done here because the
     download outlasts puppet's old 300s exec timeout (ronin #1470 raises it)."""
@@ -491,9 +511,12 @@ def step_install_clt(h: BuilderHost) -> None:
         return
     _sh(h, f"sudo touch {CLT_TRIGGER}")
     try:
-        _sh(h, f"sudo softwareupdate -i {shlex.quote(CLT_LABEL)} --verbose", timeout=3600)
+        # check=False: "No such update" is the case the dmg below covers.
+        _sh(h, f"sudo softwareupdate -i {shlex.quote(CLT_LABEL)} --verbose", check=False, timeout=3600)
     finally:
         _sh(h, f"sudo rm -f {CLT_TRIGGER}", check=False)
+    if not clt_ready(h):
+        _install_clt_dmg(h)
     if not clt_ready(h):
         raise ReprovisionError("Command Line Tools still missing usr/bin/git after the install")
     ui.ok("installed")

orchestrator/tests/test_builder_upgrade.py

@@ -477,3 +477,54 @@ def test_runner_maps_builder_upgrade_action():
     assert cmd[1:] == ["builder-upgrade", "macmini-r8-124", "--job", "7", "--unquarantine"]
     dry = runner._job_cmd("macmini-r8-124", {"id": 8, "action": "builder-upgrade", "params": {"dry_run": True}}, None)
     assert "--dry-run" in dry and "--unquarantine" not in dry
+
+
+def _clt_fake(*, softwareupdate_installs: bool, sig: str = "Status: signed Apple Software"):
+    state = {"installed": False}
+
+    def softwareupdate():
+        state["installed"] = softwareupdate_installs
+        return _cp("" if softwareupdate_installs else "Command Line Tools for Xcode-16.4: No such update", rc=0 if softwareupdate_installs else 1)
+
+    def installer():
+        state["installed"] = True
+        return _cp("installer: The install was successful.")
+
+    def ready():
+        return _cp("", rc=0 if state["installed"] else 1)
+
+    return FakeSSH({
+        "CommandLineTools/usr/bin/git": ready,
+        "softwareupdate -i": softwareupdate,
+        "ls /private/var/tmp/clt-mnt": "/private/var/tmp/clt-mnt/Command Line Tools.pkg\n",
+        "pkgutil --check-signature": sig,
+        "installer -pkg": installer,
+    })
+
+
+def test_install_clt_uses_softwareupdate_when_offered():
+    fake = _clt_fake(softwareupdate_installs=True)
+    with patch("orchestrator.builder_upgrade.ssh.run", fake.run):
+        bu.step_install_clt(_host())
+    assert not any("curl" in c for c in fake.commands())
+
+
+def test_install_clt_falls_back_to_the_ronin_dmg():
+    # r8-198: softwareupdate listed no CLT at all.
+    fake = _clt_fake(softwareupdate_installs=False)
+    with patch("orchestrator.builder_upgrade.ssh.run", fake.run):
+        bu.step_install_clt(_host())
+    cmds = fake.commands()
+    assert any(bu.CLT_DMG_URL in c for c in cmds)
+    assert any("installer -pkg" in c for c in cmds)
+    assert "hdiutil detach" in cmds[-2] or any("hdiutil detach" in c for c in cmds[-3:])
+
+
+def test_install_clt_refuses_a_dmg_pkg_not_signed_by_apple():
+    fake = _clt_fake(softwareupdate_installs=False, sig="Status: no signature")
+    with patch("orchestrator.builder_upgrade.ssh.run", fake.run):
+        with pytest.raises(ReprovisionError, match="not Apple-signed"):
+            bu.step_install_clt(_host())
+    cmds = fake.commands()
+    assert not any("installer -pkg" in c for c in cmds)
+    assert any("hdiutil detach" in c for c in cmds)