From a9772bb016d158bf85a741026f1b2a954a32d3bb Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 6 Oct 2026 20:13:48 +1100 Subject: [PATCH 1/2] fix: realign hyperi-update with how tools install The Linux updater now refreshes every release binary the roles put in /usr/local/bin and nothing at the OS level carries. One refetch helper replaces the three per-format ones. It takes the asset's own arch spelling, a forge other than GitHub (tea from Gitea, macbash from downloads.hyperi.io), tar, nested tar and zip assets, a published checksum, and a minimum release age that matches the role's 7-day cooldown for aws-vault and Fedora's kubectx and kubens. It only touches a binary that lives under /usr/local, and only on the distro whose role installs it there. A stamp in /var/lib/hyperi-update records the source and digest of each install, so a release that has not moved is not downloaded again. A GITHUB_TOKEN or GH_TOKEN in the environment is sent to api.github.com from a 0600 file. Newly covered: lazygit, actionlint, osv-scanner, aws-vault, git-scrub, sccache, fnm, tea and macbash on both distros, hadolint, gitleaks and act on Ubuntu, and sd, kubectx and kubens on Fedora. Both updaters now refresh pnpm globals with `pnpm update -g --latest`, which `npm update -g` never reached, and uv-managed Pythons with `uv python upgrade`, which installs no python or python3 shim. Go tools are refreshed only when their binary is in ~/go/bin and the module has a newer release, which adds gosec and flarectl on Linux without doubling a dnf copy. The macOS updater drops `claude update`, since the claude-code cask rides `brew upgrade --cask --greedy`, and finds brew's keg-only rustup. developer-go appends ~/go/bin to the login PATH, where gopls, govulncheck, gosec and flarectl were installed but unreachable. Comments that claimed hyperi-update refreshed git-scrub on macOS and Apple container now say what does, and the install matrix lists the binaries the updater covers. --- ansible/roles/contributor/tasks/git_scrub.yml | 10 +- ansible/roles/developer-go/tasks/go.yml | 9 +- .../developer-rust/files/hyperi-rust-setup | 4 +- .../files/update/hyperi-update-linux.sh | 489 +++++++++++++----- .../files/update/hyperi-update-macos.sh | 90 ++-- ansible/roles/soe/tasks/colima.yml | 5 +- docs/install-matrix.md | 8 +- 7 files changed, 433 insertions(+), 182 deletions(-) diff --git a/ansible/roles/contributor/tasks/git_scrub.yml b/ansible/roles/contributor/tasks/git_scrub.yml index 93275d0..c695922 100644 --- a/ansible/roles/contributor/tasks/git_scrub.yml +++ b/ansible/roles/contributor/tasks/git_scrub.yml @@ -7,10 +7,12 @@ # failure: gitleaks scans FULL history, so a secret removed from HEAD still # fails `hyperi-ci check`. soe inherits it through meta/dependencies. # -# GitHub release tarball on every platform (Tier 3 -- hyperi-update pulls the -# latest on each run). It is the only rung: not on crates.io, no git-scrub path -# on downloads.hyperi.io, and the release's git-scrub.rb is not in the hyperi-io -# tap. Move macOS to community.general.homebrew once that formula is tapped. +# GitHub release tarball on every platform (Tier 3). hyperi-update pulls the +# latest on Linux. On macOS only a re-run of this role does, because the macOS +# updater leaves everything to brew. It is the only rung: not on crates.io, no +# git-scrub path on downloads.hyperi.io, and the release's git-scrub.rb is not +# in the hyperi-io tap. Move macOS to community.general.homebrew once that +# formula is tapped. # # The asset unpacks into a directory named after itself, so the extracted binary # path carries the version. The tag has a leading `v`; the filename does not. diff --git a/ansible/roles/developer-go/tasks/go.yml b/ansible/roles/developer-go/tasks/go.yml index e3b062d..4b9bbf3 100644 --- a/ansible/roles/developer-go/tasks/go.yml +++ b/ansible/roles/developer-go/tasks/go.yml @@ -88,8 +88,10 @@ path: "/tmp/go{{ go_version }}.linux-{{ hyperi_arch_deb }}.tar.gz" state: absent - # /etc/profile is re-read by every login shell, so the prepend must test + # /etc/profile is re-read by every login shell, so each entry must test # PATH first or it stacks a copy per nested shell, per tmux pane, per `su -`. + # ~/go/bin is appended, not prepended, so a go-installed copy never shadows + # a packaged tool of the same name. - name: Add the Go toolchain to PATH (system-wide) ansible.builtin.copy: content: | @@ -99,6 +101,11 @@ *":/usr/local/go/bin:"*) ;; *) export PATH="/usr/local/go/bin:$PATH" ;; esac + # gopls, govulncheck, gosec and flarectl are go-installed here. + case ":$PATH:" in + *":$HOME/go/bin:"*) ;; + *) export PATH="$PATH:$HOME/go/bin" ;; + esac dest: /etc/profile.d/hyperi-go.sh owner: root group: root diff --git a/ansible/roles/developer-rust/files/hyperi-rust-setup b/ansible/roles/developer-rust/files/hyperi-rust-setup index 6d0de74..b1ce311 100644 --- a/ansible/roles/developer-rust/files/hyperi-rust-setup +++ b/ansible/roles/developer-rust/files/hyperi-rust-setup @@ -366,8 +366,8 @@ def install_sccache_linux(dry_run: bool, rep: Reporter) -> None: """Put the latest upstream sccache release at SCCACHE_BIN. Re-runs are a no-op once the installed version matches the latest tag, so - the same call both installs and upgrades and hyperi-update needs no special - case. + the same call both installs and upgrades. Between converges hyperi-update + refreshes the same binary from the same release assets. """ target = _sccache_target() if target is None: diff --git a/ansible/roles/developer/files/update/hyperi-update-linux.sh b/ansible/roles/developer/files/update/hyperi-update-linux.sh index 2f83f6b..de711b8 100644 --- a/ansible/roles/developer/files/update/hyperi-update-linux.sh +++ b/ansible/roles/developer/files/update/hyperi-update-linux.sh @@ -6,22 +6,23 @@ # * System packages (apt or dnf, incl. 3rd-party repos: docker, vscode, # chrome, brave, git, gh, node, k8s, azure, gcloud, # opentofu, ...) -# * Snap (if installed) — needs sudo -# * Flatpak (apps + runtimes) — user -# * Firmware (fwupd) — needs sudo -# * uv tools (gnome-extensions-cli, ...) — user -# * rustup (Rust toolchains) — user -# * Go toolchain (/usr/local/go, no upstream repo) — needs sudo -# * fnm Node majors (the n-1 Node, per user) — user -# * Claude Code CLI (self-installed under ~/.local) — user -# * Codex CLI (re-run of the installer) - user -# * Codex plugin (claude plugin update) - user +# * Snap (if installed) -- needs sudo +# * Flatpak (apps + runtimes) -- user +# * Firmware (fwupd) -- needs sudo +# * uv tools (gnome-extensions-cli, ...) -- user +# * uv Pythons (patch releases of each minor) -- user +# * rustup (Rust toolchains) -- user +# * cargo, go, npm and pnpm global tools -- user +# * Go toolchain (/usr/local/go, no upstream repo) -- needs sudo +# * fnm Node majors (the n-1 Node, per user) -- user +# * Release binaries (/usr/local/bin, no repo/snap) -- needs sudo +# * Claude Code CLI (self-installed under ~/.local) -- user +# * Codex CLI (re-run of the installer) -- user +# * Codex plugin (claude plugin update) -- user # -# Note on the dev stacks: node, gh, docker and git come from UPSTREAM signed -# repos, so "System packages" above already carries them to latest -- there is -# nothing stack-specific to do for them here. Only the two that have no -# upstream repo (the Go toolchain, and fnm's Node majors) need their own -# sections, plus rustup, which manages its own toolchains. +# Anything a package repo, snap or flatpak carries is left to the system-package +# sections, and the sections after them cover only what nothing at the OS level +# refreshes. # # Each section is independent and self-guarding: a tool that isn't installed is # skipped (printed, not fatal), and a failing step is recorded and reported in @@ -58,10 +59,13 @@ fi CARGO_BIN="${CARGO_HOME:-$HOME/.cargo}/bin" # Make user-level tools reachable even when launched from a GUI/.desktop entry -# that doesn't source the login shell (rustup and the cargo tools live in the -# cargo home, uv and claude in ~/.local/bin, Go in /usr/local/go/bin and the -# go-installed tools in ~/go/bin). -export PATH="$HOME/.local/bin:$CARGO_BIN:/usr/local/go/bin:$HOME/go/bin:$PATH" +# or the systemd unit, neither of which sources the login shell (rustup and the +# cargo tools live in the cargo home, uv and claude in ~/.local/bin, Go in +# /usr/local/go/bin, the go-installed tools in ~/go/bin and the pnpm globals in +# PNPM_HOME). +export PNPM_HOME="${PNPM_HOME:-$HOME/.local/share/pnpm}" +export FNM_DIR="${FNM_DIR:-$HOME/.local/share/fnm}" +export PATH="$HOME/.local/bin:$CARGO_BIN:/usr/local/go/bin:$HOME/go/bin:$PNPM_HOME:$PNPM_HOME/bin:$PATH" ASSUME_YES=0 @@ -149,12 +153,13 @@ if [[ "$ASSUME_YES" -eq 0 ]]; then have snap && printf ' - snap packages\n' have flatpak && printf ' - flatpak apps and runtimes\n' have fwupdmgr && printf ' - device firmware\n' - have uv && printf ' - uv tools\n' + have uv && printf ' - uv tools and uv-managed Pythons\n' have rustup && printf ' - rust toolchains\n' have cargo-install-update && printf ' - cargo-installed tools\n' - have go && printf ' - go-installed tools (gopls, govulncheck)\n' + have go && printf ' - go-installed tools in ~/go/bin (gopls, govulncheck, gosec, flarectl)\n' have npm && printf ' - npm global tools + pnpm\n' - printf ' - static release binaries (kind, argocd, kubeconform, kube-linter, dive, kustomize, k9s, yq, terraform-docs, golangci-lint)\n' + have pnpm && printf ' - pnpm global tools\n' + printf ' - release binaries in /usr/local/bin that no package covers\n' have claude && printf ' - Claude Code CLI\n' have codex && printf ' - Codex CLI (re-run of the official installer)\n' have claude && printf ' - the Codex plugin for Claude Code, if installed\n' @@ -176,6 +181,14 @@ fi # # The keepalive only matters when a password was actually entered: NOPASSWD # leaves no timestamp to refresh. +# Stops the keepalive and removes the GitHub token file, however the run ends. +cleanup() { + [[ -n "${SUDO_KEEPALIVE_PID:-}" ]] && kill "$SUDO_KEEPALIVE_PID" 2>/dev/null + [[ -n "${GH_AUTH_HEADER:-}" ]] && rm -f "$GH_AUTH_HEADER" + return 0 +} +trap cleanup EXIT + section "Authenticating (sudo)" if sudo -n true 2>/dev/null; then ok "sudo authenticated (passwordless)" @@ -183,7 +196,6 @@ elif [[ -t 0 ]] && sudo -v; then ok "sudo authenticated" ( while true; do sudo -n true 2>/dev/null; sleep 50; kill -0 "$$" 2>/dev/null || exit; done ) & SUDO_KEEPALIVE_PID=$! - trap '[[ -n "${SUDO_KEEPALIVE_PID:-}" ]] && kill "$SUDO_KEEPALIVE_PID" 2>/dev/null' EXIT else printf '%s \xe2\x9c\x97 sudo authentication failed — aborting%s\n' "$RED" "$RESET" printf '%s No passwordless sudo and no terminal to prompt on.%s\n' "$RED" "$RESET" @@ -277,9 +289,22 @@ else skip "uv not found" fi +# --- uv-managed Pythons ---------------------------------------------------- +# Nothing else moves a uv-installed Python to a newer patch. `uv python upgrade` +# touches only the minors already installed and, without --default, adds no +# python or python3 shim. +section "uv Pythons" +if ! have uv; then + skip "uv not found" +elif [[ "$(uv python list --only-installed --managed-python --output-format json 2>/dev/null)" != *'"version"'* ]]; then + skip "uv manages no Pythons" +else + run "uv python upgrade" uv python upgrade +fi + # --- rustup toolchains ----------------------------------------------------- -# Updates the Rust toolchains. NOTE: cargo-installed binaries (nextest, deny, -# bacon, ...) are not refreshed by rustup; reinstall them with cargo if needed. +# Updates the Rust toolchains and rustup itself. The cargo-installed binaries +# are the next section's job. section "rustup toolchains" # rustup proxies in the cargo home with no rustup on PATH means the toolchain is # not where the environment says it is, which is a fault to report, not a skip. @@ -308,23 +333,39 @@ else fi # --- go-installed tools ---------------------------------------------------- -# No bulk updater for `go install` tools, so re-install @latest the ones that -# are already present (this adds nothing that was not there before). +# No bulk updater for `go install` tools, so each one the roles put in ~/go/bin +# is re-installed @latest. Keyed on the binary being in ~/go/bin rather than on +# PATH, because a dnf gosec or govulncheck would otherwise gain a second copy. section "go tools" +GO_HOME="$HOME/go" if have go; then + go_found=0 for gt in \ "gopls:golang.org/x/tools/gopls@latest" \ - "govulncheck:golang.org/x/vuln/cmd/govulncheck@latest"; do + "govulncheck:golang.org/x/vuln/cmd/govulncheck@latest" \ + "gosec:github.com/securego/gosec/v2/cmd/gosec@latest" \ + "flarectl:github.com/cloudflare/cloudflare-go/cmd/flarectl@latest"; do bin="${gt%%:*}"; mod="${gt#*:}" - have "$bin" && run "go install $bin" go install "$mod" + [[ -x "$GO_HOME/bin/$bin" ]] || continue + go_found=1 + # " " as built into the binary, so a tool already at the + # module's latest release is not rebuilt. + built="$(go version -m "$GO_HOME/bin/$bin" 2>/dev/null | awk '$1 == "mod" {print $2, $3; exit}')" + if [[ -n "$built" ]] && [[ "$(go list -m -f '{{.Version}}' "${built% *}@latest" 2>/dev/null)" == "${built#* }" ]]; then + ok "$bin ${built#* } is current" + continue + fi + run "go install $bin" env GOPATH="$GO_HOME" GOBIN="$GO_HOME/bin" go install "$mod" done + [[ "$go_found" -eq 1 ]] || skip "no go-installed tools in $GO_HOME/bin" else skip "go not found" fi -# --- npm global tools ------------------------------------------------------ -# maid, semantic-release, typescript, tsx, ts-node -- global npm packages; plus -# pnpm via corepack (it rides Node, but pin it to latest here). +# --- npm and pnpm global tools --------------------------------------------- +# semantic-release, maid and wrangler are npm globals. eslint, prettier, +# typescript, tsx and ts-node are pnpm globals, which `npm update -g` never +# sees. pnpm itself is corepack's, so it is re-activated at latest first. section "npm global tools" if have npm; then run "npm update -g" npm update -g @@ -333,6 +374,15 @@ else skip "npm not found" fi +section "pnpm global tools" +if have pnpm; then + # --latest, because the roles install each global at its latest release and + # the ranges pnpm recorded would otherwise hold them at that major. + run "pnpm update -g --latest" pnpm update -g --latest +else + skip "pnpm not found" +fi + # --- Go toolchain ---------------------------------------------------------- # Go publishes no apt/dnf repo, so `apt/dnf upgrade` never moves it -- the # playbook installs the current tarball into /usr/local/go and this section is @@ -409,24 +459,41 @@ else skip "fnm not found" fi -# --- Tier 3: static binaries with no repo/snap/lang-manager ---------------- -# These ship only as a GitHub release asset, so nothing above refreshes them -- -# re-fetch the latest here. Each downloads to a temp path, is checked for the ELF -# magic, and is only moved into place on success, so a failed or corrupt fetch -# never breaks the working copy. Only tools already installed are touched, and -# k9s is Ubuntu-only (Fedora's k9s is the dnf package -- re-fetching would shadow -# it). Still uncovered: aws-vault and tea -- re-run the playbook to refresh those. -section "Static binaries (GitHub releases)" - -gh_latest_tag() { # -> newest release tag (empty on failure) - curl -fsSL "https://api.github.com/repos/$1/releases/latest" 2>/dev/null \ - | sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1 -} +# --- Release binaries ------------------------------------------------------ +# These ship only as a release asset, with no package repo, snap or language +# manager to carry them, so the latest is fetched here. A download replaces the +# working copy only after it matches any published digest and yields an ELF +# binary, so a failed or corrupt fetch never breaks what is installed. +# +# Only a binary already in /usr/local/bin is touched, and only on the distro +# whose role installs it there, so a tool the other distro packages is never +# shadowed by a second copy. +section "Release binaries" + +# Holds the source and digest of each binary installed here, so a release that +# has not moved is recognised without downloading it again. +STAMP_DIR=/var/lib/hyperi-update + +# A GitHub release's download directory, as a refetch template. +GH_DL='https://github.com/{REPO}/releases/download/{TAG}' + +# api.github.com allows 60 anonymous requests an hour per IP, so a token in the +# environment is sent when there is one, from a 0600 file rather than argv. +gh_token="${GITHUB_TOKEN:-${GH_TOKEN:-}}" +if [[ -n "$gh_token" ]]; then + GH_AUTH_HEADER="$(mktemp)" + printf 'Authorization: Bearer %s\n' "$gh_token" >"$GH_AUTH_HEADER" +fi +unset gh_token is_elf() { # -> 0 if it begins with the ELF magic (7f 45 4c 46) [[ "$(head -c 4 "$1" 2>/dev/null | od -An -tx1 | tr -d ' \n')" == "7f454c46" ]] } +sha256_of() { # -> its sha256, empty when unreadable + sha256sum "$1" 2>/dev/null | cut -d' ' -f1 +} + # installed_local -> 0 if /usr/local/bin/ is a file under # /usr/local. A copy found elsewhere on PATH (~/.local/bin, a package) is not # the one these helpers manage, and refetching for it would add a second copy. @@ -435,116 +502,272 @@ installed_local() { real="$(readlink -e "/usr/local/bin/$1" 2>/dev/null)" && [[ -f "$real" && "$real" == /usr/local/* ]] } -# refetch_raw -- a bare executable asset. -# Template may use {TAG} and {ARCH}. -refetch_raw() { - local name="$1" repo="$2" tmpl="$3" tag asset url tmp - installed_local "$name" || { skip "$name not installed in /usr/local/bin"; return; } - tag="$(gh_latest_tag "$repo")" - [[ -n "$tag" ]] || { FAILURES+=("$name (no release tag)"); return; } - asset="${tmpl//\{TAG\}/$tag}"; asset="${asset//\{ARCH\}/$ARCH_DEB}" - url="https://github.com/$repo/releases/download/$tag/$asset" - tmp="$(mktemp)" - if curl -fsSL "$url" -o "$tmp" && [[ -s "$tmp" ]] && is_elf "$tmp"; then - sudo install -m 0755 "$tmp" "/usr/local/bin/$name" && ok "$name -> $tag" \ - || FAILURES+=("$name (install)") - else - FAILURES+=("$name (download)") +# api_get : fetch a release document, sending the token to GitHub only. +api_get() { + local auth=() + if [[ -n "${GH_AUTH_HEADER:-}" && "$1" == https://api.github.com/* ]]; then + auth=(-H "@$GH_AUTH_HEADER") fi - rm -f "$tmp" + curl -fsSL "${auth[@]}" "$1" -o "$2" 2>/dev/null } -# refetch_targz -- extract -# from a .tar.gz release asset. Template may use {TAG}, {VER} (tag minus a -# leading v) and {ARCH}. -refetch_targz() { - local name="$1" repo="$2" tmpl="$3" member="$4" tag ver asset url tmp dir - installed_local "$name" || { skip "$name not installed in /usr/local/bin"; return; } - tag="$(gh_latest_tag "$repo")" - [[ -n "$tag" ]] || { FAILURES+=("$name (no release tag)"); return; } - ver="${tag#v}" - asset="${tmpl//\{TAG\}/$tag}"; asset="${asset//\{VER\}/$ver}"; asset="${asset//\{ARCH\}/$ARCH_DEB}" - url="https://github.com/$repo/releases/download/$tag/$asset" - tmp="$(mktemp)"; dir="$(mktemp -d)" - if curl -fsSL "$url" -o "$tmp" && tar -xzf "$tmp" -C "$dir" "$member" 2>/dev/null \ - && [[ -s "$dir/$member" ]] && is_elf "$dir/$member"; then - sudo install -m 0755 "$dir/$member" "/usr/local/bin/$name" && ok "$name -> $tag" \ - || FAILURES+=("$name (install)") - else - FAILURES+=("$name (download)") +# release_tag [] : the tag of a single release document, +# or, given an age, of the newest release in a list that is not a draft or a +# prerelease and was published at least that many days ago. The age rule is the +# one the roles apply through infrastructure_min_release_age_days. +release_tag() { + local doc tag='' + doc="$(mktemp)" + if api_get "$1" "$doc"; then + tag="$(python3 - "$doc" "${2:-0}" 2>/dev/null <<'PY' +import json +import sys +import time + +with open(sys.argv[1], encoding="utf-8") as fh: + data = json.load(fh) +if isinstance(data, dict): + print(data.get("tag_name", "")) +else: + cutoff = time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - int(sys.argv[2]) * 86400) + ) + ready = sorted( + ( + r + for r in data + if not r.get("draft") + and not r.get("prerelease") + and r.get("published_at") + and r["published_at"] <= cutoff + ), + key=lambda r: r["published_at"], + reverse=True, + ) + print(ready[0]["tag_name"] if ready else "") +PY +)" fi - rm -rf "$tmp" "$dir" + rm -f "$doc" + printf '%s' "$tag" +} + +# published_sum : the asset's sha256 from a checksum file of +# " [*]" lines, or from a file holding one bare digest. +published_sum() { + local sums + sums="$(curl -fsSL "$1" 2>/dev/null)" || return 1 + awk -v a="$2" ' + { n = $2; sub(/^\*/, "", n); sub(/^\.\//, "", n) } + length($1) == 64 && $1 ~ /^[0-9a-f]+$/ { + if (n == a) { print $1; found = 1; exit } + if (NR == 1 && NF == 1) bare = $1 + } + END { if (!found && bare != "") print bare }' <<<"$sums" } -# refetch_targz_nested -- as refetch_targz, but -# the binary sits one directory deep in the tarball. -refetch_targz_nested() { - local name="$1" repo="$2" tmpl="$3" tag ver asset url tmp dir +# is_current / record : read or write the stamp that +# ties the installed binary to the release it came from. +is_current() { + local line + line="$(cat "$STAMP_DIR/$1" 2>/dev/null)" || return 1 + [[ "$line" == "$2 $(sha256_of "/usr/local/bin/$1")" ]] +} + +record() { + sudo mkdir -p "$STAMP_DIR" && + printf '%s %s\n' "$2" "$(sha256_of "/usr/local/bin/$1")" | sudo tee "$STAMP_DIR/$1" >/dev/null +} + +# unpack : print the path of the binary unpacked from +# /asset, failing when the archive does not hold . +unpack() { + local out="$2/out" + [[ "$1" == raw ]] && { printf '%s' "$2/asset"; return 0; } + mkdir -p "$out" + case "$1" in + tar) tar -xzf "$2/asset" -C "$out" "$3" ;; + tar-nested) tar -xzf "$2/asset" -C "$out" --strip-components=1 --wildcards "*/$3" ;; + zip) unzip -q -o "$2/asset" "$3" -d "$out" ;; + *) return 1 ;; + esac >/dev/null 2>&1 && [[ -s "$out/$3" ]] && printf '%s' "$out/$3" +} + +# fill