diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 5e7e51c5c..d624512a8 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -77,8 +77,9 @@ export MAPLE_HOOK_STAGED="$STAGED_FILE" export MAPLE_HOOK_SELECTED="$SELECTED" export MAPLE_HOOK_FULL="${MAPLE_HOOK_FULL:-0}" -# run_component NAME DIR FLAKE_REF SCRIPT [ENV=VALUE ...] +# run_component NAME DIR FLAKE_REF SCRIPT [--bash] [ENV=VALUE ...] # DIR is where the script and `nix develop` run; FLAKE_REF is relative to DIR. +# CI entrypoints may opt into Bash; component hooks use POSIX sh by default. # ENV=VALUE pairs are set for `nix develop` itself, not only for the script: # a flake's shellHook runs before the `-c` command, and the opensecret shell # starts PostgreSQL and writes .env unless OPENSECRET_DEV_* tell it not to. @@ -86,6 +87,11 @@ export MAPLE_HOOK_FULL="${MAPLE_HOOK_FULL:-0}" run_component() { name=$1; dir=$2; flake_ref=$3; script=$4 shift 4 + component_shell=sh + if [ "${1:-}" = "--bash" ]; then + component_shell=bash + shift + fi hook_log "=== $name ===" started=$(date +%s) # Git exports GIT_INDEX_FILE, GIT_DIR, and friends to hooks. They must not @@ -94,10 +100,10 @@ run_component() { # shellcheck disable=SC2086 # GIT_ENV_UNSET is a deliberate word list if [ -n "$NIX_BIN" ]; then (cd "$dir" && env $GIT_ENV_UNSET "$@" "$NIX_BIN" develop --no-update-lock-file "$flake_ref" \ - -c env MAPLE_HOOK_IN_NIX=1 sh "$script") + -c env MAPLE_HOOK_IN_NIX=1 "$component_shell" "$script") else # shellcheck disable=SC2086 - (cd "$dir" && env $GIT_ENV_UNSET MAPLE_HOOK_IN_NIX=0 "$@" sh "$script") + (cd "$dir" && env $GIT_ENV_UNSET MAPLE_HOOK_IN_NIX=0 "$@" "$component_shell" "$script") fi status=$? hook_log "=== $name finished in $(( $(date +%s) - started ))s ===" @@ -116,6 +122,9 @@ fi if hook_selected updates; then run_component updates "$REPO_ROOT" ".#ci" "$REPO_ROOT/services/updates/.githooks/pre-commit" fi +if hook_selected agent_updates; then + run_component agent-updates "$REPO_ROOT" ".#ci" "$REPO_ROOT/scripts/ci/agent-updates.sh" --bash +fi if hook_selected sdk_rust || hook_selected sdk_ts; then run_component sdk "$REPO_ROOT/sdk" "." "$REPO_ROOT/sdk/.githooks/pre-commit" fi diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cce1379f3..7870d9221 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -28,6 +28,7 @@ /apps/maple-agent/release_profile.rs @AnthonyRonning /apps/maple-agent/release-profiles.json @AnthonyRonning /apps/maple-agent/scripts/ @AnthonyRonning +/apps/maple-agent/crates/maple-agent-updater/ @AnthonyRonning /apps/maple-agent/app/packaging/ @AnthonyRonning /apps/maple-agent/app/macos/ @AnthonyRonning /apps/maple-research/frontend/package.json @AnthonyRonning @@ -75,4 +76,5 @@ # Retain the existing owner boundary for these distribution components. /services/updates/ @AnthonyRonning +/services/agent-updates/ @AnthonyRonning /proxy/ @AnthonyRonning diff --git a/.github/workflows/agent-updates-publish.yml b/.github/workflows/agent-updates-publish.yml new file mode 100644 index 000000000..2f49a4be5 --- /dev/null +++ b/.github/workflows/agent-updates-publish.yml @@ -0,0 +1,248 @@ +name: Publish Agent Updates +run-name: Publish Agent ${{ inputs.version }} from verified build ${{ inputs.build_run_id }} + +on: + workflow_dispatch: + inputs: + build_run_id: + description: Exact completed trusted master run with verified Prod packages + required: true + type: string + version: + description: Stable Agent version matching the selected Prod packages + required: true + type: string + notes: + description: Plain-text release notes (must remain identical on retries) + required: true + type: string + include_linux: + description: Include Linux only after its packaged update launch gate passes + required: true + default: false + type: boolean + +permissions: + contents: read + actions: read + attestations: read + +concurrency: + # Operator pointer changes must use this same serialization boundary. + group: agent-updates-production + cancel-in-progress: false + +jobs: + verify: + if: ${{ github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Install pinned Nix + uses: DeterminateSystems/determinate-nix-action@96c5ee93a9dddf2830d31e1da6ec719077336c28 # v3.23.3 + with: + source-tag: v3.23.3 + github-token: "" + - name: Verify pinned Nix runtime + run: bash scripts/ci/verify-nix-runtime.sh + - name: Require separately approved production setup + env: + ACTIVATED: ${{ vars.AGENT_UPDATES_ENABLED }} + PUBLIC_KEY: ${{ vars.AGENT_UPDATE_PUBLIC_KEY }} + run: | + set -euo pipefail + [ "$ACTIVATED" = true ] || { echo 'Agent update production setup has not been activated by its owner.' >&2; exit 1; } + [ -n "$PUBLIC_KEY" ] || { echo 'Agent public update key is not configured.' >&2; exit 1; } + printf '%s\n' "$PUBLIC_KEY" > public-key.pub + - name: Verify exact build, download final packages and prepare without secrets + env: + GH_TOKEN: ${{ github.token }} + BUILD_RUN_ID: ${{ inputs.build_run_id }} + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_NOTES: ${{ inputs.notes }} + INCLUDE_LINUX: ${{ inputs.include_linux }} + run: | + nix develop --no-update-lock-file .#ci -c bash -euo pipefail <<'PREPARE' + linux_args=() + if [ "$INCLUDE_LINUX" = true ]; then linux_args=(--linux); fi + python3 services/agent-updates/publisher/publisher.py trust \ + --run-id "$BUILD_RUN_ID" "${linux_args[@]}" --output trust.json + gh run download "$BUILD_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --name "$(jq -er .artifacts.macos trust.json)" --dir packages/macos + linux_args=() + if [ "$INCLUDE_LINUX" = true ]; then + gh run download "$BUILD_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --name "$(jq -er .artifacts.linux trust.json)" --dir packages/linux + linux_args=(--linux packages/linux) + fi + printf '%s' "$RELEASE_NOTES" > notes.txt + python3 services/agent-updates/publisher/publisher.py prepare \ + --run-id "$BUILD_RUN_ID" --version "$RELEASE_VERSION" --build "$(jq -er .build trust.json)" \ + --macos packages/macos "${linux_args[@]}" --notes-file notes.txt --public-key public-key.pub --output bundle + PREPARE + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: agent-update-prepared-${{ github.run_id }} + path: bundle/ + overwrite: true + if-no-files-found: error + retention-days: 7 + + reserve: + needs: verify + if: ${{ github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' }} + environment: updates-production + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Install pinned Nix + uses: DeterminateSystems/determinate-nix-action@96c5ee93a9dddf2830d31e1da6ec719077336c28 # v3.23.3 + with: + source-tag: v3.23.3 + github-token: "" + - name: Verify pinned Nix runtime + run: bash scripts/ci/verify-nix-runtime.sh + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: agent-update-prepared-${{ github.run_id }} + path: bundle + - name: Realize pinned publisher tools before retrieving credentials + run: nix develop --no-update-lock-file .#ci -c true + - name: Resolve Agent bucket credentials from Bitwarden Secrets Manager + id: r2-credentials + uses: bitwarden/sm-action@1238aae8fc64b212641190a9227c8a734ab1a793 # v3.0.1 + with: + access_token: ${{ secrets.AGENT_UPDATES_BWS_ACCESS_TOKEN }} + secrets: | + ${{ vars.AGENT_R2_ACCESS_KEY_ID_SECRET_ID }} > AWS_ACCESS_KEY_ID + ${{ vars.AGENT_R2_SECRET_ACCESS_KEY_SECRET_ID }} > AWS_SECRET_ACCESS_KEY + set_env: false + - name: Reserve immutable exact manifest and reuse prior signed envelope + env: + AWS_ACCESS_KEY_ID: ${{ steps.r2-credentials.outputs.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ steps.r2-credentials.outputs.AWS_SECRET_ACCESS_KEY }} + R2_ACCOUNT: ${{ vars.AGENT_R2_ACCOUNT_ID }} + R2_BUCKET: ${{ vars.AGENT_R2_BUCKET }} + run: | + set -euo pipefail + [[ "$R2_ACCOUNT" =~ ^[0-9a-f]{32}$ ]] + [ -n "$R2_BUCKET" ] + nix develop --no-update-lock-file .#ci -c python3 services/agent-updates/publisher/publisher.py reserve \ + --bundle bundle --public-key bundle/public-key.pub \ + --endpoint "https://${R2_ACCOUNT}.r2.cloudflarestorage.com" --bucket "$R2_BUCKET" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: agent-update-reserved-${{ github.run_id }} + path: bundle/ + overwrite: true + if-no-files-found: error + retention-days: 7 + + sign: + needs: reserve + if: ${{ github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' }} + environment: updates-production + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Install pinned Nix + uses: DeterminateSystems/determinate-nix-action@96c5ee93a9dddf2830d31e1da6ec719077336c28 # v3.23.3 + with: + source-tag: v3.23.3 + github-token: "" + - name: Verify pinned Nix runtime + run: bash scripts/ci/verify-nix-runtime.sh + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: agent-update-reserved-${{ github.run_id }} + path: bundle + - name: Realize pinned publisher tools before retrieving credentials + run: nix develop --no-update-lock-file .#ci -c true + - name: Resolve Agent update key from Bitwarden Secrets Manager + id: signing-key + uses: bitwarden/sm-action@1238aae8fc64b212641190a9227c8a734ab1a793 # v3.0.1 + with: + access_token: ${{ secrets.AGENT_UPDATES_BWS_ACCESS_TOKEN }} + secrets: | + ${{ vars.AGENT_UPDATE_SIGNING_KEY_ID }} > AGENT_UPDATE_SIGNING_KEY + set_env: false + - name: Sign exact reserved manifest with Agent-only key + env: + AGENT_UPDATE_SIGNING_KEY: ${{ steps.signing-key.outputs.AGENT_UPDATE_SIGNING_KEY }} + run: | + set -euo pipefail + umask 077 + key_file="$(mktemp)" + trap 'rm -f "$key_file"' EXIT + [ -n "$AGENT_UPDATE_SIGNING_KEY" ] + printf '%s\n' "$AGENT_UPDATE_SIGNING_KEY" > "$key_file" + unset AGENT_UPDATE_SIGNING_KEY + nix develop --no-update-lock-file .#ci -c python3 services/agent-updates/publisher/publisher.py sign \ + --bundle bundle --public-key bundle/public-key.pub --secret-key "$key_file" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: agent-update-signed-${{ github.run_id }} + path: bundle/ + overwrite: true + if-no-files-found: error + retention-days: 7 + + publish: + needs: sign + if: ${{ github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' }} + environment: updates-production + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Install pinned Nix + uses: DeterminateSystems/determinate-nix-action@96c5ee93a9dddf2830d31e1da6ec719077336c28 # v3.23.3 + with: + source-tag: v3.23.3 + github-token: "" + - name: Verify pinned Nix runtime + run: bash scripts/ci/verify-nix-runtime.sh + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: agent-update-signed-${{ github.run_id }} + path: bundle + - name: Realize pinned publisher tools before retrieving credentials + run: nix develop --no-update-lock-file .#ci -c true + - name: Resolve Agent bucket credentials from Bitwarden Secrets Manager + id: r2-credentials + uses: bitwarden/sm-action@1238aae8fc64b212641190a9227c8a734ab1a793 # v3.0.1 + with: + access_token: ${{ secrets.AGENT_UPDATES_BWS_ACCESS_TOKEN }} + secrets: | + ${{ vars.AGENT_R2_ACCESS_KEY_ID_SECRET_ID }} > AWS_ACCESS_KEY_ID + ${{ vars.AGENT_R2_SECRET_ACCESS_KEY_SECRET_ID }} > AWS_SECRET_ACCESS_KEY + set_env: false + - name: Upload immutably, conditionally promote, and verify public delivery + env: + AWS_ACCESS_KEY_ID: ${{ steps.r2-credentials.outputs.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ steps.r2-credentials.outputs.AWS_SECRET_ACCESS_KEY }} + R2_ACCOUNT: ${{ vars.AGENT_R2_ACCOUNT_ID }} + R2_BUCKET: ${{ vars.AGENT_R2_BUCKET }} + run: | + set -euo pipefail + [[ "$R2_ACCOUNT" =~ ^[0-9a-f]{32}$ ]] + [ -n "$R2_BUCKET" ] + nix develop --no-update-lock-file .#ci -c python3 services/agent-updates/publisher/publisher.py publish \ + --bundle bundle --public-key bundle/public-key.pub \ + --endpoint "https://${R2_ACCOUNT}.r2.cloudflarestorage.com" --bucket "$R2_BUCKET" \ + --base-url https://updates.trymaple.ai diff --git a/.github/workflows/agent-updates-tests.yml b/.github/workflows/agent-updates-tests.yml new file mode 100644 index 000000000..d92e7e286 --- /dev/null +++ b/.github/workflows/agent-updates-tests.yml @@ -0,0 +1,47 @@ +name: Agent Update Delivery Tests + +permissions: + contents: read + +on: + push: + branches: [master] + paths: + - ".github/workflows/agent-updates-*.yml" + - "services/agent-updates/**" + - "scripts/ci/agent-updates.sh" + - "scripts/ci/test_agent_updates_workflows.py" + - "scripts/ci/_common.sh" + - "apps/maple-agent/scripts/release-info.py" + - "apps/maple-agent/release-profiles.json" + - "flake.nix" + - "flake.lock" + pull_request: + paths: + - ".github/workflows/agent-updates-*.yml" + - "services/agent-updates/**" + - "scripts/ci/agent-updates.sh" + - "scripts/ci/test_agent_updates_workflows.py" + - "scripts/ci/_common.sh" + - "apps/maple-agent/scripts/release-info.py" + - "apps/maple-agent/release-profiles.json" + - "flake.nix" + - "flake.lock" + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - name: Install pinned Nix + uses: DeterminateSystems/determinate-nix-action@96c5ee93a9dddf2830d31e1da6ec719077336c28 # v3.23.3 + with: + source-tag: v3.23.3 + github-token: "" + - name: Verify pinned Nix runtime + run: bash scripts/ci/verify-nix-runtime.sh + - name: Check isolated Agent delivery + run: nix develop --no-update-lock-file .#ci -c ./scripts/ci/agent-updates.sh diff --git a/apps/maple-agent/scripts/test_release_packaging.py b/apps/maple-agent/scripts/test_release_packaging.py index 8b6321eaa..ca955803f 100644 --- a/apps/maple-agent/scripts/test_release_packaging.py +++ b/apps/maple-agent/scripts/test_release_packaging.py @@ -9,6 +9,7 @@ import plistlib import shlex import shutil +from signal import SIGKILL import subprocess import sys import tempfile @@ -383,6 +384,32 @@ def run_packager(self, unsigned=False, failure="", signal=False, search_list=Non log.unlink(missing_ok=True) search_state = self.directory / "keychain-search.json" search_state.write_text(json.dumps(self.search_list if search_list is None else search_list)) + # Bash 5.3 uses a pipe for this heredoc, but the Darwin pipe capacity + # can be smaller than its threshold. Keep this longer mock as a file + # so a command substitution cannot block before Python starts. + security_list = self.directory / "mock-security-list.py" + security_list.write_text("""import json,os,sys +from pathlib import Path +state=Path(os.environ['FIXTURE_KEYCHAIN_SEARCH']) +args=sys.argv[1:] +if args == ['list-keychains','-d','user']: + if os.environ.get('FIXTURE_FAILURE') == 'malformed-search': + print('not a quoted path') + else: + for path in json.loads(state.read_text()): + print(' "'+path+'"') +else: + assert args[:4] == ['list-keychains','-d','user','-s'], args + paths=args[4:] + adding=any('/.macos-release.' in path and path.endswith('/signing.keychain-db') for path in paths) + failure=os.environ.get('FIXTURE_FAILURE') + if failure == 'restore-search' and not adding: + raise SystemExit(1) + state.write_text(json.dumps(paths)) + # Simulate a setter that changed state before reporting failure. + if failure == 'set-search' and adding: + raise SystemExit(1) +""") wrapper = self.directory / "mock-packager.sh" wrapper.write_text(f"""#!/usr/bin/env bash set -euo pipefail @@ -419,29 +446,7 @@ def run_packager(self, unsigned=False, failure="", signal=False, search_list=Non if [[ "$1" == find-identity ]]; then echo '1) 1111111111111111111111111111111111111111 "Developer ID Application: Fixture (TEAM123456)"'; elif [[ "$1" == create-keychain ]]; then touch "${{@:$#}}"; elif [[ "$1" == list-keychains ]]; then - python3 - "$@" <<'PY' -import json,os,sys -from pathlib import Path -state=Path(os.environ['FIXTURE_KEYCHAIN_SEARCH']) -args=sys.argv[1:] -if args == ['list-keychains','-d','user']: - if os.environ.get('FIXTURE_FAILURE') == 'malformed-search': - print('not a quoted path') - else: - for path in json.loads(state.read_text()): - print(' "'+path+'"') -else: - assert args[:4] == ['list-keychains','-d','user','-s'], args - paths=args[4:] - adding=any('/.macos-release.' in path and path.endswith('/signing.keychain-db') for path in paths) - failure=os.environ.get('FIXTURE_FAILURE') - if failure == 'restore-search' and not adding: - raise SystemExit(1) - state.write_text(json.dumps(paths)) - # Simulate a setter that changed state before reporting failure. - if failure == 'set-search' and adding: - raise SystemExit(1) -PY + python3 {shlex.quote(str(security_list))} "$@" fi ;; codesign) if [[ "$1" == --force && "$3" != - ]]; then @@ -483,7 +488,20 @@ def run_packager(self, unsigned=False, failure="", signal=False, search_list=Non "APPLE_ID": "fixture-apple-id", "APPLE_ID_PASSWORD": "fixture-notary-password", "APPLE_TEAM_ID": "TEAM123456", }) - result = subprocess.run(["bash", str(wrapper)], env=env, capture_output=True, text=True) + process = subprocess.Popen(["bash", str(wrapper)], env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + stdout, stderr = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + # The fixture owns its process group, including nested mock-tool + # children. Reap all of them when it times out, not unrelated tests. + try: + os.killpg(process.pid, SIGKILL) + except ProcessLookupError: + pass + process.communicate() + self.fail("mock macOS packager exceeded its 60-second timeout") + result = subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] return result, calls diff --git a/docs/app-signing-environments.md b/docs/app-signing-environments.md index 53f8b14e7..ea445d5866 100644 --- a/docs/app-signing-environments.md +++ b/docs/app-signing-environments.md @@ -63,6 +63,14 @@ merged, remove this environment's required reviewer while retaining its exact setting, not a new approval for each release. See the [updater rollout checks](../services/updates/README.md). +The independent Agent publisher reuses `updates-production` for manual trusted-master +reservation, signing and publication, with Agent-specific BWS item-ID variables +and an existing BWS access credential stored as `AGENT_UPDATES_BWS_ACCESS_TOKEN`. +The pinned action passes outputs only to the consuming step, without exporting +them to subsequent steps. This shared access does not isolate the environment's +secret context or narrow the BWS credential's grants. Agent serialization, keys +and bucket remain independent; Research's workflow and configuration are unchanged. + `apple-signing` remains automatic for now because Maple Dev and production iOS share its credentials. Gating this environment would also interrupt the automatic Maple Dev TestFlight lane, potentially at both its build and upload diff --git a/docs/repository-workflows.md b/docs/repository-workflows.md index 4723e7aa2..dceaeb5f5 100644 --- a/docs/repository-workflows.md +++ b/docs/repository-workflows.md @@ -9,7 +9,8 @@ root. For everyday setup, use the owning component guide and skill. `./setup-hooks.sh` installs `.githooks/pre-commit`. It classifies staged paths with `scripts/ci/hook_change_detection.py` and runs each affected component's own `.githooks/pre-commit` inside that component's Nix flake, so the tools match -CI: the root `.#ci` shell for Research, Auth, `services/updates/`, and repository +CI: the root `.#ci` shell for Research, Auth, `services/updates/`, the independent +Agent delivery checks in `scripts/ci/agent-updates.sh`, and repository checks; the component flakes for `apps/maple-agent/`, `sdk/`, `proxy/`, and `services/opensecret?submodules=1`. Without Nix it runs the same commands from `PATH` and warns that results may differ. It runs each selected component's format, lint, type-check, and unit-test gates; diff --git a/flake.nix b/flake.nix index b5e8db9f4..114a47e39 100644 --- a/flake.nix +++ b/flake.nix @@ -198,7 +198,7 @@ ]; # Frontend scripts run tsc and Vite through their Node shebangs; pin # that runtime instead of relying on a hosted runner's global Node. - ciPackages = [ rustupShim pkgs.nodejs ] ++ commonPackages; + ciPackages = [ rustupShim pkgs.nodejs pkgs.gh ] ++ commonPackages; linuxTauriPackages = with pkgs; @@ -743,7 +743,7 @@ ''; change-detection = pkgs.runCommand "maple-ci-change-detection-check" { - nativeBuildInputs = [ pkgs.python3 ]; + nativeBuildInputs = with pkgs; [ bash git python3 ]; src = ./.; } '' cd "$src" @@ -790,6 +790,16 @@ touch "$out" ''; + agent-updates = pkgs.runCommand "maple-agent-updates-check" { + nativeBuildInputs = with pkgs; [ git minisign python3 yq-go ]; + src = ./.; + } '' + cd "$src" + python3 scripts/ci/test_agent_updates_workflows.py + python3 -I -m unittest discover -s services/agent-updates/publisher -p 'test_*.py' + touch "$out" + ''; + ios-build-profile = pkgs.runCommand "maple-ios-build-profile-check" { nativeBuildInputs = with pkgs; [ bash python3 unzip ]; src = ./.; diff --git a/scripts/ci/agent-updates.sh b/scripts/ci/agent-updates.sh new file mode 100755 index 000000000..b4c9de1c1 --- /dev/null +++ b/scripts/ci/agent-updates.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=_common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh" + +print_source_provenance +export WRANGLER_SEND_METRICS=false +cd "${REPO_ROOT}" +python3 scripts/ci/test_agent_updates_workflows.py +python3 -I -m unittest discover -s services/agent-updates/publisher -p 'test_*.py' +cd services/agent-updates +bun install --frozen-lockfile --ignore-scripts +bun run check diff --git a/scripts/ci/hook_change_detection.py b/scripts/ci/hook_change_detection.py index 862ce201d..59a525ee6 100644 --- a/scripts/ci/hook_change_detection.py +++ b/scripts/ci/hook_change_detection.py @@ -23,6 +23,7 @@ "proxy", "opensecret", "updates", + "agent_updates", "repo", ) ALL_OUTPUTS = frozenset(OUTPUTS) @@ -50,6 +51,10 @@ AGENT_PREFIX = "apps/maple-agent/" AGENT_INERT_PREFIXES = ("docs/", ".githooks/") +AGENT_PUBLICATION_INPUTS = frozenset({ + "apps/maple-agent/scripts/release-info.py", + "apps/maple-agent/release-profiles.json", +}) SDK_TS_PREFIXES = ("sdk/src/",) SDK_TS_FILES = frozenset( @@ -95,6 +100,7 @@ ) UPDATES_PREFIX = "services/updates/" +AGENT_UPDATES_PREFIX = "services/agent-updates/" REPO_PREFIXES = (".github/", "scripts/ci/") REPO_FILES = frozenset({"flake.nix", "flake.lock"}) @@ -133,6 +139,8 @@ def classify_path(path: str) -> frozenset[str]: relative = path.removeprefix(AGENT_PREFIX) if relative.startswith(AGENT_INERT_PREFIXES): return frozenset() + if path in AGENT_PUBLICATION_INPUTS: + return frozenset({"agent", "agent_updates"}) return frozenset({"agent"}) if path in SDK_SHARED_FILES: @@ -161,6 +169,10 @@ def classify_path(path: str) -> frozenset[str]: return frozenset() if path.startswith(UPDATES_PREFIX): return frozenset({"updates"}) + if path.startswith(AGENT_UPDATES_PREFIX + ".githooks/"): + return frozenset() + if path.startswith(AGENT_UPDATES_PREFIX): + return frozenset({"agent_updates"}) if path in REPO_FILES or path.startswith(REPO_PREFIXES): return frozenset({"repo"}) diff --git a/scripts/ci/test_agent_updates_workflows.py b/scripts/ci/test_agent_updates_workflows.py new file mode 100644 index 000000000..6769b7f60 --- /dev/null +++ b/scripts/ci/test_agent_updates_workflows.py @@ -0,0 +1,176 @@ +"""Keep Agent data publication separate from Research and signing authority.""" + +import functools +import json +from pathlib import Path +import re +import subprocess +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +MASTER = "github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'" + + +@functools.cache +def workflow(name): + return json.loads(subprocess.check_output( + ["yq", "-o=json", ".", str(ROOT / ".github/workflows" / name)], text=True, + )) + + +class AgentUpdatesWorkflows(unittest.TestCase): + def test_delivery_checks_follow_imported_profile_and_package_contracts(self): + config = workflow("agent-updates-tests.yml") + for event in ("push", "pull_request"): + self.assertTrue({ + "apps/maple-agent/scripts/release-info.py", + "apps/maple-agent/release-profiles.json", + "services/agent-updates/**", + } <= set(config["on"][event]["paths"])) + + def test_manual_master_only_and_separate_authority(self): + config = workflow("agent-updates-publish.yml") + self.assertEqual(set(config["on"]), {"workflow_dispatch"}) + self.assertEqual(config["permissions"], { + "contents": "read", "actions": "read", "attestations": "read", + }) + self.assertEqual(config["concurrency"], { + "group": "agent-updates-production", "cancel-in-progress": False, + }) + jobs = config["jobs"] + self.assertEqual(set(jobs), {"verify", "reserve", "sign", "publish"}) + self.assertNotIn("environment", jobs["verify"]) + self.assertEqual(jobs["reserve"]["needs"], "verify") + self.assertEqual(jobs["sign"]["needs"], "reserve") + self.assertEqual(jobs["publish"]["needs"], "sign") + for name, job in jobs.items(): + self.assertIn(MASTER, " ".join(job["if"].split())) + environment = job.get("environment") + if isinstance(environment, dict): + environment = environment["name"] + if name in {"reserve", "sign", "publish"}: + self.assertEqual(environment, "updates-production") + self.assertNotIn("permissions", job) + for step in job["steps"]: + self.assertNotIn("continue-on-error", step) + for value in step.get("env", {}).values(): + if "secrets." in str(value): + secret = re.search(r"secrets\.([A-Z0-9_]+)", value)[1] + self.fail(f"credential must be passed from a step output: {secret}") + if step.get("uses", "").startswith("actions/checkout@"): + self.assertEqual(step["with"]["ref"], "${{ github.sha }}") + self.assertIs(step["with"]["persist-credentials"], False) + + def test_bws_reuses_existing_access_without_exporting_credentials_to_later_steps(self): + jobs = workflow("agent-updates-publish.yml")["jobs"] + bws_action = workflow("opensecret-eif-release.yml")["jobs"]["sign"]["steps"] + bws_action = next(step["uses"] for step in bws_action + if step.get("uses", "").startswith("bitwarden/sm-action@")) + self.assertEqual(bws_action, "bitwarden/sm-action@1238aae8fc64b212641190a9227c8a734ab1a793") + for name in ("reserve", "sign", "publish"): + with self.subTest(job=name): + job = jobs[name] + self.assertNotIn("env", job) + steps = job["steps"] + fetches = [i for i, step in enumerate(steps) + if step.get("uses", "").startswith("bitwarden/sm-action@")] + self.assertEqual(len(fetches), 1) + index = fetches[0] + fetch = steps[index] + self.assertEqual(fetch["uses"], bws_action) + self.assertEqual(fetch["with"]["access_token"], "${{ secrets.AGENT_UPDATES_BWS_ACCESS_TOKEN }}") + self.assertIs(fetch["with"]["set_env"], False) + realization = next(i for i, step in enumerate(steps) + if step.get("run") == "nix develop --no-update-lock-file .#ci -c true") + download = next(i for i, step in enumerate(steps) + if step.get("uses", "").startswith("actions/download-artifact@")) + self.assertLess(download, realization) + self.assertLess(realization, index) + for step in steps: + references = re.findall(r"secrets\.([A-Z0-9_]+)", json.dumps(step)) + self.assertEqual(references, ["AGENT_UPDATES_BWS_ACCESS_TOKEN"] if step is fetch else []) + if name == "sign": + self.assertEqual(fetch["with"]["secrets"].splitlines(), [ + "${{ vars.AGENT_UPDATE_SIGNING_KEY_ID }} > AGENT_UPDATE_SIGNING_KEY", + ]) + expected = {"AGENT_UPDATE_SIGNING_KEY": "${{ steps.signing-key.outputs.AGENT_UPDATE_SIGNING_KEY }}"} + else: + self.assertEqual(fetch["with"]["secrets"].splitlines(), [ + "${{ vars.AGENT_R2_ACCESS_KEY_ID_SECRET_ID }} > AWS_ACCESS_KEY_ID", + "${{ vars.AGENT_R2_SECRET_ACCESS_KEY_SECRET_ID }} > AWS_SECRET_ACCESS_KEY", + ]) + expected = { + "AWS_ACCESS_KEY_ID": "${{ steps.r2-credentials.outputs.AWS_ACCESS_KEY_ID }}", + "AWS_SECRET_ACCESS_KEY": "${{ steps.r2-credentials.outputs.AWS_SECRET_ACCESS_KEY }}", + } + consuming = steps[index + 1] + for key, value in expected.items(): + self.assertEqual(consuming["env"][key], value) + for other in steps[:index] + steps[index + 2:]: + self.assertNotRegex(json.dumps(other), r"steps\.(?:signing-key|r2-credentials)\.outputs") + self.assertNotRegex(other.get("run", ""), r"GITHUB_ENV|BWS_ACCESS_TOKEN") + + def test_preparation_verifies_provenance_before_any_secrets(self): + verify = workflow("agent-updates-publish.yml")["jobs"]["verify"] + source = "\n".join(step.get("run", "") for step in verify["steps"]) + self.assertIn("AGENT_UPDATES_ENABLED", str(verify)) + self.assertIn('"$ACTIVATED" = true', source) + self.assertIn("nix develop --no-update-lock-file .#ci -c bash -euo pipefail <<'PREPARE'", source) + self.assertIn("publisher.py trust", source) + self.assertIn("publisher.py prepare", source) + self.assertIn("--public-key public-key.pub", source) + self.assertIn("--version", source) + self.assertIn("--run-id", source) + self.assertIn("--linux", source) + self.assertNotIn("secrets.", str(verify)) + self.assertNotIn("--unsigned", source) + self.assertNotIn("gh release", source) + + def test_retry_transfers_keep_stable_artifact_names(self): + jobs = workflow("agent-updates-publish.yml")["jobs"] + for producer, consumer in (("verify", "reserve"), ("reserve", "sign"), ("sign", "publish")): + upload = next(step["with"] for step in jobs[producer]["steps"] + if step.get("uses", "").startswith("actions/upload-artifact@")) + download = next(step["with"] for step in jobs[consumer]["steps"] + if step.get("uses", "").startswith("actions/download-artifact@")) + self.assertEqual(upload["name"], download["name"]) + self.assertNotIn("run_attempt", upload["name"]) + self.assertIs(upload["overwrite"], True) + self.assertEqual(upload["if-no-files-found"], "error") + + def test_credentialed_steps_never_execute_packages_or_deploy(self): + jobs = workflow("agent-updates-publish.yml")["jobs"] + for name in ("reserve", "sign", "publish"): + source = "\n".join(step.get("run", "") for step in jobs[name]["steps"]) + self.assertNotRegex(source, r"(?:\.AppImage|tar\s|hdiutil|codesign|package-release|verify-release|gh\s+release|git\s+(?:push|tag)|wrangler|workers/domains)") + self.assertNotIn("services/updates", source) + self.assertNotIn("CLOUDFLARE_API_TOKEN", source) + publish = str(jobs["publish"]) + self.assertIn("--base-url https://updates.trymaple.ai", publish) + self.assertNotIn("AGENT_UPDATES_ORIGIN", publish) + + def test_routes_and_worker_resources_require_separate_setup(self): + source = (ROOT / "services/agent-updates/wrangler.jsonc").read_text() + config = json.loads(re.sub(r",(\s*[}\]])", r"\1", re.sub(r"//[^\n]*", "", source))) + self.assertFalse(config["workers_dev"]) + self.assertFalse(config["preview_urls"]) + self.assertNotIn("routes", config) + self.assertNotIn("route", config) + self.assertNotIn("assets", config) + self.assertNotIn("env", config) + self.assertEqual(config["r2_buckets"], [{ + "binding": "AGENT_RELEASES", "bucket_name": "maple-agent-updates", + }]) + # Keep an accidental deploy out of the ordinary package commands. + package = json.loads((ROOT / "services/agent-updates/package.json").read_text()) + self.assertNotIn("deploy", package["scripts"]) + self.assertEqual(package["scripts"]["build:dry-run"], "bun src/dry-run.ts") + dry_run = (ROOT / "services/agent-updates/src/dry-run.ts").read_text() + self.assertIn('"--dry-run"', dry_run) + self.assertIn('"deploy"', dry_run) + self.assertIn('bucket_name: "agent-updates-local-dry-run"', dry_run) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_hook_change_detection.py b/scripts/ci/test_hook_change_detection.py index 478b314b7..06c8d7b1b 100644 --- a/scripts/ci/test_hook_change_detection.py +++ b/scripts/ci/test_hook_change_detection.py @@ -1,6 +1,11 @@ #!/usr/bin/env python3 """Unit tests for the pre-commit hook component selector.""" +import os +from pathlib import Path +import shutil +import subprocess +import tempfile import unittest from hook_change_detection import OUTPUTS, classify_paths @@ -38,6 +43,7 @@ def test_documentation_and_hook_metadata_select_nothing(self) -> None: "services/opensecret/AGENTS.md", "services/opensecret/.githooks/pre-commit", "services/updates/.githooks/pre-commit", + "services/agent-updates/.githooks/pre-commit", "services/opensecret/docs/nitro-deploy.md", "services/opensecret/pcrProd.json", "scripts/prepare-frontend-deps.sh", @@ -71,6 +77,11 @@ def test_agent_component(self) -> None: with self.subTest(path=path): self.assert_selects([path], "agent") + def test_agent_publication_inputs_select_both_consumers_without_research(self) -> None: + for path in ("apps/maple-agent/scripts/release-info.py", "apps/maple-agent/release-profiles.json"): + with self.subTest(path=path): + self.assert_selects([path], "agent", "agent_updates") + def test_sdk_components(self) -> None: self.assert_selects(["sdk/rust/src/client.rs"], "sdk_rust") self.assert_selects(["sdk/rust/tests/crypto.rs"], "sdk_rust") @@ -91,6 +102,17 @@ def test_proxy_opensecret_and_updates(self) -> None: self.assert_selects(["services/opensecret/nix/eif.nix"]) self.assert_selects(["services/updates/src/index.ts"], "updates") + def test_agent_updates_component_is_isolated_from_research(self) -> None: + for path in ( + "services/agent-updates/src/worker.ts", + "services/agent-updates/publisher/publisher.py", + "services/agent-updates/publisher/test_publisher.py", + "services/agent-updates/bun.lock", + "services/agent-updates/wrangler.jsonc", + ): + with self.subTest(path=path): + self.assert_selects([path], "agent_updates") + def test_shared_crates_do_not_fan_out_to_consumers(self) -> None: self.assert_selects(["proxy/src/proxy.rs", "sdk/rust/src/client.rs"], "proxy", "sdk_rust") self.assert_selects( @@ -108,5 +130,53 @@ def test_unknown_root_selects_everything(self) -> None: self.assert_selects(["newservice/main.rs"], *OUTPUTS) +class AgentUpdatesHookTests(unittest.TestCase): + def test_service_paths_run_the_bash_ci_entrypoint_in_the_root_shell(self) -> None: + source = Path(__file__).resolve().parents[2] + for use_nix in (True, False): + with self.subTest(nix=use_nix), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "repository with spaces" + root.mkdir() + for relative in ( + ".githooks/pre-commit", ".githooks/lib/common.sh", + "scripts/ci/hook_change_detection.py", + ): + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source / relative, destination) + script = root / "scripts/ci/agent-updates.sh" + script.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'selected=("$MAPLE_HOOK_SELECTED")\n' + 'printf "%s\\n%s\\n%s\\n" "${selected[0]}" "$PWD" ' + '"$MAPLE_HOOK_IN_NIX" > hook-result\n' + ) + tools = root / "tools" + tools.mkdir() + nix = tools / "nix" + nix.write_text( + '#!/bin/sh\n' + '[ "$1" = develop ] && [ "$2" = --no-update-lock-file ] ' + '&& [ "$3" = ".#ci" ] && [ "$4" = -c ] || exit 70\n' + 'shift 4\nexec "$@"\n' + ) + nix.chmod(0o755) + changed = root / "services/agent-updates/src/worker.ts" + changed.parent.mkdir(parents=True) + changed.write_text("export default {};\n") + environment = {key: value for key, value in os.environ.items() + if not key.startswith(("GIT_", "MAPLE_HOOK_"))} + environment["PATH"] = f"{tools}{os.pathsep}{environment['PATH']}" + environment["MAPLE_HOOK_NO_NIX"] = "0" if use_nix else "1" + subprocess.run(["git", "init", "-q"], cwd=root, env=environment, check=True) + subprocess.run(["git", "add", str(changed)], cwd=root, env=environment, check=True) + subprocess.run(["sh", ".githooks/pre-commit"], cwd=root, env=environment, + check=True, capture_output=True, text=True) + selected, directory, in_nix = (root / "hook-result").read_text().splitlines() + self.assertEqual(selected.strip(), "agent_updates") + self.assertEqual(Path(directory).resolve(), root.resolve()) + self.assertEqual(in_nix, "1" if use_nix else "0") + + if __name__ == "__main__": unittest.main() diff --git a/scripts/ci/test_nix_workflows.py b/scripts/ci/test_nix_workflows.py index c020d8ee7..eef5be86c 100644 --- a/scripts/ci/test_nix_workflows.py +++ b/scripts/ci/test_nix_workflows.py @@ -18,7 +18,7 @@ "name": "Verify pinned Nix runtime", "run": "bash scripts/ci/verify-nix-runtime.sh", } -EXPECTED_INSTALLS = 69 +EXPECTED_INSTALLS = 74 @functools.cache diff --git a/scripts/ci/test_signing_workflows.py b/scripts/ci/test_signing_workflows.py index 60d957abe..a3520e536 100644 --- a/scripts/ci/test_signing_workflows.py +++ b/scripts/ci/test_signing_workflows.py @@ -36,10 +36,15 @@ "AZURE_SUBSCRIPTION_ID", "AZURE_TENANT_ID", }, "zapstore-publishing": {"ZAPSTORE_SIGN_WITH"}, + "updates-production": {"AGENT_UPDATES_BWS_ACCESS_TOKEN"}, } SIGNING_SECRETS = set.union(*ENVIRONMENTS.values()) CONSUMERS = { "agent-desktop-build.yml": {"macos": "desktop-signing"}, + "agent-updates-publish.yml": { + "reserve": "updates-production", "sign": "updates-production", + "publish": "updates-production", + }, "desktop-build.yml": { "build-macos": "desktop-signing", "build-linux": "desktop-signing", "build-windows": "windows-signing", @@ -56,7 +61,12 @@ "build-android": "android-signing", "build-ios": "apple-signing", }, "zapstore-publish.yml": {"publish": "zapstore-publishing"}, - "signing-credentials-check.yml": {name: name for name in ENVIRONMENTS}, + "signing-credentials-check.yml": { + name: name for name in ( + "desktop-signing", "apple-signing", "android-signing", + "windows-signing", "zapstore-publishing", + ) + }, } @@ -283,6 +293,21 @@ def test_missing_wrong_and_dynamic_environments_fail_closed(self): check_boundary({**reference, "environment": environment}) check_boundary({**reference, "environment": "apple-signing"}) + def test_agent_update_bws_access_is_exclusive_to_its_publisher_jobs(self): + consumers = [] + for name, config in workflows().items(): + for job_name, job in config["jobs"].items(): + if "AGENT_UPDATES_BWS_ACCESS_TOKEN" in signing_references(job): + consumers.append((name, job_name)) + self.assertEqual(consumers, [("agent-updates-publish.yml", job) + for job in ("reserve", "sign", "publish")]) + reference = {"steps": [{"with": {"access_token": "${{ secrets.AGENT_UPDATES_BWS_ACCESS_TOKEN }}"}}]} + for environment in (None, "desktop-signing", "agent-updates-production", + "${{ inputs.environment }}"): + with self.subTest(environment=environment), self.assertRaises(ValueError): + check_boundary({**reference, "environment": environment}) + check_boundary({**reference, "environment": "updates-production"}) + if __name__ == "__main__": unittest.main() diff --git a/services/agent-updates/.gitignore b/services/agent-updates/.gitignore new file mode 100644 index 000000000..0dcc8a419 --- /dev/null +++ b/services/agent-updates/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +.wrangler/ diff --git a/services/agent-updates/README.md b/services/agent-updates/README.md new file mode 100644 index 000000000..553543b9d --- /dev/null +++ b/services/agent-updates/README.md @@ -0,0 +1,146 @@ +# Maple Agent update delivery + +Independent Agent Worker and R2 publisher. Research's `services/updates/`, +`maple-updates` Worker, domain mapping, publishing workflow and installed clients +are unchanged. This code creates no Cloudflare resources and starts no releases. +See [publisher tooling](publisher/README.md) for exact commands and retry behavior. + +The checked-in binding names the owner-provisioned private `maple-agent-updates` +bucket. This is configuration, not evidence of Worker deployment or activated +delivery. There is no deploy script or automatic deployment workflow. +`workers_dev` and preview URLs are off; routine configuration must never contain +`route`, `routes` or a Custom Domain. Local checks use a separate fixture bucket. + +## Local verification + +From the monorepo root: + +```sh +nix develop --no-update-lock-file .#ci -c ./scripts/ci/agent-updates.sh +``` + +This runs schema and Worker tests, real workerd/R2 fixture requests, publisher +failure/retry tests and a deploy dry run. Fixtures use fake signatures: they +prove delivery and parsing, while publisher tests and the client own pinned-key +authentication. None of these tests reads a production bucket or executes a +downloaded app. Source checks do not establish Cloudflare permissions or a +successful packaged update. + +## Public contract + +Only GET and HEAD are supported. Metadata and errors use `no-store` at both +browser and CDN layers. No origin fetch is made on failure. + +| Path | Behavior | +| ---------------------------------------- | -------------------------------------------------------------------------------- | +| `/agent/stable/latest.json` | Exact signed envelope from `stable/latest.json` | +| `/agent/stable/releases/V/manifest.json` | Immutable signed envelope | +| `/agent/stable/releases/V/FILE` | Signed, enumerated artifact; GET/HEAD, single byte ranges, ETag, immutable cache | +| `/download/agent/stable/macos` | Uncached 302 to the current DMG | +| `/download/agent/stable/linux-appimage` | Uncached 302 only when Linux is present | + +Missing objects/targets return 404. Malformed metadata, mismatched stored sizes +and bucket failures return bounded 503 responses. The Worker checks schema and +minisign structure; it has no signing key. Clients must authenticate exact +manifest bytes with their pinned keys before trusting fields or hashes. + +The envelope has exactly `manifest` (JSON text) and `signature` (minisign text). +The manifest has `format: maple-agent-update-v1`, `product: maple-agent`, +`channel: stable`, canonical stable SemVer `version`, positive decimal-string +`build` (up to 18 digits), lowercase `source_sha`, UTC second-resolution +`pub_date`, plain-text `notes`, `artifacts`, and `downloads`. + +`artifacts.darwin-aarch64` is the `.app.tar.gz`; `downloads.macos` is the DMG. +Optional `artifacts.linux-x86_64` and `downloads.linux-appimage` must describe the +same `.AppImage`. Each artifact contains exactly `path`, positive `size` (at most +4 GiB) and lowercase SHA-256 `sha256`. Paths must start +`/agent/stable/releases//` and end in one ASCII filename using letters, +digits, `.`, `_`, `-`; traversal and encoded separators are rejected. Supporting +build-info and provenance objects in R2 are private, not public artifact routes. + +Limits: 128 KiB envelope, 64 KiB manifest, 4 KiB minisign signature, 16 KiB notes. +Signature format is prehashed minisign Ed25519 with exact trusted comment +`product=maple-agent channel=stable version=`. No expiry is claimed. + +## Publication and authority + +`agent-updates-publish.yml` is manual, trusted-master-only, and serialized with +operator pointer changes through `agent-updates-production`. It selects an exact +completed trusted-master `agent-desktop-build.yml` run, requires the successful +Prod producers and final download verifiers for every included platform (not +merely overall workflow success), verifies every final file's +GitHub attestation against master and the source commit, and validates build +profile, version, inventory and hashes before receiving signing/bucket secrets. +Unrelated Dev or omitted-Linux failures do not block a verified macOS-only +publication. Downloaded executable/package content is never executed in +credentialed jobs. + +Publication uses create-only immutable objects, reuses identical bytes, resumes +missing objects and rejects differing bytes for a reserved version. The complete +signed manifest is created after payload verification. Only the latest pointer +is mutable, using an atomic compare-and-swap. A retry of the identical promoted +release verifies it as a no-op, including after a failed live verification. +Never delete a reserved release to make a conflicting retry pass. Publish a new +version or resolve the exact failure with the owner. + +The owner must prepare and approve these inputs before activating delivery: + +- A private existing Agent bucket, no `r2.dev`/bucket custom domain, retention + and immutable-object overwrite/delete policy protecting release prefixes. +- Reuse the existing `desktop-signing` environment for macOS package signing + and `updates-production` for Agent reservation, update signing and publication. + Preserve their current protection policies and Research configuration. Update + signing uses new Agent primary/offline backup keys; never Research's key. +- Bucket-scoped S3 Object Read & Write credentials, an approved account/bucket, + public primary key and exact HTTPS origin. R2 credentials cannot deploy Workers. +- An existing Agent Worker and scoped Worker Editor deployment credential; + confirm a pinned code deployment with the R2 binding preserves Routes. No + Route or Custom Domain management is part of routine code deployment. +- Exactly `https://updates.trymaple.ai/agent/stable/*` and + `https://updates.trymaple.ai/download/agent/stable/*`, separately owner-managed. + Confirm zone/account/proxied DNS, HTTP redirects and edge allowlisting first. + +Agent-specific workflow inputs are explicit. Reuse the +existing BWS access credential; no new machine accounts or access tokens are +required by this workflow: + +| Scope | Required input | +| ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| Repository variables | `AGENT_UPDATES_ENABLED=true`, `AGENT_UPDATE_PUBLIC_KEY` (public minisign key) | +| `updates-production` environment variables | `AGENT_R2_ACCOUNT_ID`, `AGENT_R2_BUCKET`, `AGENT_UPDATE_SIGNING_KEY_ID`, `AGENT_R2_ACCESS_KEY_ID_SECRET_ID`, `AGENT_R2_SECRET_ACCESS_KEY_SECRET_ID` | +| `updates-production` environment secret | `AGENT_UPDATES_BWS_ACCESS_TOKEN` (existing BWS access credential) | +| BWS items selected by the environment variables | Dedicated unencrypted Agent minisign key-file contents and Agent-bucket S3 access-key pair | + +The pinned BWS action follows the existing PCR workflow: `set_env: false`, with +outputs passed only to the consuming step after the pinned tools are realized. +The sign step maps only the Agent update key; reservation/publication map only +the Agent bucket credentials. These mappings do not reduce the existing BWS +credential's grants or the GitHub environment's secret context. Shared access +therefore remains a trusted workflow boundary, not separate credential authority. + +The public key must also be pinned in both the chosen build's Prod source profile +and the publisher's trusted master profile. An empty source key list fails closed; +setting workflow variables cannot replace those pins. Production verification uses +the fixed `https://updates.trymaple.ai` origin. No Worker deployment token belongs +in the Agent publication steps. Preserve the existing environment's policies; +the Agent workflow additionally enforces manual dispatch from trusted `master`. + +Before/after route activation compare Research's metadata bytes and all six +GET/HEAD download redirects, accounting for legitimate Research releases. In an +isolated rehearsal verify Research-style Worker redeploy/domain reattachment +preserves Agent routes; recheck after the next authorized Research publication. +Do not create a Research release for this check. + +An unadvertised signed `0.0.1` to `0.0.2` rehearsal through real packages, feed, +keys and UI is required before invitations. Include Linux only after its own +packaged launch/update/relaunch gates pass. This service does not prove those +client gates or authorize a deploy/publication. +The separately authorized bucket rehearsal must also demonstrate identical +retries under its actual release-prefix lock policy, differing-byte refusal, +missing-object repair and conditional pointer-race rejection. Mock storage and +request-signature tests do not establish those live R2 properties. + +Pointer rollback changes discovery, not installed apps. Install-time client +revalidation is required. Freeze/replay protection is limited in v1; a compromised +accepted key may require manual trusted reinstall. Keep the offline backup +separate and document key retirement/manual recovery before production signing. diff --git a/services/agent-updates/bun.lock b/services/agent-updates/bun.lock new file mode 100644 index 000000000..6d751a6d5 --- /dev/null +++ b/services/agent-updates/bun.lock @@ -0,0 +1,210 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@trymaple/agent-updates-worker", + "devDependencies": { + "@types/bun": "1.3.13", + "prettier": "3.9.6", + "typescript": "5.9.3", + "wrangler": "4.125.0", + }, + }, + }, + "packages": { + "@cloudflare/kv-asset-handler": ["@cloudflare/kv-asset-handler@0.5.0", "", {}, "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg=="], + + "@cloudflare/unenv-preset": ["@cloudflare/unenv-preset@2.16.1", "", { "peerDependencies": { "unenv": "2.0.0-rc.24", "workerd": ">1.20260305.0 <2.0.0-0" }, "optionalPeers": ["workerd"] }, "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw=="], + + "@cloudflare/workerd-darwin-64": ["@cloudflare/workerd-darwin-64@1.20260820.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-5F2/t7SVnugG3rscSe9da1LoHst+GiuVGPaE9BP6j5AonlFpiYi0eoJrl3wof9zDBIIgJZ87NjRj9TKjbbYgHg=="], + + "@cloudflare/workerd-darwin-arm64": ["@cloudflare/workerd-darwin-arm64@1.20260820.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jFnG7715+r9FXRZPpuWWe5Ayd9v/IJKDODARg56ffFJWWtte6bFNi5VY3GazBM04hEmxny6OjXQBh3j2E/wNZw=="], + + "@cloudflare/workerd-linux-64": ["@cloudflare/workerd-linux-64@1.20260820.1", "", { "os": "linux", "cpu": "x64" }, "sha512-TbTYaCBht0OaOWmnVpg43hXVYtIti/Kg6lvO819H2DwbxPpK1BH0z2C+Y7ySrVJLlxZQeic5eN7/noqbP80hJg=="], + + "@cloudflare/workerd-linux-arm64": ["@cloudflare/workerd-linux-arm64@1.20260820.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-FQmri1UF7hBnnpeyC5SZJCAnsSRs3+Ykn9wlO5zxmE2qIgKfbJZ+toJ6qrQyugWlxE65juT33HU6aYLtutLJHw=="], + + "@cloudflare/workerd-windows-64": ["@cloudflare/workerd-windows-64@1.20260820.1", "", { "os": "win32", "cpu": "x64" }, "sha512-BPvCuMxIQfA47wtYsGbBG6Bcar57Qs7yHqU2jWkT1+sRnL/741/ZbQGP9kVRMQ5fwBMuqNFmQRzAu8gSm7ri/w=="], + + "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + + "@emnapi/runtime": ["@emnapi/runtime@1.11.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA=="], + + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], + + "@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], + + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], + + "@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], + + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], + + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], + + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], + + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], + + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], + + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], + + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], + + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], + + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], + + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], + + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], + + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], + + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], + + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], + + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], + + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], + + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], + + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], + + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.3.1" }, "os": "darwin", "cpu": "arm64" }, "sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.3.1" }, "os": "darwin", "cpu": "x64" }, "sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw=="], + + "@img/sharp-freebsd-wasm32": ["@img/sharp-freebsd-wasm32@0.35.2", "", { "dependencies": { "@img/sharp-wasm32": "0.35.2" }, "os": "freebsd" }, "sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.3.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.3.1", "", { "os": "linux", "cpu": "none" }, "sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.3.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.3.1" }, "os": "linux", "cpu": "arm" }, "sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.3.1" }, "os": "linux", "cpu": "arm64" }, "sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.3.1" }, "os": "linux", "cpu": "ppc64" }, "sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.3.1" }, "os": "linux", "cpu": "none" }, "sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.3.1" }, "os": "linux", "cpu": "s390x" }, "sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.3.1" }, "os": "linux", "cpu": "x64" }, "sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.3.1" }, "os": "linux", "cpu": "arm64" }, "sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.3.1" }, "os": "linux", "cpu": "x64" }, "sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.2", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw=="], + + "@img/sharp-webcontainers-wasm32": ["@img/sharp-webcontainers-wasm32@0.35.2", "", { "dependencies": { "@img/sharp-wasm32": "0.35.2" }, "cpu": "none" }, "sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.35.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.35.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.35.2", "", { "os": "win32", "cpu": "x64" }, "sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.9", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ=="], + + "@poppinss/colors": ["@poppinss/colors@4.1.6", "", { "dependencies": { "kleur": "^4.1.5" } }, "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg=="], + + "@poppinss/dumper": ["@poppinss/dumper@0.6.5", "", { "dependencies": { "@poppinss/colors": "^4.1.5", "@sindresorhus/is": "^7.0.2", "supports-color": "^10.0.0" } }, "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw=="], + + "@poppinss/exception": ["@poppinss/exception@1.2.3", "", {}, "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw=="], + + "@sindresorhus/is": ["@sindresorhus/is@7.2.0", "", {}, "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw=="], + + "@speed-highlight/core": ["@speed-highlight/core@1.2.24", "", {}, "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw=="], + + "@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], + + "@types/node": ["@types/node@26.3.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-L3fgrnchriRC2ExBflb8j4uZZURHZfQsmQeyVzhjcHW4kkwVyo8/0h1B2MVzMTrYUJYu6G7EWs14hW/L9putqw=="], + + "blake3-wasm": ["blake3-wasm@2.1.5", "", {}, "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g=="], + + "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], + + "cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], + + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + + "error-stack-parser-es": ["error-stack-parser-es@1.0.5", "", {}, "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA=="], + + "esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], + + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + + "kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="], + + "miniflare": ["miniflare@5.20260820.0-alpha", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.2", "undici": "7.29.0", "workerd": "1.20260820.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" } }, "sha512-Bv1j2kcKKNwXLWuCx+j0xGt7z318mqQkJmEN6ellM9sCESbPBDTM9ofZMbKqx47jSnoGA3CiaUkAmzGVXUa/wQ=="], + + "path-to-regexp": ["path-to-regexp@6.3.0", "", {}, "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ=="], + + "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "prettier": ["prettier@3.9.6", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g=="], + + "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + + "sharp": ["sharp@0.35.2", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.4" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.2", "@img/sharp-darwin-x64": "0.35.2", "@img/sharp-freebsd-wasm32": "0.35.2", "@img/sharp-libvips-darwin-arm64": "1.3.1", "@img/sharp-libvips-darwin-x64": "1.3.1", "@img/sharp-libvips-linux-arm": "1.3.1", "@img/sharp-libvips-linux-arm64": "1.3.1", "@img/sharp-libvips-linux-ppc64": "1.3.1", "@img/sharp-libvips-linux-riscv64": "1.3.1", "@img/sharp-libvips-linux-s390x": "1.3.1", "@img/sharp-libvips-linux-x64": "1.3.1", "@img/sharp-libvips-linuxmusl-arm64": "1.3.1", "@img/sharp-libvips-linuxmusl-x64": "1.3.1", "@img/sharp-linux-arm": "0.35.2", "@img/sharp-linux-arm64": "0.35.2", "@img/sharp-linux-ppc64": "0.35.2", "@img/sharp-linux-riscv64": "0.35.2", "@img/sharp-linux-s390x": "0.35.2", "@img/sharp-linux-x64": "0.35.2", "@img/sharp-linuxmusl-arm64": "0.35.2", "@img/sharp-linuxmusl-x64": "0.35.2", "@img/sharp-webcontainers-wasm32": "0.35.2", "@img/sharp-win32-arm64": "0.35.2", "@img/sharp-win32-ia32": "0.35.2", "@img/sharp-win32-x64": "0.35.2" } }, "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w=="], + + "supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici": ["undici@7.29.0", "", {}, "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "unenv": ["unenv@2.0.0-rc.24", "", { "dependencies": { "pathe": "^2.0.3" } }, "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw=="], + + "workerd": ["workerd@1.20260820.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260820.1", "@cloudflare/workerd-darwin-arm64": "1.20260820.1", "@cloudflare/workerd-linux-64": "1.20260820.1", "@cloudflare/workerd-linux-arm64": "1.20260820.1", "@cloudflare/workerd-windows-64": "1.20260820.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-/wk4rFNHH6IVMXFe6aPEZsT5YWpWtfKTUMt7+rFkyeGbXcGCy4yxODmJatbqYj0jmqHETdz0+m2mT+vNjXnF7w=="], + + "wrangler": ["wrangler@4.125.0", "", { "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", "miniflare": "5.20260820.0-alpha", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", "workerd": "1.20260820.1" }, "optionalDependencies": { "fsevents": "2.3.3" }, "peerDependencies": { "@cloudflare/workers-types": "^5.20260820.1" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js", "cf-wrangler": "bin/cf-wrangler.js" } }, "sha512-yFpvggu+xk1Hdm/Uxwaqa19bb7GArME4CrCS3Vov68a2TZq2MPO+wLocKbbnIC9K0oLowcdau7/ycxbbNHKCEg=="], + + "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], + + "youch": ["youch@4.1.0-beta.10", "", { "dependencies": { "@poppinss/colors": "^4.1.5", "@poppinss/dumper": "^0.6.4", "@speed-highlight/core": "^1.2.7", "cookie": "^1.0.2", "youch-core": "^0.3.3" } }, "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ=="], + + "youch-core": ["youch-core@0.3.3", "", { "dependencies": { "@poppinss/exception": "^1.2.2", "error-stack-parser-es": "^1.0.5" } }, "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA=="], + } +} diff --git a/services/agent-updates/package.json b/services/agent-updates/package.json new file mode 100644 index 000000000..774d367b2 --- /dev/null +++ b/services/agent-updates/package.json @@ -0,0 +1,20 @@ +{ + "name": "@trymaple/agent-updates-worker", + "version": "0.0.0", + "private": true, + "packageManager": "bun@1.3.5", + "scripts": { + "check": "bun run format:check && bun run typecheck && bun run test && bun run build:dry-run", + "build:dry-run": "bun src/dry-run.ts", + "format": "prettier --write \"src/**/*.ts\" README.md package.json tsconfig.json wrangler.jsonc", + "format:check": "prettier --check \"src/**/*.ts\" README.md package.json tsconfig.json wrangler.jsonc", + "test": "bun --no-env-file test", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/bun": "1.3.13", + "prettier": "3.9.6", + "typescript": "5.9.3", + "wrangler": "4.125.0" + } +} diff --git a/services/agent-updates/publisher/README.md b/services/agent-updates/publisher/README.md new file mode 100644 index 000000000..8ca33c1f3 --- /dev/null +++ b/services/agent-updates/publisher/README.md @@ -0,0 +1,134 @@ +# Agent release publisher + +This tooling publishes the independent Agent feed. It does not change Research, +create Cloudflare resources, generate keys, create GitHub releases or execute +downloaded applications. Run it from a trusted checkout with the root pinned +`nix develop --no-update-lock-file .#ci` shell (Python, GitHub CLI and minisign). + +The workflow separates read-only package verification, bucket reservation, +signing and publication. Reservation/publication use bucket-scoped S3 credentials; +the signing process receives a dedicated unencrypted minisign secret-key file. +Credentialed jobs reuse the existing `updates-production` environment and BWS +access credential. The pinned BWS action does not export to later steps; each +consuming step explicitly maps only the values it needs. This does not narrow +the underlying BWS grants or the environment's existing secret context. Pinned +tools are realized before retrieval. No new machine accounts or access tokens +are required; the exact item-ID inputs are listed in the [service setup](../README.md#publication-and-authority). +The public key must already be pinned both in the selected build's source profile +and in the publisher's trusted source profile. An empty key list blocks publication. + +```sh +publisher=services/agent-updates/publisher/publisher.py +python3 "$publisher" trust --run-id BUILD_RUN_ID --output trust.json +# Use --linux on trust and --linux linux-artifacts on prepare only when the +# Linux package and installed-update acceptance gates have passed. +python3 "$publisher" prepare --run-id BUILD_RUN_ID --version 0.0.1 \ + --build BUILD_RUN_NUMBER --macos macos-artifacts --notes-file notes.txt \ + --public-key public-key.pub --output bundle +python3 "$publisher" reserve --bundle bundle --public-key bundle/public-key.pub \ + --endpoint https://ACCOUNT.r2.cloudflarestorage.com --bucket AGENT_BUCKET +python3 "$publisher" sign --bundle bundle --public-key bundle/public-key.pub \ + --secret-key /private/temporary/signing-key +python3 "$publisher" publish --bundle bundle --public-key bundle/public-key.pub \ + --endpoint https://ACCOUNT.r2.cloudflarestorage.com --bucket AGENT_BUCKET \ + --base-url https://updates.trymaple.ai +``` + +`trust` returns the source SHA, selected artifact names, exact run attempt, +publication date and build number as JSON. Only completed trusted master runs of +`agent-desktop-build.yml` qualify. The selected Prod signing/package and downloaded +consumer validation jobs must have succeeded; an aggregate success or skipped job +is insufficient, while an unrelated Dev failure does not block a qualified Prod +package. The latest job inventory retains successful producer jobs when only +failed consumers were rerun. `prepare` repeats this check and verifies every final downloaded file's +GitHub attestation against the exact source SHA, master ref and signer workflow. +The self-hosted-runner denial applies to the job generating that attestation. +Final package attestations are generated on GitHub-hosted jobs; their input +binaries are compiled on the workflow's Blacksmith runners. +Package metadata, public Prod profile, inventory, notarization results and streaming +checksums are validated before copying the binary payloads into the bundle. The +macOS archive's bounded regular `Info.plist` member is read without extraction to +match the selected run's build number, version and application identity. Sparse +archives are rejected; extension headers and cumulative metadata are bounded. + +The manifest build number is the selected build run's `run_number`, matching the +packaging workflow's `CFBundleVersion` input. Its application version must match +the selected packages and be a canonical stable SemVer. The default publication +date is the build run's creation date, making preparation deterministic across +retries. `--pub-date` can select another UTC date, but retries must preserve it, +the notes and every other reserved byte. No timestamps are inserted when signing. + +`reserve` creates `stable/releases/V/publication-manifest.json` using +`If-None-Match: *`. This private object reserves the exact unsigned bytes before +the signing stage. It also reserves `publication-metadata.json`, an exact hash and +size inventory of the private audit sidecars. Both digests are bound by the +reservation receipt. It recovers an existing complete signed manifest into the +bundle, preserving its envelope on a retry. `sign` requires the reservation +receipt, signs the exact bytes with prehashed minisign and the exact trusted +comment `product=maple-agent channel=stable version=V`, then verifies the signature. + +`publish` revalidates the envelope and local payload hashes, confirms the durable +reservation and snapshots the current feed's ETag. It creates immutable payloads, +and the bounded private audit sidecars, verifies each by reading its content back, +then creates the complete signed `stable/releases/V/manifest.json` and promotes +`stable/latest.json` using `If-Match` (or `If-None-Match: *` for the first release). +Existing immutable content is streamed and reused only if its bytes match, +before attempting a write against a possibly locked release object. Missing +objects still use create-only writes; a competing creation must match exactly. +A partially uploaded release resumes without replacing completed objects. A +concurrent pointer change stops promotion. Re-running an already published exact +release verifies the objects, repairs missing objects with create-only writes and +leaves the pointer unchanged; a different envelope at the same or lower version is +rejected. Failed attempts never delete objects or automatically restore a pointer. + +The bundle contains raw, provenance-checked build-info, package manifest, +SHA256SUMS and macOS notarization logs under `metadata/`, prefixed by platform, +plus the stable run/source/artifact identity and attestation policy in +`provenance.json`. The current diagnostic run attempt stays in CI's `trust.json`; +it is omitted from durable metadata so a successful verifier-only rerun can reuse +the exact release. A finite filename allowlist and a 4 MiB limit per file +bound this audit inventory. Reservation, signing and publication require exactly +the five macOS sidecars and provenance, plus the three Linux sidecars when Linux +is included; a missing, empty, partial or extra inventory is rejected. These +objects are stored immutably under +`stable/releases/V/metadata/` and verified before the complete manifest appears. +They remain private because the Worker serves only artifacts enumerated in the +signed wire manifest. The publication reservation and metadata inventory are also +private. Missing sidecars can be repaired on an identical retry; differing bytes +are a conflict. Original CI artifacts retain the independently verifiable package +attestations. +R2 credentials use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`; the publisher +signs every S3 request with SigV4 and includes conditional headers in that signature. +All network requests use HTTPS, fixed timeouts and no implicit redirects. Errors +do not print subprocess output, signing-key paths, request headers or credentials. + +Publication finishes by checking the live latest and immutable signed manifest, +signature, full artifact hashes, HEAD lengths, one-byte ranges, and installer +redirect destinations. Repeat that separately with `verify-live --bundle bundle +--public-key bundle/public-key.pub --base-url https://updates.trymaple.ai` after a +transport failure or operational change. If this final check fails after promotion, +inspect the live state and retry the identical bundle; publication may already +have completed. This check does not install or relaunch an application. + +Run offline tests: + +```sh +nix develop --no-update-lock-file .#ci -c python3 -m unittest discover \ + -s services/agent-updates/publisher -p 'test_*.py' +``` + +The fake-store tests cover interrupted writes, reservation/envelope reuse, +immutable conflicts, pointer races and already-published recovery. Their synthetic +signatures exercise envelope plumbing without making a cryptographic claim. +The pinned-shell minisign tests verify the shared public wire fixture and an +existing signed pointer using raw or commented public-key files and pinned-key +rotation, including an identical publication retry and missing-object repair. They generate +disposable offline keys only inside a private temporary directory, sign a reserved +manifest, and exercise verification, tampering and deterministic signing retries. +Mock-transport tests exercise fixed-time SigV4 vectors, signed conditional headers, +bounded reads, streamed uploads and hashes, HTTP failures and redirect refusal. +These offline checks do not establish the configured R2 bucket's behavior. Before +activation, its rehearsal must include locked-prefix identical reservation and +publication retries, a different-byte immutable conflict, missing-object repair, +and a competing pointer promotion that leaves the competing bytes unchanged. +No production credentials, resources or live release are used by these tests. diff --git a/services/agent-updates/publisher/fixtures/README.md b/services/agent-updates/publisher/fixtures/README.md new file mode 100644 index 000000000..8995dbb52 --- /dev/null +++ b/services/agent-updates/publisher/fixtures/README.md @@ -0,0 +1,8 @@ +These are public test fixtures, signed with the deterministic test-only seed +`[7; 32]` also used by `src/tests.rs`. This key is public knowledge and must never +be pinned in an application profile or used to sign a production release. + +`manifest.json` contains the exact bytes authenticated by `manifest.minisig`. +`signed-envelope.json` embeds the same bytes and signature for client, Worker +and publisher contract tests. The artifact bytes are the ASCII string +`signed payload fixture bytes`; they are not an application archive. diff --git a/services/agent-updates/publisher/fixtures/manifest.json b/services/agent-updates/publisher/fixtures/manifest.json new file mode 100644 index 000000000..62fc084c9 --- /dev/null +++ b/services/agent-updates/publisher/fixtures/manifest.json @@ -0,0 +1 @@ +{"format":"maple-agent-update-v1","product":"maple-agent","channel":"stable","version":"0.0.2","build":"2","source_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","pub_date":"2026-10-21T15:00:00Z","notes":"A test update.\nPlain text.","artifacts":{"darwin-aarch64":{"path":"/agent/stable/releases/0.0.2/Maple-Agent.app.tar.gz","size":28,"sha256":"106cfb1b4d93e07f8c02c2a3440aafd7dd2f5b0889bbb7ba77bc99ca85ef84da"}},"downloads":{"macos":{"path":"/agent/stable/releases/0.0.2/Maple-Agent.dmg","size":28,"sha256":"106cfb1b4d93e07f8c02c2a3440aafd7dd2f5b0889bbb7ba77bc99ca85ef84da"}}} \ No newline at end of file diff --git a/services/agent-updates/publisher/fixtures/manifest.minisig b/services/agent-updates/publisher/fixtures/manifest.minisig new file mode 100644 index 000000000..a2ef4d8c8 --- /dev/null +++ b/services/agent-updates/publisher/fixtures/manifest.minisig @@ -0,0 +1,4 @@ +untrusted comment: TEST ONLY +RUQxMjM0NTY3OCUVO3EkRz9emd4Vq7qd4IR+x9DelvoFJkcVwb/q0ODwQHQxK7yxsqUtoOA++iMkjIiVg3y+EKA964mP5WQwdw0= +trusted comment: product=maple-agent channel=stable version=0.0.2 +4Mo/xlTkXa0LMc4N+SJyNbTqErz0CLZg4vQMSCz4KzR1MdAITbAmXq9Vy4aXRhSXtFoeuTBSe3w4IF8I+aMXDQ== diff --git a/services/agent-updates/publisher/fixtures/public-test-key.pub b/services/agent-updates/publisher/fixtures/public-test-key.pub new file mode 100644 index 000000000..9d4de7dda --- /dev/null +++ b/services/agent-updates/publisher/fixtures/public-test-key.pub @@ -0,0 +1,2 @@ +untrusted comment: PUBLIC TEST KEY; NEVER SHIP +RWQxMjM0NTY3OOpKbGPinFIKvvVQexMuxfmVR3auvr57kkIe6mkURtIs diff --git a/services/agent-updates/publisher/fixtures/signed-envelope.json b/services/agent-updates/publisher/fixtures/signed-envelope.json new file mode 100644 index 000000000..385f99a0f --- /dev/null +++ b/services/agent-updates/publisher/fixtures/signed-envelope.json @@ -0,0 +1,4 @@ +{ + "manifest": "{\"format\":\"maple-agent-update-v1\",\"product\":\"maple-agent\",\"channel\":\"stable\",\"version\":\"0.0.2\",\"build\":\"2\",\"source_sha\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"pub_date\":\"2026-10-21T15:00:00Z\",\"notes\":\"A test update.\\nPlain text.\",\"artifacts\":{\"darwin-aarch64\":{\"path\":\"/agent/stable/releases/0.0.2/Maple-Agent.app.tar.gz\",\"size\":28,\"sha256\":\"106cfb1b4d93e07f8c02c2a3440aafd7dd2f5b0889bbb7ba77bc99ca85ef84da\"}},\"downloads\":{\"macos\":{\"path\":\"/agent/stable/releases/0.0.2/Maple-Agent.dmg\",\"size\":28,\"sha256\":\"106cfb1b4d93e07f8c02c2a3440aafd7dd2f5b0889bbb7ba77bc99ca85ef84da\"}}}", + "signature": "untrusted comment: TEST ONLY\nRUQxMjM0NTY3OCUVO3EkRz9emd4Vq7qd4IR+x9DelvoFJkcVwb/q0ODwQHQxK7yxsqUtoOA++iMkjIiVg3y+EKA964mP5WQwdw0=\ntrusted comment: product=maple-agent channel=stable version=0.0.2\n4Mo/xlTkXa0LMc4N+SJyNbTqErz0CLZg4vQMSCz4KzR1MdAITbAmXq9Vy4aXRhSXtFoeuTBSe3w4IF8I+aMXDQ==\n" +} diff --git a/services/agent-updates/publisher/publisher.py b/services/agent-updates/publisher/publisher.py new file mode 100644 index 000000000..f14f1dd39 --- /dev/null +++ b/services/agent-updates/publisher/publisher.py @@ -0,0 +1,864 @@ +#!/usr/bin/env python3 +"""Prepare, sign and conditionally publish independently delivered Agent updates. + +Only Python's standard library is used. GitHub provenance is checked with gh; +cryptography uses the repository's pinned minisign, never downloaded executables. +""" + +from __future__ import annotations + +import argparse +import base64 +from contextlib import contextmanager +from datetime import datetime, timezone +import hashlib +import hmac +import importlib.util +import json +import os +from pathlib import Path +import plistlib +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import tarfile +import urllib.error +import urllib.parse +import urllib.request +from xml.parsers.expat import ExpatError + + +REPOSITORY = "MaplePrivacyLabs/Maple" +WORKFLOW = ".github/workflows/agent-desktop-build.yml" +MAX_ARTIFACT = 4 * 1024**3 +MAX_MANIFEST = 64 * 1024 +MAX_ENVELOPE = 128 * 1024 +MAX_SIGNATURE = 4 * 1024 +MAX_NOTES = 16 * 1024 +MAX_JSON = 4 * 1024 * 1024 +CHUNK = 1024 * 1024 +NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*\Z") +SEMVER = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\Z") +SHA = re.compile(r"[0-9a-f]{40}\Z") +DIGEST = re.compile(r"[0-9a-f]{64}\Z") +ROOT = Path(__file__).resolve().parents[3] +FEED_URL = "https://updates.trymaple.ai/agent/stable/latest.json" +MACOS_METADATA_NAMES = {"macos-" + name for name in ("build-info.json", "manifest.json", "SHA256SUMS", "notarization-log.json", "dmg-notarization-log.json")} +LINUX_METADATA_NAMES = {"linux-" + name for name in ("build-info.json", "manifest.json", "SHA256SUMS")} +METADATA_NAMES = {"provenance.json"} | MACOS_METADATA_NAMES | LINUX_METADATA_NAMES + + +class PublishError(ValueError): + """A bounded, credential-free diagnostic safe to print in CI.""" + + +class StoreError(PublishError): + def __init__(self, status): + super().__init__(f"R2 request failed (HTTP {status})") + self.status = status + + +def need(condition, message): + if not condition: + raise PublishError(message) + + +def unique_object(pairs): + result = {} + for key, value in pairs: + need(key not in result, "duplicate JSON field") + result[key] = value + return result + + +def decode_json(data, maximum=MAX_JSON): + need(len(data) <= maximum, "oversized JSON") + try: + return json.loads(data.decode("utf-8"), object_pairs_hook=unique_object, + parse_constant=lambda _: (_ for _ in ()).throw(PublishError("non-finite JSON number"))) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError) as error: + raise PublishError("invalid JSON") from error + + +def encode_json(value): + return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + + +@contextmanager +def regular_file(path): + # NOFOLLOW and NONBLOCK reject symlinks/FIFOs before a privileged job reads. + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as source: + need(stat.S_ISREG(os.fstat(source.fileno()).st_mode), "expected a regular file") + yield source + + +def read_bytes(path, maximum): + with regular_file(path) as source: + need(os.fstat(source.fileno()).st_size <= maximum, "oversized input file") + data = source.read(maximum + 1) + need(len(data) <= maximum, "oversized input file") + return data + + +def hash_stream(source, maximum=MAX_ARTIFACT): + digest, size = hashlib.sha256(), 0 + while block := source.read(min(CHUNK, maximum - size + 1)): + size += len(block) + need(size <= maximum, "oversized artifact") + digest.update(block) + return digest.hexdigest(), size + + +def hash_file(path, maximum=MAX_ARTIFACT): + with regular_file(path) as source: + need(0 < os.fstat(source.fileno()).st_size <= maximum, "invalid artifact size") + return hash_stream(source, maximum) + + +def write_once(path, data): + path = Path(path) + if path.exists() or path.is_symlink(): + need(read_bytes(path, max(MAX_JSON, len(data))) == data, "existing prepared bytes differ; use the original bundle") + return + with path.open("xb") as target: + target.write(data) + + +def version_tuple(value): + need(isinstance(value, str) and len(value) <= 64 and SEMVER.fullmatch(value), "version must be canonical stable SemVer") + numbers = tuple(int(component) for component in value.split(".")) + need(all(number <= 2**64 - 1 for number in numbers), "SemVer component exceeds u64") + return numbers + + +def trusted_comment(version): + return f"product=maple-agent channel=stable version={version}" + + +def release_key(version, filename): + version_tuple(version) + need(isinstance(filename, str) and len(filename) <= 200 and ".." not in filename and NAME.fullmatch(filename), "invalid release filename") + return f"stable/releases/{version}/{filename}" + + +def public_path(version, filename): + return "/agent/" + release_key(version, filename) + + +def validate_manifest(data): + manifest = decode_json(data, MAX_MANIFEST) + fields = {"format", "product", "channel", "version", "build", "source_sha", "pub_date", "notes", "artifacts", "downloads"} + need(isinstance(manifest, dict) and set(manifest) == fields, "invalid manifest fields") + need(manifest["format"] == "maple-agent-update-v1" and manifest["product"] == "maple-agent" and manifest["channel"] == "stable", "wrong update identity") + version_tuple(manifest["version"]) + build = manifest["build"] + need(isinstance(build, str) and re.fullmatch(r"[1-9][0-9]{0,17}", build), "invalid build number") + need(isinstance(manifest["source_sha"], str) and SHA.fullmatch(manifest["source_sha"]), "invalid source SHA") + date = manifest["pub_date"] + need(isinstance(date, str) and re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", date), "invalid publication date") + try: + datetime.strptime(date, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as error: + raise PublishError("invalid publication date") from error + notes = manifest["notes"] + need(isinstance(notes, str) and len(notes.encode("utf-8")) <= MAX_NOTES, "oversized notes") + need(all(ord(char) >= 32 or char in "\n\t" for char in notes) and "\x7f" not in notes, "notes contain ASCII controls") + artifacts, downloads = manifest["artifacts"], manifest["downloads"] + need(isinstance(artifacts, dict) and {"darwin-aarch64"} <= set(artifacts) <= {"darwin-aarch64", "linux-x86_64"}, "invalid update targets") + need(isinstance(downloads, dict) and set(downloads) == ({"macos", "linux-appimage"} if "linux-x86_64" in artifacts else {"macos"}), "downloads do not match update targets") + for kind, entries in (("artifacts", artifacts), ("downloads", downloads)): + for target, entry in entries.items(): + need(isinstance(entry, dict) and set(entry) == {"path", "size", "sha256"}, "invalid artifact record") + path = entry["path"] + need(isinstance(path, str), "invalid artifact path") + prefix = f"/agent/stable/releases/{manifest['version']}/" + need(path.startswith(prefix), "artifact path has wrong release root") + filename = path[len(prefix):] + need(public_path(manifest["version"], filename) == path, "invalid artifact path") + suffix = ".app.tar.gz" if target == "darwin-aarch64" else ".dmg" if target == "macos" else ".AppImage" + need(filename.endswith(suffix), "wrong artifact suffix") + need(type(entry["size"]) is int and 0 < entry["size"] <= MAX_ARTIFACT, "invalid artifact size") + need(isinstance(entry["sha256"], str) and DIGEST.fullmatch(entry["sha256"]), "invalid artifact digest") + if "linux-x86_64" in artifacts: + need(artifacts["linux-x86_64"] == downloads["linux-appimage"], "Linux download and update differ") + return manifest + + +def signature_text(signature, version): + need(isinstance(signature, str) and len(signature.encode("utf-8")) <= MAX_SIGNATURE, "invalid or oversized signature") + lines = signature.splitlines() + need(len(lines) == 4 and lines[0].startswith("untrusted comment: ") and lines[2] == "trusted comment: " + trusted_comment(version), "wrong minisign comment or signature structure") + try: + packet = base64.b64decode(lines[1], validate=True) + global_signature = base64.b64decode(lines[3], validate=True) + except ValueError as error: + raise PublishError("invalid minisign encoding") from error + need(len(packet) == 74 and packet[:2] == b"ED" and len(global_signature) == 64, "signature must use prehashed minisign") + + +def parse_envelope(data): + envelope = decode_json(data, MAX_ENVELOPE) + need(isinstance(envelope, dict) and set(envelope) == {"manifest", "signature"} and isinstance(envelope["manifest"], str), "invalid signed envelope") + manifest = validate_manifest(envelope["manifest"].encode("utf-8")) + signature_text(envelope["signature"], manifest["version"]) + return envelope, manifest + + +def minisign_verify(data, public_key): + envelope, manifest = parse_envelope(data) + # Never print tool stderr: it may contain untrusted data or a secret path. + with tempfile.TemporaryDirectory(prefix="maple-agent-verify-") as temporary: + message = Path(temporary) / "manifest.json" + signature = Path(temporary) / "signature.minisig" + message.write_bytes(envelope["manifest"].encode("utf-8")) + signature.write_text(envelope["signature"], encoding="utf-8") + # Configured/public pinned keys may be raw base64 or a commented file. + # minisign -p only accepts the latter, while -P accepts the normalized + # public key packet in either case. No secret key is passed in argv. + result = subprocess.run(["minisign", "-V", "-H", "-q", "-m", str(message), "-x", str(signature), "-P", public_key_text(public_key)], stdin=subprocess.DEVNULL, capture_output=True, timeout=30) + need(result.returncode == 0, "minisign verification failed") + return manifest + + +def gh_json(path): + result = subprocess.run(["gh", "api", "--method", "GET", path], capture_output=True, timeout=60) + need(result.returncode == 0, "GitHub API request failed") + return decode_json(result.stdout) + + +def validate_run(run, jobs, linux=False): + need(isinstance(run, dict) and run.get("repository", {}).get("full_name") == REPOSITORY and run.get("head_repository", {}).get("full_name") == REPOSITORY, "untrusted build repository") + need(run.get("path") == WORKFLOW and run.get("head_branch") == "master" and run.get("event") in {"push", "workflow_dispatch"}, "untrusted build workflow or ref") + need(run.get("status") == "completed", "build run has not completed") + need(isinstance(run.get("head_sha"), str) and SHA.fullmatch(run["head_sha"]), "invalid build commit") + need(type(run.get("id")) is int and run["id"] > 0 and type(run.get("run_attempt")) is int and run["run_attempt"] > 0, "invalid build run identity") + need(type(run.get("run_number")) is int and 0 < run["run_number"] < 10**18, "invalid build run number") + need(isinstance(jobs, dict) and jobs.get("total_count") == len(jobs.get("jobs", [])), "incomplete build job inventory") + required = {"Signed macOS (prod)", "Verify downloaded macOS (prod)"} + if linux: + required |= {"Linux AppImage (prod)", "Attest Linux packages (prod)", "Verify downloaded Linux (prod)"} + for name in required: + matches = [job for job in jobs["jobs"] if job.get("name") == name] + need(len(matches) == 1 and matches[0].get("status") == "completed" and matches[0].get("conclusion") == "success", "required Prod package job did not succeed: " + name) + # An unrelated Dev or platform failure is not an Agent Prod release gate. + # Overall run success is intentionally not treated as package proof. + return run + + +def trust_run(run_id, linux=False): + need(str(run_id).isdecimal() and int(run_id) > 0, "invalid run ID") + run = gh_json(f"repos/{REPOSITORY}/actions/runs/{int(run_id)}") + attempt = run.get("run_attempt") + need(type(attempt) is int and attempt > 0, "invalid build attempt") + # GitHub's latest job inventory preserves successful producer jobs when + # only failed consumers were re-run. An attempt-only inventory would omit + # those producers even though their exact artifacts remain authoritative. + jobs = gh_json(f"repos/{REPOSITORY}/actions/runs/{int(run_id)}/jobs?filter=latest&per_page=100") + validate_run(run, jobs, linux) + need(run["id"] == int(run_id), "build run ID differs") + artifacts = gh_json(f"repos/{REPOSITORY}/actions/runs/{int(run_id)}/artifacts?per_page=100") + need(artifacts.get("total_count") == len(artifacts.get("artifacts", [])), "incomplete artifact inventory") + names = {"macos": f"maple-agent-prod-macos-aarch64-{int(run_id)}"} + if linux: + names["linux"] = f"maple-agent-prod-linux-x86_64-{int(run_id)}" + for name in names.values(): + matches = [item for item in artifacts["artifacts"] if item.get("name") == name] + need(len(matches) == 1 and matches[0].get("expired") is False, "required package artifact missing, duplicated or expired") + return {"run_id": run["id"], "run_attempt": attempt, "source_sha": run["head_sha"], "build": str(run["run_number"]), "pub_date": run["created_at"], "artifacts": names} + + +def attest(path, source_sha): + result = subprocess.run(["gh", "attestation", "verify", str(path), "--repo", REPOSITORY, "--signer-workflow", f"{REPOSITORY}/{WORKFLOW}", "--source-ref", "refs/heads/master", "--source-digest", source_sha, "--deny-self-hosted-runners"], stdin=subprocess.DEVNULL, capture_output=True, timeout=120) + need(result.returncode == 0, "final package attestation verification failed") + + +def public_key_text(path): + lines = read_bytes(path, MAX_SIGNATURE).decode("ascii").splitlines() + if len(lines) == 2 and lines[0].startswith("untrusted comment: "): + lines = lines[1:] + need(len(lines) == 1, "invalid minisign public key file") + try: + packet = base64.b64decode(lines[0], validate=True) + except ValueError as error: + raise PublishError("invalid minisign public key encoding") from error + need(len(packet) == 42 and packet[:2] == b"Ed", "invalid minisign public key") + return lines[0] + + +def validate_pins(key, local_profile, source_profile): + for profile in (local_profile, source_profile): + need(isinstance(profile, dict) and isinstance(profile.get("prod"), dict), "missing Prod release profile") + prod = profile["prod"] + need(prod.get("update_feed_url") == FEED_URL, "Prod update feed does not match the production feed") + keys = prod.get("update_public_keys") + need(isinstance(keys, list) and all(isinstance(item, str) for item in keys) and key in keys, "signing key is not pinned by the selected source and trusted publisher") + + +def check_source_pins(public_key, source_sha): + key = public_key_text(public_key) + local = decode_json(read_bytes(ROOT / "apps/maple-agent/release-profiles.json", MAX_MANIFEST)) + response = gh_json(f"repos/{REPOSITORY}/contents/apps/maple-agent/release-profiles.json?ref={source_sha}") + need(isinstance(response, dict) and response.get("encoding") == "base64" and isinstance(response.get("content"), str), "selected source profile is unavailable") + try: + source_bytes = base64.b64decode(response["content"].replace("\n", ""), validate=True) + except ValueError as error: + raise PublishError("selected source profile encoding is invalid") from error + validate_pins(key, local, decode_json(source_bytes, MAX_MANIFEST)) + return key + + +def durable_provenance(proof): + # A verifier-only rerun advances the diagnostic attempt while retaining + # the exact source, packages, hashes and release identity. + provenance = {name: value for name, value in proof.items() if name != "run_attempt"} + provenance["attestation_policy"] = {"repository": REPOSITORY, "signer_workflow": WORKFLOW, "source_ref": "refs/heads/master", "source_sha": proof["source_sha"], "deny_self_hosted_runners": True} + return encode_json(provenance) + + +def verify_current(data, public_key): + """Accept an existing feed signed by either currently trusted pinned key. + + The new release still has to use the specifically selected key. This + fallback only verifies the pointer being replaced during normal rotation. + Removing the old key deliberately stops accepting its feed as authority. + """ + selected = public_key_text(public_key) + profile = decode_json(read_bytes(ROOT / "apps/maple-agent/release-profiles.json", MAX_MANIFEST)) + validate_pins(selected, profile, profile) + for key in profile["prod"]["update_public_keys"]: + with tempfile.TemporaryDirectory(prefix="maple-agent-current-key-") as temporary: + path = Path(temporary) / "public-key.pub" + path.write_text(key + "\n", encoding="ascii") + public_key_text(path) + try: + return minisign_verify(data, path) + except PublishError: + continue + raise PublishError("current feed signature is not trusted by the pinned key set") + + +class BoundedTarInfo(tarfile.TarInfo): + def _proc_member(self, archive): + # tarfile processes PAX/GNU extension records before yielding a member. + # Cap them here, before its parser can allocate a header-sized buffer. + archive.maple_headers = getattr(archive, "maple_headers", 0) + 1 + need(archive.maple_headers <= 100_000, "app archive exceeds bounded header inventory") + need(0 <= self.size <= MAX_ARTIFACT, "app archive contains an oversized member") + if self.type in {tarfile.XHDTYPE, tarfile.XGLTYPE, tarfile.SOLARIS_XHDTYPE, tarfile.GNUTYPE_LONGNAME, tarfile.GNUTYPE_LONGLINK}: + need(self.size <= 64 * 1024, "app archive extension metadata exceeds bounds") + archive.maple_metadata_bytes = getattr(archive, "maple_metadata_bytes", 0) + self.size + need(archive.maple_metadata_bytes <= MAX_JSON, "app archive cumulative extension metadata exceeds bounds") + need(self.type != tarfile.GNUTYPE_SPARSE, "sparse app archive members are unsupported") + return super()._proc_member(archive) + + def _proc_gnusparse_00(self, *arguments): + raise PublishError("sparse app archive members are unsupported") + + def _proc_gnusparse_01(self, *arguments): + raise PublishError("sparse app archive members are unsupported") + + def _proc_gnusparse_10(self, *arguments): + raise PublishError("sparse app archive members are unsupported") + + +def _archive_plist(archive): + expected = "Maple Agent.app/Contents/Info.plist" + found, count, expanded = None, 0, 0 + with regular_file(archive) as compressed: + with tarfile.open(fileobj=compressed, mode="r|gz", tarinfo=BoundedTarInfo) as source: + for member in source: + count += 1 + expanded += member.size + need(count <= 100_000 and 0 <= member.size <= MAX_ARTIFACT and expanded <= 8 * 1024**3, "app archive exceeds bounded inventory") + if member.name != expected: + continue + need(found is None and member.isfile() and 0 < member.size <= 64 * 1024, "invalid or duplicate app Info.plist member") + with source.extractfile(member) as payload: + data = payload.read(64 * 1024 + 1) + need(len(data) == member.size, "truncated app Info.plist") + try: + found = plistlib.loads(data) + except (plistlib.InvalidFileException, ValueError, TypeError, ExpatError, RecursionError) as error: + raise PublishError("invalid app Info.plist") from error + return found + + +def validate_archive_build(archive, version, build, bundle_id="cloud.opensecret.maple.agent"): + """Read the small plist member without extracting or executing the app.""" + try: + found = _archive_plist(archive) + except (tarfile.TarError, EOFError, OSError, RecursionError) as error: + raise PublishError("invalid app archive") from error + need(isinstance(found, dict), "app archive has no regular Info.plist") + need(found.get("CFBundleVersion") == build and found.get("CFBundleShortVersionString") == version and found.get("CFBundleIdentifier") == bundle_id, "app Info.plist build, version or identity differs from selected run") + + +def validate_package(directory, platform, source_sha, version, attestor=attest, expected_build=None, metadata_records=None): + directory = Path(directory) + need(directory.is_dir() and not directory.is_symlink(), "package input must be a directory") + info = decode_json(read_bytes(directory / "build-info.json", 16 * 1024)) + # Reuse the checked-in public profile contract, never package-supplied code. + script = ROOT / "apps/maple-agent/scripts/release-info.py" + spec = importlib.util.spec_from_file_location("agent_release_info", script) + release_info = importlib.util.module_from_spec(spec) + spec.loader.exec_module(release_info) + try: + release_info.validate_info("prod", info, source_sha) + except (ValueError, KeyError, TypeError) as error: + raise PublishError("package does not match the Prod build identity") from error + need(info["version"] == version, "package version differs from publication version") + stem = f"maple-agent-prod-{version}-{info['git_revision']}-{platform}" + package = stem + (".dmg" if platform == "macos-aarch64" else ".AppImage") + binaries = [package, stem + ".app.tar.gz"] if platform == "macos-aarch64" else [package] + expected = {"build-info.json", "manifest.json", "SHA256SUMS", *binaries} + if platform == "macos-aarch64": + expected |= {"notarization-log.json", "dmg-notarization-log.json"} + need(set(path.name for path in directory.iterdir()) == expected, "unexpected or missing final package files") + records = {} + for name in sorted(expected): + path = directory / name + digest, size = hash_file(path, MAX_ARTIFACT if name in binaries else MAX_JSON) + records[name] = {"sha256": digest, "size": size} + metadata = decode_json(read_bytes(directory / "manifest.json", MAX_MANIFEST)) + expected_metadata = {"schema_version": 1, "profile": "prod", "platform": platform, "source_sha": source_sha, "version": version, "package": package, "distribution": "master", "macos_signing": "developer-id-notarized" if platform == "macos-aarch64" else None} + if platform == "linux-x86_64": + expected_metadata["linux_integrity"] = "sha256" + need(isinstance(metadata, dict) and type(metadata.get("schema_version")) is int and metadata == expected_metadata, "invalid final package metadata") + checked = set() + for line in read_bytes(directory / "SHA256SUMS", 16 * 1024).decode("ascii").splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) ([A-Za-z0-9._-]+)", line) + need(match is not None, "invalid package checksum record") + digest, name = match.groups() + need(name not in checked and name != "SHA256SUMS" and name in records and records[name]["sha256"] == digest, "package checksum mismatch or duplicate") + checked.add(name) + need(checked == expected - {"SHA256SUMS"}, "package checksum inventory differs") + if platform == "macos-aarch64": + for name in ("notarization-log.json", "dmg-notarization-log.json"): + notary = decode_json(read_bytes(directory / name, MAX_JSON)) + need(isinstance(notary, dict) and notary.get("status") == "Accepted" and notary.get("issues") in (None, []), "package notarization was not accepted") + if expected_build is not None: + validate_archive_build(directory / binaries[1], version, expected_build, info["bundle_id"]) + for name in sorted(expected): + attestor(directory / name, source_sha) + if metadata_records is not None: + metadata_records.update({name: record for name, record in records.items() if name not in binaries}) + return [(directory / name, {"path": public_path(version, name), **records[name]}) for name in binaries] + + +def prepare(run_id, version, build, macos, linux, notes_file, output, public_key, pub_date=None): + version_tuple(version) + proof = trust_run(run_id, bool(linux)) + need(build == proof["build"], "build number must match the package run number") + key = check_source_pins(public_key, proof["source_sha"]) + notes = read_bytes(notes_file, MAX_NOTES).decode("utf-8") + mac_metadata, linux_metadata = {}, {} + mac = validate_package(macos, "macos-aarch64", proof["source_sha"], version, expected_build=proof["build"], metadata_records=mac_metadata) + lin = validate_package(linux, "linux-x86_64", proof["source_sha"], version, metadata_records=linux_metadata) if linux else [] + manifest = {"format": "maple-agent-update-v1", "product": "maple-agent", "channel": "stable", "version": version, "build": build, "source_sha": proof["source_sha"], "pub_date": pub_date or proof["pub_date"], "notes": notes, "artifacts": {"darwin-aarch64": mac[1][1]}, "downloads": {"macos": mac[0][1]}} + if lin: + manifest["artifacts"]["linux-x86_64"] = lin[0][1] + manifest["downloads"]["linux-appimage"] = lin[0][1] + data = encode_json(manifest) + validate_manifest(data) + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + need(not output.is_symlink(), "bundle directory must not be a symlink") + write_once(output / "manifest.json", data) + write_once(output / "trust.json", encode_json(proof)) + write_once(output / "public-key.pub", ("untrusted comment: Maple Agent update key\n" + key + "\n").encode("ascii")) + metadata = output / "metadata" + metadata.mkdir(exist_ok=True) + need(not metadata.is_symlink(), "metadata directory must not be a symlink") + inventory = {} + for prefix, directory, source_records in (("macos", macos, mac_metadata), ("linux", linux, linux_metadata)): + for name, record in sorted(source_records.items()): + data = read_bytes(Path(directory) / name, MAX_JSON) + need((hashlib.sha256(data).hexdigest(), len(data)) == (record["sha256"], record["size"]), "package metadata changed after provenance verification") + destination = prefix + "-" + name + write_once(metadata / destination, data) + inventory[destination] = record + # A validator-only build retry advances run_attempt without changing any + # release bytes. Retain the stable build identity in durable audit data; + # the current attempt remains available in the CI trust.json diagnostic. + provenance_data = durable_provenance(proof) + write_once(metadata / "provenance.json", provenance_data) + inventory["provenance.json"] = {"sha256": hashlib.sha256(provenance_data).hexdigest(), "size": len(provenance_data)} + write_once(output / "metadata-inventory.json", encode_json(inventory)) + payloads = output / "payloads" + payloads.mkdir(exist_ok=True) + need(not payloads.is_symlink(), "payload directory must not be a symlink") + for source, record in mac + lin: + destination = payloads / source.name + if destination.exists() or destination.is_symlink(): + need(hash_file(destination) == (record["sha256"], record["size"]), "existing prepared artifact differs") + else: + with regular_file(source) as input_file, destination.open("xb") as target: + shutil.copyfileobj(input_file, target, CHUNK) + need(hash_file(destination) == (record["sha256"], record["size"]), "artifact changed during preparation") + return manifest + + +def bundle_records(bundle, manifest): + payloads = Path(bundle) / "payloads" + need(payloads.is_dir() and not payloads.is_symlink(), "invalid prepared payload directory") + records = {record["path"].rsplit("/", 1)[1]: record for record in [*manifest["artifacts"].values(), *manifest["downloads"].values()]} + need(set(path.name for path in payloads.iterdir()) == set(records), "prepared payload inventory differs") + for name, record in records.items(): + need(hash_file(payloads / name, record["size"]) == (record["sha256"], record["size"]), "prepared artifact hash differs") + return [(payloads / name, record) for name, record in sorted(records.items())] + + +def bundle_metadata(bundle, manifest): + bundle = Path(bundle) + index = bundle / "metadata-inventory.json" + need(index.exists() or index.is_symlink(), "metadata has no inventory") + data = read_bytes(index, MAX_MANIFEST) + inventory = decode_json(data, MAX_MANIFEST) + expected = {"provenance.json"} | MACOS_METADATA_NAMES + if "linux-x86_64" in manifest["artifacts"]: + expected |= LINUX_METADATA_NAMES + need(isinstance(inventory, dict) and set(inventory) == expected, "private metadata inventory must contain the exact sidecars for every included target") + directory = bundle / "metadata" + need(directory.is_dir() and not directory.is_symlink() and {path.name for path in directory.iterdir()} == set(inventory), "private metadata inventory differs") + result = [] + for name, record in sorted(inventory.items()): + need(isinstance(record, dict) and set(record) == {"sha256", "size"} and isinstance(record["sha256"], str) and DIGEST.fullmatch(record["sha256"]), "invalid private metadata record") + need(type(record["size"]) is int and 0 < record["size"] <= MAX_JSON, "invalid private metadata size") + path = directory / name + need(hash_file(path, record["size"]) == (record["sha256"], record["size"]), "private metadata hash differs") + result.append((path, record)) + return data, result + + +def metadata_key(version, filename): + need(filename in METADATA_NAMES, "invalid private metadata name") + return release_key(version, "metadata") + "/" + filename + + +def publish_metadata(store, version, records): + for path, record in records: + key = metadata_key(version, path.name) + immutable_put(store, key, path, "application/json" if path.suffix == ".json" else "text/plain", maximum=record["size"]) + need(store.same(key, record["sha256"], record["size"]), "private metadata read-back differs") + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, file, code, message, headers, new_url): + return None + + +def https_origin(value): + parsed = urllib.parse.urlsplit(value) + need(parsed.scheme == "https" and parsed.hostname and not parsed.username and not parsed.password and not parsed.query and not parsed.fragment and parsed.path in {"", "/"} and parsed.port in {None, 443}, "expected a plain HTTPS origin") + return f"https://{parsed.netloc}" + + +class R2Store: + """Path-style S3 requests with signed conditional headers; no redirects.""" + + def __init__(self, endpoint, bucket, access_key=None, secret_key=None): + self.endpoint = https_origin(endpoint) + need(re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket), "invalid bucket name") + self.bucket = bucket + self.access_key = access_key or os.environ.get("AWS_ACCESS_KEY_ID", "") + self.secret_key = secret_key or os.environ.get("AWS_SECRET_ACCESS_KEY", "") + need(self.access_key and self.secret_key, "R2 credentials are not configured") + self.opener = urllib.request.build_opener(NoRedirect) + + def request(self, method, key, data=None, content_hash=None, size=None, condition=None, content_type=None, maximum=MAX_JSON, consumer=None): + need(not key.startswith("/") and all(NAME.fullmatch(part) for part in key.split("/")), "invalid R2 object key") + path = "/" + self.bucket + "/" + urllib.parse.quote(key, safe="/-_.~") + now = datetime.now(timezone.utc) + timestamp, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d") + digest = content_hash or hashlib.sha256(data or b"").hexdigest() + headers = {"host": urllib.parse.urlsplit(self.endpoint).netloc, "x-amz-content-sha256": digest, "x-amz-date": timestamp} + if condition: + headers[condition[0].lower()] = condition[1] + if content_type: + headers["content-type"] = content_type + if size is not None: + headers["content-length"] = str(size) + names = ";".join(sorted(headers)) + canonical_headers = "".join(f"{name}:{headers[name].strip()}\n" for name in sorted(headers)) + canonical = f"{method}\n{path}\n\n{canonical_headers}\n{names}\n{digest}" + scope = f"{day}/auto/s3/aws4_request" + to_sign = f"AWS4-HMAC-SHA256\n{timestamp}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}" + signing = ("AWS4" + self.secret_key).encode() + for part in (day, "auto", "s3", "aws4_request"): + signing = hmac.new(signing, part.encode(), hashlib.sha256).digest() + signature = hmac.new(signing, to_sign.encode(), hashlib.sha256).hexdigest() + headers["authorization"] = f"AWS4-HMAC-SHA256 Credential={self.access_key}/{scope}, SignedHeaders={names}, Signature={signature}" + request = urllib.request.Request(self.endpoint + path, method=method, data=data, headers=headers) + try: + with self.opener.open(request, timeout=120) as response: + response_headers = dict(response.headers.items()) + if consumer is not None: + return consumer(response), response_headers + if method == "HEAD": + return b"", response_headers + body = response.read(maximum + 1) + need(len(body) <= maximum, "oversized R2 response") + return body, response_headers + except urllib.error.HTTPError as error: + raise StoreError(error.code) from None + except (OSError, urllib.error.URLError) as error: + raise PublishError("R2 transport failed") from None + + def get(self, key, maximum=MAX_JSON): + try: + data, headers = self.request("GET", key, maximum=maximum) + except StoreError as error: + if error.status == 404: + return None + raise + etag = next((value for name, value in headers.items() if name.lower() == "etag"), None) + need(etag is not None and len(etag) < 256, "R2 object has no valid ETag") + return data, etag + + def same(self, key, digest, size): + # Read existing content, rather than trusting mutable custom metadata + # or multipart ETags as SHA-256. This is only needed on retries/races. + try: + return self.stream_hash(key, maximum=size) == (digest, size) + except StoreError as error: + if error.status == 404: + return False + raise + + def stream_hash(self, key, maximum=MAX_ARTIFACT): + # A GET can be large: hash a bounded stream, rather than buffering it. + return self.request("GET", key, consumer=lambda response: hash_stream(response, maximum))[0] + + def put(self, key, data, *, condition, content_type="application/octet-stream"): + if isinstance(data, Path): + digest, size = hash_file(data, MAX_JSON if "/metadata/" in key else MAX_ARTIFACT) + with regular_file(data) as source: + self.request("PUT", key, source, digest, size, condition, content_type) + else: + self.request("PUT", key, data, hashlib.sha256(data).hexdigest(), len(data), condition, content_type) + + +def immutable_put(store, key, data, content_type="application/octet-stream", maximum=MAX_ARTIFACT): + if isinstance(data, Path): + digest, size = hash_file(data, maximum) + else: + digest, size = hashlib.sha256(data).hexdigest(), len(data) + # A locked release object may reject an attempted write before evaluating + # its conditional header. Reuse matching bytes without any write attempt. + # Missing/different objects still require a create-only PUT; the second + # comparison handles a writer winning the race after this first read. + if store.same(key, digest, size): + return + try: + store.put(key, data, condition=("If-None-Match", "*"), content_type=content_type) + except StoreError as error: + if error.status != 412: + raise + need(store.same(key, digest, size), "immutable object differs; choose a new release version") + + +def reserve(bundle, store, public_key=None): + bundle = Path(bundle) + data = read_bytes(bundle / "manifest.json", MAX_MANIFEST) + manifest = validate_manifest(data) + bundle_records(bundle, manifest) + metadata_inventory, _ = bundle_metadata(bundle, manifest) + # Reserve exact unsigned bytes before any signing job can use the version. + immutable_put(store, release_key(manifest["version"], "publication-manifest.json"), data, "application/json") + reserved = store.get(release_key(manifest["version"], "publication-manifest.json"), MAX_MANIFEST) + need(reserved is not None and reserved[0] == data, "reserved manifest read-back differs") + immutable_put(store, release_key(manifest["version"], "publication-metadata.json"), metadata_inventory, "application/json") + reserved_metadata = store.get(release_key(manifest["version"], "publication-metadata.json"), MAX_MANIFEST) + need(reserved_metadata is not None and reserved_metadata[0] == metadata_inventory, "reserved metadata read-back differs") + existing = store.get(release_key(manifest["version"], "manifest.json"), MAX_ENVELOPE) + if existing: + envelope, old_manifest = parse_envelope(existing[0]) + need(envelope["manifest"].encode("utf-8") == data, "reserved envelope has different manifest bytes") + if public_key: + minisign_verify(existing[0], public_key) + write_once(bundle / "envelope.json", existing[0]) + receipt = {"manifest_sha256": hashlib.sha256(data).hexdigest(), "metadata_inventory_sha256": hashlib.sha256(metadata_inventory).hexdigest(), "version": manifest["version"]} + write_once(bundle / "reservation.json", encode_json(receipt)) + return manifest + + +def sign(bundle, secret_key, public_key): + bundle = Path(bundle) + need(public_key_text(public_key) == public_key_text(bundle / "public-key.pub"), "signing key differs from the prepared pinned key") + data = read_bytes(bundle / "manifest.json", MAX_MANIFEST) + manifest = validate_manifest(data) + bundle_records(bundle, manifest) + metadata_inventory, _ = bundle_metadata(bundle, manifest) + receipt = decode_json(read_bytes(bundle / "reservation.json", MAX_MANIFEST)) + need(receipt == {"manifest_sha256": hashlib.sha256(data).hexdigest(), "metadata_inventory_sha256": hashlib.sha256(metadata_inventory).hexdigest(), "version": manifest["version"]}, "manifest has not been reserved before signing") + if (bundle / "envelope.json").exists(): + existing = read_bytes(bundle / "envelope.json", MAX_ENVELOPE) + need(parse_envelope(existing)[0]["manifest"].encode("utf-8") == data, "existing envelope differs") + minisign_verify(existing, public_key) + return manifest + with tempfile.TemporaryDirectory(prefix="maple-agent-sign-") as temporary: + message = Path(temporary) / "manifest.json" + signature = Path(temporary) / "signature.minisig" + message.write_bytes(data) + result = subprocess.run(["minisign", "-S", "-W", "-s", str(secret_key), "-m", str(message), "-x", str(signature), "-c", "Maple Agent update", "-t", trusted_comment(manifest["version"])], stdin=subprocess.DEVNULL, capture_output=True, timeout=30) + need(result.returncode == 0, "minisign signing failed") + signature_data = read_bytes(signature, MAX_SIGNATURE).decode("utf-8") + envelope = encode_json({"manifest": data.decode("utf-8"), "signature": signature_data}) + minisign_verify(envelope, public_key) + write_once(bundle / "envelope.json", envelope) + return manifest + + +def publish(bundle, store, public_key, verifier=minisign_verify, previous_verifier=verify_current): + bundle = Path(bundle) + need(public_key_text(public_key) == public_key_text(bundle / "public-key.pub"), "publication key differs from the prepared pinned key") + data = read_bytes(bundle / "manifest.json", MAX_MANIFEST) + envelope = read_bytes(bundle / "envelope.json", MAX_ENVELOPE) + manifest = verifier(envelope, public_key) + need(parse_envelope(envelope)[0]["manifest"].encode("utf-8") == data, "signed and reserved manifest bytes differ") + records = bundle_records(bundle, manifest) + metadata_inventory, metadata_records = bundle_metadata(bundle, manifest) + reserved = store.get(release_key(manifest["version"], "publication-manifest.json"), MAX_MANIFEST) + need(reserved is not None and reserved[0] == data, "version must be reserved before signing and publication") + reserved_metadata = store.get(release_key(manifest["version"], "publication-metadata.json"), MAX_MANIFEST) + need(reserved_metadata is not None and reserved_metadata[0] == metadata_inventory, "metadata must be reserved before signing and publication") + # Snapshot before uploading. An operator changing the pointer while this + # publisher runs must win over this attempt; retry requires fresh review. + pointer = store.get("stable/latest.json", MAX_ENVELOPE) + if pointer: + previous = previous_verifier(pointer[0], public_key) + if pointer[0] == envelope: + # Repair deleted/missing objects with create-only writes; existing + # different bytes remain a hard conflict, even on a feed retry. + # The already matching pointer never needs to be rewritten. + for path, record in records: + immutable_put(store, record["path"][len("/agent/"):], path, maximum=record["size"]) + need(store.same(record["path"][len("/agent/"):], record["sha256"], record["size"]), "published artifact differs or is missing") + publish_metadata(store, manifest["version"], metadata_records) + immutable_put(store, release_key(manifest["version"], "manifest.json"), envelope, "application/json") + immutable = store.get(release_key(manifest["version"], "manifest.json"), MAX_ENVELOPE) + need(immutable is not None and immutable[0] == envelope, "published envelope differs or is missing") + return "already-published" + need(version_tuple(manifest["version"]) > version_tuple(previous["version"]), "release version must advance the published version") + for path, record in records: + immutable_put(store, record["path"][len("/agent/"):], path, maximum=record["size"]) + need(store.same(record["path"][len("/agent/"):], record["sha256"], record["size"]), "uploaded artifact read-back differs") + publish_metadata(store, manifest["version"], metadata_records) + immutable_put(store, release_key(manifest["version"], "manifest.json"), envelope, "application/json") + complete = store.get(release_key(manifest["version"], "manifest.json"), MAX_ENVELOPE) + need(complete is not None and complete[0] == envelope, "complete manifest read-back differs") + condition = ("If-Match", pointer[1]) if pointer else ("If-None-Match", "*") + try: + store.put("stable/latest.json", envelope, condition=condition, content_type="application/json") + except StoreError as error: + if error.status == 412: + raise PublishError("publication pointer changed concurrently; inspect it before retrying") from None + raise + return "published" + + +def live_request(opener, url, *, method="GET", headers=None, maximum=MAX_ENVELOPE): + request = urllib.request.Request(url, method=method, headers=headers or {}) + try: + response = opener.open(request, timeout=120) + except urllib.error.HTTPError as error: + response = error + except (OSError, urllib.error.URLError): + raise PublishError("live endpoint transport failed") from None + with response: + data = response.read(maximum + 1) + need(len(data) <= maximum, "oversized live response") + return response.status, response.headers, data + + +def verify_live(bundle, public_key, base_url): + bundle = Path(bundle) + origin = https_origin(base_url) + envelope = read_bytes(bundle / "envelope.json", MAX_ENVELOPE) + manifest = minisign_verify(envelope, public_key) + bundle_records(bundle, manifest) + opener = urllib.request.build_opener(NoRedirect) + for path, expected in (("/agent/stable/latest.json", envelope), (public_path(manifest["version"], "manifest.json"), envelope)): + status, _, body = live_request(opener, origin + path) + need(status == 200 and body == expected, "live signed metadata differs") + for record in {item["path"]: item for item in [*manifest["artifacts"].values(), *manifest["downloads"].values()]}.values(): + status, headers, _ = live_request(opener, origin + record["path"], method="HEAD", maximum=0) + need(status == 200 and headers.get("Content-Length") == str(record["size"]), "live artifact HEAD failed") + status, headers, part = live_request(opener, origin + record["path"], headers={"Range": "bytes=0-0"}, maximum=1) + need(status == 206 and len(part) == 1 and headers.get("Content-Range") == f"bytes 0-0/{record['size']}", "live artifact range request failed") + request = urllib.request.Request(origin + record["path"]) + try: + with opener.open(request, timeout=120) as response: + need(response.status == 200 and hash_stream(response, record["size"]) == (record["sha256"], record["size"]), "live artifact digest differs") + except (urllib.error.URLError, OSError): + raise PublishError("live artifact download failed") from None + for target, record in manifest["downloads"].items(): + status, headers, _ = live_request(opener, origin + f"/download/agent/stable/{target}", maximum=1024) + need(status in {302, 307} and headers.get("Location") in {record["path"], origin + record["path"]}, "live download redirect differs") + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + trust = commands.add_parser("trust") + trust.add_argument("--run-id", required=True) + trust.add_argument("--linux", action="store_true") + trust.add_argument("--output", required=True) + prep = commands.add_parser("prepare") + prep.add_argument("--run-id", required=True) + prep.add_argument("--version", required=True) + prep.add_argument("--build", required=True) + prep.add_argument("--macos", required=True) + prep.add_argument("--linux") + prep.add_argument("--notes-file", required=True) + prep.add_argument("--pub-date") + prep.add_argument("--output", required=True) + prep.add_argument("--public-key", required=True) + for name in ("reserve", "sign", "publish", "verify-live"): + command = commands.add_parser(name) + command.add_argument("--bundle", required=True) + command.add_argument("--public-key", required=True) + if name in {"reserve", "publish"}: + command.add_argument("--endpoint", required=True) + command.add_argument("--bucket", required=True) + if name == "sign": + command.add_argument("--secret-key", required=True) + if name in {"publish", "verify-live"}: + command.add_argument("--base-url", required=True) + args = parser.parse_args() + if args.command == "trust": + proof = trust_run(args.run_id, args.linux) + write_once(args.output, encode_json(proof)) + print(json.dumps(proof, sort_keys=True)) + elif args.command == "prepare": + manifest = prepare(args.run_id, args.version, args.build, args.macos, args.linux, args.notes_file, args.output, args.public_key, args.pub_date) + print(f"prepared Agent {manifest['version']} from {manifest['source_sha']}") + elif args.command == "reserve": + manifest = reserve(args.bundle, R2Store(args.endpoint, args.bucket), args.public_key) + print(f"reserved Agent {manifest['version']} manifest") + elif args.command == "sign": + manifest = sign(args.bundle, args.secret_key, args.public_key) + print(f"signed Agent {manifest['version']}") + elif args.command == "publish": + result = publish(args.bundle, R2Store(args.endpoint, args.bucket), args.public_key) + verify_live(args.bundle, args.public_key, args.base_url) + print(result + "; live metadata, signature, artifacts, ranges and redirects verified") + elif args.command == "verify-live": + manifest = verify_live(args.bundle, args.public_key, args.base_url) + print(f"verified live Agent {manifest['version']}") + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, KeyError, TypeError, UnicodeError, tarfile.TarError, RecursionError, subprocess.SubprocessError): + # Do not expose subprocess stderr, request URLs/headers, or secret paths. + # PublishError messages are authored here and contain no credentials. + error = sys.exc_info()[1] + print(str(error) if isinstance(error, PublishError) else "publisher operation failed; inspect trusted inputs and credentials", file=sys.stderr) + raise SystemExit(1) diff --git a/services/agent-updates/publisher/test_minisign.py b/services/agent-updates/publisher/test_minisign.py new file mode 100644 index 000000000..64cd4ac09 --- /dev/null +++ b/services/agent-updates/publisher/test_minisign.py @@ -0,0 +1,176 @@ +"""Verify the shared public wire fixture with the pinned real minisign CLI.""" + +import base64 +import json +import hashlib +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import publisher + + +FIXTURES = Path(__file__).with_name("fixtures") + + +@unittest.skipUnless(shutil.which("minisign"), "run in the pinned root CI shell for cryptographic tests") +class MinisignTests(unittest.TestCase): + def setUp(self): + self.data = (FIXTURES / "signed-envelope.json").read_bytes() + self.key = FIXTURES / "public-test-key.pub" + + def test_shared_client_fixture_verifies_prehashed_with_exact_comment(self): + manifest = publisher.minisign_verify(self.data, self.key) + self.assertEqual(manifest["version"], "0.0.2") + envelope = json.loads(self.data) + self.assertEqual(envelope["manifest"].encode(), (FIXTURES / "manifest.json").read_bytes()) + self.assertEqual(envelope["signature"].encode(), (FIXTURES / "manifest.minisig").read_bytes()) + + def test_raw_public_key_file_verifies_with_the_real_cli(self): + with tempfile.TemporaryDirectory(prefix="maple-agent-public-key-test-") as temporary: + key = Path(temporary) / "one-line.pub" + key.write_text(publisher.public_key_text(self.key) + "\n") + self.assertEqual(publisher.minisign_verify(self.data, key)["version"], "0.0.2") + + def test_existing_signed_pointer_rotation_retry_and_missing_object_repair(self): + class Store: + def __init__(self): + self.objects = {} + self.writes = [] + + def get(self, key, maximum=publisher.MAX_JSON): + if key not in self.objects: + return None + data = self.objects[key] + return data, '"' + hashlib.sha256(data).hexdigest() + '"' + + def same(self, key, digest, size): + value = self.objects.get(key) + return value is not None and len(value) == size and hashlib.sha256(value).hexdigest() == digest + + def put(self, key, data, *, condition, content_type): + current = self.get(key) + if condition == ("If-None-Match", "*"): + if current is not None: + raise publisher.StoreError(412) + elif current is None or condition != ("If-Match", current[1]): + raise publisher.StoreError(412) + self.writes.append(key) + self.objects[key] = data.read_bytes() if isinstance(data, Path) else data + + with tempfile.TemporaryDirectory(prefix="maple-agent-pointer-test-") as temporary: + root = Path(temporary) + bundle = root / "bundle" + bundle.mkdir() + (bundle / "manifest.json").write_bytes((FIXTURES / "manifest.json").read_bytes()) + (bundle / "envelope.json").write_bytes(self.data) + # Match the workflow variable's raw one-line public-key input. + (bundle / "public-key.pub").write_text(publisher.public_key_text(self.key) + "\n") + payloads = bundle / "payloads" + payloads.mkdir() + for name in ("Maple-Agent.app.tar.gz", "Maple-Agent.dmg"): + (payloads / name).write_bytes(b"signed payload fixture bytes") + metadata = bundle / "metadata" + metadata.mkdir() + inventory = {} + for name in sorted({"provenance.json"} | publisher.MACOS_METADATA_NAMES): + data = ("offline public signing fixture: " + name + "\n").encode() + (metadata / name).write_bytes(data) + inventory[name] = {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)} + (bundle / "metadata-inventory.json").write_bytes(publisher.encode_json(inventory)) + profile = root / "apps/maple-agent/release-profiles.json" + profile.parent.mkdir(parents=True) + # The first valid public key does not sign this envelope, exercising + # the pinned-key fallback used during rotation with the real CLI. + other_key = base64.b64encode(b"Ed" + bytes(40)).decode() + profile.write_bytes(publisher.encode_json({"prod": { + "update_feed_url": publisher.FEED_URL, + "update_public_keys": [other_key, publisher.public_key_text(self.key)], + }})) + store = Store() + with patch.object(publisher, "ROOT", root): + publisher.reserve(bundle, store, self.key) + self.assertEqual(publisher.publish(bundle, store, self.key), "published") + store.writes.clear() + self.assertEqual(publisher.publish(bundle, store, self.key), "already-published") + self.assertEqual(store.writes, []) + missing = "stable/releases/0.0.2/Maple-Agent.app.tar.gz" + del store.objects[missing] + self.assertEqual(publisher.publish(bundle, store, self.key), "already-published") + self.assertEqual(store.writes, [missing]) + self.assertEqual(store.objects["stable/latest.json"], self.data) + tampered = json.loads(self.data) + tampered["manifest"] = tampered["manifest"].replace("A test update.", "A fake update.") + with self.assertRaisesRegex(publisher.PublishError, "not trusted"): + publisher.verify_current(publisher.encode_json(tampered), self.key) + + def test_tampered_signed_bytes_and_comment_are_rejected(self): + envelope = json.loads(self.data) + envelope["manifest"] = envelope["manifest"].replace("A test update.", "A fake update.") + with self.assertRaisesRegex(publisher.PublishError, "verification failed"): + publisher.minisign_verify(publisher.encode_json(envelope), self.key) + envelope = json.loads(self.data) + envelope["signature"] = envelope["signature"].replace("channel=stable", "channel=preview") + with self.assertRaisesRegex(publisher.PublishError, "comment"): + publisher.minisign_verify(publisher.encode_json(envelope), self.key) + + def test_real_temporary_key_reservation_signing_verification_and_retry(self): + class ReservationStore: + def __init__(self): + self.objects = {} + + def put(self, key, data, *, condition, content_type): + self.objects[key] = data + + def get(self, key, maximum=publisher.MAX_JSON): + return (self.objects[key], '"fixture-etag"') if key in self.objects else None + + def same(self, key, digest, size): + data = self.objects.get(key) + return data is not None and len(data) == size and hashlib.sha256(data).hexdigest() == digest + + # These disposable, offline test keys never leave the private tempdir. + # Production key generation is an owner operation outside this tool. + with tempfile.TemporaryDirectory(prefix="maple-agent-minisign-test-") as temporary: + bundle = Path(temporary) + key = bundle / "public-key.pub" + secret = bundle / "test-only-secret.key" + result = subprocess.run(["minisign", "-G", "-W", "-p", str(key), "-s", str(secret)], stdin=subprocess.DEVNULL, capture_output=True, timeout=30) + self.assertEqual(result.returncode, 0, "offline temporary test key generation failed") + (bundle / "manifest.json").write_bytes((FIXTURES / "manifest.json").read_bytes()) + payloads = bundle / "payloads" + payloads.mkdir() + for name in ("Maple-Agent.app.tar.gz", "Maple-Agent.dmg"): + (payloads / name).write_bytes(b"signed payload fixture bytes") + metadata = bundle / "metadata" + metadata.mkdir() + inventory = {} + for name in sorted({"provenance.json"} | publisher.MACOS_METADATA_NAMES): + data = ("offline signing test sidecar: " + name + "\n").encode() + (metadata / name).write_bytes(data) + inventory[name] = {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)} + (bundle / "metadata-inventory.json").write_bytes(publisher.encode_json(inventory)) + store = ReservationStore() + publisher.reserve(bundle, store) + self.assertIn("stable/releases/0.0.2/publication-manifest.json", store.objects) + publisher.sign(bundle, secret, key) + original = (bundle / "envelope.json").read_bytes() + self.assertEqual(publisher.minisign_verify(original, key)["version"], "0.0.2") + publisher.sign(bundle, secret, key) + self.assertEqual((bundle / "envelope.json").read_bytes(), original) + # A repeated signing attempt is deterministic: the exact trusted + # comment and untrusted comment contain no implicit timestamp. + (bundle / "envelope.json").unlink() + publisher.sign(bundle, secret, key) + self.assertEqual((bundle / "envelope.json").read_bytes(), original) + envelope = json.loads(original) + envelope["manifest"] = envelope["manifest"].replace("A test update.", "A fake update.") + with self.assertRaisesRegex(publisher.PublishError, "verification failed"): + publisher.minisign_verify(publisher.encode_json(envelope), key) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/agent-updates/publisher/test_publisher.py b/services/agent-updates/publisher/test_publisher.py new file mode 100644 index 000000000..058069ff8 --- /dev/null +++ b/services/agent-updates/publisher/test_publisher.py @@ -0,0 +1,1023 @@ +"""Offline publication recovery and untrusted artifact boundary tests. + +The fake store models conditional writes, interrupted uploads and concurrent +pointer changes. Its synthetic signatures exercise envelope plumbing only; +cryptographic verification remains an explicitly injected boundary here. +""" + +import base64 +import gzip +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import plistlib +import tarfile +import tempfile +import unittest +from unittest.mock import patch +from urllib.parse import urlsplit + + +SPEC = importlib.util.spec_from_file_location("agent_publisher", Path(__file__).with_name("publisher.py")) +publisher = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(publisher) +SOURCE_SHA = "a" * 40 +PUBLIC_KEY = base64.b64encode(b"Ed" + bytes(40)).decode() +MACOS_METADATA = ( + "macos-SHA256SUMS", "macos-build-info.json", "macos-dmg-notarization-log.json", + "macos-manifest.json", "macos-notarization-log.json", "provenance.json", +) +LINUX_METADATA = ("linux-SHA256SUMS", "linux-build-info.json", "linux-manifest.json") + + +def synthetic_envelope(manifest): + signature = "\n".join(( + "untrusted comment: offline fixture, not a real signature", + base64.b64encode(b"ED" + bytes(72)).decode(), + "trusted comment: " + publisher.trusted_comment(manifest["version"]), + base64.b64encode(bytes(64)).decode(), + "", + )) + return publisher.encode_json({ + "manifest": publisher.encode_json(manifest).decode(), + "signature": signature, + }) + + +def fixture_verifier(data, public_key): + """A structural verifier: intentionally no cryptographic claims.""" + if publisher.public_key_text(public_key) != PUBLIC_KEY: + raise publisher.PublishError("unexpected verification key") + return publisher.parse_envelope(data)[1] + + +class FakeStore: + def __init__(self): + self.objects = {} + self.revisions = {} + self.calls = [] + self.fail_once = None + self.corrupt_once = None + self.before_pointer = None + + def set(self, key, data): + self.objects[key] = data + self.revisions[key] = self.revisions.get(key, 0) + 1 + + def etag(self, key): + return '"' + str(self.revisions[key]) + '"' + + def get(self, key, maximum=publisher.MAX_JSON): + self.calls.append(("get", key)) + if key not in self.objects: + return None + publisher.need(len(self.objects[key]) <= maximum, "oversized fake object") + return self.objects[key], self.etag(key) + + def same(self, key, digest, size): + self.calls.append(("same", key)) + if key not in self.objects: + return False + data = self.objects[key] + return len(data) == size and hashlib.sha256(data).hexdigest() == digest + + def put(self, key, data, *, condition, content_type="application/octet-stream"): + self.calls.append(("put", key, condition, content_type)) + if key == "stable/latest.json" and self.before_pointer: + callback, self.before_pointer = self.before_pointer, None + callback() + if key == self.fail_once: + self.fail_once = None + raise publisher.StoreError(503) + header, value = condition + if header == "If-None-Match": + if value != "*" or key in self.objects: + raise publisher.StoreError(412) + elif header == "If-Match": + if key not in self.objects or value != self.etag(key): + raise publisher.StoreError(412) + else: + raise AssertionError("unconditional write") + value = data.read_bytes() if isinstance(data, Path) else data + if key == self.corrupt_once: + self.corrupt_once = None + value = b"acknowledged but corrupted object" + self.set(key, value) + + +class FakeResponse(io.BytesIO): + def __init__(self, data, status=200, headers=None): + super().__init__(data) + self.status = status + self.headers = headers or {} + + +class FixtureOrigin: + """The exact public route contract, with no network or executable input.""" + + def __init__(self, bundle, manifest, envelope): + self.bundle, self.manifest, self.envelope = bundle, manifest, envelope + self.calls = [] + self.redirect = None + self.range_header = None + + def open(self, request, timeout): + self.assert_origin(request.full_url) + path = urlsplit(request.full_url).path + method = request.get_method() + range_header = request.get_header("Range") + self.calls.append((method, path, range_header)) + immutable = publisher.public_path(self.manifest["version"], "manifest.json") + if path in {"/agent/stable/latest.json", immutable} and method == "GET": + return FakeResponse(self.envelope) + records = [*self.manifest["artifacts"].values(), *self.manifest["downloads"].values()] + record = next((item for item in records if item["path"] == path), None) + if record: + data = (self.bundle / "payloads" / path.rsplit("/", 1)[1]).read_bytes() + if method == "HEAD": + return FakeResponse(b"", headers={"Content-Length": str(len(data))}) + if method == "GET" and range_header == "bytes=0-0": + return FakeResponse(data[:1], status=206, headers={ + "Content-Range": self.range_header or f"bytes 0-0/{len(data)}", + }) + if method == "GET" and range_header is None: + return FakeResponse(data) + if path == "/download/agent/stable/macos" and method == "GET": + return FakeResponse(b"", status=302, headers={ + "Location": self.redirect or self.manifest["downloads"]["macos"]["path"], + }) + return FakeResponse(b"404\n", status=404) + + def assert_origin(self, url): + if not url.startswith("https://updates.example/"): + raise AssertionError("unexpected live verification origin") + + +class PublicationTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.store = FakeStore() + self.bundle, self.manifest, self.envelope = self.make_bundle("1.2.3", "first") + + def make_bundle(self, version, name, notes="Release fixture", linux=False): + bundle = self.root / name + payloads = bundle / "payloads" + payloads.mkdir(parents=True) + artifacts = {} + binaries = [("agent.app.tar.gz", b"archive fixture"), ("agent.dmg", b"disk image fixture")] + if linux: + binaries.append(("agent.AppImage", b"AppImage fixture")) + for filename, data in binaries: + (payloads / filename).write_bytes(data) + artifacts[filename] = { + "path": publisher.public_path(version, filename), + "size": len(data), "sha256": hashlib.sha256(data).hexdigest(), + } + manifest = { + "format": "maple-agent-update-v1", "product": "maple-agent", + "channel": "stable", "version": version, "build": "42", + "source_sha": SOURCE_SHA, "pub_date": "2026-10-21T18:00:00Z", + "notes": notes, "artifacts": {"darwin-aarch64": artifacts["agent.app.tar.gz"]}, + "downloads": {"macos": artifacts["agent.dmg"]}, + } + if linux: + manifest["artifacts"]["linux-x86_64"] = artifacts["agent.AppImage"] + manifest["downloads"]["linux-appimage"] = artifacts["agent.AppImage"] + envelope = synthetic_envelope(manifest) + (bundle / "manifest.json").write_bytes(publisher.encode_json(manifest)) + (bundle / "envelope.json").write_bytes(envelope) + (bundle / "public-key.pub").write_text(PUBLIC_KEY + "\n") + metadata = bundle / "metadata" + metadata.mkdir() + for filename in (*MACOS_METADATA, *(LINUX_METADATA if linux else ())): + data = b"offline checksum fixture\n" if filename.endswith("SHA256SUMS") else \ + publisher.encode_json({"fixture": filename, "version": version}) + (metadata / filename).write_bytes(data) + self.index_metadata(bundle) + return bundle, manifest, envelope + + def reserve(self): + return publisher.reserve(self.bundle, self.store) + + def publish(self, bundle=None): + selected = bundle or self.bundle + return publisher.publish(selected, self.store, selected / "public-key.pub", + fixture_verifier, previous_verifier=fixture_verifier) + + def release_key(self, filename): + return publisher.release_key("1.2.3", filename) + + def metadata_key(self, filename="macos-build-info.json"): + return self.release_key("metadata") + "/" + filename + + def add_metadata(self, bundle=None, data=b'{"profile":"prod","version":"1.2.3"}\n'): + bundle = bundle or self.bundle + directory = bundle / "metadata" + directory.mkdir(exist_ok=True) + (directory / "macos-build-info.json").write_bytes(data) + return self.index_metadata(bundle) + + def index_metadata(self, bundle): + inventory = {path.name: {"sha256": hashlib.sha256(path.read_bytes()).hexdigest(), "size": path.stat().st_size} + for path in sorted((bundle / "metadata").iterdir())} + (bundle / "metadata-inventory.json").write_bytes(publisher.encode_json(inventory)) + return inventory + + def test_reservation_precedes_artifacts_and_complete_manifest_precedes_pointer(self): + self.reserve() + self.assertEqual(self.publish(), "published") + writes = [call[1] for call in self.store.calls if call[0] == "put"] + self.assertEqual(writes, [ + self.release_key("publication-manifest.json"), + self.release_key("publication-metadata.json"), + self.release_key("agent.app.tar.gz"), self.release_key("agent.dmg"), + *[self.metadata_key(name) for name in MACOS_METADATA], + self.release_key("manifest.json"), "stable/latest.json", + ]) + self.assertEqual(self.store.objects["stable/latest.json"], self.envelope) + self.assertTrue(all(call[2] == ("If-None-Match", "*") for call in self.store.calls if call[0] == "put")) + + def test_each_payload_and_complete_manifest_are_read_back_before_promotion(self): + self.reserve() + self.publish() + complete = next(index for index, call in enumerate(self.store.calls) + if call[:2] == ("put", self.release_key("manifest.json"))) + pointer = next(index for index, call in enumerate(self.store.calls) + if call[:2] == ("put", "stable/latest.json")) + for name in ("agent.app.tar.gz", "agent.dmg"): + verification = next(index for index, call in enumerate(self.store.calls) + if call == ("same", self.release_key(name))) + self.assertLess(verification, complete) + marker_readback = [index for index, call in enumerate(self.store.calls) + if call == ("get", self.release_key("manifest.json")) and index > complete] + self.assertTrue(marker_readback) + self.assertLess(marker_readback[0], pointer) + + def test_acknowledged_corrupt_payload_cannot_be_marked_complete_or_promoted(self): + self.reserve() + self.store.corrupt_once = self.release_key("agent.app.tar.gz") + with self.assertRaises(publisher.PublishError): + self.publish() + self.assertNotIn(self.release_key("manifest.json"), self.store.objects) + self.assertNotIn("stable/latest.json", self.store.objects) + + def test_acknowledged_corrupt_complete_manifest_cannot_be_promoted(self): + self.reserve() + self.store.corrupt_once = self.release_key("manifest.json") + with self.assertRaises(publisher.PublishError): + self.publish() + self.assertNotIn("stable/latest.json", self.store.objects) + + def test_acknowledged_corrupt_reservation_cannot_receive_signing_receipt(self): + self.store.corrupt_once = self.release_key("publication-manifest.json") + with self.assertRaises(publisher.PublishError): + self.reserve() + self.assertFalse((self.bundle / "reservation.json").exists()) + + def test_private_metadata_inventory_is_reserved_and_files_verified_before_complete(self): + inventory = self.add_metadata() + self.reserve() + self.assertEqual(self.publish(), "published") + index = publisher.encode_json(inventory) + self.assertEqual(self.store.objects[self.release_key("publication-metadata.json")], index) + self.assertEqual(self.store.objects[self.metadata_key()], (self.bundle / "metadata/macos-build-info.json").read_bytes()) + receipt = json.loads((self.bundle / "reservation.json").read_text()) + self.assertEqual(receipt["metadata_inventory_sha256"], hashlib.sha256(index).hexdigest()) + complete = next(position for position, call in enumerate(self.store.calls) + if call[:2] == ("put", self.release_key("manifest.json"))) + for name in MACOS_METADATA: + verification = next(position for position, call in enumerate(self.store.calls) + if call == ("same", self.metadata_key(name))) + self.assertLess(verification, complete) + # Audit sidecars remain outside the signed public artifact inventory. + self.assertFalse(any("/metadata/" in record["path"] for record in + [*self.manifest["artifacts"].values(), *self.manifest["downloads"].values()])) + + def test_empty_metadata_inventory_cannot_reserve_a_production_release(self): + for path in (self.bundle / "metadata").iterdir(): + path.unlink() + (self.bundle / "metadata-inventory.json").write_bytes(publisher.encode_json({})) + with self.assertRaisesRegex(publisher.PublishError, "exact sidecars"): + self.reserve() + self.assertFalse(self.store.objects) + + def test_missing_or_subset_metadata_inventory_cannot_bypass_required_audit_data(self): + inventory = json.loads((self.bundle / "metadata-inventory.json").read_text()) + (self.bundle / "metadata-inventory.json").unlink() + with self.assertRaisesRegex(publisher.PublishError, "metadata has no inventory"): + self.reserve() + (self.bundle / "metadata").rename(self.bundle / "saved-metadata") + with self.assertRaisesRegex(publisher.PublishError, "metadata has no inventory"): + self.reserve() + (self.bundle / "saved-metadata").rename(self.bundle / "metadata") + for name in MACOS_METADATA: + with self.subTest(missing=name): + subset = {key: record for key, record in inventory.items() if key != name} + (self.bundle / "metadata-inventory.json").write_bytes(publisher.encode_json(subset)) + with self.assertRaisesRegex(publisher.PublishError, "exact sidecars"): + self.reserve() + self.assertFalse(self.store.objects) + + def test_metadata_inventory_exactly_tracks_linux_presence_in_signed_manifest(self): + for variant in ("missing-linux", "partial-linux", "unexpected-linux"): + with self.subTest(variant=variant): + bundle, _, _ = self.make_bundle("1.2.3", variant, linux=variant != "unexpected-linux") + if variant == "unexpected-linux": + for name in LINUX_METADATA: + (bundle / "metadata" / name).write_text("offline Linux metadata\n") + else: + for name in (LINUX_METADATA if variant == "missing-linux" else ("linux-manifest.json",)): + (bundle / "metadata" / name).unlink() + self.index_metadata(bundle) + with self.assertRaisesRegex(publisher.PublishError, "exact sidecars"): + publisher.reserve(bundle, self.store) + self.assertFalse(self.store.objects) + + def test_linux_bundle_includes_and_publishes_all_required_private_sidecars(self): + bundle, _, _ = self.make_bundle("1.2.3", "complete-linux", linux=True) + publisher.reserve(bundle, self.store) + self.assertEqual(self.publish(bundle), "published") + for name in (*MACOS_METADATA, *LINUX_METADATA): + self.assertEqual(self.store.objects[self.metadata_key(name)], (bundle / "metadata" / name).read_bytes()) + + def test_metadata_missing_corrupt_unchecked_and_symlink_files_fail_before_reservation(self): + for variant in ("missing", "corrupt", "unchecked", "symlink"): + with self.subTest(variant=variant): + bundle, _, _ = self.make_bundle("1.2.3", "metadata-" + variant) + self.add_metadata(bundle) + metadata = bundle / "metadata/macos-build-info.json" + if variant == "missing": + metadata.unlink() + elif variant == "corrupt": + metadata.write_bytes(b"different bytes") + elif variant == "unchecked": + (bundle / "metadata/linux-build-info.json").write_text("{}") + else: + metadata.unlink() + metadata.symlink_to(bundle / "manifest.json") + with self.assertRaises((publisher.PublishError, OSError)): + publisher.reserve(bundle, self.store) + self.assertFalse(self.store.objects) + + def test_private_metadata_directory_requires_index_and_cannot_be_a_symlink(self): + self.add_metadata() + index = (self.bundle / "metadata-inventory.json").read_bytes() + (self.bundle / "metadata-inventory.json").unlink() + with self.assertRaisesRegex(publisher.PublishError, "metadata has no inventory"): + self.reserve() + (self.bundle / "metadata-inventory.json").write_bytes(index) + directory = self.bundle / "metadata" + directory.rename(self.bundle / "other-metadata") + directory.symlink_to(self.bundle / "other-metadata", target_is_directory=True) + with self.assertRaises(publisher.PublishError): + self.reserve() + self.assertFalse(self.store.objects) + + def test_acknowledged_corrupt_metadata_index_cannot_receive_signing_receipt(self): + self.add_metadata() + self.store.corrupt_once = self.release_key("publication-metadata.json") + with self.assertRaisesRegex(publisher.PublishError, "reserved metadata read-back differs"): + self.reserve() + self.assertFalse((self.bundle / "reservation.json").exists()) + self.assertNotIn(self.release_key("manifest.json"), self.store.objects) + self.assertNotIn("stable/latest.json", self.store.objects) + + def test_publication_requires_reserved_metadata_index_before_writing_payloads(self): + self.add_metadata() + self.reserve() + del self.store.objects[self.release_key("publication-metadata.json")] + self.store.calls.clear() + with self.assertRaisesRegex(publisher.PublishError, "metadata must be reserved"): + self.publish() + self.assertFalse(any(call[0] == "put" for call in self.store.calls)) + + def test_unallowlisted_metadata_or_invalid_hash_index_is_rejected_before_reservation(self): + valid = self.add_metadata() + invalid = ( + {**valid, "unexpected.json": valid["macos-build-info.json"]}, + {**valid, "../macos-build-info.json": valid["macos-build-info.json"]}, + {**valid, "macos-build-info.json": {**valid["macos-build-info.json"], "sha256": "not-a-digest"}}, + {**valid, "macos-build-info.json": {**valid["macos-build-info.json"], "size": True}}, + {**valid, "macos-build-info.json": {**valid["macos-build-info.json"], "extra": "unexpected"}}, + ) + for inventory in invalid: + with self.subTest(inventory=inventory): + (self.bundle / "metadata-inventory.json").write_bytes(publisher.encode_json(inventory)) + with self.assertRaises(publisher.PublishError): + self.reserve() + self.assertFalse(self.store.objects) + + def test_metadata_index_cannot_change_after_reservation_even_with_same_signed_manifest(self): + self.add_metadata() + self.reserve() + self.add_metadata(data=b'{"profile":"prod","version":"1.2.3","changed":true}\n') + self.store.calls.clear() + with self.assertRaises(publisher.PublishError): + self.publish() + self.assertFalse(any(call[0] == "put" for call in self.store.calls)) + self.assertNotIn("stable/latest.json", self.store.objects) + + def test_metadata_upload_failure_leaves_no_complete_release_and_resumes_identically(self): + self.add_metadata() + self.reserve() + self.store.fail_once = self.metadata_key() + with self.assertRaises(publisher.StoreError): + self.publish() + self.assertNotIn(self.release_key("manifest.json"), self.store.objects) + self.assertNotIn("stable/latest.json", self.store.objects) + self.assertEqual(self.publish(), "published") + self.assertEqual(self.store.revisions[self.release_key("agent.app.tar.gz")], 1) + self.assertEqual(self.store.objects[self.metadata_key()], (self.bundle / "metadata/macos-build-info.json").read_bytes()) + + def test_acknowledged_corrupt_private_metadata_cannot_be_marked_complete(self): + self.add_metadata() + self.reserve() + self.store.corrupt_once = self.metadata_key() + with self.assertRaises(publisher.PublishError): + self.publish() + self.assertNotIn(self.release_key("manifest.json"), self.store.objects) + self.assertNotIn("stable/latest.json", self.store.objects) + + def test_published_retry_repairs_missing_private_metadata_without_pointer_rewrite(self): + self.add_metadata() + self.reserve() + self.publish() + del self.store.objects[self.metadata_key()] + self.store.calls.clear() + self.assertEqual(self.publish(), "already-published") + self.assertEqual(self.store.objects[self.metadata_key()], (self.bundle / "metadata/macos-build-info.json").read_bytes()) + self.assertEqual(self.store.revisions["stable/latest.json"], 1) + self.assertFalse(any(call[:2] == ("put", "stable/latest.json") for call in self.store.calls)) + + def test_published_retry_rejects_existing_corrupt_private_metadata(self): + self.add_metadata() + self.reserve() + self.publish() + self.store.set(self.metadata_key(), b"corrupt audit metadata") + self.store.calls.clear() + with self.assertRaisesRegex(publisher.PublishError, "immutable object differs"): + self.publish() + self.assertEqual(self.store.objects[self.metadata_key()], b"corrupt audit metadata") + self.assertEqual(self.store.revisions["stable/latest.json"], 1) + self.assertFalse(any(call[:2] == ("put", "stable/latest.json") for call in self.store.calls)) + + def test_retry_resumes_partial_upload_without_discoverable_incomplete_release(self): + self.reserve() + self.store.fail_once = self.release_key("agent.dmg") + with self.assertRaises(publisher.StoreError): + self.publish() + first = self.store.objects[self.release_key("agent.app.tar.gz")] + self.assertNotIn(self.release_key("manifest.json"), self.store.objects) + self.assertNotIn("stable/latest.json", self.store.objects) + self.assertEqual(self.publish(), "published") + self.assertEqual(self.store.objects[self.release_key("agent.app.tar.gz")], first) + self.assertEqual(self.store.revisions[self.release_key("agent.app.tar.gz")], 1) + self.assertIn(("same", self.release_key("agent.app.tar.gz")), self.store.calls) + + def test_retry_after_complete_manifest_before_promotion_reuses_exact_envelope(self): + self.reserve() + self.store.fail_once = "stable/latest.json" + with self.assertRaises(publisher.StoreError): + self.publish() + self.assertEqual(self.store.objects[self.release_key("manifest.json")], self.envelope) + self.assertNotIn("stable/latest.json", self.store.objects) + self.assertEqual(self.publish(), "published") + self.assertEqual(self.store.revisions[self.release_key("manifest.json")], 1) + + def test_existing_immutable_different_bytes_fail_before_pointer(self): + self.reserve() + self.store.set(self.release_key("agent.app.tar.gz"), b"different archive") + with self.assertRaisesRegex(publisher.PublishError, "immutable object differs"): + self.publish() + self.assertNotIn("stable/latest.json", self.store.objects) + self.assertEqual(self.store.objects[self.release_key("agent.app.tar.gz")], b"different archive") + + def test_existing_complete_manifest_different_bytes_are_not_overwritten(self): + self.reserve() + self.store.set(self.release_key("manifest.json"), b"other envelope") + with self.assertRaisesRegex(publisher.PublishError, "immutable object differs"): + self.publish() + self.assertNotIn("stable/latest.json", self.store.objects) + + def test_pointer_race_preserves_competing_promotion(self): + _, old_manifest, old_envelope = self.make_bundle("1.2.2", "old") + _, newer_manifest, newer_envelope = self.make_bundle("1.2.4", "newer") + self.assertLess(publisher.version_tuple(old_manifest["version"]), publisher.version_tuple(newer_manifest["version"])) + self.store.set("stable/latest.json", old_envelope) + self.reserve() + self.store.before_pointer = lambda: self.store.set("stable/latest.json", newer_envelope) + with self.assertRaisesRegex(publisher.PublishError, "pointer changed concurrently"): + self.publish() + self.assertEqual(self.store.objects["stable/latest.json"], newer_envelope) + pointer_write = [call for call in self.store.calls if call[:2] == ("put", "stable/latest.json")][-1] + self.assertEqual(pointer_write[2], ("If-Match", '"1"')) + + def test_creation_race_preserves_operator_pointer(self): + _, _, newer_envelope = self.make_bundle("1.2.4", "newer") + self.reserve() + self.store.before_pointer = lambda: self.store.set("stable/latest.json", newer_envelope) + with self.assertRaisesRegex(publisher.PublishError, "pointer changed concurrently"): + self.publish() + self.assertEqual(self.store.objects["stable/latest.json"], newer_envelope) + + def test_identical_published_release_is_verified_no_op(self): + self.reserve() + self.publish() + revisions = self.store.revisions.copy() + self.store.calls.clear() + self.assertEqual(self.publish(), "already-published") + self.assertFalse(any(call[:2] == ("put", "stable/latest.json") for call in self.store.calls)) + self.assertEqual(self.store.revisions, revisions) + self.assertTrue({ + self.release_key("agent.app.tar.gz"), self.release_key("agent.dmg"), + } <= {call[1] for call in self.store.calls if call[0] == "same"}) + + def test_identical_locked_release_retry_never_attempts_an_overwrite(self): + self.reserve() + self.publish() + self.store.calls.clear() + original_put = self.store.put + + def locked_put(key, data, **kwargs): + if key in self.store.objects: + raise publisher.StoreError(403) + return original_put(key, data, **kwargs) + + with patch.object(self.store, "put", side_effect=locked_put): + publisher.reserve(self.bundle, self.store) + self.assertEqual(self.publish(), "already-published") + self.assertFalse(any(call[0] == "put" for call in self.store.calls)) + missing = self.release_key("agent.app.tar.gz") + del self.store.objects[missing] + self.assertEqual(self.publish(), "already-published") + self.assertEqual([call[1] for call in self.store.calls if call[0] == "put"], [missing]) + self.assertEqual(self.store.revisions["stable/latest.json"], 1) + + def test_immutable_create_race_reuses_only_identical_bytes(self): + key, data = self.release_key("race.bin"), b"release fixture" + original_put = self.store.put + for winning in (data, b"different content"): + with self.subTest(winning=winning): + self.store.objects.pop(key, None) + def racing_put(name, value, **kwargs): + self.store.set(name, winning) + return original_put(name, value, **kwargs) + with patch.object(self.store, "put", side_effect=racing_put): + if winning == data: + publisher.immutable_put(self.store, key, data) + else: + with self.assertRaisesRegex(publisher.PublishError, "immutable object differs"): + publisher.immutable_put(self.store, key, data) + self.assertEqual(self.store.objects[key], winning) + + def test_immutable_transport_or_authorization_failure_never_becomes_success(self): + for status in (403, 409, 500): + with self.subTest(status=status), patch.object(self.store, "put", side_effect=publisher.StoreError(status)): + with self.assertRaises(publisher.StoreError) as error: + publisher.immutable_put(self.store, self.release_key("missing.bin"), b"fixture") + self.assertEqual(error.exception.status, status) + with patch.object(self.store, "same", side_effect=publisher.StoreError(403)), \ + patch.object(self.store, "put") as write: + with self.assertRaises(publisher.StoreError): + publisher.immutable_put(self.store, self.release_key("missing.bin"), b"fixture") + write.assert_not_called() + + def test_identical_pointer_repairs_missing_artifact_without_rewriting_pointer(self): + self.reserve() + self.publish() + del self.store.objects[self.release_key("agent.app.tar.gz")] + self.store.calls.clear() + self.assertEqual(self.publish(), "already-published") + self.assertEqual(self.store.objects[self.release_key("agent.app.tar.gz")], + (self.bundle / "payloads/agent.app.tar.gz").read_bytes()) + self.assertEqual(self.store.objects["stable/latest.json"], self.envelope) + self.assertEqual(self.store.revisions["stable/latest.json"], 1) + self.assertFalse(any(call[:2] == ("put", "stable/latest.json") for call in self.store.calls)) + + def test_identical_pointer_does_not_overwrite_corrupt_existing_artifact(self): + self.reserve() + self.publish() + self.store.set(self.release_key("agent.app.tar.gz"), b"corrupt archive") + self.store.calls.clear() + with self.assertRaisesRegex(publisher.PublishError, "immutable object differs"): + self.publish() + self.assertEqual(self.store.objects[self.release_key("agent.app.tar.gz")], b"corrupt archive") + self.assertEqual(self.store.objects["stable/latest.json"], self.envelope) + self.assertEqual(self.store.revisions["stable/latest.json"], 1) + self.assertFalse(any(call[:2] == ("put", "stable/latest.json") for call in self.store.calls)) + + def test_identical_pointer_repairs_missing_complete_manifest_without_pointer_rewrite(self): + self.reserve() + self.publish() + del self.store.objects[self.release_key("manifest.json")] + self.store.calls.clear() + self.assertEqual(self.publish(), "already-published") + self.assertEqual(self.store.objects[self.release_key("manifest.json")], self.envelope) + self.assertEqual(self.store.objects["stable/latest.json"], self.envelope) + self.assertEqual(self.store.revisions["stable/latest.json"], 1) + self.assertFalse(any(call[:2] == ("put", "stable/latest.json") for call in self.store.calls)) + + def test_identical_pointer_does_not_overwrite_corrupt_complete_manifest(self): + self.reserve() + self.publish() + self.store.set(self.release_key("manifest.json"), b"corrupt complete manifest") + self.store.calls.clear() + with self.assertRaisesRegex(publisher.PublishError, "immutable object differs"): + self.publish() + self.assertEqual(self.store.objects[self.release_key("manifest.json")], b"corrupt complete manifest") + self.assertEqual(self.store.revisions["stable/latest.json"], 1) + self.assertFalse(any(call[:2] == ("put", "stable/latest.json") for call in self.store.calls)) + + def test_equal_or_lower_version_with_different_bytes_is_rejected(self): + self.reserve() + self.publish() + for version in ("1.2.3", "1.2.2"): + with self.subTest(version=version): + bundle, _, _ = self.make_bundle(version, "other-" + version, "changed notes") + if version == "1.2.2": + publisher.reserve(bundle, self.store) + self.store.calls.clear() + with self.assertRaisesRegex(publisher.PublishError, "reserved before|must advance"): + self.publish(bundle) + self.assertFalse(any(call[0] == "put" for call in self.store.calls)) + + def test_equal_version_different_envelope_is_not_treated_as_published_no_op(self): + self.reserve() + self.publish() + envelope = json.loads(self.envelope) + envelope["signature"] = envelope["signature"].replace("offline fixture, not a real signature", "different fixture comment") + (self.bundle / "envelope.json").write_bytes(publisher.encode_json(envelope)) + self.store.calls.clear() + with self.assertRaisesRegex(publisher.PublishError, "release version must advance"): + self.publish() + self.assertFalse(any(call[0] == "put" for call in self.store.calls)) + + def test_reservation_rejects_changed_notes_or_timestamp(self): + self.reserve() + other, _, _ = self.make_bundle("1.2.3", "changed", "changed notes") + with self.assertRaisesRegex(publisher.PublishError, "immutable object differs"): + publisher.reserve(other, self.store) + self.assertFalse((other / "reservation.json").exists()) + + def test_reserved_complete_envelope_is_reused_on_retry(self): + self.reserve() + self.publish() + retry, _, _ = self.make_bundle("1.2.3", "retry") + (retry / "envelope.json").unlink() + publisher.reserve(retry, self.store) + self.assertEqual((retry / "envelope.json").read_bytes(), self.envelope) + self.assertEqual(self.publish(retry), "already-published") + + def test_publish_requires_reservation_and_verification_before_mutations(self): + with self.assertRaisesRegex(publisher.PublishError, "must be reserved"): + self.publish() + self.assertFalse(self.store.objects) + self.reserve() + self.store.calls.clear() + wrong_key = self.root / "other-key.pub" + wrong_key.write_text(base64.b64encode(b"Ed" + bytes([1]) * 40).decode() + "\n") + with self.assertRaisesRegex(publisher.PublishError, "key differs"): + publisher.publish(self.bundle, self.store, wrong_key, fixture_verifier) + self.assertFalse(any(call[0] == "put" for call in self.store.calls)) + + def test_manifest_byte_identity_is_checked_not_only_json_equivalence(self): + self.reserve() + (self.bundle / "manifest.json").write_text(json.dumps(self.manifest, indent=2)) + with self.assertRaisesRegex(publisher.PublishError, "manifest bytes differ"): + self.publish() + self.assertNotIn("stable/latest.json", self.store.objects) + + def test_invalid_prepared_inventory_is_rejected_before_reservation(self): + (self.bundle / "payloads/unexpected.txt").write_text("extra") + with self.assertRaisesRegex(publisher.PublishError, "inventory differs"): + self.reserve() + self.assertFalse(self.store.objects) + + def test_corrupt_or_symlink_payload_is_rejected_before_reservation(self): + payload = self.bundle / "payloads/agent.app.tar.gz" + payload.write_bytes(b"corrupt") + with self.assertRaisesRegex(publisher.PublishError, "hash differs"): + self.reserve() + payload.unlink() + payload.symlink_to(self.bundle / "manifest.json") + with self.assertRaises(OSError): + self.reserve() + self.assertFalse(self.store.objects) + + def test_live_verification_covers_public_feed_immutable_head_range_and_download_routes(self): + origin = FixtureOrigin(self.bundle, self.manifest, self.envelope) + with patch.object(publisher, "minisign_verify", fixture_verifier), \ + patch.object(publisher.urllib.request, "build_opener", return_value=origin): + result = publisher.verify_live(self.bundle, self.bundle / "public-key.pub", "https://updates.example") + self.assertEqual(result, self.manifest) + self.assertIn(("GET", "/agent/stable/latest.json", None), origin.calls) + self.assertIn(("GET", publisher.public_path("1.2.3", "manifest.json"), None), origin.calls) + self.assertIn(("GET", "/download/agent/stable/macos", None), origin.calls) + for record in [*self.manifest["artifacts"].values(), *self.manifest["downloads"].values()]: + for method, range_header in (("HEAD", None), ("GET", "bytes=0-0"), ("GET", None)): + self.assertIn((method, record["path"], range_header), origin.calls) + + def test_live_verification_rejects_foreign_or_wrong_download_redirect(self): + for redirect in ("https://foreign.example/app.dmg", "/agent/stable/releases/1.2.4/app.dmg"): + with self.subTest(redirect=redirect): + origin = FixtureOrigin(self.bundle, self.manifest, self.envelope) + origin.redirect = redirect + with patch.object(publisher, "minisign_verify", fixture_verifier), \ + patch.object(publisher.urllib.request, "build_opener", return_value=origin), \ + self.assertRaisesRegex(publisher.PublishError, "download redirect differs"): + publisher.verify_live(self.bundle, self.bundle / "public-key.pub", "https://updates.example") + + def test_live_verification_rejects_wrong_range_response(self): + origin = FixtureOrigin(self.bundle, self.manifest, self.envelope) + origin.range_header = "bytes 0-0/999999" + with patch.object(publisher, "minisign_verify", fixture_verifier), \ + patch.object(publisher.urllib.request, "build_opener", return_value=origin), \ + self.assertRaisesRegex(publisher.PublishError, "range request failed"): + publisher.verify_live(self.bundle, self.bundle / "public-key.pub", "https://updates.example") + + def test_release_filenames_match_worker_ambiguity_and_length_limits(self): + for filename in ("agent..app.tar.gz", "a" * 201 + ".dmg", "../app.dmg", "app%2fdmg"): + with self.subTest(filename=filename), self.assertRaises(publisher.PublishError): + publisher.release_key("1.2.3", filename) + + +class PackageBoundaryTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + profiles = json.loads((publisher.ROOT / "apps/maple-agent/release-profiles.json").read_text()) + metadata_spec = importlib.util.spec_from_file_location( + "fixture_release_info", publisher.ROOT / "apps/maple-agent/scripts/release-info.py") + release_info = importlib.util.module_from_spec(metadata_spec) + metadata_spec.loader.exec_module(release_info) + self.info = {field: profiles["prod"][field] for field in release_info.PUBLIC_FIELDS} + self.info.update({"profile": "prod", "version": "1.2.3", + "source_sha": SOURCE_SHA, "git_revision": SOURCE_SHA[:9]}) + self.stem = "maple-agent-prod-1.2.3-" + SOURCE_SHA[:9] + "-macos-aarch64" + self.metadata = {"schema_version": 1, "profile": "prod", "platform": "macos-aarch64", + "source_sha": SOURCE_SHA, "version": "1.2.3", "package": self.stem + ".dmg", + "distribution": "master", "macos_signing": "developer-id-notarized"} + (self.directory / "build-info.json").write_bytes(publisher.encode_json(self.info)) + (self.directory / "manifest.json").write_bytes(publisher.encode_json(self.metadata)) + (self.directory / (self.stem + ".dmg")).write_bytes(b"fake disk image; never execute") + self.write_archive() + for name in ("notarization-log.json", "dmg-notarization-log.json"): + (self.directory / name).write_bytes(publisher.encode_json({"status": "Accepted", "issues": None})) + self.checksums() + self.attested = [] + + def write_archive(self, plist=None, entries=None): + if plist is None: + plist = {"CFBundleIdentifier": "cloud.opensecret.maple.agent", + "CFBundleShortVersionString": "1.2.3", "CFBundleVersion": "42"} + if entries is None: + entries = [("Maple Agent.app/Contents/Info.plist", tarfile.REGTYPE, plistlib.dumps(plist))] + with tarfile.open(self.directory / (self.stem + ".app.tar.gz"), "w:gz") as archive: + for name, kind, data in entries: + member = tarfile.TarInfo(name) + member.type = kind + member.mode = 0o644 + if kind == tarfile.REGTYPE: + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + else: + member.linkname = "Other.plist" + archive.addfile(member) + + def checksums(self): + (self.directory / "SHA256SUMS").write_text("".join( + hashlib.sha256(path.read_bytes()).hexdigest() + " " + path.name + "\n" + for path in sorted(self.directory.iterdir()) if path.name != "SHA256SUMS" and path.is_file() + )) + + def validate(self): + return publisher.validate_package(self.directory, "macos-aarch64", SOURCE_SHA, "1.2.3", + attestor=lambda path, source: self.attested.append((path.name, source)), + expected_build="42") + + def test_exact_final_inventory_is_hashed_and_each_file_attested_without_execution(self): + result = self.validate() + self.assertEqual([path.name for path, _ in result], [self.stem + ".dmg", self.stem + ".app.tar.gz"]) + self.assertEqual({name for name, _ in self.attested}, {path.name for path in self.directory.iterdir()}) + self.assertTrue(all(source == SOURCE_SHA for _, source in self.attested)) + + def test_extra_files_missing_files_and_symlinks_are_rejected(self): + extra = self.directory / "unexpected.json" + extra.write_text("{}") + with self.assertRaisesRegex(publisher.PublishError, "unexpected or missing"): + self.validate() + extra.unlink() + target = self.directory / "notarization-log.json" + target.unlink() + with self.assertRaisesRegex(publisher.PublishError, "unexpected or missing"): + self.validate() + target.symlink_to(self.directory / "dmg-notarization-log.json") + with self.assertRaises(OSError): + self.validate() + self.assertFalse(self.attested) + + def test_corrupt_checksum_duplicate_and_traversal_records_are_rejected(self): + checksums = self.directory / "SHA256SUMS" + original = checksums.read_text() + for value in (original.replace(original[:64], "b" * 64, 1), + original + original.splitlines()[0] + "\n", + "a" * 64 + " ../outside\n"): + with self.subTest(value=value): + checksums.write_text(value) + with self.assertRaises(publisher.PublishError): + self.validate() + self.assertFalse(self.attested) + + def test_unsigned_or_dev_identity_is_rejected_even_with_valid_checksums(self): + self.metadata["distribution"] = "unsigned-pr" + (self.directory / "manifest.json").write_bytes(publisher.encode_json(self.metadata)) + self.checksums() + with self.assertRaisesRegex(publisher.PublishError, "invalid final package metadata"): + self.validate() + self.info["profile"] = "dev" + (self.directory / "build-info.json").write_bytes(publisher.encode_json(self.info)) + self.checksums() + with self.assertRaisesRegex(publisher.PublishError, "Prod build identity"): + self.validate() + self.assertFalse(self.attested) + + def test_rejected_notarization_and_duplicate_metadata_keys_are_rejected(self): + (self.directory / "notarization-log.json").write_bytes(publisher.encode_json({"status": "Rejected", "issues": []})) + self.checksums() + with self.assertRaisesRegex(publisher.PublishError, "notarization was not accepted"): + self.validate() + (self.directory / "build-info.json").write_bytes(b'{"profile":"prod","profile":"prod"}') + with self.assertRaisesRegex(publisher.PublishError, "duplicate JSON field"): + self.validate() + self.assertFalse(self.attested) + + def test_archive_build_version_and_bundle_identity_match_trusted_publication(self): + for field, value in (("CFBundleVersion", "43"), ("CFBundleShortVersionString", "1.2.4"), + ("CFBundleIdentifier", "cloud.opensecret.maple")): + with self.subTest(field=field): + plist = {"CFBundleIdentifier": "cloud.opensecret.maple.agent", + "CFBundleShortVersionString": "1.2.3", "CFBundleVersion": "42", field: value} + self.write_archive(plist) + self.checksums() + with self.assertRaises(publisher.PublishError): + self.validate() + self.assertFalse(self.attested) + + def test_archive_duplicate_or_missing_info_plist_is_rejected_without_extraction(self): + entry = ("Maple Agent.app/Contents/Info.plist", tarfile.REGTYPE, + plistlib.dumps({"CFBundleIdentifier": "cloud.opensecret.maple.agent", + "CFBundleShortVersionString": "1.2.3", "CFBundleVersion": "42"})) + for entries in ([entry, entry], [("Other.app/Contents/Info.plist", *entry[1:])], []): + with self.subTest(entries=len(entries)): + self.write_archive(entries=entries) + self.checksums() + with self.assertRaises(publisher.PublishError): + self.validate() + self.assertFalse(self.attested) + + def test_archive_nonregular_or_oversized_info_plist_is_rejected(self): + for kind, data in ((tarfile.SYMTYPE, b""), (tarfile.LNKTYPE, b""), + (tarfile.FIFOTYPE, b""), (tarfile.REGTYPE, bytes(64 * 1024 + 1))): + with self.subTest(kind=kind, size=len(data)): + self.write_archive(entries=[("Maple Agent.app/Contents/Info.plist", kind, data)]) + self.checksums() + with self.assertRaises(publisher.PublishError): + self.validate() + self.assertFalse(self.attested) + + def test_archive_invalid_plist_or_nondictionary_plist_is_rejected(self): + for data in (b"invalid plist", b'broken', + plistlib.dumps(["unexpected", "plist", "shape"])): + with self.subTest(data=data[:20]): + self.write_archive(entries=[("Maple Agent.app/Contents/Info.plist", tarfile.REGTYPE, data)]) + self.checksums() + with self.assertRaises(publisher.PublishError): + self.validate() + self.assertFalse(self.attested) + + def test_archive_invalid_container_is_rejected_with_bounded_error(self): + (self.directory / (self.stem + ".app.tar.gz")).write_bytes(b"not a gzip or tar archive") + self.checksums() + with self.assertRaises(publisher.PublishError): + self.validate() + self.assertFalse(self.attested) + + def test_archive_oversized_member_is_rejected_before_reading_its_body(self): + member = tarfile.TarInfo("Maple Agent.app/Contents/oversized") + member.size = publisher.MAX_ARTIFACT + 1 + # Only the header exists. The size gate must reject it before a stream + # parser tries to skip/decompress the declared multi-gigabyte body. + data = gzip.compress(member.tobuf() + bytes(1024)) + (self.directory / (self.stem + ".app.tar.gz")).write_bytes(data) + self.checksums() + with self.assertRaisesRegex(publisher.PublishError, "bounded inventory|oversized member"): + self.validate() + self.assertFalse(self.attested) + + def test_archive_extension_headers_are_bounded_before_tarfile_parses_them(self): + for kind in (tarfile.XHDTYPE, tarfile.XGLTYPE, tarfile.SOLARIS_XHDTYPE, + tarfile.GNUTYPE_LONGNAME, tarfile.GNUTYPE_LONGLINK): + with self.subTest(kind=kind): + member = tarfile.TarInfo("extension") + member.type, member.size = kind, 64 * 1024 + 1 + data = gzip.compress(member.tobuf() + bytes(1024)) + (self.directory / (self.stem + ".app.tar.gz")).write_bytes(data) + self.checksums() + with self.assertRaisesRegex(publisher.PublishError, "extension metadata exceeds bounds"): + self.validate() + self.assertFalse(self.attested) + + def test_archive_sparse_member_is_not_interpreted_as_an_app_payload(self): + member = tarfile.TarInfo("sparse") + member.type = tarfile.GNUTYPE_SPARSE + data = gzip.compress(member.tobuf() + bytes(1024)) + (self.directory / (self.stem + ".app.tar.gz")).write_bytes(data) + self.checksums() + with self.assertRaisesRegex(publisher.PublishError, "sparse app archive members"): + self.validate() + self.assertFalse(self.attested) + + def test_pax_sparse_variants_are_rejected_before_stdlib_sparse_map_parsing(self): + cases = ( + ({"GNU.sparse.major": "1", "GNU.sparse.minor": "0", "GNU.sparse.realsize": "999999999"}, + "_proc_gnusparse_10", b"999999999\n"), + ({"GNU.sparse.map": "0,999999999", "GNU.sparse.size": "999999999"}, + "_proc_gnusparse_01", b""), + ({"GNU.sparse.size": "999999999", "GNU.sparse.numblocks": "999999999"}, + "_proc_gnusparse_00", b""), + ) + for headers, parser, body in cases: + with self.subTest(parser=parser): + member = tarfile.TarInfo("Maple Agent.app/Contents/sparse") + member.pax_headers, member.size = headers, len(body) + with tarfile.open(self.directory / (self.stem + ".app.tar.gz"), "w:gz") as archive: + archive.addfile(member, io.BytesIO(body)) + self.checksums() + # Keep even a regression safe: reaching the standard library's + # sparse parser fails immediately before attacker map counts + # can drive its list allocation or block-reading loops. + with patch.object(tarfile.TarInfo, parser, side_effect=AssertionError("stdlib sparse parser reached")), \ + self.assertRaisesRegex(publisher.PublishError, "sparse app archive members are unsupported"): + self.validate() + self.assertFalse(self.attested) + + +class TrustedRunTests(unittest.TestCase): + def setUp(self): + self.run = { + "id": 123, "run_attempt": 2, "run_number": 42, + "repository": {"full_name": publisher.REPOSITORY}, + "head_repository": {"full_name": publisher.REPOSITORY}, + "path": publisher.WORKFLOW, "head_branch": "master", "event": "push", + "status": "completed", "conclusion": "success", "head_sha": SOURCE_SHA, + } + self.names = {"Signed macOS (prod)", "Verify downloaded macOS (prod)"} + + def jobs(self, names=None): + jobs = [{"name": name, "status": "completed", "conclusion": "success"} for name in sorted(names or self.names)] + return {"total_count": len(jobs), "jobs": jobs} + + def test_macos_requires_concrete_completed_prod_jobs(self): + self.assertEqual(publisher.validate_run(self.run, self.jobs()), self.run) + for conclusion in ("skipped", "failure", "cancelled", None): + jobs = self.jobs() + jobs["jobs"][0]["conclusion"] = conclusion + with self.subTest(conclusion=conclusion), self.assertRaises(publisher.PublishError): + publisher.validate_run(self.run, jobs) + with self.assertRaisesRegex(publisher.PublishError, "required Prod package job"): + publisher.validate_run(self.run, self.jobs({"Select Agent packaging"})) + + def test_linux_is_optional_but_each_linux_gate_is_required_when_selected(self): + with self.assertRaises(publisher.PublishError): + publisher.validate_run(self.run, self.jobs(), linux=True) + names = self.names | {"Linux AppImage (prod)", "Attest Linux packages (prod)", "Verify downloaded Linux (prod)"} + publisher.validate_run(self.run, self.jobs(names), linux=True) + for name in names - self.names: + with self.subTest(name=name), self.assertRaises(publisher.PublishError): + publisher.validate_run(self.run, self.jobs(names - {name}), linux=True) + + def test_incomplete_duplicate_or_untrusted_job_inventory_fails_closed(self): + jobs = self.jobs() + jobs["total_count"] += 1 + with self.assertRaisesRegex(publisher.PublishError, "incomplete"): + publisher.validate_run(self.run, jobs) + jobs = self.jobs() + jobs["jobs"].append(jobs["jobs"][0].copy()) + jobs["total_count"] += 1 + with self.assertRaises(publisher.PublishError): + publisher.validate_run(self.run, jobs) + for field, value in (("head_branch", "feature"), ("event", "pull_request"), + ("path", ".github/workflows/release.yml"), ("head_sha", "a" * 7)): + with self.subTest(field=field), self.assertRaises(publisher.PublishError): + publisher.validate_run({**self.run, field: value}, self.jobs()) + with self.assertRaisesRegex(publisher.PublishError, "untrusted build repository"): + publisher.validate_run({**self.run, "head_repository": {"full_name": "other/Maple"}}, self.jobs()) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/agent-updates/publisher/test_r2_store.py b/services/agent-updates/publisher/test_r2_store.py new file mode 100644 index 000000000..4ab26e941 --- /dev/null +++ b/services/agent-updates/publisher/test_r2_store.py @@ -0,0 +1,272 @@ +"""Offline S3 signing and transport boundary tests; no bucket or credentials. + +The fixed signing vectors use AWS's published example access/secret key pair, +an explicitly written canonical request, and R2's ``auto`` signing region. +The expected digests and signatures are literals derived independently of +R2Store. Mock openers consume file streams without making network requests. +""" + +from datetime import datetime, timezone +from email.message import Message +import hashlib +import hmac +import importlib.util +import io +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import urllib.error +from urllib.response import addinfourl + + +SPEC = importlib.util.spec_from_file_location("agent_r2_publisher", Path(__file__).with_name("publisher.py")) +publisher = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(publisher) + +ENDPOINT = "https://example.r2.cloudflarestorage.com" +ACCESS_KEY = "AKIDEXAMPLE" +SECRET_KEY = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" +KEY = "stable/releases/0.0.2/agent.app.tar.gz" +PAYLOAD = b"test payload" +PAYLOAD_SHA = "813ca5285c28ccee5cab8b10ebda9c908fd6d78ed9dc94cc65ea6cb67a7f13ae" +FIXED_TIME = datetime(2026, 10, 22, 3, 4, 5, tzinfo=timezone.utc) +CANONICAL_CREATE = ( + "PUT\n/agent-updates/stable/releases/0.0.2/agent.app.tar.gz\n\n" + "content-length:12\n" + "content-type:application/octet-stream\n" + "host:example.r2.cloudflarestorage.com\n" + "if-none-match:*\n" + "x-amz-content-sha256:813ca5285c28ccee5cab8b10ebda9c908fd6d78ed9dc94cc65ea6cb67a7f13ae\n" + "x-amz-date:20261022T030405Z\n\n" + "content-length;content-type;host;if-none-match;x-amz-content-sha256;x-amz-date\n" + "813ca5285c28ccee5cab8b10ebda9c908fd6d78ed9dc94cc65ea6cb67a7f13ae" +) +CREATE_CANONICAL_SHA = "b862834295f34ed57bb95db458cb4798c77e2f103834662c801972ff78d52f06" +CREATE_SIGNATURE = "f4bdc56620f024468268b63c28ce74c623374223950021e28dca54adae79565a" + + +class Response(io.BytesIO): + def __init__(self, data=b"", headers=None): + super().__init__(data) + self.headers = headers or {} + self.read_sizes = [] + self.bytes_read = 0 + + def read(self, size=-1): + self.read_sizes.append(size) + data = super().read(size) + self.bytes_read += len(data) + return data + + +class OfflineOpener: + """Always intercept requests, including when an assertion fails.""" + + def __init__(self): + self.requests = [] + self.responses = [] + self.error = None + self.capture = None + + def open(self, request, timeout): + self.requests.append((request, timeout)) + if self.error is not None: + raise self.error + if self.capture is not None: + self.capture(request) + return self.responses.pop(0) if self.responses else Response() + + +def headers(request): + return {name.lower(): value for name, value in request.header_items()} + + +class R2StoreTests(unittest.TestCase): + def setUp(self): + self.opener = OfflineOpener() + self.opener_factory = self.enterContext(patch.object(publisher.urllib.request, "build_opener", return_value=self.opener)) + self.clock = self.enterContext(patch.object(publisher, "datetime")) + self.clock.now.return_value = FIXED_TIME + self.store = publisher.R2Store(ENDPOINT, "agent-updates", ACCESS_KEY, SECRET_KEY) + + def test_fixed_create_only_sigv4_vector(self): + # Capture the actual final string-to-sign as well as the wire header. + # Expectations are fixed, not generated by the production signer. + original_hmac = hmac.new + with patch.object(publisher.hmac, "new", wraps=original_hmac) as signer: + self.store.put(KEY, PAYLOAD, condition=("If-None-Match", "*")) + request, timeout = self.opener.requests[0] + actual = headers(request) + self.assertEqual(hashlib.sha256(CANONICAL_CREATE.encode()).hexdigest(), CREATE_CANONICAL_SHA) + self.assertEqual(signer.call_args.args[1], ( + "AWS4-HMAC-SHA256\n20261022T030405Z\n20261022/auto/s3/aws4_request\n" + CREATE_CANONICAL_SHA + ).encode()) + self.assertEqual(actual["authorization"], ( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261022/auto/s3/aws4_request, " + "SignedHeaders=content-length;content-type;host;if-none-match;x-amz-content-sha256;x-amz-date, " + "Signature=" + CREATE_SIGNATURE + )) + self.assertEqual(actual["if-none-match"], "*") + self.assertEqual(actual["x-amz-content-sha256"], PAYLOAD_SHA) + self.assertEqual(actual["content-length"], "12") + self.assertEqual(request.full_url, ENDPOINT + "/agent-updates/" + KEY) + self.assertEqual(request.get_method(), "PUT") + self.assertEqual(request.data, PAYLOAD) + self.assertEqual(timeout, 120) + self.clock.now.assert_called_once_with(timezone.utc) + + def test_fixed_pointer_compare_and_swap_sigv4_vector(self): + self.store.put("stable/latest.json", PAYLOAD, condition=("If-Match", '"old-etag"'), content_type="application/json") + actual = headers(self.opener.requests[0][0]) + self.assertEqual(actual["if-match"], '"old-etag"') + self.assertNotIn("if-none-match", actual) + self.assertEqual(actual["authorization"], ( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261022/auto/s3/aws4_request, " + "SignedHeaders=content-length;content-type;host;if-match;x-amz-content-sha256;x-amz-date, " + "Signature=dad3f950c5030af4557aa5a4c05452af3d7351eeaf4f19898fb49e3e5342f462" + )) + + def test_file_put_streams_signed_size_and_sha256(self): + data = b"bounded offline upload\n" * 100000 + digest = hashlib.sha256(data).hexdigest() + uploaded = bytearray() + source = [] + + def consume(request): + self.assertNotIsInstance(request.data, (bytes, bytearray)) + source.append(request.data) + self.assertFalse(request.data.closed) + while block := request.data.read(65536): + uploaded.extend(block) + + self.opener.capture = consume + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "agent.app.tar.gz" + path.write_bytes(data) + self.store.put(KEY, path, condition=("If-None-Match", "*")) + actual = headers(self.opener.requests[0][0]) + self.assertEqual(uploaded, data) + self.assertTrue(source[0].closed) + self.assertEqual(actual["content-length"], str(len(data))) + self.assertEqual(actual["x-amz-content-sha256"], digest) + self.assertIn("content-length;content-type;host;if-none-match;x-amz-content-sha256;x-amz-date", actual["authorization"]) + + def test_get_preserves_quoted_etag_case_insensitively(self): + response = Response(PAYLOAD, {"eTaG": '"opaque-multipart-3"'}) + self.opener.responses.append(response) + self.assertEqual(self.store.get(KEY), (PAYLOAD, '"opaque-multipart-3"')) + request, _ = self.opener.requests[0] + self.assertEqual(request.get_method(), "GET") + self.assertIsNone(request.data) + self.assertEqual(headers(request)["x-amz-content-sha256"], hashlib.sha256(b"").hexdigest()) + self.assertTrue(response.closed) + + def test_get_rejects_missing_or_oversized_etag(self): + for response_headers in ({}, {"ETag": "x" * 256}): + with self.subTest(headers=response_headers): + self.opener.responses.append(Response(PAYLOAD, response_headers)) + with self.assertRaisesRegex(publisher.PublishError, "no valid ETag"): + self.store.get(KEY) + + def test_head_returns_headers_without_reading_a_body(self): + response = Response(b"body must never be read", {"Content-Length": "999999999"}) + self.opener.responses.append(response) + self.assertEqual(self.store.request("HEAD", KEY, maximum=1), (b"", {"Content-Length": "999999999"})) + self.assertEqual(self.opener.requests[0][0].get_method(), "HEAD") + self.assertEqual(response.read_sizes, []) + self.assertTrue(response.closed) + + def test_buffered_response_is_bounded_and_closed(self): + response = Response(b"1234567890", {"ETag": '"version"'}) + self.opener.responses.append(response) + with self.assertRaisesRegex(publisher.PublishError, "oversized R2 response"): + self.store.get(KEY, maximum=4) + self.assertEqual(response.read_sizes, [5]) + self.assertEqual(response.bytes_read, 5) + self.assertTrue(response.closed) + + def test_stream_hash_accepts_exact_bound_without_buffering_entire_object(self): + data = b"large fixture\n" * 170000 + response = Response(data) + self.opener.responses.append(response) + self.assertEqual(self.store.stream_hash(KEY, maximum=len(data)), (hashlib.sha256(data).hexdigest(), len(data))) + self.assertGreater(len(response.read_sizes), 2) + self.assertTrue(all(0 < size <= publisher.CHUNK for size in response.read_sizes)) + self.assertTrue(response.closed) + + def test_stream_hash_stops_after_one_byte_beyond_bound(self): + response = Response(b"0123456789" * 100) + self.opener.responses.append(response) + with self.assertRaisesRegex(publisher.PublishError, "oversized artifact"): + self.store.stream_hash(KEY, maximum=7) + self.assertEqual(response.read_sizes, [8]) + self.assertEqual(response.bytes_read, 8) + self.assertTrue(response.closed) + + def test_get_and_same_treat_only_http_404_as_missing(self): + self.opener.error = urllib.error.HTTPError(ENDPOINT, 404, "not found", {}, None) + self.assertIsNone(self.store.get(KEY)) + self.assertFalse(self.store.same(KEY, PAYLOAD_SHA, len(PAYLOAD))) + with self.assertRaises(publisher.StoreError) as raised: + self.store.request("HEAD", KEY) + self.assertEqual(raised.exception.status, 404) + + def test_http_conflict_and_forbidden_preserve_status_without_error_body(self): + for status in (412, 403): + with self.subTest(status=status): + self.opener.error = urllib.error.HTTPError(ENDPOINT, status, "private upstream diagnostic", {}, None) + with self.assertRaises(publisher.StoreError) as raised: + self.store.put(KEY, PAYLOAD, condition=("If-None-Match", "*")) + self.assertEqual(raised.exception.status, status) + self.assertEqual(str(raised.exception), f"R2 request failed (HTTP {status})") + with self.assertRaises(publisher.StoreError) as raised: + self.store.get(KEY) + self.assertEqual(raised.exception.status, status) + + def test_transport_errors_are_bounded_and_do_not_expose_upstream_diagnostics(self): + for error in (OSError("private path or credential"), urllib.error.URLError("private upstream URL")): + with self.subTest(error=type(error).__name__): + self.opener.error = error + with self.assertRaises(publisher.PublishError) as raised: + self.store.get(KEY) + self.assertEqual(str(raised.exception), "R2 transport failed") + + def test_redirect_handler_refuses_following_all_supported_redirects(self): + handler = self.opener_factory.call_args.args[0] + if isinstance(handler, type): + handler = handler() + self.assertIsInstance(handler, publisher.NoRedirect) + for status in (301, 302, 303, 307, 308): + with self.subTest(status=status): + requests = [] + + class OfflineHTTPS(publisher.urllib.request.BaseHandler): + def https_open(self, request): + requests.append(request.full_url) + response_headers = Message() + response_headers["Location"] = "https://other.example/secret" + response = addinfourl(io.BytesIO(b""), response_headers, request.full_url, status) + response.msg = "Redirect" + return response + + # Exercise urllib's real redirect/error processing. There is + # no network-capable handler in this director, even on failure. + opener = publisher.urllib.request.OpenerDirector() + for component in (OfflineHTTPS(), handler, publisher.urllib.request.HTTPErrorProcessor(), publisher.urllib.request.HTTPDefaultErrorHandler()): + opener.add_handler(component) + self.store.opener = opener + with self.assertRaises(publisher.StoreError) as raised: + self.store.get(KEY) + self.assertEqual(raised.exception.status, status) + self.assertEqual(requests, [ENDPOINT + "/agent-updates/" + KEY]) + + def test_invalid_object_keys_never_reach_transport(self): + for key in ("/absolute", "stable//latest.json", "../secret", "stable/%2fsecret", "stable/key?query", "stable/key#fragment"): + with self.subTest(key=key), self.assertRaisesRegex(publisher.PublishError, "invalid R2 object key"): + self.store.get(key) + self.assertEqual(self.opener.requests, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/agent-updates/publisher/test_trust.py b/services/agent-updates/publisher/test_trust.py new file mode 100644 index 000000000..bcd9bbf42 --- /dev/null +++ b/services/agent-updates/publisher/test_trust.py @@ -0,0 +1,175 @@ +"""Trust boundary tests, independent of GitHub, credentials and signing keys.""" + +import base64 +from copy import deepcopy +import json +import io +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import publisher as p + + +class TrustTests(unittest.TestCase): + def setUp(self): + self.run = { + "repository": {"full_name": p.REPOSITORY}, + "head_repository": {"full_name": p.REPOSITORY}, + "path": p.WORKFLOW, "head_branch": "master", "event": "push", + "status": "completed", "conclusion": "failure", "head_sha": "a" * 40, + "id": 12, "run_attempt": 2, "run_number": 42, + "created_at": "2026-10-21T01:02:03Z", + } + self.jobs = {"total_count": 5, "jobs": [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in ("Signed macOS (prod)", "Verify downloaded macOS (prod)", + "Linux AppImage (prod)", "Attest Linux packages (prod)", + "Verify downloaded Linux (prod)") + ]} + + def test_actual_prod_consumers_gate_even_if_dev_failed(self): + self.assertEqual(p.validate_run(self.run, self.jobs, True), self.run) + + def test_skipped_failed_missing_duplicate_and_incomplete_jobs_fail(self): + cases = [] + for conclusion in ("skipped", "failure", "cancelled", "neutral", None): + jobs = deepcopy(self.jobs) + jobs["jobs"][1]["conclusion"] = conclusion + cases.append(jobs) + missing = deepcopy(self.jobs) + missing["jobs"].pop(1) + missing["total_count"] -= 1 + cases.append(missing) + duplicate = deepcopy(self.jobs) + duplicate["jobs"].append(deepcopy(duplicate["jobs"][1])) + duplicate["total_count"] += 1 + cases.append(duplicate) + incomplete = deepcopy(self.jobs) + incomplete["total_count"] += 1 + cases.append(incomplete) + for jobs in cases: + with self.subTest(jobs=jobs), self.assertRaises(p.PublishError): + p.validate_run(self.run, jobs) + + def test_linux_is_required_only_when_requested(self): + self.jobs["jobs"][4]["conclusion"] = "failure" + p.validate_run(self.run, self.jobs, False) + with self.assertRaises(p.PublishError): + p.validate_run(self.run, self.jobs, True) + + def test_fork_pull_request_branch_and_other_workflow_fail(self): + replacements = { + "repository": {"full_name": "elsewhere/Maple"}, + "head_repository": {"full_name": "elsewhere/Maple"}, + "event": "pull_request", "head_branch": "feature", + "path": ".github/workflows/other.yml", "head_sha": "A" * 40, + "status": "in_progress", "run_attempt": 0, + } + for key, value in replacements.items(): + run = {**self.run, key: value} + with self.subTest(key=key), self.assertRaises(p.PublishError): + p.validate_run(run, self.jobs) + + def test_trust_uses_latest_jobs_including_successful_prior_attempt_and_final_artifact(self): + artifacts = {"total_count": 1, "artifacts": [ + {"name": "maple-agent-prod-macos-aarch64-12", "expired": False} + ]} + with patch.object(p, "gh_json", side_effect=[self.run, self.jobs, artifacts]) as api: + proof = p.trust_run("12") + self.assertEqual(proof["build"], "42") + self.assertIn("/jobs?filter=latest", api.call_args_list[1].args[0]) + self.assertEqual(proof["run_attempt"], 2) + self.assertEqual(proof["artifacts"]["macos"], "maple-agent-prod-macos-aarch64-12") + artifacts["artifacts"][0]["name"] += "-unsigned" + with patch.object(p, "gh_json", side_effect=[self.run, self.jobs, artifacts]), self.assertRaises(p.PublishError): + p.trust_run("12") + + def test_durable_audit_ignores_diagnostic_attempt_but_binds_build_identity(self): + proof = {"run_id": 12, "run_attempt": 1, "source_sha": "a" * 40, + "build": "42", "pub_date": "2026-10-21T01:02:03Z", + "artifacts": {"macos": "maple-agent-prod-macos-aarch64-12"}} + original = p.durable_provenance(proof) + self.assertEqual(original, p.durable_provenance({**proof, "run_attempt": 2})) + self.assertNotEqual(original, p.durable_provenance({**proof, "source_sha": "b" * 40})) + self.assertNotIn("run_attempt", json.loads(original)) + + def test_stream_hash_reads_only_expected_size_plus_one_on_oversized_response(self): + stream = io.BytesIO(b"oversized response body") + with self.assertRaisesRegex(p.PublishError, "oversized"): + p.hash_stream(stream, maximum=3) + self.assertEqual(stream.tell(), 4) + + +class PinTests(unittest.TestCase): + def setUp(self): + self.key = base64.b64encode(b"Ed" + b"a" * 40).decode("ascii") + self.profile = {"prod": {"update_feed_url": p.FEED_URL, "update_public_keys": [self.key]}} + + def test_key_must_be_in_both_source_and_trusted_profiles(self): + p.validate_pins(self.key, self.profile, deepcopy(self.profile)) + for side in (0, 1): + for change in ([], [self.key + "x"]): + profiles = [deepcopy(self.profile), deepcopy(self.profile)] + profiles[side]["prod"]["update_public_keys"] = change + with self.subTest(side=side, change=change), self.assertRaises(p.PublishError): + p.validate_pins(self.key, *profiles) + + def test_feed_mismatch_and_missing_pins_fail(self): + for bad in ({"prod": {}}, {"prod": {"update_feed_url": "https://elsewhere.invalid", "update_public_keys": [self.key]}}): + with self.assertRaises(p.PublishError): + p.validate_pins(self.key, self.profile, bad) + + def test_source_profile_is_read_at_exact_source_sha(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "apps/maple-agent").mkdir(parents=True) + (root / "apps/maple-agent/release-profiles.json").write_bytes(p.encode_json(self.profile)) + key = root / "public-key.pub" + key.write_text("untrusted comment: test\n" + self.key + "\n") + response = {"encoding": "base64", "content": base64.b64encode(p.encode_json(self.profile)).decode()} + with patch.object(p, "ROOT", root), patch.object(p, "gh_json", return_value=response) as api: + self.assertEqual(p.check_source_pins(key, "b" * 40), self.key) + self.assertTrue(api.call_args.args[0].endswith("?ref=" + "b" * 40)) + + def test_public_key_format_is_bounded_and_algorithm_checked(self): + with tempfile.TemporaryDirectory() as temporary: + key = Path(temporary) / "key.pub" + key.write_text(self.key + "\n") + self.assertEqual(p.public_key_text(key), self.key) + for value in ("invalid", self.key + "\n" + self.key, base64.b64encode(b"ED" + b"a" * 40).decode(), "x" * (p.MAX_SIGNATURE + 1)): + key.write_text(value) + with self.subTest(value=value[:20]), self.assertRaises(p.PublishError): + p.public_key_text(key) + + def test_existing_pointer_accepts_primary_when_new_release_selects_backup(self): + backup = base64.b64encode(b"Ed" + b"b" * 40).decode("ascii") + profile = deepcopy(self.profile) + profile["prod"]["update_public_keys"].append(backup) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "apps/maple-agent").mkdir(parents=True) + (root / "apps/maple-agent/release-profiles.json").write_bytes(p.encode_json(profile)) + selected = root / "selected.pub" + selected.write_text(backup + "\n") + attempted = [] + + def verify(data, path): + candidate = p.public_key_text(path) + attempted.append(candidate) + if candidate == self.key: + return {"version": "1.0.0"} + raise p.PublishError("fixture signature mismatch") + + with patch.object(p, "ROOT", root), patch.object(p, "minisign_verify", side_effect=verify): + self.assertEqual(p.verify_current(b"signed-by-primary", selected), {"version": "1.0.0"}) + self.assertEqual(attempted, [self.key]) + profile["prod"]["update_public_keys"] = [backup] + (root / "apps/maple-agent/release-profiles.json").write_bytes(p.encode_json(profile)) + with patch.object(p, "ROOT", root), patch.object(p, "minisign_verify", side_effect=verify), self.assertRaisesRegex(p.PublishError, "not trusted"): + p.verify_current(b"signed-by-retired-primary", selected) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/agent-updates/src/dry-run.ts b/services/agent-updates/src/dry-run.ts new file mode 100644 index 000000000..d5f7b3f4f --- /dev/null +++ b/services/agent-updates/src/dry-run.ts @@ -0,0 +1,63 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +// Keep local validation independent of the owner-provisioned production bucket. +// Bundle and validate code using Wrangler's actual dry-run path. +// This checked-in config uses whole-line comments and trailing commas only. +// Unsupported JSONC syntax fails closed instead of weakening deployment guards. +const config: unknown = JSON.parse( + (await Bun.file("wrangler.jsonc").text()) + .replace(/^\s*\/\/.*$/gm, "") + .replace(/,(\s*[}\]])/g, "$1"), +); +if (!config || typeof config !== "object" || Array.isArray(config)) { + throw new Error("invalid Worker configuration"); +} +const workerConfig = config as Record; +if ( + "route" in workerConfig || + "routes" in workerConfig || + "env" in workerConfig +) { + throw new Error( + "Agent dry run cannot manage routing or environment overrides", + ); +} +const directory = await mkdtemp(join(tmpdir(), "agent-updates-dry-run-")); +try { + const path = join(directory, "wrangler.json"); + await Bun.write( + path, + JSON.stringify({ + ...workerConfig, + main: resolve("src/index.ts"), + r2_buckets: [ + { + binding: "AGENT_RELEASES", + bucket_name: "agent-updates-local-dry-run", + }, + ], + }), + ); + const result = Bun.spawn( + [ + "node", + resolve("node_modules/wrangler/bin/wrangler.js"), + "deploy", + "--dry-run", + "--config", + path, + "--outdir", + resolve(".wrangler/dry-run"), + ], + { + stdout: "inherit", + stderr: "inherit", + env: { ...process.env, WRANGLER_SEND_METRICS: "false" }, + }, + ); + if ((await result.exited) !== 0) process.exitCode = 1; +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/services/agent-updates/src/fixtures.ts b/services/agent-updates/src/fixtures.ts new file mode 100644 index 000000000..3c2fdcbc9 --- /dev/null +++ b/services/agent-updates/src/fixtures.ts @@ -0,0 +1,39 @@ +import type { Manifest } from "./manifest"; + +export const PAYLOAD = new TextEncoder().encode("a signed package fixture\n"); +export function fixture(linux = false): { + manifest: Manifest; + bytes: Uint8Array; +} { + const prefix = "/agent/stable/releases/0.0.2/"; + const artifact = (name: string) => ({ + path: prefix + name, + size: PAYLOAD.length, + sha256: "a".repeat(64), + }); + const manifest: Manifest = { + format: "maple-agent-update-v1", + product: "maple-agent", + channel: "stable", + version: "0.0.2", + build: "2", + source_sha: "b".repeat(40), + pub_date: "2026-10-21T12:00:00Z", + notes: "Preview update\nFixes.", + artifacts: { "darwin-aarch64": artifact("Maple-Agent.app.tar.gz") }, + downloads: { macos: artifact("Maple-Agent.dmg") }, + }; + if (linux) + manifest.artifacts["linux-x86_64"] = manifest.downloads["linux-appimage"] = + artifact("Maple-Agent.AppImage"); + return { manifest, bytes: encodeFixture(manifest) }; +} +export function encodeFixture(manifest: Manifest): Uint8Array { + const signature = btoa("ED" + "\0".repeat(72)); + return new TextEncoder().encode( + JSON.stringify({ + manifest: JSON.stringify(manifest), + signature: `untrusted comment: fixture\n${signature}\ntrusted comment: product=maple-agent channel=stable version=${manifest.version}\n${btoa("\0".repeat(64))}\n`, + }), + ); +} diff --git a/services/agent-updates/src/index.ts b/services/agent-updates/src/index.ts new file mode 100644 index 000000000..737a7b980 --- /dev/null +++ b/services/agent-updates/src/index.ts @@ -0,0 +1,2 @@ +// Validation helpers are not workerd service entrypoints. +export { default } from "./worker"; diff --git a/services/agent-updates/src/manifest.ts b/services/agent-updates/src/manifest.ts new file mode 100644 index 000000000..e41376def --- /dev/null +++ b/services/agent-updates/src/manifest.ts @@ -0,0 +1,194 @@ +export const MAX_ENVELOPE_BYTES = 128 * 1024; +export const MAX_MANIFEST_BYTES = 64 * 1024; +export const MAX_SIGNATURE_BYTES = 4096; +export const MAX_ARTIFACT_BYTES = 4 * 1024 ** 3; +export const VERSION = /^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/; +const encoder = new TextEncoder(); + +export interface Artifact { + path: string; + size: number; + sha256: string; +} +export interface Manifest { + format: "maple-agent-update-v1"; + product: "maple-agent"; + channel: "stable"; + version: string; + build: string; + source_sha: string; + pub_date: string; + notes: string; + artifacts: { "darwin-aarch64": Artifact; "linux-x86_64"?: Artifact }; + downloads: { macos: Artifact; "linux-appimage"?: Artifact }; +} +export interface Envelope { + manifest: string; + signature: string; +} + +function record(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function keys(value: Record, allowed: string[]): boolean { + return Object.keys(value).every((key) => allowed.includes(key)); +} +function fullMatch(pattern: RegExp, value: string): boolean { + // JavaScript's `$` also matches before a final line terminator. + return pattern.exec(value)?.[0] === value; +} +function boundedText(value: unknown, max: number): value is string { + return typeof value === "string" && encoder.encode(value).length <= max; +} +function scalarText(value: string): boolean { + // JSON may contain escaped lone UTF-16 surrogates despite valid UTF-8 bytes. + for (const character of value) { + const code = character.codePointAt(0)!; + if (code >= 0xd800 && code <= 0xdfff) return false; + } + return true; +} +function artifact( + value: unknown, + version: string, + suffix: string, +): value is Artifact { + if (!record(value) || !keys(value, ["path", "size", "sha256"])) return false; + const prefix = `/agent/stable/releases/${version}/`; + if (typeof value.path !== "string" || !value.path.startsWith(prefix)) + return false; + const file = value.path.slice(prefix.length); + return ( + file.length <= 200 && + fullMatch(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, file) && + file.endsWith(suffix) && + !file.includes("..") && + typeof value.size === "number" && + Number.isSafeInteger(value.size) && + value.size > 0 && + value.size <= MAX_ARTIFACT_BYTES && + typeof value.sha256 === "string" && + fullMatch(/^[0-9a-f]{64}$/, value.sha256) + ); +} + +export function parseManifest(value: unknown): Manifest { + if ( + !record(value) || + !keys(value, [ + "format", + "product", + "channel", + "version", + "build", + "source_sha", + "pub_date", + "notes", + "artifacts", + "downloads", + ]) || + value.format !== "maple-agent-update-v1" || + value.product !== "maple-agent" || + value.channel !== "stable" || + typeof value.version !== "string" || + value.version.length > 64 || + !fullMatch(VERSION, value.version) || + typeof value.build !== "string" || + !fullMatch(/^[1-9][0-9]{0,17}$/, value.build) || + typeof value.source_sha !== "string" || + !fullMatch(/^[0-9a-f]{40}$/, value.source_sha) || + typeof value.pub_date !== "string" || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/.test(value.pub_date) || + !Number.isFinite(Date.parse(value.pub_date)) || + new Date(value.pub_date).toISOString().replace(".000Z", "Z") !== + value.pub_date || + !boundedText(value.notes, 16 * 1024) || + !scalarText(value.notes) || + /[\x00-\x08\x0b-\x1f\x7f]/.test(value.notes) || + !record(value.artifacts) || + !keys(value.artifacts, ["darwin-aarch64", "linux-x86_64"]) || + !record(value.downloads) || + !keys(value.downloads, ["macos", "linux-appimage"]) || + !artifact( + value.artifacts["darwin-aarch64"], + value.version, + ".app.tar.gz", + ) || + !artifact(value.downloads.macos, value.version, ".dmg") + ) + throw new Error("invalid manifest"); + const linux = value.artifacts["linux-x86_64"]; + const download = value.downloads["linux-appimage"]; + if ( + (linux === undefined) !== (download === undefined) || + (linux !== undefined && + (!artifact(linux, value.version, ".AppImage") || + !artifact(download, value.version, ".AppImage") || + linux.path !== download.path || + linux.size !== download.size || + linux.sha256 !== download.sha256)) + ) { + throw new Error("invalid Linux artifact"); + } + return value as unknown as Manifest; +} + +// The Worker validates public metadata structure. Clients and the publisher +// authenticate exact bytes with pinned keys; R2 write authority is not signing authority. +export function parseEnvelope(bytes: Uint8Array): { + envelope: Envelope; + manifest: Manifest; +} { + if (bytes.length === 0 || bytes.length > MAX_ENVELOPE_BYTES) + throw new Error("oversized envelope"); + const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + const envelope: unknown = JSON.parse(text); + if ( + !record(envelope) || + !keys(envelope, ["manifest", "signature"]) || + !boundedText(envelope.manifest, MAX_MANIFEST_BYTES) || + !boundedText(envelope.signature, MAX_SIGNATURE_BYTES) + ) + throw new Error("invalid envelope"); + const manifest = parseManifest(JSON.parse(envelope.manifest)); + const lines = envelope.signature.trimEnd().split("\n"); + if ( + lines.length !== 4 || + !lines[0].startsWith("untrusted comment: ") || + lines[2] !== + `trusted comment: product=maple-agent channel=stable version=${manifest.version}` || + !fullMatch(/^[A-Za-z0-9+/]+={0,2}$/, lines[1]) || + !fullMatch(/^[A-Za-z0-9+/]+={0,2}$/, lines[3]) + ) + throw new Error("invalid signature format"); + const signature = atob(lines[1]); + if ( + signature.length !== 74 || + signature.slice(0, 2) !== "ED" || + atob(lines[3]).length !== 64 + ) + throw new Error("unsupported signature format"); + return { envelope: envelope as unknown as Envelope, manifest }; +} + +export function releasePath( + path: string, +): { version: string; file: string; key: string } | null { + const match = + /^\/agent\/stable\/releases\/([^/]+)\/([A-Za-z0-9][A-Za-z0-9._-]{0,199})$/.exec( + path, + ); + if ( + !match || + match[1].length > 64 || + !fullMatch(VERSION, match[1]) || + match[0] !== path || + match[2].includes("..") + ) + return null; + return { + version: match[1], + file: match[2], + key: `stable/releases/${match[1]}/${match[2]}`, + }; +} diff --git a/services/agent-updates/src/runtime.test.ts b/services/agent-updates/src/runtime.test.ts new file mode 100644 index 000000000..6e638cda4 --- /dev/null +++ b/services/agent-updates/src/runtime.test.ts @@ -0,0 +1,125 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "bun:test"; +import { fixture, PAYLOAD } from "./fixtures"; + +// Miniflare is Wrangler's exact locked runtime dependency. Node hosts the +// harness; assertions make real requests through workerd and its R2 binding. +const server = ` + import { Miniflare, convertV4MiniflareOptions } from 'miniflare'; + import { readFileSync } from 'node:fs'; + const mf = new Miniflare(convertV4MiniflareOptions({modules:true, name:'agent-updates-test', script:readFileSync(process.env.MAPLE_TEST_WORKER,'utf8'), compatibilityDate:'2026-08-24', r2Buckets:['AGENT_RELEASES'], port:0, inspectorPort:0})); + process.on('SIGTERM', async()=>{await mf.dispose(); process.exit(0);}); + process.send({origin:(await mf.ready).origin}); +`; +test("real workerd R2 binding streams exact immutable bytes, ranges, HEAD and redirects", async () => { + const directory = await mkdtemp(join(tmpdir(), "agent-updates-workerd-")); + let child: ReturnType | undefined; + let timer: ReturnType | undefined; + try { + const wrapper = join(directory, "entry.ts"); + await Bun.write( + wrapper, + ` + import worker from ${JSON.stringify(join(import.meta.dir, "index.ts"))}; + export default { async fetch(request,env) { + if (new URL(request.url).pathname === '/__seed' && request.method === 'POST') { + const {metadata,payload} = await request.json(); + const bytes = Uint8Array.from(metadata); + await env.AGENT_RELEASES.put('stable/latest.json',bytes); + await env.AGENT_RELEASES.put('stable/releases/0.0.2/manifest.json',bytes); + await env.AGENT_RELEASES.put('stable/releases/0.0.2/Maple-Agent.app.tar.gz',Uint8Array.from(payload)); + await env.AGENT_RELEASES.put('stable/releases/0.0.2/Maple-Agent.dmg',Uint8Array.from(payload)); + return new Response('seeded'); + } + return worker.fetch(request,env); + }}; + `, + ); + const build = await Bun.build({ + entrypoints: [wrapper], + outdir: directory, + target: "browser", + format: "esm", + }); + if (!build.success) throw new Error("fixture Worker bundle failed"); + const origin = await new Promise((resolve, reject) => { + timer = setTimeout( + () => reject(new Error("workerd startup timeout")), + 20000, + ); + child = Bun.spawn(["node", "--input-type=module", "-e", server], { + env: { + ...process.env, + MAPLE_TEST_WORKER: join(directory, "entry.js"), + MAPLE_TEST_METADATA: Buffer.from(fixture().bytes).toString("base64"), + MAPLE_TEST_PAYLOAD: Buffer.from(PAYLOAD).toString("base64"), + }, + stdout: "ignore", + stderr: "inherit", + serialization: "json", + ipc(value: unknown) { + if ( + value && + typeof value === "object" && + "origin" in value && + typeof value.origin === "string" + ) + resolve(value.origin); + }, + onExit(_child, code) { + reject(new Error(`workerd exited ${code}`)); + }, + }); + }); + clearTimeout(timer); + const seeded = await fetch(`${origin}/__seed`, { + method: "POST", + body: JSON.stringify({ + metadata: Array.from(fixture().bytes), + payload: Array.from(PAYLOAD), + }), + }); + expect(seeded.status).toBe(200); + const latest = await fetch(`${origin}/agent/stable/latest.json`); + expect(latest.status).toBe(200); + expect(new Uint8Array(await latest.arrayBuffer())).toEqual(fixture().bytes); + const redirect = await fetch(`${origin}/download/agent/stable/macos`, { + redirect: "manual", + }); + expect(redirect.status).toBe(302); + expect(redirect.headers.get("location")).toBe( + "/agent/stable/releases/0.0.2/Maple-Agent.dmg", + ); + const artifact = `${origin}/agent/stable/releases/0.0.2/Maple-Agent.app.tar.gz`; + const full = await fetch(artifact); + expect(full.status).toBe(200); + expect(new Uint8Array(await full.arrayBuffer())).toEqual(PAYLOAD); + const range = await fetch(artifact, { headers: { range: "bytes=2-7" } }); + expect(range.status).toBe(206); + expect(range.headers.get("content-range")).toBe( + `bytes 2-7/${PAYLOAD.length}`, + ); + expect(new Uint8Array(await range.arrayBuffer())).toEqual( + PAYLOAD.slice(2, 8), + ); + const head = await fetch(artifact, { method: "HEAD" }); + expect(head.status).toBe(200); + expect(head.headers.get("content-length")).toBe(String(PAYLOAD.length)); + expect(await head.text()).toBe(""); + expect( + (await fetch(`${origin}/download/agent/stable/linux-appimage`)).status, + ).toBe(404); + expect((await fetch(`${origin}/latest.json`)).status).toBe(404); + } finally { + clearTimeout(timer); + child?.kill("SIGTERM"); + if (child) { + const forceStop = setTimeout(() => child?.kill("SIGKILL"), 3000); + await child.exited; + clearTimeout(forceStop); + } + await rm(directory, { recursive: true, force: true }); + } +}, 45000); diff --git a/services/agent-updates/src/worker.test.ts b/services/agent-updates/src/worker.test.ts new file mode 100644 index 000000000..682a09867 --- /dev/null +++ b/services/agent-updates/src/worker.test.ts @@ -0,0 +1,332 @@ +import { expect, test } from "bun:test"; +import worker, { type Bucket } from "./worker"; +import { fixture, encodeFixture, PAYLOAD } from "./fixtures"; +import { + MAX_ENVELOPE_BYTES, + parseEnvelope, + parseManifest, + releasePath, +} from "./manifest"; + +class MemoryBucket implements Bucket { + objects = new Map>(); + calls: string[] = []; + fail = false; + async get( + key: string, + options?: { range: { offset: number; length: number } }, + ) { + this.calls.push(`get:${key}`); + if (this.fail) throw new Error("private fixture error"); + const bytes = this.objects.get(key); + if (!bytes) return null; + const range = options?.range; + const body = range + ? bytes.slice(range.offset, range.offset + range.length) + : bytes; + return { + size: bytes.length, + httpEtag: '"fixture"', + body: new Response(body).body!, + }; + } + async head(key: string) { + this.calls.push(`head:${key}`); + if (this.fail) throw new Error("private fixture error"); + const bytes = this.objects.get(key); + return bytes ? { size: bytes.length, httpEtag: '"fixture"' } : null; + } +} +function setup(linux = false) { + const bucket = new MemoryBucket(); + const current = fixture(linux); + bucket.objects.set("stable/latest.json", current.bytes); + bucket.objects.set("stable/releases/0.0.2/manifest.json", current.bytes); + for (const artifact of [ + ...Object.values(current.manifest.artifacts), + ...Object.values(current.manifest.downloads), + ]) + bucket.objects.set(artifact.path.replace("/agent/", ""), PAYLOAD); + const fetch = (path: string, init?: RequestInit) => + worker.fetch(new Request(`https://updates.example${path}`, init), { + AGENT_RELEASES: bucket, + }); + return { bucket, fetch, ...current }; +} + +test("latest exact envelope and HEAD are never cached", async () => { + const { fetch, bytes } = setup(); + for (const method of ["GET", "HEAD"]) { + const result = await fetch("/agent/stable/latest.json", { method }); + expect(result.status).toBe(200); + expect(result.headers.get("cache-control")).toBe("no-store"); + expect(result.headers.get("cdn-cache-control")).toBe("no-store"); + expect(result.headers.get("content-length")).toBe(String(bytes.length)); + expect(new Uint8Array(await result.arrayBuffer())).toEqual( + method === "HEAD" ? new Uint8Array() : bytes, + ); + } +}); +test("downloads are uncached origin-relative redirects; omitted Linux is isolated", async () => { + for (const linux of [false, true]) { + const { fetch } = setup(linux); + for (const method of ["GET", "HEAD"]) { + const mac = await fetch("/download/agent/stable/macos", { method }); + expect(mac.status).toBe(302); + expect(mac.headers.get("location")).toBe( + "/agent/stable/releases/0.0.2/Maple-Agent.dmg", + ); + expect(mac.headers.get("cache-control")).toBe("no-store"); + expect( + (await fetch("/download/agent/stable/linux-appimage", { method })) + .status, + ).toBe(linux ? 302 : 404); + } + } +}); +test("immutable artifact GET, HEAD, ranges, validators and If-Range", async () => { + const { fetch, bucket } = setup(); + const path = "/agent/stable/releases/0.0.2/Maple-Agent.app.tar.gz"; + const result = await fetch(path); + expect(await result.text()).toBe(new TextDecoder().decode(PAYLOAD)); + expect(result.headers.get("cache-control")).toContain("immutable"); + for (const [range, expected] of [ + ["bytes=2-6", PAYLOAD.slice(2, 7)], + ["bytes=2-", PAYLOAD.slice(2)], + ["bytes=-4", PAYLOAD.slice(-4)], + ["bytes=0-9999", PAYLOAD], + ]) { + const result = await fetch(path, { headers: { range: range as string } }); + expect(result.status).toBe(206); + expect(new Uint8Array(await result.arrayBuffer())).toEqual( + expected as Uint8Array, + ); + } + for (const range of [ + "bytes=9999-", + "bytes=-0", + "bytes=3-2", + "bytes=0-1,4-5", + "bytes=-", + "items=0-1", + "bytes=999999999999999999999-9999999999999999999999", + ]) + expect((await fetch(path, { headers: { range } })).status).toBe(416); + expect( + ( + await fetch(path, { + headers: { range: "bytes=0-2", "if-range": '"changed"' }, + }) + ).status, + ).toBe(200); + expect( + (await fetch(path, { headers: { "if-none-match": '"fixture"' } })).status, + ).toBe(304); + bucket.calls = []; + const head = await fetch(path, { + method: "HEAD", + headers: { range: "bytes=0-2" }, + }); + expect(head.status).toBe(200); + expect(head.headers.get("content-length")).toBe(String(PAYLOAD.length)); + expect(head.headers.get("content-range")).toBeNull(); + expect(await head.text()).toBe(""); + expect(bucket.calls).not.toContain( + "get:stable/releases/0.0.2/Maple-Agent.app.tar.gz", + ); +}); +test("immutable manifest is exact, but unsigned auxiliary objects are private", async () => { + const { fetch, bucket, bytes } = setup(); + const response = await fetch("/agent/stable/releases/0.0.2/manifest.json"); + expect(response.status).toBe(200); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(bytes); + for (const file of [ + "private.json", + "publication-manifest.json", + "publication-metadata.json", + "metadata/macos-build-info.json", + ]) { + bucket.objects.set(`stable/releases/0.0.2/${file}`, PAYLOAD); + expect((await fetch(`/agent/stable/releases/0.0.2/${file}`)).status).toBe( + 404, + ); + } +}); +test("unknown paths, methods and ambiguous encodings cannot reach arbitrary bucket keys", async () => { + const { fetch, bucket } = setup(); + for (const path of [ + "/latest.json", + "/download/research/stable/macos", + "/agent/stable/random", + "/agent/stable/latest.json?secret=value", + "/agent/stable/releases/0.0.2/x%2fy.dmg", + "/agent/stable/releases/00.0.2/a.dmg", + "/agent/stable//latest.json", + "/agent/stable/releases/0.0.2/a..dmg", + "/download/agent/stable/unknown", + ]) + expect((await fetch(path)).status).toBe(404); + expect(bucket.calls).toEqual([]); + const post = await fetch("/agent/stable/latest.json", { method: "POST" }); + expect(post.status).toBe(405); + expect(post.headers.get("allow")).toBe("GET, HEAD"); +}); +test("missing and malformed data and R2 failures fail only Agent with bounded errors", async () => { + const { fetch, bucket } = setup(); + bucket.objects.delete("stable/latest.json"); + expect((await fetch("/agent/stable/latest.json")).status).toBe(404); + for (const bad of [ + new Uint8Array(MAX_ENVELOPE_BYTES + 1), + new TextEncoder().encode("{}"), + new Uint8Array([0xff]), + ]) { + bucket.objects.set("stable/latest.json", bad); + for (const path of [ + "/agent/stable/latest.json", + "/download/agent/stable/macos", + ]) { + const result = await fetch(path); + expect(result.status).toBe(503); + expect(await result.text()).toBe("503\n"); + } + } + bucket.fail = true; + expect((await fetch("/agent/stable/latest.json")).status).toBe(503); + expect((await fetch("/latest.json")).status).toBe(404); +}); +test("size mismatch and incomplete publication cannot stream corrupt or partial artifacts", async () => { + const { fetch, bucket } = setup(); + const path = "/agent/stable/releases/0.0.2/Maple-Agent.dmg"; + bucket.objects.set( + "stable/releases/0.0.2/Maple-Agent.dmg", + new Uint8Array([1]), + ); + expect((await fetch(path)).status).toBe(503); + bucket.objects.delete("stable/releases/0.0.2/manifest.json"); + expect((await fetch(path)).status).toBe(404); +}); +test("metadata stream size is enforced independently of the bucket's declared size", async () => { + for (const oversized of [false, true]) { + const { fetch, bucket, bytes } = setup(); + let cancelled = false; + const stream = oversized + ? new ReadableStream>({ + start(controller) { + controller.enqueue(new Uint8Array(MAX_ENVELOPE_BYTES + 1)); + }, + cancel() { + cancelled = true; + }, + }) + : new Response(bytes).body!; + bucket.get = async () => ({ + size: oversized ? MAX_ENVELOPE_BYTES : bytes.length + 1, + httpEtag: '"fixture"', + body: stream, + }); + const result = await fetch("/agent/stable/latest.json"); + expect(result.status).toBe(503); + if (oversized) expect(cancelled).toBe(true); + } +}); +test("schema binds product, channel, version, target, path, size, hashes and signature comment", () => { + const mutations = [ + (v: any) => { + v.product = "research"; + }, + (v: any) => { + v.channel = "dev"; + }, + (v: any) => { + v.version = "0.0.2-beta"; + }, + (v: any) => { + v.build = "01"; + }, + (v: any) => { + v.build = "2\n"; + }, + (v: any) => { + v.source_sha += "\n"; + }, + (v: any) => { + v.downloads.macos.sha256 += "\n"; + }, + (v: any) => { + v.version += "\n"; + }, + (v: any) => { + v.pub_date = "2026-02-30T00:00:00Z"; + }, + (v: any) => { + v.notes = "\u0000"; + }, + (v: any) => { + v.notes = "\ud800"; + }, + (v: any) => { + v.notes = "é".repeat(8193); + }, + (v: any) => { + v.extra = true; + }, + (v: any) => { + v.artifacts["windows-x86_64"] = v.artifacts["darwin-aarch64"]; + }, + (v: any) => { + delete v.artifacts["darwin-aarch64"]; + }, + (v: any) => { + v.downloads.macos.path = "https://evil.example/a.dmg"; + }, + (v: any) => { + v.downloads.macos.path = "/agent/stable/releases/0.0.1/a.dmg"; + }, + (v: any) => { + v.downloads.macos.path += "?x=y"; + }, + (v: any) => { + v.downloads.macos.path = "/agent/stable/releases/0.0.2/../a.dmg"; + }, + (v: any) => { + v.downloads.macos.path = "/agent/stable/releases/0.0.2/a%2fb.dmg"; + }, + (v: any) => { + v.downloads.macos.size = 4 * 1024 ** 3 + 1; + }, + (v: any) => { + v.downloads.macos.sha256 = "A".repeat(64); + }, + (v: any) => { + v.artifacts["linux-x86_64"] = v.downloads.macos; + }, + ]; + for (const change of mutations) { + const { manifest } = fixture(); + change(manifest); + expect(() => parseEnvelope(encodeFixture(manifest))).toThrow(); + } + const envelope = JSON.parse(new TextDecoder().decode(fixture().bytes)); + envelope.signature = envelope.signature.replace( + "version=0.0.2", + "version=0.0.3", + ); + expect(() => + parseEnvelope(new TextEncoder().encode(JSON.stringify(envelope))), + ).toThrow(); + expect(releasePath("/agent/stable/releases/0.0.2/manifest.json")?.key).toBe( + "stable/releases/0.0.2/manifest.json", + ); + expect( + releasePath("/agent/stable/releases/0.0.2\n/manifest.json"), + ).toBeNull(); + const noncanonical = fixture().manifest; + noncanonical.version += "\n"; + for (const artifact of [ + ...Object.values(noncanonical.artifacts), + ...Object.values(noncanonical.downloads), + ]) { + artifact.path = artifact.path.replace("/0.0.2/", "/0.0.2\n/"); + } + expect(() => parseManifest(noncanonical)).toThrow(); +}); diff --git a/services/agent-updates/src/worker.ts b/services/agent-updates/src/worker.ts new file mode 100644 index 000000000..a899f838e --- /dev/null +++ b/services/agent-updates/src/worker.ts @@ -0,0 +1,261 @@ +import { + MAX_ENVELOPE_BYTES, + parseEnvelope, + releasePath, + type Artifact, + type Manifest, +} from "./manifest"; + +interface StoredObject { + size: number; + httpEtag: string; + body?: ReadableStream; +} +export interface Bucket { + get( + key: string, + options?: { range: { offset: number; length: number } }, + ): Promise; + head(key: string): Promise; +} +export interface Env { + AGENT_RELEASES: Bucket; +} + +const NO_STORE = { + "cache-control": "no-store", + "cdn-cache-control": "no-store", + "x-content-type-options": "nosniff", +}; +function response( + status: number, + method: string, + extra?: HeadersInit, +): Response { + return new Response(method === "HEAD" ? null : `${status}\n`, { + status, + headers: { + ...NO_STORE, + "content-type": "text/plain; charset=utf-8", + ...extra, + }, + }); +} +async function metadata( + bucket: Bucket, + key: string, +): Promise<{ bytes: Uint8Array; manifest: Manifest } | null> { + const object = await bucket.get(key); + if (!object) return null; + if (object.size === 0 || object.size > MAX_ENVELOPE_BYTES || !object.body) { + await object.body?.cancel().catch(() => {}); + throw new Error("metadata size"); + } + const reader = object.body.getReader(); + let length = 0; + const chunks: Uint8Array[] = []; + try { + while (true) { + const part = await reader.read(); + if (part.done) break; + length += part.value.length; + if (length > MAX_ENVELOPE_BYTES || length > object.size) + throw new Error("metadata size"); + chunks.push(part.value); + } + } catch (error) { + await reader.cancel().catch(() => {}); + throw error; + } + if (length !== object.size) throw new Error("truncated metadata"); + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return { bytes, manifest: parseEnvelope(bytes).manifest }; +} + +function requestedRange( + header: string, + size: number, +): { offset: number; length: number } | null { + const match = /^bytes=(\d*)-(\d*)$/.exec(header); + if (!match || (!match[1] && !match[2])) return null; + if (!match[1]) { + const suffix = Number(match[2]); + return Number.isSafeInteger(suffix) && suffix > 0 + ? { offset: Math.max(0, size - suffix), length: Math.min(size, suffix) } + : null; + } + const start = Number(match[1]); + const end = match[2] ? Number(match[2]) : size - 1; + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start >= size || + end < start + ) + return null; + return { offset: start, length: Math.min(end, size - 1) - start + 1 }; +} + +function contentType(path: string): string { + if (path.endsWith(".dmg")) return "application/x-apple-diskimage"; + if (path.endsWith(".tar.gz")) return "application/gzip"; + return "application/octet-stream"; +} +async function artifactResponse( + request: Request, + bucket: Bucket, + key: string, + artifact: Artifact, +): Promise { + const object = await bucket.head(key); + if (!object) return response(404, request.method); + if (object.size !== artifact.size) throw new Error("artifact size mismatch"); + const headers = new Headers({ + "cache-control": "public, max-age=31536000, immutable", + "content-type": contentType(artifact.path), + "x-content-type-options": "nosniff", + "accept-ranges": "bytes", + etag: object.httpEtag, + "content-length": String(object.size), + }); + if ( + request.headers + .get("if-none-match") + ?.split(",") + .some((value) => value.trim() === object.httpEtag || value.trim() === "*") + ) + return new Response(null, { status: 304, headers }); + // RFC 9110: Range modifies GET only; HEAD describes the full representation. + const rangeHeader = + request.method === "GET" ? request.headers.get("range") : null; + const ifRange = request.headers.get("if-range"); + const range = + rangeHeader && (!ifRange || ifRange === object.httpEtag) + ? requestedRange(rangeHeader, object.size) + : undefined; + if (range === null) + return response(416, request.method, { + "content-range": `bytes */${object.size}`, + "accept-ranges": "bytes", + }); + if (range) { + headers.set( + "content-range", + `bytes ${range.offset}-${range.offset + range.length - 1}/${object.size}`, + ); + headers.set("content-length", String(range.length)); + } + const status = range ? 206 : 200; + if (request.method === "HEAD") return new Response(null, { status, headers }); + const full = await bucket.get(key, range ? { range } : undefined); + if ( + !full || + !full.body || + full.size !== object.size || + full.httpEtag !== object.httpEtag + ) { + await full?.body?.cancel().catch(() => {}); + throw new Error("artifact changed"); + } + return new Response(full.body, { status, headers }); +} + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + const path = url.pathname; + // No origin fetch: missing data and exceptions remain inside this Worker. + if ( + !path.startsWith("/agent/stable/") && + !path.startsWith("/download/agent/stable/") + ) + return response(404, request.method); + if (!["GET", "HEAD"].includes(request.method)) + return response(405, request.method, { allow: "GET, HEAD" }); + if ( + url.search || + path.includes("%") || + path.includes("\\") || + path.includes("//") + ) + return response(404, request.method); + try { + if ( + path === "/agent/stable/latest.json" || + path.startsWith("/download/agent/stable/") + ) { + if ( + path.startsWith("/download/") && + ![ + "/download/agent/stable/macos", + "/download/agent/stable/linux-appimage", + ].includes(path) + ) + return response(404, request.method); + const current = await metadata( + env.AGENT_RELEASES, + "stable/latest.json", + ); + if (!current) return response(404, request.method); + if (path.startsWith("/download/")) { + const target = path.endsWith("/macos") + ? current.manifest.downloads.macos + : current.manifest.downloads["linux-appimage"]; + return target + ? new Response(null, { + status: 302, + headers: { ...NO_STORE, location: target.path }, + }) + : response(404, request.method); + } + return new Response(request.method === "HEAD" ? null : current.bytes, { + headers: { + ...NO_STORE, + "content-type": "application/json; charset=utf-8", + "content-length": String(current.bytes.length), + }, + }); + } + const release = releasePath(path); + if (!release) return response(404, request.method); + const published = await metadata( + env.AGENT_RELEASES, + `stable/releases/${release.version}/manifest.json`, + ); + if (!published) return response(404, request.method); + if (published.manifest.version !== release.version) + throw new Error("version mismatch"); + if (release.file === "manifest.json") + return new Response( + request.method === "HEAD" ? null : published.bytes, + { + headers: { + "cache-control": "public, max-age=31536000, immutable", + "content-type": "application/json; charset=utf-8", + "content-length": String(published.bytes.length), + "x-content-type-options": "nosniff", + }, + }, + ); + const artifact = [ + ...Object.values(published.manifest.artifacts), + ...Object.values(published.manifest.downloads), + ].find((value) => value.path === path); + return artifact + ? await artifactResponse( + request, + env.AGENT_RELEASES, + release.key, + artifact, + ) + : response(404, request.method); + } catch { + return response(503, request.method); + } + }, +}; diff --git a/services/agent-updates/tsconfig.json b/services/agent-updates/tsconfig.json new file mode 100644 index 000000000..a8f94721f --- /dev/null +++ b/services/agent-updates/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["ES2022", "WebWorker"], + "module": "ESNext", + "moduleResolution": "Bundler", + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["bun"] + }, + "include": ["src/**/*.ts"] +} diff --git a/services/agent-updates/wrangler.jsonc b/services/agent-updates/wrangler.jsonc new file mode 100644 index 000000000..557fe145c --- /dev/null +++ b/services/agent-updates/wrangler.jsonc @@ -0,0 +1,16 @@ +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "maple-agent-updates", + "main": "src/index.ts", + "compatibility_date": "2026-08-24", + "workers_dev": false, + "preview_urls": false, + // Owner-provisioned private Agent bucket. This config does not activate delivery; + // Worker deployment and owner-managed Routes require separate authorization. + "r2_buckets": [ + { + "binding": "AGENT_RELEASES", + "bucket_name": "maple-agent-updates", + }, + ], +}