From 1e32bd94624bad5cc0a3b05cb9ed24e7b2e03a82 Mon Sep 17 00:00:00 2001 From: Anthony Ronning <101225832+AnthonyRonning@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:35:28 +0000 Subject: [PATCH] Recover Agent dependency caches and remove CI setup bottlenecks --- .../validate-opensecret/references/checks.md | 6 +- .github/workflows/agent-ci.yml | 221 ++++- .github/workflows/agent-desktop-build.yml | 148 +++- .github/workflows/opensecret-ci.yml | 6 +- .github/workflows/sdk-integration.yml | 8 +- .../crates/maple-agent/src/agent.rs | 28 +- .../maple-agent/src/agent/maintenance.rs | 2 +- .../crates/maple-agent/src/agent/projects.rs | 51 ++ .../maple-agent/src/agent/shutdown_tests.rs | 18 +- docs/agent-ci-performance.md | 201 ++++- docs/repository-workflows.md | 89 +- flake.nix | 2 + scripts/ci/agent_cache_scope.py | 262 ++++++ scripts/ci/agent_change_detection.py | 2 + scripts/ci/agent_dependency_archive.py | 822 ++++++++++++++++++ scripts/ci/agent_master_selection.py | 29 +- scripts/ci/test_agent_cache_scope.py | 248 ++++++ scripts/ci/test_agent_change_detection.py | 1 + scripts/ci/test_agent_dependency_archive.py | 575 ++++++++++++ scripts/ci/test_agent_master_selection.py | 82 +- scripts/ci/test_agent_workflows.py | 296 +++++-- scripts/ci/test_opensecret_workflows.py | 13 +- services/opensecret/docs/dev-shell.md | 8 + services/opensecret/flake.nix | 49 +- 24 files changed, 2948 insertions(+), 219 deletions(-) create mode 100644 scripts/ci/agent_cache_scope.py create mode 100644 scripts/ci/agent_dependency_archive.py create mode 100644 scripts/ci/test_agent_cache_scope.py create mode 100644 scripts/ci/test_agent_dependency_archive.py diff --git a/.agents/skills/validate-opensecret/references/checks.md b/.agents/skills/validate-opensecret/references/checks.md index 986f45ae1..3f7f3fe98 100644 --- a/.agents/skills/validate-opensecret/references/checks.md +++ b/.agents/skills/validate-opensecret/references/checks.md @@ -28,14 +28,14 @@ side-effect-disabled component Nix environment: git -C ../.. submodule update --init --recursive -- services/opensecret/privatemode-public OPENSECRET_DEV_POSTGRES=0 OPENSECRET_DEV_ENV=0 OPENSECRET_DEV_CONTAINERS=0 \ - nix develop --no-write-lock-file '.?submodules=1' -c cargo fmt --all -- --check + nix develop --no-write-lock-file '.?submodules=1#ci' -c cargo fmt --all -- --check OPENSECRET_DEV_POSTGRES=0 OPENSECRET_DEV_ENV=0 OPENSECRET_DEV_CONTAINERS=0 \ - nix develop --no-write-lock-file '.?submodules=1' -c env RUSTFLAGS='-D warnings' \ + nix develop --no-write-lock-file '.?submodules=1#ci' -c env RUSTFLAGS='-D warnings' \ cargo clippy --locked --all-targets --all-features -- -D warnings OPENSECRET_DEV_POSTGRES=0 OPENSECRET_DEV_ENV=0 OPENSECRET_DEV_CONTAINERS=0 \ - nix develop --no-write-lock-file '.?submodules=1' -c env RUSTFLAGS='-D warnings' \ + nix develop --no-write-lock-file '.?submodules=1#ci' -c env RUSTFLAGS='-D warnings' \ cargo test --locked --all-features ``` diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml index 753036e1f..64096499f 100644 --- a/.github/workflows/agent-ci.yml +++ b/.github/workflows/agent-ci.yml @@ -33,6 +33,9 @@ jobs: name: Select Agent checks runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read + actions: read outputs: agent: ${{ steps.classify.outputs.agent }} steps: @@ -41,18 +44,37 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Retain Agent changes since the last validated master run + id: master + if: ${{ github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && github.event_name == 'push' }} + working-directory: . + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/ci/agent_master_selection.py --lane ci >> "$GITHUB_OUTPUT" + - name: Classify changed paths id: classify working-directory: . env: BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + MASTER_AGENT: ${{ steps.master.outputs.agent }} run: | set -euo pipefail enable_agent() { echo 'agent=true' >> "$GITHUB_OUTPUT"; } case "$GITHUB_EVENT_NAME" in pull_request) separator='...' ;; push) + # A later unrelated push must not consume Agent work cancelled + # before every workspace, provider and release check completed. + if [ "$GITHUB_REPOSITORY" = MaplePrivacyLabs/Maple ] && [ "$GITHUB_REF" = refs/heads/master ]; then + if [ "${MASTER_AGENT:-}" = false ]; then + echo 'agent=false' >> "$GITHUB_OUTPUT" + else + enable_agent + fi + exit 0 + fi separator='..' if [[ "$BASE_SHA" =~ ^0+$ ]]; then enable_agent @@ -79,11 +101,16 @@ jobs: # A failed selector must never silently omit validation. if: ${{ always() && !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.agent != 'false') }} runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: read timeout-minutes: 120 strategy: fail-fast: false matrix: os: [blacksmith-8vcpu-ubuntu-2404, macos-26, windows-latest] + env: + DEPENDENCY_CACHE_JOB: Agent (${{ matrix.os }}) steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -137,13 +164,24 @@ jobs: id: environment run: | if [ "$RUNNER_OS" = Windows ]; then - python3 ../../scripts/ci/agent_cache.py --profile ci --output "$RUNNER_TEMP/agent-environment.json" + python3 ../../scripts/ci/agent_cache_scope.py --profile ci --output "$RUNNER_TEMP/agent-environment.json" else - nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache.py --profile ci --output "$RUNNER_TEMP/agent-environment.json" + nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache_scope.py --profile ci --output "$RUNNER_TEMP/agent-environment.json" fi + - name: Restore validated master dependency archive + id: dependency-archive + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py restore \ + --environment "$RUNNER_TEMP/agent-environment.json" --target "target" \ + --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-ci.yml \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + - name: Cache Agent Rust dependencies id: rust-cache + if: ${{ steps.dependency-archive.outputs.restored != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: workspaces: apps/maple-agent -> target @@ -151,26 +189,36 @@ jobs: # Dependency hashes belong in the action's exact suffix, not its restore prefix. key: maple-agent-ci-v2-${{ steps.environment.outputs.compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save-if: "false" - name: Check whether Cargo targets were restored id: restored-target run: | - if [ -f "$GITHUB_WORKSPACE/apps/maple-agent/target/CACHEDIR.TAG" ]; then - echo 'present=true' >> "$GITHUB_OUTPUT" - else - echo 'present=false' >> "$GITHUB_OUTPUT" - fi + python3 - <<'PYTHON' + import os + from pathlib import Path + target = Path(os.environ["GITHUB_WORKSPACE"]) / "apps/maple-agent/target" + present = any( + any(fingerprint.iterdir()) and any( + output.is_file() and output.stat().st_size > 0 + for pattern in ("*.rlib", "*.rmeta") + for output in (fingerprint.parent / "deps").glob(pattern) + ) + for fingerprint in target.rglob(".fingerprint") if fingerprint.is_dir() + ) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"present={str(present).lower()}\n") + PYTHON - name: Restore compatible legacy Cargo targets on a cache miss - if: ${{ steps.rust-cache.outputs.cache-hit != 'true' && steps.restored-target.outputs.present != 'true' }} + if: ${{ steps.dependency-archive.outputs.restored != 'true' && steps.restored-target.outputs.present != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: workspaces: apps/maple-agent -> target # Read only within the existing actual compiler/native ABI prefix. - # The primary cache retains its exact dependency suffix and owns saves. + # Native bootstrap is read-only; durable artifacts own cache writes. add-rust-environment-hash-key: "false" - key: maple-agent-ci-v2-${{ steps.environment.outputs.compatibility }} + key: maple-agent-ci-v2-${{ steps.environment.outputs.legacy_compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} save-if: "false" @@ -190,8 +238,7 @@ jobs: - name: Verify the canonical SDK and proxy dependency graph run: | - metadata="$(mktemp)" - trap 'rm -f "$metadata"' EXIT + metadata="$RUNNER_TEMP/agent-workspace-metadata.json" if [ "$RUNNER_OS" = Windows ]; then cargo metadata --locked --format-version 1 > "$metadata" else @@ -242,6 +289,29 @@ jobs: nix develop --no-update-lock-file . -c bash -c 'cargo metadata --locked --format-version 1 > "$1"' -- "$metadata" nix develop --no-update-lock-file . -c python3 dependencies/sqlcipher/verify_sqlcipher.py --metadata "$metadata" + - name: Prepare trusted dependency archive after validation + id: dependency-seed + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py pack \ + --environment "$RUNNER_TEMP/agent-environment.json" --metadata "$RUNNER_TEMP/agent-workspace-metadata.json" \ + --target "target" --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-ci.yml \ + --output-dir "$RUNNER_TEMP/agent-dependency-seed" \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + + - name: Upload durable dependency archive + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.dependency-seed.outputs.needs-upload == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ${{ steps.dependency-seed.outputs.artifact-name }} + path: | + ${{ runner.temp }}/agent-dependency-seed/manifest.json + ${{ runner.temp }}/agent-dependency-seed/dependencies.tar.gz + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 3 + - name: Summarize phase timings and target size if: always() run: python3 ../../scripts/ci/agent_measure.py --summarize target @@ -260,11 +330,16 @@ jobs: needs: changes if: ${{ always() && !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.agent != 'false') }} runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: read timeout-minutes: 90 strategy: fail-fast: false matrix: os: [blacksmith-8vcpu-ubuntu-2404, macos-26, windows-latest] + env: + DEPENDENCY_CACHE_JOB: Agent provider (${{ matrix.os }}) steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -335,17 +410,30 @@ jobs: id: environment run: | if [ "$RUNNER_OS" = Windows ]; then - python3 ../../scripts/ci/agent_cache.py --profile provider --cargo-config-dir "$PROVIDER_WORK" --output target/ci-metrics/environment.json + python3 ../../scripts/ci/agent_cache_scope.py --profile provider --cargo-config-dir "$PROVIDER_WORK" --output target/ci-metrics/environment.json else - nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache.py --profile provider --cargo-config-dir "$PROVIDER_WORK" --output target/ci-metrics/environment.json + nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache_scope.py --profile provider --cargo-config-dir "$PROVIDER_WORK" --output target/ci-metrics/environment.json fi + - name: Restore validated master dependency archive + id: dependency-archive + env: + GH_TOKEN: ${{ github.token }} + PROVIDER_DEPENDENCY_FINGERPRINT: ${{ steps.provider.outputs.dependency-fingerprint }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py restore \ + --environment "target/ci-metrics/environment.json" --target "$PROVIDER_TARGET" \ + --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-ci.yml \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" \ + --dependency-fingerprint "$PROVIDER_DEPENDENCY_FINGERPRINT" + - name: Restore only matching verified provider artifacts id: provider-cache + if: ${{ steps.dependency-archive.outputs.restored != 'true' }} uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: ${{ env.PROVIDER_TARGET }} - key: maple-agent-provider-v2-${{ steps.environment.outputs.compatibility }}-${{ steps.provider.outputs.dependency-fingerprint }} + key: maple-agent-provider-v2-${{ steps.environment.outputs.legacy_compatibility }}-${{ steps.provider.outputs.dependency-fingerprint }} - name: Test released rusqlite against the static provider run: | @@ -356,12 +444,31 @@ jobs: nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_measure.py --phase provider-test --target "$PROVIDER_TARGET" -- python3 scripts/test-rusqlite-provider.py "$metadata" --phase run --work-dir "$PROVIDER_WORK" --target-dir "$PROVIDER_TARGET" fi - - name: Save trusted provider artifacts - if: ${{ success() && github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.provider-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + - name: Prepare trusted dependency archive after validation + id: dependency-seed + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + PROVIDER_DEPENDENCY_FINGERPRINT: ${{ steps.provider.outputs.dependency-fingerprint }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py pack \ + --environment "target/ci-metrics/environment.json" --metadata "$PROVIDER_WORK/maple-provider-metadata.json" \ + --target "$PROVIDER_TARGET" --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-ci.yml \ + --output-dir "$RUNNER_TEMP/agent-dependency-seed" \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" \ + --dependency-fingerprint "$PROVIDER_DEPENDENCY_FINGERPRINT" + + - name: Upload durable dependency archive + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.dependency-seed.outputs.needs-upload == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - path: ${{ env.PROVIDER_TARGET }} - key: ${{ steps.provider-cache.outputs.cache-primary-key }} + name: ${{ steps.dependency-seed.outputs.artifact-name }} + path: | + ${{ runner.temp }}/agent-dependency-seed/manifest.json + ${{ runner.temp }}/agent-dependency-seed/dependencies.tar.gz + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 3 - name: Summarize phase timings and target size if: always() @@ -381,7 +488,12 @@ jobs: needs: changes if: ${{ always() && !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.agent != 'false') }} runs-on: blacksmith-8vcpu-ubuntu-2404 + permissions: + contents: read + actions: read timeout-minutes: 90 + env: + DEPENDENCY_CACHE_JOB: Agent Linux release binary steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -403,13 +515,24 @@ jobs: id: environment run: | if [ "$RUNNER_OS" = Windows ]; then - python3 ../../scripts/ci/agent_cache.py --profile release --output "$RUNNER_TEMP/agent-environment.json" + python3 ../../scripts/ci/agent_cache_scope.py --profile release --output "$RUNNER_TEMP/agent-environment.json" else - nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache.py --profile release --output "$RUNNER_TEMP/agent-environment.json" + nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache_scope.py --profile release --output "$RUNNER_TEMP/agent-environment.json" fi + - name: Restore validated master dependency archive + id: dependency-archive + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py restore \ + --environment "$RUNNER_TEMP/agent-environment.json" --target "target" \ + --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-ci.yml \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + - name: Cache Agent release dependencies id: rust-cache + if: ${{ steps.dependency-archive.outputs.restored != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: workspaces: apps/maple-agent -> target @@ -417,24 +540,34 @@ jobs: # Dependency hashes belong in the action's exact suffix, not its restore prefix. key: maple-agent-release-v2-${{ steps.environment.outputs.compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save-if: "false" - name: Check whether Cargo targets were restored id: restored-target run: | - if [ -f "$GITHUB_WORKSPACE/apps/maple-agent/target/CACHEDIR.TAG" ]; then - echo 'present=true' >> "$GITHUB_OUTPUT" - else - echo 'present=false' >> "$GITHUB_OUTPUT" - fi + python3 - <<'PYTHON' + import os + from pathlib import Path + target = Path(os.environ["GITHUB_WORKSPACE"]) / "apps/maple-agent/target" + present = any( + any(fingerprint.iterdir()) and any( + output.is_file() and output.stat().st_size > 0 + for pattern in ("*.rlib", "*.rmeta") + for output in (fingerprint.parent / "deps").glob(pattern) + ) + for fingerprint in target.rglob(".fingerprint") if fingerprint.is_dir() + ) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"present={str(present).lower()}\n") + PYTHON - name: Restore compatible legacy Cargo targets on a cache miss - if: ${{ steps.rust-cache.outputs.cache-hit != 'true' && steps.restored-target.outputs.present != 'true' }} + if: ${{ steps.dependency-archive.outputs.restored != 'true' && steps.restored-target.outputs.present != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: workspaces: apps/maple-agent -> target add-rust-environment-hash-key: "false" - key: maple-agent-release-v2-${{ steps.environment.outputs.compatibility }} + key: maple-agent-release-v2-${{ steps.environment.outputs.legacy_compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} save-if: "false" @@ -445,8 +578,7 @@ jobs: - name: Verify the canonical SDK and proxy dependency graph run: | - metadata="$(mktemp)" - trap 'rm -f "$metadata"' EXIT + metadata="$RUNNER_TEMP/agent-workspace-metadata.json" if [ "$RUNNER_OS" = Windows ]; then cargo metadata --locked --format-version 1 > "$metadata" else @@ -482,6 +614,29 @@ jobs: if-no-files-found: error retention-days: 5 + - name: Prepare trusted dependency archive after validation + id: dependency-seed + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py pack \ + --environment "$RUNNER_TEMP/agent-environment.json" --metadata "$RUNNER_TEMP/agent-workspace-metadata.json" \ + --target "target" --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-ci.yml \ + --output-dir "$RUNNER_TEMP/agent-dependency-seed" \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + + - name: Upload durable dependency archive + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.dependency-seed.outputs.needs-upload == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ${{ steps.dependency-seed.outputs.artifact-name }} + path: | + ${{ runner.temp }}/agent-dependency-seed/manifest.json + ${{ runner.temp }}/agent-dependency-seed/dependencies.tar.gz + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 3 + - name: Summarize phase timings and target size if: always() run: python3 ../../scripts/ci/agent_measure.py --summarize target diff --git a/.github/workflows/agent-desktop-build.yml b/.github/workflows/agent-desktop-build.yml index dc5300974..ce3124cb3 100644 --- a/.github/workflows/agent-desktop-build.yml +++ b/.github/workflows/agent-desktop-build.yml @@ -81,6 +81,9 @@ jobs: needs: changes if: ${{ always() && !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.agent != 'false') && (github.event_name == 'pull_request' || (github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch'))) }} runs-on: blacksmith-6vcpu-macos-26 + permissions: + contents: read + actions: read timeout-minutes: 120 strategy: fail-fast: false @@ -88,6 +91,7 @@ jobs: variant: ${{ fromJSON(github.event_name == 'pull_request' && '["dev"]' || '["dev","prod"]') }} env: MAPLE_RELEASE_PROFILE: ${{ matrix.variant }} + DEPENDENCY_CACHE_JOB: Build macOS binary (${{ matrix.variant }}) steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: @@ -107,37 +111,66 @@ jobs: run: python3 scripts/ci/select-xcode.py - name: Fingerprint the actual release environment id: environment - run: nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache.py --profile release --output "$RUNNER_TEMP/agent-environment.json" + run: nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache_scope.py --profile release --output "$RUNNER_TEMP/agent-environment.json" + - name: Restore validated master dependency archive + id: dependency-archive + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py restore \ + --environment "$RUNNER_TEMP/agent-environment.json" --target "target" \ + --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-desktop-build.yml \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + - name: Cache Agent release dependencies id: rust-cache + if: ${{ steps.dependency-archive.outputs.restored != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: workspaces: apps/maple-agent -> target add-rust-environment-hash-key: "true" key: maple-agent-release-v2-${{ matrix.variant }}-${{ steps.environment.outputs.compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save-if: "false" - name: Check whether Cargo targets were restored id: restored-target run: | - if [ -f "$GITHUB_WORKSPACE/apps/maple-agent/target/CACHEDIR.TAG" ]; then - echo 'present=true' >> "$GITHUB_OUTPUT" - else - echo 'present=false' >> "$GITHUB_OUTPUT" - fi + python3 - <<'PYTHON' + import os + from pathlib import Path + target = Path(os.environ["GITHUB_WORKSPACE"]) / "apps/maple-agent/target" + present = any( + any(fingerprint.iterdir()) and any( + output.is_file() and output.stat().st_size > 0 + for pattern in ("*.rlib", "*.rmeta") + for output in (fingerprint.parent / "deps").glob(pattern) + ) + for fingerprint in target.rglob(".fingerprint") if fingerprint.is_dir() + ) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"present={str(present).lower()}\n") + PYTHON + - name: Restore compatible legacy Cargo targets on a cache miss - if: ${{ steps.rust-cache.outputs.cache-hit != 'true' && steps.restored-target.outputs.present != 'true' }} + if: ${{ steps.dependency-archive.outputs.restored != 'true' && steps.restored-target.outputs.present != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: workspaces: apps/maple-agent -> target add-rust-environment-hash-key: "false" - key: maple-agent-release-v2-${{ matrix.variant }}-${{ steps.environment.outputs.compatibility }} + key: maple-agent-release-v2-${{ matrix.variant }}-${{ steps.environment.outputs.legacy_compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} save-if: "false" - name: Record the current release environment after restore run: | mkdir -p target/ci-metrics cp "$RUNNER_TEMP/agent-environment.json" target/ci-metrics/environment.json + - name: Verify the canonical SDK and proxy dependency graph + run: | + metadata="$RUNNER_TEMP/agent-workspace-metadata.json" + # shellcheck disable=SC2016 # The child shell expands its own positional argument. + nix develop --no-update-lock-file . -c bash -c 'cargo metadata --locked --format-version 1 > "$1"' -- "$metadata" + python3 ../../scripts/ci/verify-agent-rust-deps.py "$metadata" + nix develop --no-update-lock-file . -c python3 scripts/verify-sqlcipher-build.py "$metadata" - name: Build optimized Agent with embedded profile run: nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_measure.py --phase release-build --target target -- ./scripts/build-release.sh "$MAPLE_RELEASE_PROFILE" - name: Upload prebuilt Agent binary and notices @@ -151,6 +184,29 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Prepare trusted dependency archive after validation + id: dependency-seed + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py pack \ + --environment "$RUNNER_TEMP/agent-environment.json" --metadata "$RUNNER_TEMP/agent-workspace-metadata.json" \ + --target "target" --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-desktop-build.yml \ + --output-dir "$RUNNER_TEMP/agent-dependency-seed" \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + + - name: Upload durable dependency archive + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.dependency-seed.outputs.needs-upload == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ${{ steps.dependency-seed.outputs.artifact-name }} + path: | + ${{ runner.temp }}/agent-dependency-seed/manifest.json + ${{ runner.temp }}/agent-dependency-seed/dependencies.tar.gz + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 3 + - name: Summarize release phase timings and target size if: always() run: python3 ../../scripts/ci/agent_measure.py --summarize target @@ -291,6 +347,9 @@ jobs: needs: changes if: ${{ always() && !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.agent != 'false') && (github.event_name == 'pull_request' || (github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch'))) }} runs-on: blacksmith-8vcpu-ubuntu-2404 + permissions: + contents: read + actions: read timeout-minutes: 120 strategy: fail-fast: false @@ -298,6 +357,7 @@ jobs: variant: ${{ fromJSON(github.event_name == 'pull_request' && '["dev"]' || '["dev","prod"]') }} env: MAPLE_RELEASE_PROFILE: ${{ matrix.variant }} + DEPENDENCY_CACHE_JOB: Linux bundle (${{ matrix.variant }}) steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: @@ -314,37 +374,66 @@ jobs: run: bash scripts/ci/verify-nix-runtime.sh - name: Fingerprint the actual release environment id: environment - run: nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache.py --profile release --output "$RUNNER_TEMP/agent-environment.json" + run: nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache_scope.py --profile release --output "$RUNNER_TEMP/agent-environment.json" + - name: Restore validated master dependency archive + id: dependency-archive + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py restore \ + --environment "$RUNNER_TEMP/agent-environment.json" --target "target" \ + --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-desktop-build.yml \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + - name: Cache Agent release dependencies id: rust-cache + if: ${{ steps.dependency-archive.outputs.restored != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: workspaces: apps/maple-agent -> target add-rust-environment-hash-key: "true" key: maple-agent-release-v2-${{ matrix.variant }}-${{ steps.environment.outputs.compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save-if: "false" - name: Check whether Cargo targets were restored id: restored-target run: | - if [ -f "$GITHUB_WORKSPACE/apps/maple-agent/target/CACHEDIR.TAG" ]; then - echo 'present=true' >> "$GITHUB_OUTPUT" - else - echo 'present=false' >> "$GITHUB_OUTPUT" - fi + python3 - <<'PYTHON' + import os + from pathlib import Path + target = Path(os.environ["GITHUB_WORKSPACE"]) / "apps/maple-agent/target" + present = any( + any(fingerprint.iterdir()) and any( + output.is_file() and output.stat().st_size > 0 + for pattern in ("*.rlib", "*.rmeta") + for output in (fingerprint.parent / "deps").glob(pattern) + ) + for fingerprint in target.rglob(".fingerprint") if fingerprint.is_dir() + ) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"present={str(present).lower()}\n") + PYTHON + - name: Restore compatible legacy Cargo targets on a cache miss - if: ${{ steps.rust-cache.outputs.cache-hit != 'true' && steps.restored-target.outputs.present != 'true' }} + if: ${{ steps.dependency-archive.outputs.restored != 'true' && steps.restored-target.outputs.present != 'true' }} uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: workspaces: apps/maple-agent -> target add-rust-environment-hash-key: "false" - key: maple-agent-release-v2-${{ matrix.variant }}-${{ steps.environment.outputs.compatibility }} + key: maple-agent-release-v2-${{ matrix.variant }}-${{ steps.environment.outputs.legacy_compatibility }} cmd-format: python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0} save-if: "false" - name: Record the current release environment after restore run: | mkdir -p target/ci-metrics cp "$RUNNER_TEMP/agent-environment.json" target/ci-metrics/environment.json + - name: Verify the canonical SDK and proxy dependency graph + run: | + metadata="$RUNNER_TEMP/agent-workspace-metadata.json" + # shellcheck disable=SC2016 # The child shell expands its own positional argument. + nix develop --no-update-lock-file . -c bash -c 'cargo metadata --locked --format-version 1 > "$1"' -- "$metadata" + python3 ../../scripts/ci/verify-agent-rust-deps.py "$metadata" + nix develop --no-update-lock-file . -c python3 scripts/verify-sqlcipher-build.py "$metadata" - name: Build optimized Agent with embedded profile run: nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_measure.py --phase release-build --target target -- ./scripts/build-release.sh "$MAPLE_RELEASE_PROFILE" - name: Package Linux installed bundle @@ -366,6 +455,29 @@ jobs: if-no-files-found: error retention-days: 30 + - name: Prepare trusted dependency archive after validation + id: dependency-seed + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: | + python3 ../../scripts/ci/agent_dependency_archive.py pack \ + --environment "$RUNNER_TEMP/agent-environment.json" --metadata "$RUNNER_TEMP/agent-workspace-metadata.json" \ + --target "target" --job-name "$DEPENDENCY_CACHE_JOB" --workflow agent-desktop-build.yml \ + --output-dir "$RUNNER_TEMP/agent-dependency-seed" \ + --receipt "$RUNNER_TEMP/agent-dependency-receipt.json" + + - name: Upload durable dependency archive + if: ${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.dependency-seed.outputs.needs-upload == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ${{ steps.dependency-seed.outputs.artifact-name }} + path: | + ${{ runner.temp }}/agent-dependency-seed/manifest.json + ${{ runner.temp }}/agent-dependency-seed/dependencies.tar.gz + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 3 + - name: Summarize release phase timings and target size if: always() run: python3 ../../scripts/ci/agent_measure.py --summarize target diff --git a/.github/workflows/opensecret-ci.yml b/.github/workflows/opensecret-ci.yml index 354cd7123..c40281cfc 100644 --- a/.github/workflows/opensecret-ci.yml +++ b/.github/workflows/opensecret-ci.yml @@ -67,13 +67,13 @@ jobs: save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - name: Check formatting - run: nix develop --no-update-lock-file '.?submodules=1' -c cargo fmt --all -- --check + run: nix develop --no-update-lock-file '.?submodules=1#ci' -c cargo fmt --all -- --check - name: Lint all backend targets and features - run: nix develop --no-update-lock-file '.?submodules=1' -c cargo clippy --locked --all-targets --all-features -- -D warnings + run: nix develop --no-update-lock-file '.?submodules=1#ci' -c cargo clippy --locked --all-targets --all-features -- -D warnings - name: Run backend tests - run: nix develop --no-update-lock-file '.?submodules=1' -c cargo test --locked --all-features + run: nix develop --no-update-lock-file '.?submodules=1#ci' -c cargo test --locked --all-features nix: name: OpenSecret Nix checks and application package diff --git a/.github/workflows/sdk-integration.yml b/.github/workflows/sdk-integration.yml index 184caf606..4c3624ebe 100644 --- a/.github/workflows/sdk-integration.yml +++ b/.github/workflows/sdk-integration.yml @@ -81,19 +81,19 @@ jobs: - name: Migrate the disposable database and build OpenSecret run: | - nix develop --no-update-lock-file './services/opensecret?submodules=1' -c bash -lc ' + nix develop --no-update-lock-file './services/opensecret?submodules=1#ci' -c bash -lc ' set -euo pipefail cd services/opensecret diesel migration run cargo build --locked ' - nix develop --no-update-lock-file './services/opensecret?submodules=1' -c \ + nix develop --no-update-lock-file './services/opensecret?submodules=1#ci' -c \ psql "${DATABASE_URL}" -X -v ON_ERROR_STOP=1 \ -f sdk/test/integration/bootstrap.sql - name: Verify OAuth persistence and encrypted callback flows run: | - nix develop --no-update-lock-file './services/opensecret?submodules=1' -c bash -lc ' + nix develop --no-update-lock-file './services/opensecret?submodules=1#ci' -c bash -lc ' set -euo pipefail cd services/opensecret bash ../../.agents/skills/validate-opensecret/scripts/disposable_db_tests.sh @@ -101,7 +101,7 @@ jobs: - name: Start and verify OpenSecret run: | - nix develop --no-update-lock-file './services/opensecret?submodules=1' -c \ + nix develop --no-update-lock-file './services/opensecret?submodules=1#ci' -c \ services/opensecret/target/debug/opensecret \ > "${RUNNER_TEMP}/opensecret.log" 2>&1 & echo "$!" > "${RUNNER_TEMP}/opensecret.pid" diff --git a/apps/maple-agent/crates/maple-agent/src/agent.rs b/apps/maple-agent/crates/maple-agent/src/agent.rs index 044e4fb64..97f692efa 100644 --- a/apps/maple-agent/crates/maple-agent/src/agent.rs +++ b/apps/maple-agent/crates/maple-agent/src/agent.rs @@ -257,7 +257,9 @@ struct ServiceState { runtime: tokio::sync::Mutex>>, /// A stopped runtime retains its sessions until their final writes succeed. /// It is never returned to callers as a runnable runtime. - shutdown_pending: tokio::sync::Mutex>, + /// Observers share access so a maintenance read of `None` cannot look like + /// pending history to synchronous admission checks. Transitions are exclusive. + shutdown_pending: tokio::sync::RwLock>, /// Orders start, stop and restart. lifecycle: tokio::sync::Mutex<()>, /// Held while the account's settings are read and saved, so a save cannot @@ -324,7 +326,7 @@ impl MapleAgentService { host, state: Arc::new(ServiceState { runtime: tokio::sync::Mutex::new(None), - shutdown_pending: tokio::sync::Mutex::new(None), + shutdown_pending: tokio::sync::RwLock::new(None), lifecycle: tokio::sync::Mutex::new(()), settings: tokio::sync::Mutex::new(()), stores: std::sync::Mutex::new(HashMap::new()), @@ -500,7 +502,7 @@ impl MapleAgentService { if let Some(runtime) = runtime { runtime.flush_loaded_sessions().await?; } - let pending = self.state.shutdown_pending.lock().await.clone(); + let pending = self.state.shutdown_pending.read().await.clone(); let replacement = if let Some(pending) = &pending { // The inner error is the first flush failure; the completed drain // has already stopped tools and notified extensions exactly once. @@ -528,7 +530,7 @@ impl MapleAgentService { // Acquire both publication slots before clearing the failure latch. // Cancellation at either await leaves the old non-runnable owner and // notice intact; no await separates the successful publication below. - let mut pending_slot = self.state.shutdown_pending.lock().await; + let mut pending_slot = self.state.shutdown_pending.write().await; let mut runtime_slot = self.state.runtime.lock().await; match failure { Some(failure) if !store.clear_persistence_failure(failure.revision) => { @@ -561,7 +563,7 @@ impl MapleAgentService { /// Called with the service lifecycle lock. Publish retained ownership before /// the first await that can leave a drain running without its caller. async fn begin_runtime_shutdown_locked(&self) -> Option<(PendingRuntimeShutdown, bool)> { - let mut pending_slot = self.state.shutdown_pending.lock().await; + let mut pending_slot = self.state.shutdown_pending.write().await; if let Some(pending) = pending_slot.as_ref() { return Some((pending.clone(), true)); } @@ -587,7 +589,7 @@ impl MapleAgentService { first_flush?; } pending.runtime.finish_shutdown().await?; - self.state.shutdown_pending.lock().await.take(); + self.state.shutdown_pending.write().await.take(); Ok(()) } @@ -625,11 +627,11 @@ impl MapleAgentService { if self.state.runtime.lock().await.is_some() { return Err("Stop Agent work before finishing the local store reset".into()); } - let pending = self.state.shutdown_pending.lock().await.clone(); + let pending = self.state.shutdown_pending.read().await.clone(); if let Some(pending) = pending { let _save_result = pending.wait().await?; pending.runtime.discard_drained_sessions().await; - self.state.shutdown_pending.lock().await.take(); + self.state.shutdown_pending.write().await.take(); } Ok(()) } @@ -691,7 +693,9 @@ impl MapleAgentService { // A stopped runtime can still own unsaved source entries. Mutations // such as task deletion must not remove their destination before Retry // saves them. Status and committed-history reads remain available. - match self.state.shutdown_pending.try_lock() { + // A writer is publishing or removing retained ownership. Keep that + // transition closed, while concurrent status/maintenance reads coexist. + match self.state.shutdown_pending.try_read() { Ok(pending) if pending.is_none() => Ok(()), _ => Err(RUNTIME_SAVE_PENDING_ERROR.to_string()), } @@ -832,7 +836,7 @@ impl AgentRuntimeHandle { /// The account's running runtime, or an error when it is not running. async fn runtime(&self) -> Result, String> { - if self.service.state.shutdown_pending.lock().await.is_some() { + if self.service.state.shutdown_pending.read().await.is_some() { return Err(RUNTIME_SAVE_PENDING_ERROR.to_string()); } self.current_runtime() @@ -870,7 +874,7 @@ impl AgentRuntimeHandle { maple_api_session: Arc, request: Option, ) -> Result { - if self.service.state.shutdown_pending.lock().await.is_some() { + if self.service.state.shutdown_pending.read().await.is_some() { return Err(RUNTIME_SAVE_PENDING_ERROR.to_string()); } if let Some(current) = self.service.state.runtime.lock().await.as_ref() { @@ -941,7 +945,7 @@ impl AgentRuntimeHandle { if let Some(current) = self.service.state.runtime.lock().await.as_ref() { current.ensure_account(&self.account_scope)?; } - if let Some(pending) = self.service.state.shutdown_pending.lock().await.as_ref() { + if let Some(pending) = self.service.state.shutdown_pending.read().await.as_ref() { pending.runtime.ensure_account(&self.account_scope)?; } self.service.shutdown_runtime_locked().await diff --git a/apps/maple-agent/crates/maple-agent/src/agent/maintenance.rs b/apps/maple-agent/crates/maple-agent/src/agent/maintenance.rs index a27516b43..31ee7eb64 100644 --- a/apps/maple-agent/crates/maple-agent/src/agent/maintenance.rs +++ b/apps/maple-agent/crates/maple-agent/src/agent/maintenance.rs @@ -139,7 +139,7 @@ fn admitted(state: &ServiceState) -> bool { // effects yield until Retry or another completed shutdown saves them. && state .shutdown_pending - .try_lock() + .try_read() .is_ok_and(|pending| pending.is_none()) } diff --git a/apps/maple-agent/crates/maple-agent/src/agent/projects.rs b/apps/maple-agent/crates/maple-agent/src/agent/projects.rs index 517b803ce..399e04021 100644 --- a/apps/maple-agent/crates/maple-agent/src/agent/projects.rs +++ b/apps/maple-agent/crates/maple-agent/src/agent/projects.rs @@ -354,6 +354,57 @@ mod tests { ); } + #[tokio::test] + async fn pending_history_observers_do_not_reject_committed_project_renames() { + let fixture = fixture().await; + // Maintenance observes this same state from a blocking worker. A reader + // of `None` is not retained history and must not reject ordinary work. + let observed = fixture.service.state.shutdown_pending.read().await; + assert!(observed.is_none()); + let root = fixture + .handle + .rename_project_root(fixture.root.clone(), "While observed".into()) + .await + .unwrap(); + assert_eq!(root.name, "While observed"); + assert_eq!( + fixture + .handle + .project_root_metadata(fixture.root.clone()) + .await + .unwrap(), + root + ); + assert!(matches!( + fixture.recorder.events().as_slice(), + [AgentServiceEvent::ProjectsChanged { roots, .. }] + if roots.as_slice() == std::slice::from_ref(&root) + )); + drop(observed); + + // An exclusive transition still blocks admission even before it has + // installed retained history. No rename is committed or published. + let transition = fixture.service.state.shutdown_pending.write().await; + assert_eq!( + fixture + .handle + .rename_project_root(fixture.root.clone(), "During transition".into()) + .await + .unwrap_err(), + super::super::RUNTIME_SAVE_PENDING_ERROR + ); + drop(transition); + assert_eq!( + fixture + .handle + .project_root_metadata(fixture.root) + .await + .unwrap(), + root + ); + assert_eq!(fixture.recorder.events().len(), 1); + } + #[tokio::test] async fn concurrent_renames_publish_single_committed_records_in_order() { let fixture = fixture().await; diff --git a/apps/maple-agent/crates/maple-agent/src/agent/shutdown_tests.rs b/apps/maple-agent/crates/maple-agent/src/agent/shutdown_tests.rs index 0f85e8522..6590032c2 100644 --- a/apps/maple-agent/crates/maple-agent/src/agent/shutdown_tests.rs +++ b/apps/maple-agent/crates/maple-agent/src/agent/shutdown_tests.rs @@ -45,7 +45,7 @@ async fn failed_shutdown_retains_source_writes_and_retry_starts_a_fresh_runtime( .service .state .shutdown_pending - .lock() + .read() .await .is_some() ); @@ -69,7 +69,7 @@ async fn failed_shutdown_retains_source_writes_and_retry_starts_a_fresh_runtime( .service .state .shutdown_pending - .lock() + .read() .await .is_some() ); @@ -82,7 +82,7 @@ async fn failed_shutdown_retains_source_writes_and_retry_starts_a_fresh_runtime( .service .state .shutdown_pending - .lock() + .read() .await .is_none() ); @@ -152,7 +152,7 @@ async fn cancelling_a_shutdown_request_keeps_the_drain_and_retained_sessions_own .service .state .shutdown_pending - .lock() + .read() .await .is_some() { @@ -171,7 +171,7 @@ async fn cancelling_a_shutdown_request_keeps_the_drain_and_retained_sessions_own .service .state .shutdown_pending - .lock() + .read() .await .clone() .unwrap(); @@ -267,7 +267,7 @@ async fn confirmed_reset_retains_unsaved_source_until_the_store_reset_succeeds() .service .state .shutdown_pending - .lock() + .read() .await .is_some() ); @@ -293,7 +293,7 @@ async fn confirmed_reset_retains_unsaved_source_until_the_store_reset_succeeds() .service .state .shutdown_pending - .lock() + .read() .await .is_none() ); @@ -363,7 +363,7 @@ async fn confirmed_reset_intent_failure_can_stop_and_recover_without_replaying_r .service .state .shutdown_pending - .lock() + .read() .await .is_some() ); @@ -400,7 +400,7 @@ async fn confirmed_reset_intent_failure_can_stop_and_recover_without_replaying_r .service .state .shutdown_pending - .lock() + .read() .await .is_none() ); diff --git a/docs/agent-ci-performance.md b/docs/agent-ci-performance.md index f9cce03b4..93047b130 100644 --- a/docs/agent-ci-performance.md +++ b/docs/agent-ci-performance.md @@ -156,10 +156,11 @@ When neither an exact hit nor a restored Cargo target is present, a second, read-only action invocation searches the existing ABI/job/OS/architecture prefix without the supplemental host-toolchain hash. Archive paths and their cache-service version remain unchanged, allowing the existing master archives -to be used during migration. Its save step is disabled; only the primary action -can save a stable exact key after successful master validation. Checking the -restored target's `CACHEDIR.TAG` avoids downloading another large archive when -the primary already restored a different dependency suffix. +to be used during migration. Its save step was disabled; at that stage only +the primary action saved stable exact keys after successful master validation. +That migration initially checked the restored target's `CACHEDIR.TAG`. The recovery implementation below checks +actual dependency libraries/fingerprints instead, because a valid archive can +omit the tag and otherwise trigger a second download. The cold job spent **36m00s** building and **16m42s** preparing its environment. Its test phase rebuilt zero compilation units, so the Git metadata freshness @@ -181,3 +182,195 @@ Linux AppImage and downloaded Linux verifier), reducing desktop jobs from eight to four. It does not remove platform workspace/provider tests or the separate plain Linux release build. The job-count reduction saves their runner work; the slowest remaining job still determines PR wall time. + +## October 25–26, 2026: repository-wide recovery audit + +The follow-up audit covers source at +[`6d6c259d`](https://github.com/MaplePrivacyLabs/Maple/tree/6d6c259dbfc24e05fabb0d508593fecf7820fdda/.github/workflows): +**50 workflows, 125 logical jobs before matrix expansion, and 746 steps**. +The cache inventory groups **134 declared instances across 41 workflows into +31 families**, including 79 Nix installer instances. An installer archive is +not a Rust dependency cache or a populated Nix build store. Inventory counts +record configuration; they do not prove that every instance ran or restored. + +The audit includes Agent, Research, SDK, Proxy, backend, mobile, web, package +consumers and publication boundaries. It separates compilation, environment +materialization, test execution, artifact transfer and cancellation. A green +selector or aggregate with intentionally skipped children is not fresh platform +build evidence. The historical measurements above remain valid observations +of their own revisions; they do not establish present cache persistence. + +### Observed cold and warm work + +| Actual job | Whole job | Measured build phase | Cache and compiler evidence | +| --- | ---: | ---: | --- | +| [Earlier Windows workspace](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40060011728/job/120771414230) | 24m11s | 13m37s | Compatible target archive restored; native crypto reused; tests passed and a stable master seed was published. | +| [Earlier macOS workspace](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40060011728/job/120771414217) | 19m08s | 10m08s | Exact primary archive restored; native crypto reused and tests passed. | +| [`ff4c1037` Windows workspace](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40229042168/job/121276681205) | 49m43s | 35m09s | Both restores missed; 908/908 build units compiled. The SQLCipher/OpenSSL native unit took 13m24s; the new seed save was rejected. | +| [`ff4c1037` macOS workspace](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40229042168/job/121276681265) | 48m04s | 34m49s | Both restores missed; 1293/1293 build units compiled. The native unit took 11m56s; the new seed save was rejected. | +| [`ff4c1037` Linux workspace](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40229042168/job/121276681271) | 7m04s | Not separately compared here | The exact 2,784,641,540-byte archive was extracted; the log compiled 27 crates and no native crypto crates. | +| [`ff4c1037` macOS Dev producer](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40229042279/job/121276739447) | 12m51s | 10m13s | Exact release archive restored; 13/1261 units compiled, with no native crypto compilation. | + +These jobs used different revisions and hosted conditions. Native compilation +units overlap other work; their durations must not be added to the complete +build phase. The observations support a substantial recovery opportunity from +durable dependency reuse, not a controlled speedup percentage or a promised +end-to-end PR duration. + +The regression also appears in real PRs. The +[#1291 Windows workspace](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40254500033/job/121353430523) +took 44m49s and its +[macOS workspace](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40254500033/job/121353430510) +took 37m54s after both target lookups missed and native crypto compiled. +That PR was based directly on `ff4c1037` and requested the same compatibility +and dependency keys as the completed master writers. Repeated misses there +were not a new compiler/native ABI change on every PR. + +### Persistence failed despite successful build jobs + +Completed master runs +[`e5870485`](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40172748044) +and [`ff4c1037`](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40229042168) +passed their macOS/Windows builds but received the cache-service warning: + +> Your usage has exceeded the allowed rate limit. Your cache is now read-only. + +This is an explicit refusal to reserve new cache entries. It is not an observed +HTTP 429 status, proof of a concurrent writer, or an established zero-budget +diagnosis. The underlying enforcement cause remained unresolved at this audit. +The cache action reported the failed save as a warning, so a successful job or +post-cache step did not mean that the requested seed persisted. Existing +archives in older compatibility namespaces did not satisfy the current keys. + +Blacksmith persistence must be checked separately. The Linux workspace archive +restored by the completed `ff4c1037` job was absent from a subsequent provider +inventory. A Linux Prod producer also restored an exact archive in one master +run, then missed the same key in a later run and rebuilt native dependencies. +That is same-key persistence evidence, not proof of a particular eviction +cause. A provider capacity request, a displayed quota, or an inventory entry +alone is not acceptance: retain upload receipts and prove subsequent extraction +and compiler reuse on each actual backend. + +Compatibility precision remains a separate concern. Direct macOS/Windows jobs +must not inherit unrelated Linux recipe changes. The audit also found whole +flake-output hashes and cross-platform dependency-feature settings entering +compatibility decisions even when the selected compiler/native build inputs +did not change. `agent_cache_scope.py` now wraps the unchanged legacy identity +and removes only dependency declarations/feature edges proved inactive on the native +target. Unknown cfg, overrides and ambiguous features remain conservative. +Target-specific regression cases must retain invalidation for real Rust, +linker/SDK/native ABI, Cargo configuration, flags, features, profile and +SQLCipher provenance changes. Narrowing a fingerprint cannot repair a rejected +write or a vanished archive. + +### SDK integration spent time preparing the wrong environment + +The [#1291 SDK integration job](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40254500039/job/121353433324) +passed in **34m34s**. Its database/backend setup step took **27m48s**, but the +backend Cargo build reported only **3m23s**. Approximately **24m19s** elapsed +before the full development environment was ready, including source builds of +Podman, TPM tooling, GnuPG, systemd, GPGME and BWS. The development-process +flags prevented service startup; they did not remove those packages from the +Nix shell's dependency closure. + +The recovery implementation adds a separate backend `#ci` environment and +selects it in the credential-free backend and SDK integration commands. It uses +the existing pinned Rust and native-library inputs while excluding unrelated +interactive/container/credential tooling. It retains Diesel, PostgreSQL server +and client tools, and the Python dependencies used by the actual checks. +PostgreSQL server tools are necessary because the OAuth tests create disposable +databases even when the SDK backend uses the workflow's PostgreSQL service. +Migrations, OAuth persistence, TypeScript integration and Rust integration stay +selected. The default developer shell, EIF derivations, signing and deployment +inputs are separate boundaries. The 24-minute observation identifies work to +remove; hosted savings from the smaller environment are not yet verified. + +### Cancellation could discard outstanding validation + +In the seven-day master snapshot, specifically cancelled Agent jobs consumed +approximately **977 execution minutes** after deduplicating carried-over rerun +receipts. This is consumed runtime, not a claim that all of it is avoidable. +Successful jobs or artifacts from a cancelled workflow can still have value, +and the latest selected source must still receive complete validation. + +The previous workflow-level cancellation ran before immediate-push path +selection. A selected run could be interrupted by a later unrelated push whose +selector then skipped every actual Agent job. One observed sequence was +[`6d6c259d`](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40263279799), +which contained a real Linux test failure while macOS/Windows were still +running, followed by the +[PCR-only master run](https://github.com/MaplePrivacyLabs/Maple/actions/runs/40265757961), +which skipped those jobs after the earlier workflow was cancelled. Its green +selector did not certify the outstanding Agent changes. + +Agent CI now selects cumulative changes from its last verified component +baseline rather than treating every immediate push as a complete baseline. +A completed successful ancestor must contain actual passing jobs for all three +workspaces, all three providers and the plain Linux release before it can be a +CI baseline. Desktop packaging keeps its separate four-consumer baseline. +Failed, cancelled or selector-only runs cannot consume outstanding work. The +existing cancellation policy remains; the latest run reselects unfinished +changes. PR updates can still replace superseded previews. Tests must cover +selected-to-unrelated, selected-to-selected, failed-to-unrelated and +cancelled-to-unrelated transitions, along with the required platform lanes. +This scheduling correction is not a new release or publication authorization. + +### Recovery implementation and acceptance + +The recovery implementation is present in the workflow/helpers; review and +validation are still in progress. Hosted cache persistence, test completion and +speed recovery remain unverified: + +- **Trusted dependency seeds:** artifact-backed recovery is the primary read + path, with native caches retained only as read-only bootstrap. PRs remain + read-only. The consumer proves the designated repository's push/master run, + exact platform/profile producer job and attempt, source commit and artifact + creation interval. That producer job and its steps must have completed + successfully; another lane's result is not substituted for this proof, and + the seed does not certify the rest of the workflow. Manifest compatibility, + API ZIP digest, archive digest, bounded sizes/file counts and safe paths are + verified before dependency-only extraction. Git sources require a reviewed + public-repository allowlist; unknown sources and colliding output names are + excluded. Local/path crates and shipping + or test executables are excluded. Provider seeds require the exact verified + test-only graph; other compatible seeds leave locked resolution/freshness to + Cargo. Missing or rejected seeds fall back to native bootstrap or cold build. +- **Bounded refresh and visible publication:** validated master-push builders + prepare/upload seeds after their existing checks. Preparation or required + upload failure fails the job. Artifacts have three-day retention; a matching + graph restored from a proven seed created within 24 hours avoids repacking, + otherwise a selected successful master build refreshes it. There is no new + daily release/dispatch. Restore receipts identify the source run/artifact, + bytes and extraction time; publication and current product tests remain + separate evidence. Transfer and retention costs still need measurement. +- **Backend CI environment:** the backend/SDK integration calls now select the + smaller pinned shell. Prove its closure omits unused developer tooling, then + run the same backend and both SDK integration suites on a fresh hosted worker. + Compare environment materialization separately from Cargo and tests. +- **Cumulative selection:** Agent CI and Desktop use their respective verified + baselines so unrelated pushes do not consume unverified changes. Preserve the + existing cancellation/profile policies and verify the latest selected head + and each downloaded consumer explicitly. +- **Deterministic history regression:** a pending-history persistence fix is + under review for the observed concurrent project/task test failure. A retry + that happens to pass is not a fix. Require a deterministic reproduction, + focused regressions and full selected platform tests before accepting it. + +Acceptance requires a successful trusted seed-producing run and a later real +master or PR consumer for every changed cache lane. Retain exact revisions, +workflow attempts, selected compiler/native identities, restored archive or +artifact metadata, bounded Cargo timing reports and successful tests. Prove +that a version-only change reuses compatible dependencies and that a real +incompatible compiler/native/feature change does not. Compare both end-to-end +critical path and total runner time across several warmed natural runs; do not +report earlier successful seeds as proof of a later restore. + +All existing coverage remains part of the acceptance contract: the three +workspace platforms, Linux feature/security checks, three verified provider +lanes, signed SQLCipher source regeneration, notices, the optimized unpackaged +Linux binary, Dev/Prod master packages, PR Dev packages and actual downloaded +consumers. The unpackaged, Dev and Prod profiles have distinct identities and +trust settings; their final binaries are not interchangeable. No improvement +here requires reducing encryption/password costs, dropping tests, merging +profile trust roots, weakening signing/provenance, changing PCR approvals, +dispatching a release or publishing/deploying an application. diff --git a/docs/repository-workflows.md b/docs/repository-workflows.md index 338230424..be4ab08d6 100644 --- a/docs/repository-workflows.md +++ b/docs/repository-workflows.md @@ -132,28 +132,73 @@ The final `Agent CI required jobs` check rejects failed or unexpectedly skipped selected jobs. All retain fail-closed change selection. The plain Linux binary and Desktop Dev/Prod packages have separate profiles and caches. -Before restoring targets, `scripts/ci/agent_cache.py` fingerprints Rust inside -the component Nix shell, or the explicitly selected Windows MSVC environment, -plus native ABI/toolchain, flags, features, profile and provider configuration. -The component shell also pins Rustup proxies to its Nix toolchain so Cargo's -external Clippy/rustfmt commands cannot silently select a host compiler. -Windows setup also pins Cargo's linker to the selected MSVC `link.exe` and -compiles/runs an offline fixture with a Cargo build script before restoring -targets. This catches Git Bash's unrelated `link.exe` shadowing Microsoft tools. -The workspace and Desktop caches use this compatibility prefix with -`rust-cache`'s automatic dependency suffix. PRs only read; successful master -pushes write. Application version changes can reuse compatible dependencies; -Cargo still checks freshness. Cache probes use `agent_cache_command.py` to -identify only the actual selected compiler; incidental installed Rustup -toolchains do not describe these builds and are excluded from the inventory. -After a complete target-cache miss, a read-only lookup can restore an older -archive within the same actual compiler/native ABI, job, OS and architecture -prefix. A restored Cargo target marker prevents a second archive download -after an ordinary dependency-prefix hit. The primary action retains its exact -dependency hash and remains the only writer. The provider cache additionally requires the -verified, resolved upstream test-only lockfile and transformed manifest, with -no fallback across provider dependency graphs. Its temporary source and target -remain outside shipping workspace outputs. +Before restoring targets, `scripts/ci/agent_cache_scope.py` wraps the existing +`agent_cache.py` identity without changing its legacy digest. It fingerprints +Rust inside the component Nix shell, or the explicitly selected Windows MSVC +environment, plus the native ABI/toolchain, flags, profile, Cargo configuration +and provider inputs. For a known native target and modern Cargo resolver it +omits only dependency declarations and feature edges proved inactive on that +target. Unknown cfg expressions, target overrides and ambiguous feature edges +stay conservative. Real compiler/native changes still invalidate; the original +identity remains available for legacy bootstrap. + +The component shell pins Rustup proxies to its Nix toolchain so Cargo's external +Clippy/rustfmt commands cannot silently select a host compiler. Windows setup +pins Cargo's linker to the selected MSVC `link.exe` and compiles/runs an offline +Cargo build-script fixture before restore. Cache probes use +`agent_cache_command.py` to identify the selected compiler while excluding its +optional incidental installed-Rustup inventory. + +Agent workspace, provider, plain release and Desktop Dev/Prod builders first +try `agent_dependency_archive.py` through the Actions artifact service. A +consumer accepts only this repository's designated push/master workflow and +its exact matching platform/profile job. GitHub run/attempt, source commit, +artifact creation interval and completed-successful producer job/steps must +agree with the manifest. This proves that dependency seed's producer job; it +does not declare unrelated workflow jobs or product checks green. Actual +compiler/native/profile compatibility must match, and providers additionally +require their exact verified upstream test-only dependency fingerprint. +Workspace/release archives record their dependency graph; Cargo still resolves +the current locked graph and checks freshness after a compatible restore. + +The archive includes only crates.io and reviewed, pinned public Git dependencies, +registry/index data and their permitted dependency build outputs. New Git +repositories require public-source review before entering the helper's allowlist; +unknown sources and colliding package/target names are excluded. Local/path +crates, application/test executables, app bundles, notices, credentials and +Git hooks are excluded. The helper verifies the API's artifact ZIP digest, +manifest/archive digest, bounded sizes/file counts and safe dependency-only +paths before extraction; it rejects links, special files, traversal and path +aliases. Missing, incompatible or unproven artifacts fall back to native +bootstrap or a cold build. These archives remain executable dependency build +inputs, so trusted provenance is required in addition to checksums. PRs only +read them and run with no signing or publication credentials. + +Native `rust-cache` and exact provider-cache lookups are now read-only bootstrap +sources. The initial lookup retains the scoped compiler/dependency key; an +older compatible namespace can be searched with the unchanged legacy identity +only when actual dependency libraries/fingerprints were not restored. It does +not rely on `CACHEDIR.TAG`, which can be absent from an otherwise usable archive. +Provider bootstrap never falls back across verified test-only graphs. + +After its existing validation passes, only a successful master-push builder +prepares/uploads a dependency seed. Preparation or required upload failure +fails the job instead of becoming a cache-save warning. Uploads have three-day +retention. A successful master builder can reuse its proven restore receipt +without repacking only when the dependency graph still matches and the seed +was created within the previous 24 hours; otherwise it refreshes the seed. +This refresh occurs during selected master builds, not a new daily dispatch. +Each builder preserves its platform/profile/job boundary; signed packages and +publication artifacts are separate outputs. + +Agent CI also compares master cumulatively against the last successful +ancestor whose three workspace jobs, three provider jobs and plain Linux +release job actually passed. Desktop packaging retains its independent +all-four-downloaded-verifier baseline. Selector-only, failed or cancelled runs +cannot consume outstanding changes. The existing master cancellation policy +can still replace a run, but a subsequent unrelated push reselects the +unverified component changes; lookup/history uncertainty selects a build. +PRs retain their merge-base selection and superseded-preview cancellation. Unsigned builders upload seven-day `maple-agent-*-metrics-*` artifacts with actual environment identity, phase durations, logical target sizes, and bounded diff --git a/flake.nix b/flake.nix index 759a404cd..95813bc54 100644 --- a/flake.nix +++ b/flake.nix @@ -816,6 +816,8 @@ cd "$src" python3 scripts/ci/test_agent_workflows.py python3 scripts/ci/test_agent_cache.py + python3 scripts/ci/test_agent_cache_scope.py + python3 scripts/ci/test_agent_dependency_archive.py python3 scripts/ci/test_agent_windows.py python3 scripts/ci/test_agent_measure.py python3 apps/maple-agent/scripts/test_rusqlite_provider.py diff --git a/scripts/ci/agent_cache_scope.py b/scripts/ci/agent_cache_scope.py new file mode 100644 index 000000000..79c8f8ccf --- /dev/null +++ b/scripts/ci/agent_cache_scope.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Scope Agent manifest cache inputs without changing the legacy fingerprint. + +Only dependencies proven inactive on this native target can be omitted. The +original identity remains available for an exact, independently checked cache +migration; compiler, native ABI, flags, profiles and Cargo configuration retain +their existing guards. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +from pathlib import Path, PurePosixPath, PureWindowsPath +import re +import subprocess +import sys +import tomllib + +import agent_cache + + +# Bump this policy version when compatibility semantics change. Documentation +# or other source-only edits must not create a new cache namespace; normalized +# compiler/native/build inputs remain part of the canonical identity itself. +SCOPE_SCHEMA = 1 +DEPENDENCY_KINDS = ("dependencies", "dev-dependencies", "build-dependencies") +TOKEN = re.compile(r'\s*(?:(?P[A-Za-z_][A-Za-z_0-9]*)|(?P"(?:[^"\\]|\\.)*")|(?P[(),=]))') +# These are the native targets selected by the Agent workflows. A triple's +# architecture is not generally its first segment (i686 uses cfg arch=x86), +# and Python's platform name cannot identify Android or another custom target. +# Unknown native targets keep every manifest input until explicitly supported. +NATIVE_HOSTS = { + "x86_64-pc-windows-msvc": ("Windows", "windows", "x86_64"), + "x86_64-unknown-linux-gnu": ("Linux", "linux", "x86_64"), + "aarch64-apple-darwin": ("Darwin", "macos", "aarch64"), + "x86_64-apple-darwin": ("Darwin", "macos", "x86_64"), +} + + +class CfgParser: + """Evaluate a small known cfg subset, preserving unknowns in three values.""" + + def __init__(self, expression: str, values: dict[str, str | bool]): + self.tokens = [] + offset = 0 + while expression[offset:].strip(): + match = TOKEN.match(expression, offset) + if match is None: + raise ValueError("unknown cfg syntax") + self.tokens.append((match.lastgroup, match.group(match.lastgroup))) + offset = match.end() + self.index = 0 + self.values = values + + def pop(self, expected: str | None = None) -> tuple[str, str]: + if self.index >= len(self.tokens): + raise ValueError("incomplete cfg expression") + token = self.tokens[self.index] + if expected is not None and token[1] != expected: + raise ValueError("unexpected cfg token") + self.index += 1 + return token + + def peek(self) -> str | None: + return self.tokens[self.index][1] if self.index < len(self.tokens) else None + + def expression(self) -> bool | None: + kind, name = self.pop() + if kind != "word": + raise ValueError("expected cfg identifier") + if self.peek() == "=": + self.pop("=") + kind, literal = self.pop() + if kind != "string": + raise ValueError("expected cfg string") + value = self.values.get(name) + return value == json.loads(literal) if isinstance(value, str) else None + if self.peek() != "(": + value = self.values.get(name) + return value if isinstance(value, bool) else None + self.pop("(") + children = [] + while self.peek() != ")": + children.append(self.expression()) + if self.peek() != ",": + break + self.pop(",") + self.pop(")") + if name == "all": + return False if False in children else (True if all(x is True for x in children) else None) + if name == "any": + return True if True in children else (False if all(x is False for x in children) else None) + if name == "not" and len(children) == 1: + return None if children[0] is None else not children[0] + return None + + +def native_target(identity: dict) -> tuple[str, dict[str, str | bool]] | None: + """Do not infer cfg when Cargo configuration or overrides can select it.""" + configuration = identity.get("configuration", {}) + if any(configuration.get("cargo_configs", {}).values()): + return None + environment = identity.get("environment", {}) + if any(environment.get(key) for key in ( + "CARGO_BUILD_TARGET", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", + "CARGO_BUILD_RUSTC_WRAPPER", "CARGO_BUILD_RUSTC_WORKSPACE_WRAPPER", + )): + return None + for key, value in environment.items(): + if "RUSTFLAGS" in key and re.search(r"--(?:cfg|target)(?:[=\s\x1f]|$)", value): + return None + match = re.search(r"^host:\s*(\S+)\s*$", identity.get("rustc", {}).get("version", ""), re.MULTILINE) + if match is None: + return None + host = match.group(1) + native = NATIVE_HOSTS.get(host) + if native is None or native[0] != identity.get("platform"): + return None + _, target_os, architecture = native + return host, { + "target_os": target_os, + "target_arch": architecture, + "target_family": "windows" if target_os == "windows" else "unix", + "windows": target_os == "windows", + "unix": target_os != "windows", + } + + +def target_matches(target: str, context: tuple[str, dict[str, str | bool]]) -> bool | None: + host, values = context + if not target.startswith("cfg("): + # Literal target triples are Cargo's other supported target-table form. + return target == host if re.fullmatch(r"[A-Za-z0-9_.-]+", target) else None + if not target.endswith(")"): + return None + try: + parser = CfgParser(target[4:-1], values) + result = parser.expression() + return result if parser.index == len(parser.tokens) else None + except ValueError: + return None + + +def manifest_path(root: Path, relative: str) -> Path: + normalized = PurePosixPath(relative.replace("\\", "/")) + if normalized.is_absolute() or PureWindowsPath(relative).drive or ".." in normalized.parts: + raise ValueError("manifest identity must use a repository-relative path") + return root.joinpath(*normalized.parts) + + +def scoped_manifest(settings: dict, manifest: dict, context: tuple[str, dict[str, str | bool]]) -> dict: + result = copy.deepcopy(settings) + declarations: dict[str, list[tuple[bool | None, object]]] = {} + inactive = [] + for kind in DEPENDENCY_KINDS: + for name, selection in manifest.get(kind, {}).items(): + declarations.setdefault(name, []).append((True, selection)) + for target, tables in manifest.get("target", {}).items(): + matches = target_matches(target, context) + for kind in DEPENDENCY_KINDS: + for name, selection in tables.get(kind, {}).items(): + declarations.setdefault(name, []).append((matches, selection)) + if matches is False: + inactive.append((f"target.{target}.{kind}.{name}", name, + isinstance(selection, dict) and selection.get("optional") is True)) + + features = manifest.get("features", {}) + # A plain optional-dependency feature or a strong dep/feature edge may + # activate the package's own cfg(feature). Keep these cases conservative. + ambiguous = set(features) + explicit_dependencies = set() + for selections in features.values(): + for selection in selections: + if selection.startswith("dep:"): + explicit_dependencies.add(selection[4:]) + if not selection.startswith("dep:") and "?/" not in selection: + ambiguous.add(selection.split("/", 1)[0]) + inactive_optional = { + name for name, selections in declarations.items() + if name not in ambiguous and name in explicit_dependencies and all( + matches is False and isinstance(selection, dict) and selection.get("optional") is True + for matches, selection in selections + ) + } + dependency_features = result.get("dependency-features", {}) + for key, name, optional in inactive: + # Even an inactive optional declaration can create a local implicit + # feature. Do not erase its input when feature edges are ambiguous. + # Without dep:name Cargo creates an implicit local feature, including + # under --all-features, even if the target dependency is not compiled. + if name not in ambiguous and (not optional or name in inactive_optional): + dependency_features.pop(key, None) + for name, selections in result.get("features", {}).items(): + result["features"][name] = [ + selection for selection in selections + if not ( + selection.startswith("dep:") and selection[4:] in inactive_optional + or "?/" in selection and selection.split("?/", 1)[0] in inactive_optional + ) + ] + return result + + +def canonical_identity(identity: dict, root: Path = agent_cache.ROOT) -> dict: + result = copy.deepcopy(identity) + result["scope_schema"] = SCOPE_SCHEMA + context = native_target(identity) + if context is None: + return result + manifests = result.get("configuration", {}).get("manifests", {}) + workspace = next((settings for relative, settings in manifests.items() + if relative.replace("\\", "/") == "apps/maple-agent/Cargo.toml"), {}) + # Resolver 1 can unify features from inactive target dependencies. Only the + # explicitly selected modern workspace resolvers give this pruning rule. + if str(workspace.get("resolver")) not in ("2", "3"): + return result + for relative, settings in manifests.items(): + manifest = tomllib.loads(manifest_path(root, relative).read_text(encoding="utf-8")) + manifests[relative] = scoped_manifest(settings, manifest, context) + return result + + +def build_document(profile: str, root: Path = agent_cache.ROOT, + cargo_config_dir: Path | None = None) -> dict: + legacy = agent_cache.build_identity(profile, root, cargo_config_dir) + canonical = canonical_identity(legacy, root) + return { + **canonical, + "compatibility": agent_cache.digest(canonical), + "legacy_compatibility": agent_cache.digest(legacy), + "legacy_identity": legacy, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", choices=agent_cache.PROFILES, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cargo-config-dir", type=Path) + arguments = parser.parse_args() + document = build_document(arguments.profile, cargo_config_dir=arguments.cargo_config_dir) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if os.environ.get("GITHUB_OUTPUT"): + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output: + for key in ("compatibility", "legacy_compatibility"): + output.write(f"{key}={document[key]}\n") + print(f"Agent {arguments.profile} scoped cache compatibility: {document['compatibility']}") + print(f"Legacy cache compatibility: {document['legacy_compatibility']}") + print(document["rustc"]["version"]) + print(f"Native environment: {document['platform']}/{document['architecture']}; details: {arguments.output}") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error: + print(f"Agent cache scope failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/ci/agent_change_detection.py b/scripts/ci/agent_change_detection.py index 3908b54af..5e9591e1b 100644 --- a/scripts/ci/agent_change_detection.py +++ b/scripts/ci/agent_change_detection.py @@ -30,6 +30,8 @@ "scripts/ci/agent_change_detection.py", "scripts/ci/agent_master_selection.py", "scripts/ci/agent_cache.py", + "scripts/ci/agent_cache_scope.py", + "scripts/ci/agent_dependency_archive.py", "scripts/ci/agent_cache_command.py", "scripts/ci/agent_windows.py", "scripts/ci/agent_measure.py", diff --git a/scripts/ci/agent_dependency_archive.py b/scripts/ci/agent_dependency_archive.py new file mode 100644 index 000000000..d644c1480 --- /dev/null +++ b/scripts/ci/agent_dependency_archive.py @@ -0,0 +1,822 @@ +#!/usr/bin/env python3 +"""Recover trusted Cargo dependency caches through the Actions artifact service. + +This helper never uploads, executes restored files, or changes Cargo's freshness +checks. Native Cargo caches remain an independent read source. Only successful +push/master platform jobs can supply an artifact accepted by restore. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import gzip +import hashlib +import io +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile + + +SCHEMA = 1 +WORKFLOWS = {"agent-ci.yml", "agent-desktop-build.yml"} +# A clean hosted target is ~4.4 GB before dependency pruning, plus ~4.2 GB of +# public sources. Its prior zstd archive was already ~2.95 GB; allow gzip-1 +# headroom while retaining bounded downloads and three-day workflow retention. +MAX_ARCHIVE = 6 * 1024**3 +MAX_UNPACKED = 16 * 1024**3 +MAX_FILE = 3 * 1024**3 +MAX_FILES = 300_000 +MAX_MANIFEST = 512 * 1024 +MAX_CANDIDATES = 10 +MAX_AGE_DAYS = 7 +CHUNK = 1024 * 1024 +HASH = re.compile(r"[0-9a-f]{64}\Z") +SHA = re.compile(r"[0-9a-f]{40}\Z") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") +PACKAGE = re.compile(r"[A-Za-z0-9_-]+\Z") +RESERVED = {"CON", "PRN", "AUX", "NUL", *(f"COM{i}" for i in range(1, 10)), *(f"LPT{i}" for i in range(1, 10))} +LIB_EXTENSIONS = {".rlib", ".rmeta", ".d", ".so", ".dylib", ".dll", ".lib", ".a", ".pdb", ".exp", + ".dll.lib", ".dll.exp", ".dll.pdb"} +# These source repositories were independently reviewed as public. A GitHub URL +# and pinned commit alone do not establish that cached repository data is public. +REVIEWED_GIT_REPOSITORIES = { + "1password/cpace", "mapleprivacylabs/rusqlite", "proptest-rs/proptest", "trycua/cua", + "zed-industries/font-kit", "zed-industries/scap", "zed-industries/wasm_thread", + "zed-industries/xim-rs", "zed-industries/zed", +} + + +class Rejected(ValueError): + """An untrusted, incompatible, unsafe, or oversized cache is not usable.""" + + +def sha256(path: Path) -> str: + result = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(CHUNK), b""): + result.update(chunk) + return result.hexdigest() + + +def json_read(path: Path, limit: int = MAX_MANIFEST): + if path.stat().st_size > limit: + raise Rejected("JSON document exceeds the cache limit") + return json.loads(path.read_text(encoding="utf-8")) + + +def environment(path: Path): + value = json_read(path) + compatibility = value.get("compatibility", "") + payload = {key: item for key, item in value.items() if key not in {"compatibility", "legacy_compatibility", "legacy_identity"}} + calculated = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + if not HASH.fullmatch(compatibility) or compatibility != calculated: + raise Rejected("actual compiler/native compatibility document is invalid") + if value.get("profile") not in ("ci", "provider", "release"): + raise Rejected("unsupported Cargo cache profile") + for key in ("platform", "architecture"): + if not re.fullmatch(r"[A-Za-z0-9_-]+", value.get(key, "")): + raise Rejected("invalid cache platform identity") + return value + + +def artifact_name(identity, dependency=""): + if identity["profile"] == "provider" and not HASH.fullmatch(dependency): + raise Rejected("provider archives require the verified test-only dependency fingerprint") + if dependency and not HASH.fullmatch(dependency): + raise Rejected("invalid dependency fingerprint") + suffix = f"-{dependency}" if dependency else "" + return f"maple-agent-deps-v{SCHEMA}-{identity['profile']}-{identity['platform']}-{identity['architecture']}-{identity['compatibility']}{suffix}" + + +def path_parts(name: str): + path = PurePosixPath(name) + if not name or not path.parts or path.is_absolute() or path.as_posix() != name: + raise Rejected("noncanonical archive path") + for part in path.parts: + if part in (".", "..") or any(ord(c) < 32 for c in part) or any(c in part for c in "\\:"): + raise Rejected("unsafe archive path") + if part.endswith((".", " ")) or part.split(".")[0].upper() in RESERVED: + raise Rejected("archive path aliases a Windows special name") + return path.parts + + +def trusted_producer(job_name, workflow): + repository = os.environ.get("GITHUB_REPOSITORY", "") + if not REPOSITORY.fullmatch(repository): + raise Rejected("a repository-scoped Actions context is required") + if os.environ.get("GITHUB_EVENT_NAME") != "push" or os.environ.get("GITHUB_REF") != "refs/heads/master": + raise Rejected("only push/master jobs may prepare dependency archives") + head = os.environ.get("GITHUB_SHA", "") + if not SHA.fullmatch(head): + raise Rejected("producer commit identity is missing") + run_id, attempt = int(os.environ["GITHUB_RUN_ID"]), int(os.environ["GITHUB_RUN_ATTEMPT"]) + if run_id <= 0 or attempt <= 0 or not job_name or workflow not in WORKFLOWS: + raise Rejected("producer run, job or workflow identity is invalid") + return {"repository": repository, "run_id": run_id, "run_attempt": attempt, + "head_sha": head, "job_name": job_name, "workflow": workflow} + + +def packages(metadata): + """Use Cargo source identity, never workspace-directory location, as authority.""" + public, local = [], set() + for package in metadata["packages"]: + source = package.get("source") + names = {package["name"], *(target["name"] for target in package.get("targets", []) + if set(target["kind"]) & {"lib", "rlib", "proc-macro", "dylib", "staticlib", "cdylib"})} + if not all(PACKAGE.fullmatch(name) for name in names): + raise Rejected("invalid Cargo package/target name") + if source is None: + local.update(name.replace("-", "_") for name in names) + local.add(package["name"]) + continue + if source.startswith("registry+"): + allowed = source == "registry+https://github.com/rust-lang/crates.io-index" + elif source.startswith("git+"): + url = urllib.parse.urlsplit(source[4:]) + repository = url.path.strip("/").casefold().removesuffix(".git") + allowed = (url.scheme == "https" and url.hostname == "github.com" and not url.username + and not url.password and set(urllib.parse.parse_qs(url.query)) <= {"rev", "tag", "branch"} + and SHA.fullmatch(url.fragment) is not None and repository in REVIEWED_GIT_REPOSITORIES) + else: + allowed = False + if allowed: + public.append(package) + else: + # Unreviewed/private sources must not leak through target artifacts + # or through name collisions with an otherwise reviewed dependency. + local.update(name.replace("-", "_") for name in names) + local.add(package["name"]) + package_names = {package["name"] for package in public} - local + targets = {target["name"].replace("-", "_") for package in public + for target in package.get("targets", []) + if set(target["kind"]) & {"lib", "rlib", "proc-macro", "dylib", "staticlib", "cdylib"}} - local + return public, package_names, targets + + +def dependency_graph(metadata, public): + features = {node["id"]: sorted(node.get("features", [])) for node in metadata.get("resolve", {}).get("nodes", [])} + selected = sorted((package["name"], package["version"], package["source"], features.get(package["id"], [])) for package in public) + return hashlib.sha256(json.dumps(selected, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def receipt_reader(job_name): + return {"run_id": os.environ.get("GITHUB_RUN_ID"), "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), "job_name": job_name} + + +def receipt_location(receipt, target, cargo_home): + if receipt and (receipt.is_symlink() or receipt.resolve().is_relative_to(target.resolve()) or receipt.resolve().is_relative_to(cargo_home.resolve())): + raise Rejected("artifact receipts must be fresh job-local files outside restored Cargo caches") + + +def source_roots(public, cargo_home): + roots = set() + for package in public: + try: + relative = Path(package["manifest_path"]).resolve().relative_to(cargo_home) + except ValueError: + continue + parts = relative.parts + if len(parts) >= 5 and parts[:2] == ("registry", "src"): + roots.add(PurePosixPath(*parts[:4]).as_posix()) + elif len(parts) >= 5 and parts[:2] == ("git", "checkouts"): + roots.add(PurePosixPath(*parts[:4]).as_posix()) + roots.add(PurePosixPath("git", "db", parts[2]).as_posix()) + return roots + + +def tree_files(root: Path, skip=None): + """Dereference internal links; never put link entries in the portable archive.""" + if not root.exists(): + return + boundary = root.resolve() + + def walk(path, logical, ancestors): + if skip and skip(logical): + return + selected = path.resolve() + if not selected.is_relative_to(boundary): + raise Rejected("dependency tree contains a link outside its source/output root") + if selected.is_dir(): + if selected in ancestors or len(ancestors) >= 64: + raise Rejected("dependency tree contains a directory-link cycle") + for child in sorted(selected.iterdir()): + yield from walk(child, logical / child.name, ancestors | {selected}) + elif selected.is_file(): + yield selected, logical + else: + raise Rejected("dependency tree contains a nonregular file") + yield from walk(root, PurePosixPath(), set()) + + +def sanitized_git(relative, source, cargo_home): + """Return None to omit hooks, or bytes to replace local Git configuration.""" + parts = relative.parts + metadata = parts[:2] == ("git", "db") or ".git" in parts + if not metadata: + return False + if parts[0] == "git" and parts[-1] == ".git": + raise Rejected("regular .git pointers cannot enter a portable Cargo source cache") + if "hooks" in parts or source.name.endswith(".lock") or source.name == "config.worktree": + return None + if source.name == "config": + bare = parts[:2] == ("git", "db") + return f"[core]\nrepositoryformatversion = 0\nbare = {'true' if bare else 'false'}\nfilemode = false\n".encode() + if parts[-3:] == ("objects", "info", "alternates"): + lines = source.read_text(encoding="utf-8").splitlines() + replacements = [] + for line in lines: + destination = (source.parent / line).resolve() + if not destination.is_relative_to(cargo_home / "git/db") or not destination.is_dir(): + raise Rejected("Git object alternates escape the cached public repositories") + replacements.append(os.path.relpath(destination, source.parent).replace(os.sep, "/")) + return ("\n".join(replacements) + "\n").encode() + return False + + +def cargo_records(public, cargo_home): + roots = source_roots(public, cargo_home) + for root in sorted(roots): + if not (cargo_home / root).resolve().is_relative_to(cargo_home): + raise Rejected("Cargo source storage escapes its declared home") + def skip(relative): + return "hooks" in relative.parts and (root.startswith("git/db/") or ".git" in relative.parts) + for source, relative in tree_files(cargo_home / root, skip): + relative = PurePosixPath(root) / relative + replacement = sanitized_git(relative, source, cargo_home) + if replacement is not None: + yield source, "cargo/" + relative.as_posix(), replacement + # Registry archives and sparse index records are selected by Cargo metadata, + # never by copying the Cargo home (which contains credentials and binaries). + for package in public: + if not package["source"].startswith("registry+"): + continue + try: + parts = Path(package["manifest_path"]).resolve().relative_to(cargo_home).parts + except ValueError: + continue + if parts[:2] != ("registry", "src") or len(parts) < 5: + continue + registry = parts[2] + name = package["name"].lower() + index = f"1/{name}" if len(name) == 1 else f"2/{name}" if len(name) == 2 else f"3/{name[0]}/{name}" if len(name) == 3 else f"{name[:2]}/{name[2:4]}/{name}" + for relative in (f"registry/cache/{registry}/{package['name']}-{package['version']}.crate", + f"registry/index/{registry}/.cache/{index}"): + source = cargo_home / relative + if source.is_file() and not source.is_symlink(): + yield source, "cargo/" + relative, False + for registry in {PurePosixPath(root).parts[2] for root in roots if root.startswith("registry/src/")}: + source = cargo_home / "registry/index" / registry / "config.json" + if source.is_file() and not source.is_symlink(): + config = json_read(source) + # Do not persist mirror URLs, auth configuration, or private fields. + url = urllib.parse.urlsplit(config.get("dl", "")) + if url.scheme == "https" and url.hostname in {"static.crates.io", "crates.io"} and not url.username and not url.password and not url.query: + cleaned = json.dumps({"dl": config["dl"], "api": "https://crates.io"}).encode() + yield source, f"cargo/registry/index/{registry}/config.json", cleaned + + +def profile_part(parts): + for index, part in enumerate(parts): + if part in ("debug", "release") and index <= 2: + prefix = parts[:index] + if all(item == "build" or re.fullmatch(r"[A-Za-z0-9_]+(?:-[A-Za-z0-9_]+){2,}", item) for item in prefix): + return parts[index + 1:] + return () + + +def target_allowed(parts, names, targets): + tail = profile_part(parts) + if len(tail) < 2: + return False + if tail[0] in {"build", ".fingerprint"}: + package, separator, fingerprint = tail[1].rpartition("-") + return bool(separator and package in names and re.fullmatch(r"[0-9a-f]{8,64}", fingerprint) + and "incremental" not in tail) + if tail[0] == "deps" and len(tail) == 2: + name = tail[1] + extension = next((item for item in sorted(LIB_EXTENSIONS, key=len, reverse=True) if name.endswith(item)), "") + stem = name[:-len(extension)] if extension else name + stem, separator, fingerprint = stem.rpartition("-") + return bool(extension in LIB_EXTENSIONS and separator and re.fullmatch(r"[0-9a-f]{8,64}", fingerprint) + and (stem in targets or stem.removeprefix("lib") in targets)) + return False + + +def target_records(target, names, targets): + # Do not traverse app bundles, notices, incremental data or other unrelated + # target outputs merely to filter them later (macOS bundles contain links). + bases = [target, target / "build"] + for base in tuple(bases): + if base.is_dir(): + bases.extend(child for child in base.iterdir() if child.is_dir() and re.fullmatch(r"[A-Za-z0-9_]+(?:-[A-Za-z0-9_]+){2,}", child.name)) + for base in bases: + if base.exists() and (base.is_symlink() or not base.resolve().is_relative_to(target)): + raise Rejected("Cargo build layout escapes its target directory") + for profile in ("debug", "release"): + if (base / profile).is_symlink(): + raise Rejected("Cargo profile directory is a symlink") + for kind in ("deps", ".fingerprint", "build"): + directory = base / profile / kind + if not directory.is_dir() or directory.is_symlink(): + continue + for child in sorted(directory.iterdir()): + relative = PurePosixPath(*child.relative_to(target).parts) + if not target_allowed(relative.parts, names, targets): + continue + if kind == "deps": + if not child.is_file() or child.is_symlink(): + raise Rejected("dependency library is not an ordinary file") + yield child, "target/" + relative.as_posix(), False + else: + if not child.is_dir() or child.is_symlink() or not child.resolve().is_relative_to(target): + raise Rejected("dependency build output escapes its target directory") + for source, inner in tree_files(child): + yield source, "target/" + (relative / inner).as_posix(), False + + +def pack(identity, metadata, target, cargo_home, output, job_name, workflow, dependency="", receipt=None): + started = time.monotonic() + producer = trusted_producer(job_name, workflow) + name = artifact_name(identity, dependency) + public, names, targets = packages(metadata) + graph = dependency_graph(metadata, public) + receipt_location(receipt, target, cargo_home) + if receipt and receipt.is_file(): + previous = json_read(receipt) + if (previous.get("compatibility") == identity["compatibility"] and previous.get("dependency_fingerprint") == dependency + and previous.get("reader") == receipt_reader(job_name) + and previous.get("dependency_graph") == graph + and datetime.now(timezone.utc) - timedelta(days=1) < instant(previous["created_at"]) <= datetime.now(timezone.utc)): + return {"needs_upload": False, "artifact_name": name, "dependency_graph": graph, + "source_artifact_id": previous["artifact_id"], "seed_created_at": previous["created_at"]} + if target.is_symlink() or cargo_home.is_symlink(): + raise Rejected("Cargo cache roots must not be symlinks") + target, cargo_home = target.resolve(), cargo_home.resolve() + if target == cargo_home or target.is_relative_to(cargo_home) or cargo_home.is_relative_to(target): + raise Rejected("Cargo home and target roots must be separate") + if output.resolve().is_relative_to(target) or output.resolve().is_relative_to(cargo_home): + raise Rejected("dependency bundle must be outside its input cache roots") + if output.exists() and any(output.iterdir()): + raise Rejected("dependency bundle output must be new or empty") + output.mkdir(parents=True, exist_ok=True) + archive = output / "dependencies.tar.gz" + entries, total, seen = 0, 0, {} + records = cargo_records(public, cargo_home) + + def all_records(): + yield from records + yield from target_records(target, names, targets) + + with archive.open("wb") as raw, gzip.GzipFile(filename="", fileobj=raw, mode="wb", compresslevel=1, mtime=0) as compressed, tarfile.open(fileobj=compressed, mode="w|", format=tarfile.PAX_FORMAT) as stream: + for source, relative, replacement in all_records(): + path_parts(relative) + if relative.casefold() in seen: + if seen[relative.casefold()] != relative: + raise Rejected("dependency inputs contain case-alias paths") + continue + seen[relative.casefold()] = relative + attributes = source.stat() + size = len(replacement) if replacement is not False else attributes.st_size + entries, total = entries + 1, total + size + if entries > MAX_FILES or total > MAX_UNPACKED or size > MAX_FILE or raw.tell() > MAX_ARCHIVE: + raise Rejected("dependency archive exceeds bounded size/file limits") + member = tarfile.TarInfo(relative) + member.size, member.mtime = size, attributes.st_mtime + member.mode = 0o755 if attributes.st_mode & 0o111 else 0o644 + if replacement is False: + with source.open("rb") as content: + stream.addfile(member, content) + else: + stream.addfile(member, io.BytesIO(replacement)) + if not entries or archive.stat().st_size > MAX_ARCHIVE: + raise Rejected("dependency archive is empty or exceeds its compressed-size limit") + manifest = {"schema": SCHEMA, "needs_upload": True, "artifact_name": name, "compatibility": identity["compatibility"], + "profile": identity["profile"], "platform": identity["platform"], "architecture": identity["architecture"], + "dependency_fingerprint": dependency, "dependency_graph": graph, "producer": producer, + "package_names": sorted(names), "target_names": sorted(targets), + "archive_sha256": sha256(archive), "archive_bytes": archive.stat().st_size, + "unpacked_bytes": total, "files": entries, "pack_seconds": round(time.monotonic() - started, 3)} + (output / "manifest.json").write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return manifest + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, fp, code, message, headers, newurl): + return None + + +class GitHub: + def __init__(self, repository, token): + if not REPOSITORY.fullmatch(repository) or not token: + raise Rejected("repository-scoped Actions read access is required") + self.repository, self.token = repository, token + + def request(self, endpoint): + return urllib.request.Request(f"https://api.github.com/repos/{self.repository}/{endpoint}", + headers={"Authorization": f"Bearer {self.token}", "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "Maple-dependency-cache"}) + + def get(self, endpoint): + with urllib.request.build_opener(NoRedirect()).open(self.request(endpoint), timeout=30) as response: + content = response.read(4 * 1024 * 1024 + 1) + if len(content) > 4 * 1024 * 1024: + raise Rejected("Actions metadata exceeds its bounded response size") + return json.loads(content) + + def download(self, artifact_id, destination, digest): + try: + response = urllib.request.build_opener(NoRedirect()).open(self.request(f"actions/artifacts/{artifact_id}/zip"), timeout=30) + except urllib.error.HTTPError as error: + if error.code not in (301, 302, 303, 307, 308): + raise + location = error.headers.get("Location", "") + if error.fp is not None: + error.close() + url = urllib.parse.urlsplit(location) + allowed = (".blob.core.windows.net", ".actions.githubusercontent.com", ".githubusercontent.com") + if url.scheme != "https" or not url.hostname or not url.hostname.endswith(allowed) or url.username or url.password: + raise Rejected("Actions download redirect has an unexpected HTTPS origin") + # Never carry the GitHub Authorization header to the signed blob URL, + # including a second redirect to a different origin. + response = urllib.request.build_opener(NoRedirect()).open(urllib.request.Request(location), timeout=60) + observed, count = hashlib.sha256(), 0 + with response, destination.open("wb") as output: + length = response.headers.get("Content-Length") + if length and int(length) > MAX_ARCHIVE + MAX_MANIFEST + 1024**2: + raise Rejected("artifact download exceeds its size limit") + while chunk := response.read(CHUNK): + count += len(chunk) + if count > MAX_ARCHIVE + MAX_MANIFEST + 1024**2: + raise Rejected("artifact download exceeds its size limit") + observed.update(chunk) + output.write(chunk) + if observed.hexdigest() != digest: + raise Rejected("Actions artifact ZIP SHA256 differs from API digest") + return count + + +def instant(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def source_run(api, artifact, name, workflow, now): + """Establish immutable run/repository identity before downloading any ZIP.""" + if (artifact.get("name") != name or artifact.get("expired") or not isinstance(artifact.get("id"), int) + or artifact.get("size_in_bytes", MAX_ARCHIVE + 1) > MAX_ARCHIVE + MAX_MANIFEST + 1024**2 + or not re.fullmatch(r"sha256:[0-9a-f]{64}", artifact.get("digest", ""))): + raise Rejected("artifact identity, digest, expiry or size is invalid") + created = instant(artifact["created_at"]) + if not now - timedelta(days=MAX_AGE_DAYS) <= created <= now + timedelta(minutes=5): + raise Rejected("artifact is outside the bounded retention window") + origin = artifact["workflow_run"] + run = api.get(f"actions/runs/{int(origin['id'])}") + if (run.get("id") != origin["id"] or run.get("repository", {}).get("full_name") != api.repository + or run.get("head_repository", {}).get("full_name") != api.repository + or run.get("event") != "push" or run.get("head_branch") != "master" + or run.get("path") != f".github/workflows/{workflow}" + or not SHA.fullmatch(run.get("head_sha", "")) + or origin.get("head_sha") != run["head_sha"] or origin.get("head_branch") != "master" + or not isinstance(run.get("repository", {}).get("id"), int) + or not isinstance(run.get("head_repository", {}).get("id"), int) + or origin.get("repository_id") != run.get("repository", {}).get("id") + or origin.get("head_repository_id") != run.get("head_repository", {}).get("id")): + raise Rejected("artifact was not produced by this repository's push/master workflow") + if type(run.get("run_attempt")) is not int or run["run_attempt"] < 1: + raise Rejected("producer run has an invalid attempt bound") + return run + + +def source_proof(api, artifact, name, job_name, workflow, now, attempt=None, run=None): + # The latest run attempt may rerun only another platform's failed job. + # After the ZIP digest has bound its manifest, prove the declared original + # producer attempt rather than invalidating an earlier successful seed. + run = run or source_run(api, artifact, name, workflow, now) + if attempt is None: + attempt = run["run_attempt"] + if type(attempt) is not int or not 1 <= attempt <= run["run_attempt"]: + raise Rejected("manifest producer attempt is outside the trusted run bound") + created = instant(artifact["created_at"]) + jobs = api.get(f"actions/runs/{run['id']}/attempts/{attempt}/jobs?per_page=100") + if jobs.get("total_count", 101) > 100: + raise Rejected("platform job proof exceeds the bounded jobs scan") + selected = [job for job in jobs["jobs"] if job.get("name") == job_name] + if len(selected) != 1 or selected[0].get("status") != "completed" or selected[0].get("conclusion") != "success": + raise Rejected("the exact producer platform job did not complete successfully") + job = selected[0] + if not job.get("steps") or any(step.get("conclusion") not in ("success", "skipped") for step in job["steps"]): + raise Rejected("producer job has incomplete or failed checks") + if not instant(job["started_at"]) <= created <= instant(job["completed_at"]) + timedelta(seconds=30): + raise Rejected("artifact did not originate inside the proven successful job attempt") + return {"repository": api.repository, "run_id": int(run["id"]), "run_attempt": attempt, + "head_sha": run["head_sha"], "job_name": job_name, "workflow": workflow} + + +def unzip(zip_path, directory): + with zipfile.ZipFile(zip_path) as archive: + members = archive.infolist() + if len(members) != 2 or {item.filename for item in members} != {"manifest.json", "dependencies.tar.gz"}: + raise Rejected("artifact ZIP must contain exactly its manifest and dependency archive") + for member in members: + mode = member.external_attr >> 16 + limit = MAX_MANIFEST if member.filename == "manifest.json" else MAX_ARCHIVE + if (member.is_dir() or member.flag_bits & 1 or stat.S_IFMT(mode) not in (0, stat.S_IFREG) + or member.file_size < 0 or member.file_size > limit): + raise Rejected("unsafe artifact ZIP entry") + with archive.open(member) as source, (directory / member.filename).open("wb") as output: + copied = 0 + while chunk := source.read(CHUNK): + copied += len(chunk) + if copied > member.file_size or copied > limit: + raise Rejected("artifact ZIP entry exceeds its declared size") + output.write(chunk) + if copied != member.file_size: + raise Rejected("artifact ZIP entry is truncated") + + +def member_destination(name, manifest, cargo_home, target): + parts = path_parts(name) + if parts[0] == "target" and target_allowed(parts[1:], manifest["package_names"], manifest["target_names"]): + return target.joinpath(*parts[1:]) + if parts[0] == "cargo" and len(parts) >= 4: + tail = parts[1:] + registry = tail[:2] in (("registry", "src"), ("registry", "cache"), ("registry", "index")) + git = tail[:2] in (("git", "db"), ("git", "checkouts")) + if registry or git: + if git and tail[-1] == ".git": + raise Rejected("regular .git pointers cannot enter a portable Cargo source cache") + if (".git" in tail or tail[:2] == ("git", "db")) and "hooks" in tail: + raise Rejected("Git hooks cannot enter the dependency cache") + return cargo_home.joinpath(*tail) + raise Rejected("archive entry is outside the dependency-only namespaces") + + +def safe_destination(path, root): + if any(parent.is_symlink() for parent in root.parents): + raise Rejected("cache destination has a symlink ancestor") + if root.is_symlink() or root.exists() and not root.is_dir(): + raise Rejected("cache destination root is not an ordinary directory") + current = root + for part in path.relative_to(root).parts: + current /= part + if current.is_symlink(): + raise Rejected("cache destination contains a symlink") + if current != path and current.exists() and not current.is_dir(): + raise Rejected("cache destination parent is not a directory") + if path.exists() and not path.is_file(): + raise Rejected("cache destination file has an unexpected type") + + +def tar_envelope(archive): + """Bound physical TAR headers before tarfile can allocate PAX payloads.""" + headers, physical = 0, 0 + with gzip.open(archive, "rb") as stream: + while True: + header = stream.read(512) + if not header or header == bytes(512): + break + if len(header) != 512: + raise Rejected("truncated dependency TAR header") + headers += 1 + if headers > MAX_FILES * 2: + raise Rejected("dependency TAR has too many physical headers") + encoded = header[124:136] + try: + size = int.from_bytes(encoded[1:], "big") if encoded[0] == 0x80 else int(encoded.rstrip(b"\0 ").lstrip(b" ") or b"0", 8) + except ValueError as error: + raise Rejected("invalid dependency TAR size") from error + kind = header[156:157] + if size < 0 or size > MAX_FILE or kind not in (b"0", b"\0", tarfile.XHDTYPE): + raise Rejected("unsupported dependency TAR entry type/size") + if kind == tarfile.XHDTYPE: + if size > 64 * 1024: + raise Rejected("oversized dependency TAR extended metadata") + payload = stream.read(size) + remaining = payload + while remaining: + try: + length = int(remaining.split(b" ", 1)[0]) + record = remaining[:length] + key = record.split(b" ", 1)[1].split(b"=", 1)[0] + except (ValueError, IndexError) as error: + raise Rejected("invalid dependency TAR extended metadata") from error + if length <= 0 or length > len(remaining) or not record.endswith(b"\n") or key not in (b"path", b"mtime"): + raise Rejected("unsupported dependency TAR extended metadata") + remaining = remaining[length:] + skipped = (512 - size % 512) % 512 + else: + skipped = ((size + 511) // 512) * 512 + physical += 512 + ((size + 511) // 512) * 512 + if physical > MAX_UNPACKED + MAX_FILES * 2048: + raise Rejected("dependency TAR exceeds its physical decompression budget") + while skipped: + chunk = stream.read(min(CHUNK, skipped)) + if not chunk: + raise Rejected("truncated dependency TAR contents") + skipped -= len(chunk) + + +def unpack(directory, identity, producer, cargo_home, target, dependency=""): + manifest = json_read(directory / "manifest.json") + expected = artifact_name(identity, dependency) + if (manifest.get("schema") != SCHEMA or manifest.get("artifact_name") != expected + or manifest.get("compatibility") != identity["compatibility"] + or manifest.get("dependency_fingerprint") != dependency or manifest.get("producer") != producer + or any(manifest.get(key) != identity[key] for key in ("profile", "platform", "architecture"))): + raise Rejected("dependency manifest does not match the actual environment and proven producer") + if (type(manifest.get("files")) is not int or not 0 < manifest["files"] <= MAX_FILES + or type(manifest.get("unpacked_bytes")) is not int or not 0 <= manifest["unpacked_bytes"] <= MAX_UNPACKED + or not isinstance(manifest.get("package_names"), list) or not isinstance(manifest.get("target_names"), list) + or not all(isinstance(name, str) and PACKAGE.fullmatch(name) for name in manifest["package_names"] + manifest["target_names"]) + or not HASH.fullmatch(manifest.get("dependency_graph", ""))): + raise Rejected("dependency manifest has invalid size or package bounds") + archive = directory / "dependencies.tar.gz" + if (archive.stat().st_size > MAX_ARCHIVE or archive.stat().st_size != manifest["archive_bytes"] + or sha256(archive) != manifest["archive_sha256"]): + raise Rejected("dependency tar size or SHA256 differs from the trusted manifest") + tar_envelope(archive) + cargo_home, target = cargo_home.absolute(), target.absolute() + scope = {"package_names": set(manifest["package_names"]), "target_names": set(manifest["target_names"])} + seen, destinations, count, size = set(), [], 0, 0 + # Preflight every entry and every existing destination before writing any + # cache file. extractall is deliberately never used for either archive. + with tarfile.open(archive, mode="r|gz") as stream: + for member in stream: + if (not member.isreg() or member.issparse() or not 0 <= member.size <= MAX_FILE + or not math.isfinite(member.mtime) or not 0 <= member.mtime <= 4_102_444_800): + raise Rejected("dependency tar may contain only ordinary nonsparse files") + folded = member.name.casefold() + if folded in seen: + raise Rejected("dependency tar contains duplicate/case-alias paths") + seen.add(folded) + count, size = count + 1, size + member.size + if count > MAX_FILES or size > MAX_UNPACKED: + raise Rejected("dependency tar exceeds extraction bounds") + destination = member_destination(member.name, scope, cargo_home, target) + parts = PurePosixPath(member.name).parts + metadata = parts[:3] == ("cargo", "git", "db") or ".git" in parts + if metadata and parts[-1] in ("config", "config.worktree", "alternates") and member.size > 64 * 1024: + raise Rejected("Git metadata exceeds its bounded size") + root = target if member.name.startswith("target/") else cargo_home + safe_destination(destination, root) + destinations.append((member.name, destination)) + if count != manifest["files"] or size != manifest["unpacked_bytes"]: + raise Rejected("dependency tar totals differ from the trusted manifest") + staging = directory / "staged" + staging.mkdir() + with tarfile.open(archive, mode="r|gz") as stream: + extracted = 0 + for member in stream: + if extracted >= len(destinations): + raise Rejected("dependency archive gained entries between verification passes") + name, _ = destinations[extracted] + extracted += 1 + if member.name != name: + raise Rejected("dependency archive changed between verification passes") + selected = staging.joinpath(*PurePosixPath(name).parts) + selected.parent.mkdir(parents=True, exist_ok=True) + with stream.extractfile(member) as source, selected.open("wb") as output: + shutil.copyfileobj(source, output, CHUNK) + if selected.stat().st_size != member.size: + raise Rejected("dependency tar entry is truncated") + parts = PurePosixPath(name).parts + metadata = parts[:3] == ("cargo", "git", "db") or ".git" in parts + if metadata and parts[-1] == "config": + bare = parts[:3] == ("cargo", "git", "db") + expected_config = f"[core]\nrepositoryformatversion = 0\nbare = {'true' if bare else 'false'}\nfilemode = false\n".encode() + if selected.read_bytes() != expected_config: + raise Rejected("Git configuration was not sanitized by the dependency packer") + if metadata and parts[-1] == "config.worktree": + raise Rejected("Git worktree configuration cannot enter the dependency cache") + os.chmod(selected, 0o755 if member.mode & 0o111 else 0o644) + os.utime(selected, (member.mtime, member.mtime)) + if extracted != len(destinations): + raise Rejected("dependency archive lost entries between verification passes") + for name, _ in destinations: + parts = PurePosixPath(name).parts + selected = staging.joinpath(*parts) + if parts[:3] in (("cargo", "git", "db"), ("cargo", "git", "checkouts")) and parts[-3:] == ("objects", "info", "alternates"): + for line in selected.read_text(encoding="utf-8").splitlines(): + if not line or PurePosixPath(line).is_absolute() or "\\" in line or ":" in line or any(ord(c) < 32 for c in line): + raise Rejected("Git object alternates must be bounded relative paths") + referenced = (selected.parent / line).resolve() + if not referenced.is_relative_to((staging / "cargo/git/db").resolve()) or not referenced.is_dir(): + raise Rejected("Git object alternates escape the restored public object storage") + if parts[:3] == ("cargo", "registry", "index") and parts[-1] == "config.json": + config = json_read(selected, 64 * 1024) + url = urllib.parse.urlsplit(config.get("dl", "")) + if (set(config) != {"dl", "api"} or config["api"] != "https://crates.io" or url.scheme != "https" + or url.hostname not in {"static.crates.io", "crates.io"} or url.username or url.password or url.query): + raise Rejected("registry index configuration is not sanitized public configuration") + for name, destination in destinations: + root = target if name.startswith("target/") else cargo_home + safe_destination(destination, root) + destination.parent.mkdir(parents=True, exist_ok=True) + staged = staging.joinpath(*PurePosixPath(name).parts) + if destination.exists() and not os.access(destination, os.W_OK): + # Cargo Git object storage can use read-only hardlinks. Never chmod + # a restored destination through a potentially shared inode. + if sha256(destination) == sha256(staged): + continue + raise Rejected("a different read-only dependency file already exists") + os.replace(staged, destination) + return manifest + + +def outputs(values): + if os.environ.get("GITHUB_OUTPUT"): + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output: + for key, value in values.items(): + output.write(f"{key}={value}\n") + + +def restore(api, identity, target, cargo_home, job_name, workflow, dependency="", receipt=None): + started = time.monotonic() + name = artifact_name(identity, dependency) + receipt_location(receipt, target, cargo_home) + candidates = api.get("actions/artifacts?" + urllib.parse.urlencode({"name": name, "per_page": MAX_CANDIDATES})) + for listed in candidates.get("artifacts", [])[:MAX_CANDIDATES]: + try: + artifact = api.get(f"actions/artifacts/{int(listed['id'])}") + now = datetime.now(timezone.utc) + run = source_run(api, artifact, name, workflow, now) + with tempfile.TemporaryDirectory(prefix="maple-dependency-restore-") as temporary: + work = Path(temporary) + transferred = api.download(artifact["id"], work / "artifact.zip", artifact["digest"].removeprefix("sha256:")) + unzip(work / "artifact.zip", work) + declared = json_read(work / "manifest.json").get("producer", {}) + attempt = declared.get("run_attempt") + if type(attempt) is not int: + raise Rejected("manifest producer attempt is missing") + proof = source_proof(api, artifact, name, job_name, workflow, now, attempt=attempt, run=run) + manifest = unpack(work, identity, proof, cargo_home, target, dependency) + if receipt: + receipt.parent.mkdir(parents=True, exist_ok=True) + receipt.write_text(json.dumps({"compatibility": identity["compatibility"], "dependency_fingerprint": dependency, + "dependency_graph": manifest["dependency_graph"], "created_at": artifact["created_at"], + "artifact_id": artifact["id"], "artifact_digest": artifact["digest"], "producer": proof, + "reader": receipt_reader(job_name)}, sort_keys=True) + "\n", encoding="utf-8") + result = {"restored": "true", "artifact-id": artifact["id"], "source-run-id": proof["run_id"], + "archive-bytes": manifest["archive_bytes"], "download-bytes": transferred, + "restore-seconds": round(time.monotonic() - started, 3), "seed-created-at": artifact["created_at"]} + outputs(result) + print(json.dumps(result, sort_keys=True)) + return True + except (Rejected, OSError, EOFError, OverflowError, KeyError, TypeError, ValueError, tarfile.TarError, zipfile.BadZipFile) as error: + print(f"Dependency artifact {listed.get('id', 'unknown')} rejected: {type(error).__name__}", file=sys.stderr) + outputs({"restored": "false"}) + print("No compatible, proven successful master dependency artifact; Cargo will build cold") + return False + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("pack", "restore")) + parser.add_argument("--environment", type=Path, required=True) + parser.add_argument("--target", type=Path, required=True) + parser.add_argument("--cargo-home", type=Path, default=Path(os.environ.get("CARGO_HOME", str(Path.home() / ".cargo")))) + parser.add_argument("--job-name", required=True) + parser.add_argument("--workflow", choices=sorted(WORKFLOWS), default="agent-ci.yml") + parser.add_argument("--dependency-fingerprint", default="") + parser.add_argument("--metadata", type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--receipt", type=Path, help="write restored artifact proof, or reuse that proof during pack") + arguments = parser.parse_args() + identity = environment(arguments.environment) + if arguments.command == "pack": + if not arguments.metadata or not arguments.output_dir: + parser.error("pack requires --metadata and --output-dir") + manifest = pack(identity, json_read(arguments.metadata, 64 * 1024**2), arguments.target, + arguments.cargo_home, arguments.output_dir, arguments.job_name, + arguments.workflow, arguments.dependency_fingerprint, arguments.receipt) + outputs({"needs-upload": str(manifest["needs_upload"]).lower(), "packed": str(manifest["needs_upload"]).lower(), + "artifact-name": manifest["artifact_name"], "archive-bytes": manifest.get("archive_bytes", 0)}) + print(json.dumps({key: manifest[key] for key in ("needs_upload", "artifact_name", "archive_bytes", "unpacked_bytes", "files", "pack_seconds", "source_artifact_id", "seed_created_at") if key in manifest}, sort_keys=True)) + else: + try: + api = GitHub(os.environ.get("GITHUB_REPOSITORY", ""), os.environ.get("GH_TOKEN", os.environ.get("GITHUB_TOKEN", ""))) + restore(api, identity, arguments.target, arguments.cargo_home, arguments.job_name, + arguments.workflow, arguments.dependency_fingerprint, arguments.receipt) + except (Rejected, OSError, ValueError, KeyError, TypeError) as error: + outputs({"restored": "false"}) + print(f"Dependency artifact recovery unavailable ({type(error).__name__}); Cargo will build cold") + + +if __name__ == "__main__": + try: + main() + except (Rejected, OSError, ValueError, KeyError, TypeError) as error: + print(f"Dependency archive refused: {type(error).__name__}: {error}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/ci/agent_master_selection.py b/scripts/ci/agent_master_selection.py index b4933d00e..ba3731062 100644 --- a/scripts/ci/agent_master_selection.py +++ b/scripts/ci/agent_master_selection.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Compare Agent inputs with the last fully verified master package run.""" +"""Retain Agent changes until the selected master lane actually validates them.""" from __future__ import annotations @@ -21,6 +21,16 @@ f"Verify downloaded {platform} ({profile})" for platform in ("macOS", "Linux") for profile in ("dev", "prod") ) +CI_WORKFLOW = ".github/workflows/agent-ci.yml" +CI_VERIFIERS = frozenset({ + "Agent (blacksmith-8vcpu-ubuntu-2404)", + "Agent (macos-26)", + "Agent (windows-latest)", + "Agent provider (blacksmith-8vcpu-ubuntu-2404)", + "Agent provider (macos-26)", + "Agent provider (windows-latest)", + "Agent Linux release binary", +}) def api(path: str) -> dict: @@ -37,11 +47,17 @@ def git(root: Path, *args: str) -> subprocess.CompletedProcess: ) -def select_master(head: str, current_run_id: int, root: Path = ROOT) -> tuple[bool, str]: +def select_master( + head: str, current_run_id: int, root: Path = ROOT, *, lane: str = "desktop", +) -> tuple[bool, str]: if not re.fullmatch(r"[0-9a-f]{40}", head): raise ValueError("invalid current commit") + if lane not in ("desktop", "ci"): + raise ValueError("unknown Agent validation lane") + workflow = CI_WORKFLOW if lane == "ci" else WORKFLOW + verifiers = CI_VERIFIERS if lane == "ci" else VERIFIERS runs = api( - f"repos/{REPOSITORY}/actions/workflows/agent-desktop-build.yml/runs" + f"repos/{REPOSITORY}/actions/workflows/{workflow.rsplit('/', 1)[1]}/runs" "?branch=master&status=success&per_page=100" )["workflow_runs"] for run in runs: @@ -51,7 +67,7 @@ def select_master(head: str, current_run_id: int, root: Path = ROOT) -> tuple[bo or run["head_repository"]["full_name"] != REPOSITORY or run["head_branch"] != "master" or run["event"] not in ("push", "workflow_dispatch") - or run["path"] != WORKFLOW + or run["path"] != workflow or run["status"] != "completed" or run["conclusion"] != "success" ): @@ -76,7 +92,7 @@ def select_master(head: str, current_run_id: int, root: Path = ROOT) -> tuple[bo job["name"] for job in jobs["jobs"] if job["status"] == "completed" and job["conclusion"] == "success" } - if not VERIFIERS <= verified: + if not verifiers <= verified: # Selector-only successes and incomplete profiles cannot advance # the baseline past an Agent change that still needs packaging. continue @@ -93,6 +109,7 @@ def select_master(head: str, current_run_id: int, root: Path = ROOT) -> tuple[bo def main() -> int: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lane", choices=("desktop", "ci"), default="desktop") parser.add_argument("--head", default=os.environ.get("GITHUB_SHA", "")) parser.add_argument("--run-id", type=int, default=os.environ.get("GITHUB_RUN_ID", "0")) args = parser.parse_args() @@ -103,7 +120,7 @@ def main() -> int: reason = "Manual or non-master event; building without a baseline lookup." else: try: - selected, reason = select_master(args.head, args.run_id) + selected, reason = select_master(args.head, args.run_id, lane=args.lane) except (KeyError, TypeError, ValueError, OSError, subprocess.SubprocessError): reason = "Master baseline lookup or diff unavailable; building." print(reason, file=sys.stderr) diff --git a/scripts/ci/test_agent_cache_scope.py b/scripts/ci/test_agent_cache_scope.py new file mode 100644 index 000000000..f1af74bad --- /dev/null +++ b/scripts/ci/test_agent_cache_scope.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Regressions for target-aware keys and exact legacy identity migration.""" + +import copy +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import agent_cache +import agent_cache_scope as scope + + +APP = "apps/maple-agent/app/Cargo.toml" +MAC_DEPENDENCY = '\n[target.\'cfg(target_os = "macos")\'.dependencies]\nraw-window-handle = { version = "0.6", optional = true }\n' + + +class AgentCacheScopeTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + for relative in ("apps/maple-agent/Cargo.toml", APP, "sdk/rust/Cargo.toml", "proxy/Cargo.toml"): + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('[package]\nname = "fixture"\nversion = "1.0"\nedition = "2024"\n', encoding="utf-8") + self.app = self.root / APP + self.app.write_text('[package]\nname = "fixture"\n[features]\ndefault = ["desktop"]\ndesktop = []\n', encoding="utf-8") + with (self.root / "apps/maple-agent/Cargo.toml").open("a", encoding="utf-8") as manifest: + manifest.write('\n[workspace]\nresolver = "2"\n') + + def identity(self, system="Windows"): + host = {"Windows": "x86_64-pc-windows-msvc", "Linux": "x86_64-unknown-linux-gnu", "Darwin": "aarch64-apple-darwin"}[system] + return { + "schema": 1, "profile": "ci", "platform": system, "architecture": host.split("-")[0], + "rustc": {"path": "/pinned/rustc", "version": f"rustc 1.98.0\nhost: {host}\nrelease: 1.98.0"}, + "native": {"compiler": "pinned", "sdk": "pinned"}, + "environment": {"RUSTFLAGS": "-D warnings", "CARGO_PROFILE_DEV_DEBUG": "0"}, + "configuration": {"files": {"guard": "original"}, "manifests": agent_cache.manifest_settings(self.root), "cargo_configs": {"workspace": {}}}, + } + + def key(self, identity): + return agent_cache.digest(scope.canonical_identity(identity, self.root)) + + def add_mac_dependency(self, edge="dep:raw-window-handle"): + self.app.write_text(self.app.read_text().replace('desktop = []', f'desktop = ["{edge}"]') + MAC_DEPENDENCY, encoding="utf-8") + + def test_mac_only_optional_dependency_preserves_windows_and_linux_keys(self): + before = {system: self.key(self.identity(system)) for system in ("Windows", "Linux", "Darwin")} + self.add_mac_dependency() + for system in ("Windows", "Linux", "Darwin"): + with self.subTest(system=system): + after = self.key(self.identity(system)) + self.assertEqual(after == before[system], system != "Darwin") + + def test_active_dependency_features_and_other_feature_edges_remain_guarded(self): + self.add_mac_dependency() + before = self.key(self.identity("Darwin")) + self.app.write_text(self.app.read_text().replace('optional = true', 'optional = true, features = ["std"]')) + self.assertNotEqual(self.key(self.identity("Darwin")), before) + before = self.key(self.identity()) + self.app.write_text(self.app.read_text().replace('desktop = [', 'desktop = ["new-local-feature", ')) + self.assertNotEqual(self.key(self.identity()), before) + + def test_plain_optional_feature_and_strong_edge_are_conservative(self): + for edge in ("raw-window-handle", "raw-window-handle/std"): + with self.subTest(edge=edge): + self.app.write_text('[package]\nname = "fixture"\n[features]\ndefault = ["desktop"]\ndesktop = []\n', encoding="utf-8") + before = self.key(self.identity()) + self.add_mac_dependency(edge) + self.assertNotEqual(self.key(self.identity()), before) + canonical = scope.canonical_identity(self.identity(), self.root) + settings = canonical["configuration"]["manifests"][APP] + self.assertTrue(settings["dependency-features"]) + self.assertIn(edge, settings["features"]["desktop"]) + + def test_explicit_local_feature_with_dependency_name_is_not_erased(self): + self.add_mac_dependency() + self.app.write_text(self.app.read_text().replace('[features]', '[features]\nraw-window-handle = []')) + canonical = scope.canonical_identity(self.identity(), self.root) + settings = canonical["configuration"]["manifests"][APP] + self.assertIn("raw-window-handle", settings["features"]) + self.assertIn("dep:raw-window-handle", settings["features"]["desktop"]) + + def test_weak_inactive_dependency_feature_is_removed_without_dropping_feature_name(self): + self.add_mac_dependency() + self.app.write_text(self.app.read_text().replace('desktop = ["dep:raw-window-handle"]', + 'desktop = ["dep:raw-window-handle", "raw-window-handle?/std"]')) + canonical = scope.canonical_identity(self.identity(), self.root) + self.assertEqual(canonical["configuration"]["manifests"][APP]["features"]["desktop"], []) + self.assertIn("desktop", canonical["configuration"]["manifests"][APP]["features"]) + + def test_inactive_optional_dependency_implicit_local_feature_is_preserved(self): + for edge in (None, "raw-window-handle?/std"): + with self.subTest(edge=edge): + self.app.write_text('[package]\nname = "fixture"\n[features]\ndefault = ["desktop"]\ndesktop = []\n', encoding="utf-8") + before = self.key(self.identity()) + if edge is None: + self.app.write_text(self.app.read_text() + MAC_DEPENDENCY) + else: + self.add_mac_dependency(edge) + identity = self.identity() + canonical = scope.canonical_identity(identity, self.root) + self.assertNotEqual(agent_cache.digest(canonical), before) + self.assertEqual(canonical["configuration"], identity["configuration"]) + + def test_dependency_declared_active_or_unknown_elsewhere_keeps_feature_edges(self): + self.add_mac_dependency() + for table in ("dependencies", "target.'cfg(feature = \"custom\")'.dependencies"): + with self.subTest(table=table): + original = self.app.read_text() + self.app.write_text(original + f'\n[{table}]\nraw-window-handle = {{version = "0.6", optional = true}}\n') + canonical = scope.canonical_identity(self.identity(), self.root) + self.assertIn("dep:raw-window-handle", canonical["configuration"]["manifests"][APP]["features"]["desktop"]) + self.app.write_text(original) + + def test_cfg_three_value_logic_preserves_unknowns(self): + context = scope.native_target(self.identity()) + cases = { + 'cfg(target_os = "windows")': True, + 'cfg(target_os = "macos")': False, + 'cfg(unix)': False, 'cfg(windows)': True, + 'cfg(all(target_os = "macos", unknown))': False, + 'cfg(any(target_os = "macos", unknown))': None, + 'cfg(not(unknown))': None, + 'cfg(target_feature = "sse2")': None, + 'cfg(feature = "custom")': None, + 'cfg(not(windows))': False, + 'cfg(all(windows, target_arch = "x86_64",))': True, + 'cfg(unknown_function(windows))': None, + 'cfg(target_os = "macos" broken)': None, + 'aarch64-apple-darwin': False, + 'x86_64-pc-windows-msvc': True, + } + for expression, expected in cases.items(): + with self.subTest(expression=expression): + self.assertIs(scope.target_matches(expression, context), expected) + + def test_unknown_cfg_dependency_metadata_and_features_stay_strict(self): + self.app.write_text(self.app.read_text() + '\n[target.\'cfg(custom_runtime)\'.dependencies]\ncustom = {version = "1", optional = true}\n') + identity = self.identity() + canonical = scope.canonical_identity(identity, self.root) + self.assertEqual(canonical["configuration"], identity["configuration"]) + + def test_cargo_config_target_wrapper_and_cfg_overrides_disable_pruning(self): + self.add_mac_dependency() + mutations = ( + lambda x: x["configuration"]["cargo_configs"]["workspace"].update({"config.toml": "digest"}), + lambda x: x["environment"].update(CARGO_BUILD_TARGET="aarch64-apple-darwin"), + lambda x: x["environment"].update(RUSTC_WRAPPER="custom-wrapper"), + lambda x: x["environment"].update(RUSTFLAGS="--cfg windows"), + lambda x: x["environment"].update(CARGO_ENCODED_RUSTFLAGS="--cfg\x1fwindows"), + lambda x: x["environment"].update(CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS="--target=other"), + ) + for mutate in mutations: + identity = self.identity() + mutate(identity) + self.assertEqual(scope.canonical_identity(identity, self.root)["configuration"], identity["configuration"]) + + def test_legacy_or_unknown_workspace_resolver_disables_pruning(self): + self.add_mac_dependency() + for resolver in (None, "1", "future"): + with self.subTest(resolver=resolver): + identity = self.identity() + identity["configuration"]["manifests"]["apps/maple-agent/Cargo.toml"]["resolver"] = resolver + self.assertEqual(scope.canonical_identity(identity, self.root)["configuration"], identity["configuration"]) + + def test_unknown_native_triples_do_not_infer_architecture_or_target_os(self): + self.add_mac_dependency() + for system, host in ( + ("Windows", "i686-pc-windows-msvc"), + ("Linux", "armv7-unknown-linux-gnueabihf"), + ("Linux", "aarch64-linux-android"), + ): + with self.subTest(host=host): + identity = self.identity(system) + identity["rustc"]["version"] = f"rustc 1.98.0\nhost: {host}" + self.assertIsNone(scope.native_target(identity)) + self.assertEqual(scope.canonical_identity(identity, self.root)["configuration"], identity["configuration"]) + + def test_abi_toolchain_flags_profiles_and_configuration_are_not_weakened(self): + self.add_mac_dependency() + identity = self.identity() + before = self.key(identity) + for section, key, value in ( + ("native", "sdk", "changed"), ("rustc", "path", "/different/rustc"), + ("environment", "RUSTFLAGS", "-D warnings -C opt-level=2"), + ("environment", "CARGO_PROFILE_DEV_DEBUG", "1"), + ): + with self.subTest(section=section, key=key): + changed = copy.deepcopy(identity) + changed[section][key] = value + self.assertNotEqual(self.key(changed), before) + changed = copy.deepcopy(identity) + changed["configuration"]["files"]["guard"] = "changed" + self.assertNotEqual(self.key(changed), before) + self.app.write_text(self.app.read_text() + '\n[profile.dev]\nopt-level = 2\n') + self.assertNotEqual(self.key(self.identity()), before) + + def test_flat_migration_document_preserves_exact_legacy_and_canonical_hashes(self): + self.add_mac_dependency() + legacy = self.identity() + with patch.object(agent_cache, "build_identity", return_value=legacy) as build: + document = scope.build_document("ci", self.root, self.root / "provider") + build.assert_called_once_with("ci", self.root, self.root / "provider") + self.assertEqual(document["legacy_identity"], legacy) + self.assertEqual(document["legacy_compatibility"], agent_cache.digest(legacy)) + canonical = {key: value for key, value in document.items() if key not in {"compatibility", "legacy_compatibility", "legacy_identity"}} + self.assertEqual(document["compatibility"], agent_cache.digest(canonical)) + self.assertEqual(canonical["scope_schema"], 1) + self.assertIn("rustc", document) + self.assertEqual(legacy, self.identity(), "canonicalization mutated the legacy migration identity") + + def test_cli_preserves_profile_directory_and_publishes_both_hashes(self): + self.add_mac_dependency() + with patch.object(agent_cache, "build_identity", return_value=self.identity()): + document = scope.build_document("provider", self.root) + destination = self.root / "metrics/environment.json" + outputs = self.root / "outputs" + provider = self.root / "copied-provider" + arguments = ["agent_cache_scope.py", "--profile", "provider", "--output", str(destination), + "--cargo-config-dir", str(provider)] + with patch.object(scope.sys, "argv", arguments), patch.object(scope, "build_document", return_value=document) as build, \ + patch.dict(os.environ, {"GITHUB_OUTPUT": str(outputs)}), patch("sys.stdout", new_callable=io.StringIO): + scope.main() + build.assert_called_once_with("provider", cargo_config_dir=provider) + self.assertEqual(json.loads(destination.read_text()), document) + self.assertEqual(outputs.read_text().splitlines(), [ + f"compatibility={document['compatibility']}", + f"legacy_compatibility={document['legacy_compatibility']}", + ]) + + def test_windows_manifest_names_and_unsafe_paths(self): + self.add_mac_dependency() + identity = self.identity() + manifests = identity["configuration"]["manifests"] + manifests[APP.replace("/", "\\")] = manifests.pop(APP) + scope.canonical_identity(identity, self.root) + for relative in ("../outside/Cargo.toml", "/outside/Cargo.toml", "C:\\outside\\Cargo.toml"): + with self.subTest(relative=relative), self.assertRaises(ValueError): + scope.manifest_path(self.root, relative) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_agent_change_detection.py b/scripts/ci/test_agent_change_detection.py index b41d70504..07bc84c62 100644 --- a/scripts/ci/test_agent_change_detection.py +++ b/scripts/ci/test_agent_change_detection.py @@ -88,6 +88,7 @@ def test_selector_and_shared_tooling_changes_select_agent(self): "scripts/ci/agent_change_detection.py", "scripts/ci/change_detection.py", "scripts/ci/agent_master_selection.py", "scripts/ci/agent_cache.py", "scripts/ci/agent_measure.py", + "scripts/ci/agent_cache_scope.py", "scripts/ci/agent_dependency_archive.py", "scripts/ci/agent_cache_command.py", "scripts/ci/agent_windows.py", "scripts/ci/verify-agent-rust-deps.py", diff --git a/scripts/ci/test_agent_dependency_archive.py b/scripts/ci/test_agent_dependency_archive.py new file mode 100644 index 000000000..efd7bc90d --- /dev/null +++ b/scripts/ci/test_agent_dependency_archive.py @@ -0,0 +1,575 @@ +#!/usr/bin/env python3 +"""Dependency pruning, artifact provenance, transport and extraction boundaries.""" + +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import gzip +import io +import json +import os +from pathlib import Path +import shutil +import tarfile +import tempfile +import unittest +from unittest.mock import patch +import urllib.error +import zipfile + +import agent_dependency_archive as cache + + +REPO = "MaplePrivacyLabs/Maple" +JOB = "Agent (windows-latest)" +REV = "a" * 40 +REGISTRY = "registry+https://github.com/rust-lang/crates.io-index" +STAMP = "1234567890abcdef" + + +class Fixture(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.home, self.target, self.bundle = self.root / "cargo", self.root / "target", self.root / "bundle" + raw = {"profile": "ci", "platform": "Windows", "architecture": "AMD64", "native": {"compiler": "pinned"}} + self.identity = {"compatibility": hashlib.sha256(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), **raw} + self.producer = {"repository": REPO, "run_id": 41, "run_attempt": 2, + "head_sha": REV, "job_name": JOB, "workflow": "agent-ci.yml"} + self.environment = {"GITHUB_REPOSITORY": REPO, "GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/master", + "GITHUB_SHA": REV, "GITHUB_RUN_ID": "41", "GITHUB_RUN_ATTEMPT": "2"} + selected = patch.dict(os.environ, self.environment, clear=True) + selected.start() + self.addCleanup(selected.stop) + self.metadata = {"packages": [], "resolve": {"nodes": []}} + self.package("public", "1.0.0", REGISTRY, self.home / "registry/src/index-abc/public-1.0.0/Cargo.toml") + self.package("gitlib", "2.0.0", f"git+https://github.com/MaplePrivacyLabs/rusqlite?rev={REV}#{REV}", self.home / "git/checkouts/public-abc/1234567/Cargo.toml") + self.package("maple-app", "0.0.3", None, self.root / "checkout/apps/maple-agent/app/Cargo.toml") + self.package("maple-sdk", "0.4.0", None, self.root / "checkout/sdk/rust/Cargo.toml") + self.write("registry/cache/index-abc/public-1.0.0.crate", b"released archive") + self.write("registry/index/index-abc/.cache/pu/bl/public", b"public index record") + self.write("registry/index/index-abc/config.json", b'{"dl":"https://static.crates.io/crates","api":"https://crates.io","private":"secret"}') + self.write("credentials.toml", b"do not cache credentials") + self.write("config.toml", b"do not cache config") + self.write("bin/cargo-private", b"do not cache binaries") + self.write("git/db/public-abc/config", b"[credential]\nhelper = secret\n") + self.write("git/db/public-abc/objects/object", b"public Git object") + self.write("git/checkouts/public-abc/1234567/.git/config", b"[remote]\nurl = https://user:secret@example.com\n") + self.write("git/checkouts/public-abc/1234567/.git/hooks/post-checkout", b"do not execute hook") + self.write("git/checkouts/public-abc/1234567/.git/objects/info/alternates", (str(self.home / "git/db/public-abc/objects") + "\n").encode()) + for prefix in ("debug", "build/debug"): + for name in ("public", "gitlib", "maple-app", "maple-sdk"): + self.write(f"{prefix}/deps/lib{name.replace('-', '_')}-{STAMP}.rlib", name.encode(), self.target) + self.write(f"{prefix}/deps/{name.replace('-', '_')}-{STAMP}.d", b"dep-info", self.target) + self.write(f"{prefix}/.fingerprint/{name}-{STAMP}/lib-fingerprint", b"freshness", self.target) + self.write(f"{prefix}/build/{name}-{STAMP}/out/native.lib", b"native", self.target) + self.write(f"{prefix}/incremental/private/data", b"incremental", self.target) + self.write(f"{prefix}/maple-agent.exe", b"workspace executable", self.target) + + def write(self, relative, content, root=None): + path = (root or self.home) / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + os.utime(path, (1_700_000_000, 1_700_000_000)) + return path + + def package(self, name, version, source, manifest): + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text(f'[package]\nname="{name}"\nversion="{version}"\n') + package_id = f"{source or 'path+workspace'}#{name}@{version}" + self.metadata["packages"].append({"name": name, "version": version, "source": source, + "id": package_id, "manifest_path": str(manifest), "targets": [{"name": name, "kind": ["lib"]}]}) + self.metadata["resolve"]["nodes"].append({"id": package_id, "features": ["default"]}) + + def pack(self, **options): + return cache.pack(self.identity, self.metadata, self.target, self.home, self.bundle, JOB, "agent-ci.yml", **options) + + def tar_names(self): + with tarfile.open(self.bundle / "dependencies.tar.gz", "r:gz") as archive: + return archive.getnames() + + def unpack(self): + return cache.unpack(self.bundle, self.identity, self.producer, self.root / "restored-cargo", self.root / "restored-target") + + def malicious_tar(self, members): + self.pack() + with tarfile.open(self.bundle / "dependencies.tar.gz", "w:gz") as archive: + total = 0 + for name, kind, payload in members: + info = tarfile.TarInfo(name) + info.type = kind + info.size = len(payload) if kind == tarfile.REGTYPE else 0 + info.linkname = "../../outside" if kind in (tarfile.SYMTYPE, tarfile.LNKTYPE) else "" + archive.addfile(info, io.BytesIO(payload) if kind == tarfile.REGTYPE else None) + total += info.size + manifest_path = self.bundle / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest.update(files=len(members), unpacked_bytes=total, archive_bytes=(self.bundle / "dependencies.tar.gz").stat().st_size, + archive_sha256=cache.sha256(self.bundle / "dependencies.tar.gz")) + manifest_path.write_text(json.dumps(manifest)) + + +class ArchiveTests(Fixture): + def test_roundtrip_only_public_dependencies_preserving_freshness(self): + manifest = self.pack() + names = self.tar_names() + self.assertTrue(any("build/debug/build/public-" in name for name in names)) + self.assertTrue(any("debug/deps/libpublic-" in name for name in names)) + self.assertFalse(any(any(part in name for part in ("maple_app", "maple-app", "maple_sdk", "maple-sdk", "incremental", "credentials", "cargo/bin", "hooks")) for name in names)) + self.assertNotIn("cargo/config.toml", names) + self.unpack() + restored = self.root / f"restored-target/debug/deps/libpublic-{STAMP}.rlib" + self.assertEqual(restored.read_bytes(), b"public") + self.assertEqual(restored.stat().st_mtime, 1_700_000_000) + self.assertEqual(self.producer, manifest["producer"]) + self.assertNotIn("secret", (self.root / "restored-cargo/git/db/public-abc/config").read_text()) + alternate = self.root / "restored-cargo/git/checkouts/public-abc/1234567/.git/objects/info/alternates" + self.assertFalse(Path(alternate.read_text().strip()).is_absolute()) + + def test_path_package_name_collision_excludes_ambiguous_output(self): + self.package("local-public", "1.0.0", None, self.root / "checkout/local-public/Cargo.toml") + self.metadata["packages"][-1]["targets"][0]["name"] = "public" + self.pack() + self.assertFalse(any("deps/libpublic-" in name for name in self.tar_names())) + + def test_unreviewed_git_sources_outputs_and_name_collisions_are_excluded(self): + self.package("private-helper", "1.0.0", f"git+https://github.com/unreviewed/private?rev={REV}#{REV}", + self.home / "git/checkouts/private-abc/1234567/Cargo.toml") + self.metadata["packages"][-1]["targets"][0]["name"] = "public" + self.write("git/checkouts/private-abc/1234567/secret-source.rs", b"private source") + self.write(f"debug/build/private-helper-{STAMP}/out/private.lib", b"private native output", self.target) + self.write(f"debug/.fingerprint/private-helper-{STAMP}/lib-fingerprint", b"private fingerprint", self.target) + self.write(f"debug/deps/libprivate_helper-{STAMP}.rlib", b"private library", self.target) + # A private package can also collide with a reviewed package name. + self.package("gitlib", "3.0.0", f"git+https://github.com/unreviewed/another?rev={REV}#{REV}", + self.home / "git/checkouts/private-other/1234567/Cargo.toml") + self.pack() + names = self.tar_names() + self.assertFalse(any("private-abc" in name or "private-other" in name or "private-helper" in name or "private_helper" in name for name in names)) + self.assertFalse(any("deps/libpublic-" in name or "deps/libgitlib-" in name or "/build/gitlib-" in name or "/.fingerprint/gitlib-" in name for name in names)) + + def test_reviewed_git_repository_normalization_keeps_security_guards(self): + original = self.metadata["packages"][1]["source"] + for repository in ("MaplePrivacyLabs/rusqlite", "MAPLEPRIVACYLABS/RUSQLITE.git", "MaplePrivacyLabs/Rusqlite.GIT/"): + self.metadata["packages"][1]["source"] = f"git+https://GitHub.com/{repository}?rev={REV}#{REV}" + self.assertIn("gitlib", cache.packages(self.metadata)[1]) + for source in (f"git+http://github.com/MaplePrivacyLabs/rusqlite#{REV}", + f"git+https://user:secret@github.com/MaplePrivacyLabs/rusqlite#{REV}", + "git+https://github.com/MaplePrivacyLabs/rusqlite#short", + f"git+https://github.com/MaplePrivacyLabs/rusqlite?token=secret#{REV}"): + self.metadata["packages"][1]["source"] = source + self.assertNotIn("gitlib", cache.packages(self.metadata)[1]) + self.metadata["packages"][1]["source"] = original + + def test_unrelated_application_bundle_symlinks_are_not_traversed(self): + bundle = self.target / "debug/Maple.app/Contents/Frameworks" + bundle.mkdir(parents=True) + (bundle / "system").symlink_to(self.root, target_is_directory=True) + self.pack() + self.assertFalse(any("Maple.app" in name for name in self.tar_names())) + + def test_link_outside_a_selected_dependency_root_is_refused(self): + source = self.home / "registry/src/index-abc/public-1.0.0/private" + source.symlink_to(self.home / "credentials.toml") + with self.assertRaises(cache.Rejected): + self.pack() + + def test_git_pointer_files_are_refused_on_pack_and_restore(self): + checkout_git = self.home / "git/checkouts/public-abc/1234567/.git" + shutil.rmtree(checkout_git) + checkout_git.write_text("gitdir: /outside/private-object-store\n") + with self.assertRaises(cache.Rejected): + self.pack() + shutil.rmtree(self.bundle) + checkout_git.unlink() + checkout_git.mkdir() + self.malicious_tar([("cargo/git/checkouts/repo/rev/.git", tarfile.REGTYPE, b"gitdir: /outside/private-object-store\n")]) + with self.assertRaises(cache.Rejected): + self.unpack() + self.assertFalse((self.root / "restored-cargo").exists()) + + def test_pr_and_manual_producers_cannot_pack(self): + for event, ref in (("pull_request", "refs/pull/2/merge"), ("workflow_dispatch", "refs/heads/master"), ("push", "refs/heads/topic")): + with self.subTest(event=event), patch.dict(os.environ, {"GITHUB_EVENT_NAME": event, "GITHUB_REF": ref}): + with self.assertRaises(cache.Rejected): + self.pack() + + def test_recent_receipt_skips_compression_for_workspace_only_bump(self): + public, _, _ = cache.packages(self.metadata) + receipt = self.root / "receipt.json" + receipt.write_text(json.dumps({"compatibility": self.identity["compatibility"], "dependency_fingerprint": "", + "dependency_graph": cache.dependency_graph(self.metadata, public), "created_at": datetime.now(timezone.utc).isoformat(), "artifact_id": 17, + "reader": cache.receipt_reader(JOB)})) + self.metadata["packages"][-1]["version"] = "0.4.1" + result = self.pack(receipt=receipt) + self.assertFalse(result["needs_upload"]) + self.assertFalse(self.bundle.exists()) + self.metadata["resolve"]["nodes"][0]["features"].append("new-native-feature") + self.assertTrue(self.pack(receipt=receipt)["needs_upload"]) + + def test_old_receipt_and_changed_dependency_refresh_the_seed(self): + public, _, _ = cache.packages(self.metadata) + receipt = self.root / "receipt.json" + receipt.write_text(json.dumps({"compatibility": self.identity["compatibility"], "dependency_fingerprint": "", + "dependency_graph": cache.dependency_graph(self.metadata, public), "created_at": (datetime.now(timezone.utc) - timedelta(days=2)).isoformat(), "artifact_id": 17})) + self.assertTrue(self.pack(receipt=receipt)["needs_upload"]) + + def test_provider_requires_verified_test_only_graph(self): + identity = {**self.identity, "profile": "provider"} + with self.assertRaises(cache.Rejected): + cache.artifact_name(identity) + self.assertNotEqual(cache.artifact_name(identity, "b" * 64), cache.artifact_name(identity, "c" * 64)) + + def test_environment_accepts_scoped_flat_identity_without_changing_old_helper(self): + raw = {key: value for key, value in self.identity.items() if key != "compatibility"} + raw["scope_schema"] = 1 + value = {**raw, "compatibility": hashlib.sha256(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + "legacy_identity": {"old": "full raw identity"}, "legacy_compatibility": "b" * 64} + path = self.root / "environment.json" + path.write_text(json.dumps(value)) + self.assertEqual(cache.environment(path), value) + value["native"]["compiler"] = "changed" + path.write_text(json.dumps(value)) + with self.assertRaises(cache.Rejected): + cache.environment(path) + + def test_archive_digest_and_manifest_identity_are_enforced(self): + self.pack() + with (self.bundle / "dependencies.tar.gz").open("ab") as archive: + archive.write(b"tampered") + with self.assertRaises(cache.Rejected): + self.unpack() + self.assertFalse((self.root / "restored-target").exists()) + + def test_compressed_and_member_budgets_remain_separate(self): + self.assertEqual(cache.MAX_ARCHIVE, 6 * 1024**3) + self.assertEqual(cache.MAX_FILE, 3 * 1024**3) + with patch.object(cache, "MAX_ARCHIVE", 1): + with self.assertRaises(cache.Rejected): + self.pack() + shutil.rmtree(self.bundle) + # A larger compressed-archive allowance cannot admit a single member + # over its independent bound, even before tarfile reads its payload. + header = tarfile.TarInfo("target/debug/deps/libpublic-" + STAMP + ".rlib") + header.size = cache.MAX_FILE + 1 + archive = self.root / "oversized-member.tar.gz" + archive.write_bytes(gzip.compress(header.tobuf(format=tarfile.USTAR_FORMAT))) + self.assertLess(archive.stat().st_size, cache.MAX_ARCHIVE) + with self.assertRaises(cache.Rejected): + cache.tar_envelope(archive) + + def test_unsafe_paths_links_aliases_and_local_outputs_are_refused(self): + attacks = ("../escape", "/absolute", "cargo/credentials.toml", "cargo/bin/tool", "cargo/registry/src/a/../escape", + "cargo/registry/src/a/CON.txt", "cargo/registry/src/a/file:stream", "cargo/registry/src/a/back\\slash", + "cargo/registry/src/a/trailing.", f"target/debug/deps/libmaple_app-{STAMP}.rlib") + for name in attacks: + with self.subTest(name=name): + shutil.rmtree(self.bundle, ignore_errors=True) + self.malicious_tar([(name, tarfile.REGTYPE, b"payload")]) + with self.assertRaises(cache.Rejected): + self.unpack() + self.assertFalse((self.root / "restored-cargo").exists()) + for kind in (tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.FIFOTYPE, tarfile.CHRTYPE): + with self.subTest(kind=kind): + shutil.rmtree(self.bundle) + self.malicious_tar([("cargo/registry/src/a/file", kind, b"")]) + with self.assertRaises(cache.Rejected): + self.unpack() + + def test_duplicate_case_alias_and_destination_symlink_fail_before_writes(self): + self.malicious_tar([("cargo/registry/src/a/file", tarfile.REGTYPE, b"one"), ("cargo/registry/src/a/FILE", tarfile.REGTYPE, b"two")]) + with self.assertRaises(cache.Rejected): + self.unpack() + self.assertFalse((self.root / "restored-cargo").exists()) + shutil.rmtree(self.bundle) + self.pack() + destination = self.root / "restored-target/debug/deps" + destination.parent.mkdir(parents=True) + outside = self.root / "outside" + outside.mkdir() + destination.symlink_to(outside, target_is_directory=True) + with self.assertRaises(cache.Rejected): + self.unpack() + self.assertEqual(list(outside.iterdir()), []) + self.assertFalse((self.root / "restored-cargo").exists()) + + def test_untrusted_git_configuration_and_unpack_bombs_are_rejected(self): + self.malicious_tar([("cargo/git/db/public/config", tarfile.REGTYPE, b"[credential]\nhelper = malicious\n")]) + with self.assertRaises(cache.Rejected): + self.unpack() + self.assertFalse((self.root / "restored-cargo").exists()) + shutil.rmtree(self.bundle) + self.pack() + with patch.object(cache, "MAX_UNPACKED", 1): + with self.assertRaises(cache.Rejected): + self.unpack() + + def test_symlink_ancestor_and_unsafe_git_or_registry_metadata_are_rejected(self): + self.pack() + outside = self.root / "outside" + outside.mkdir() + parent = self.root / "linked-parent" + parent.symlink_to(outside, target_is_directory=True) + with self.assertRaises(cache.Rejected): + cache.unpack(self.bundle, self.identity, self.producer, self.root / "restored-cargo", parent / "target") + self.assertEqual(list(outside.iterdir()), []) + for name, payload in (("cargo/git/checkouts/repo/rev/.git/objects/info/alternates", b"/outside/objects\n"), + ("cargo/registry/index/index/config.json", b'{"dl":"https://private.example/token","api":"https://crates.io"}')): + shutil.rmtree(self.bundle) + self.malicious_tar([(name, tarfile.REGTYPE, payload)]) + with self.subTest(name=name), self.assertRaises(cache.Rejected): + self.unpack() + self.assertFalse((self.root / "restored-cargo").exists()) + + def test_oversized_pax_is_rejected_before_tarfile_reads_payload(self): + self.pack() + header = tarfile.TarInfo("PaxHeader") + header.type, header.size = tarfile.XHDTYPE, 1024**3 + archive = self.bundle / "dependencies.tar.gz" + archive.write_bytes(gzip.compress(header.tobuf(format=tarfile.USTAR_FORMAT))) + manifest = json.loads((self.bundle / "manifest.json").read_text()) + manifest.update(archive_bytes=archive.stat().st_size, archive_sha256=cache.sha256(archive)) + (self.bundle / "manifest.json").write_text(json.dumps(manifest)) + with patch.object(cache.tarfile, "open", side_effect=AssertionError("unbounded PAX parser must not run")): + with self.assertRaises(cache.Rejected): + self.unpack() + + def test_receipt_cannot_be_replayed_from_restored_cache_or_prior_reader_job(self): + with self.assertRaises(cache.Rejected): + self.pack(receipt=self.target / "receipt.json") + public, _, _ = cache.packages(self.metadata) + receipt = self.root / "receipt.json" + receipt.write_text(json.dumps({"compatibility": self.identity["compatibility"], "dependency_fingerprint": "", + "dependency_graph": cache.dependency_graph(self.metadata, public), "created_at": datetime.now(timezone.utc).isoformat(), "artifact_id": 17, + "reader": {**cache.receipt_reader(JOB), "run_id": "older-job"}})) + self.assertTrue(self.pack(receipt=receipt)["needs_upload"]) + + +class API: + repository = REPO + + def __init__(self, artifact, run, jobs, zip_path=None): + self.artifact, self.run, self.jobs, self.zip_path = artifact, run, jobs, zip_path + + def get(self, endpoint): + if endpoint.startswith("actions/artifacts?"): + return {"artifacts": [self.artifact]} + if endpoint.startswith("actions/artifacts/"): + return self.artifact + if "/attempts/" in endpoint: + return self.jobs + return self.run + + def download(self, artifact_id, destination, digest): + if cache.sha256(self.zip_path) != digest: + raise cache.Rejected("ZIP digest mismatch") + shutil.copyfile(self.zip_path, destination) + return destination.stat().st_size + + +class ProvenanceTests(Fixture): + def setUp(self): + super().setUp() + now = datetime.now(timezone.utc) + self.artifact = {"id": 7, "name": cache.artifact_name(self.identity), "expired": False, "size_in_bytes": 50, + "digest": "sha256:" + "b" * 64, "created_at": (now - timedelta(hours=1)).isoformat(), + "workflow_run": {"id": 41, "head_sha": REV, "head_branch": "master", "repository_id": 2, "head_repository_id": 2}} + self.run = {"id": 41, "run_attempt": 2, "event": "push", "head_sha": REV, "head_branch": "master", + "path": ".github/workflows/agent-ci.yml", "repository": {"id": 2, "full_name": REPO}, + "head_repository": {"id": 2, "full_name": REPO}, "conclusion": "failure"} + self.jobs = {"total_count": 1, "jobs": [{"id": 8, "name": JOB, "status": "completed", "conclusion": "success", + "started_at": (now - timedelta(hours=2)).isoformat(), "completed_at": (now - timedelta(minutes=30)).isoformat(), + "steps": [{"name": "build/test/provenance gates", "conclusion": "success"}]}]} + + def proof(self): + return cache.source_proof(API(self.artifact, self.run, self.jobs), self.artifact, + cache.artifact_name(self.identity), JOB, "agent-ci.yml", datetime.now(timezone.utc)) + + def test_exact_successful_platform_job_is_usable_despite_other_job_failure(self): + self.assertEqual(self.proof(), self.producer) + + def test_earlier_successful_platform_seed_survives_partial_later_rerun(self): + with patch.dict(os.environ, {"GITHUB_RUN_ATTEMPT": "1"}): + self.pack() + self.producer["run_attempt"] = 1 + zipped = self.root / "earlier-attempt.zip" + with zipfile.ZipFile(zipped, "w") as archive: + for name in ("manifest.json", "dependencies.tar.gz"): + archive.write(self.bundle / name, name) + self.artifact["digest"] = "sha256:" + cache.sha256(zipped) + api = API(self.artifact, self.run, self.jobs, zipped) + events = [] + get, download = api.get, api.download + + def attempted(endpoint): + events.append(endpoint) + if "/attempts/2/jobs" in endpoint: + return {"total_count": 1, "jobs": [{"name": "Agent (different-platform)", "conclusion": "failure"}]} + return get(endpoint) + + def transferred(*arguments): + events.append("verified-zip-download") + return download(*arguments) + + receipt = self.root / "receipt.json" + with patch.object(api, "get", side_effect=attempted), patch.object(api, "download", side_effect=transferred): + self.assertTrue(cache.restore(api, self.identity, self.root / "restored-target", self.root / "restored-cargo", + JOB, "agent-ci.yml", receipt=receipt)) + jobs_request = "actions/runs/41/attempts/1/jobs?per_page=100" + self.assertIn(jobs_request, events) + self.assertLess(events.index("verified-zip-download"), events.index(jobs_request)) + self.assertFalse(any("/attempts/2/jobs" in event for event in events)) + self.assertEqual(json.loads(receipt.read_text())["producer"]["run_attempt"], 1) + + def test_future_or_noninteger_manifest_attempt_is_rejected(self): + api = API(self.artifact, self.run, self.jobs) + for attempt in (0, 3, "1", True): + with self.subTest(attempt=attempt), self.assertRaises(cache.Rejected): + cache.source_proof(api, self.artifact, cache.artifact_name(self.identity), JOB, + "agent-ci.yml", datetime.now(timezone.utc), attempt=attempt) + + def test_pr_manual_fork_wrong_workflow_and_head_cannot_supply_seed(self): + for change in ({"event": "pull_request"}, {"event": "workflow_dispatch"}, {"head_branch": "topic"}, + {"head_repository": {"id": 3, "full_name": "fork/Maple"}}, {"path": ".github/workflows/other.yml"}, + {"head_sha": "c" * 40}): + with self.subTest(change=change): + original = self.run + self.run = {**self.run, **change} + with self.assertRaises(cache.Rejected): + self.proof() + self.run = original + + def test_wrong_attempt_timestamp_failed_or_duplicate_job_is_rejected(self): + for conclusion in ("failure", "skipped", "cancelled"): + self.jobs["jobs"][0]["conclusion"] = conclusion + with self.assertRaises(cache.Rejected): + self.proof() + self.jobs["jobs"][0]["conclusion"] = "success" + self.artifact["created_at"] = (datetime.now(timezone.utc) - timedelta(hours=4)).isoformat() + with self.assertRaises(cache.Rejected): + self.proof() + self.artifact["created_at"] = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat() + self.jobs["jobs"].append(copy.deepcopy(self.jobs["jobs"][0])) + with self.assertRaises(cache.Rejected): + self.proof() + + def test_missing_digest_expiry_and_age_fail_closed(self): + for change in ({"digest": None}, {"expired": True}, {"created_at": (datetime.now(timezone.utc) - timedelta(days=8)).isoformat()}, {"size_in_bytes": cache.MAX_ARCHIVE * 2}): + original = self.artifact + self.artifact = {**self.artifact, **change} + with self.subTest(change=change), self.assertRaises((cache.Rejected, TypeError)): + self.proof() + self.artifact = original + + def test_end_to_end_restore_writes_verified_receipt_and_never_executes(self): + self.pack() + zipped = self.root / "bundle.zip" + with zipfile.ZipFile(zipped, "w") as archive: + for name in ("manifest.json", "dependencies.tar.gz"): + archive.write(self.bundle / name, name) + self.artifact["digest"] = "sha256:" + cache.sha256(zipped) + receipt = self.root / "receipt.json" + with patch.object(cache.os, "system", side_effect=AssertionError("restore must not execute")): + self.assertTrue(cache.restore(API(self.artifact, self.run, self.jobs, zipped), self.identity, + self.root / "restored-target", self.root / "restored-cargo", JOB, "agent-ci.yml", receipt=receipt)) + proven = json.loads(receipt.read_text()) + self.assertEqual(proven["artifact_digest"], self.artifact["digest"]) + self.assertEqual(proven["producer"], self.producer) + self.assertTrue((self.root / f"restored-target/debug/deps/libpublic-{STAMP}.rlib").is_file()) + + def test_corrupt_service_zip_digest_is_a_cold_miss(self): + self.pack() + zipped = self.root / "corrupt.zip" + zipped.write_bytes(b"corrupt bytes") + self.assertFalse(cache.restore(API(self.artifact, self.run, self.jobs, zipped), self.identity, + self.root / "restored-target", self.root / "restored-cargo", JOB, "agent-ci.yml")) + self.assertFalse((self.root / "restored-target").exists()) + + def test_zip_accepts_only_two_regular_bounded_expected_files(self): + for name in ("../manifest.json", "unexpected", "directory/manifest.json"): + zipped = self.root / "malicious.zip" + with zipfile.ZipFile(zipped, "w") as archive: + archive.writestr(name, b"bad") + archive.writestr("dependencies.tar.gz", b"data") + with self.subTest(name=name), self.assertRaises(cache.Rejected): + cache.unzip(zipped, self.root) + + def test_scan_is_bounded_and_unavailable_service_never_blocks_build(self): + api = API(self.artifact, self.run, self.jobs) + requested = [] + + def get(endpoint): + requested.append(endpoint) + if endpoint.startswith("actions/artifacts?"): + return {"artifacts": [{"id": index} for index in range(100)]} + return {"id": 0, "name": "wrong-name"} + + with patch.object(api, "get", side_effect=get): + self.assertFalse(cache.restore(api, self.identity, self.target, self.home, JOB, "agent-ci.yml")) + self.assertEqual(len(requested), cache.MAX_CANDIDATES + 1) + identity = self.root / "identity.json" + identity.write_text(json.dumps(self.identity)) + github_output = self.root / "github-output" + with patch.dict(os.environ, {"GH_TOKEN": "read-only-token", "GITHUB_OUTPUT": str(github_output)}), \ + patch("sys.argv", ["agent_dependency_archive.py", "restore", "--environment", str(identity), "--target", str(self.root / "new-target"), "--job-name", JOB]), \ + patch.object(cache.GitHub, "get", side_effect=OSError("service unavailable")): + cache.main() + self.assertEqual(github_output.read_text(), "restored=false\n") + self.assertFalse((self.root / "new-target").exists()) + + +class Response(io.BytesIO): + def __init__(self, value): + super().__init__(value) + self.headers = {"Content-Length": str(len(value))} + + +class TransportTests(unittest.TestCase): + def test_api_auth_is_never_forwarded_to_artifact_blob(self): + api = cache.GitHub(REPO, "private-token") + observed = [] + data = b"verified ZIP bytes" + + class Opener: + def open(self, request, timeout): + observed.append(request) + if len(observed) == 1: + raise urllib.error.HTTPError(request.full_url, 302, "redirect", {"Location": "https://safe.blob.core.windows.net/artifact?signed=opaque"}, None) + return Response(data) + + with tempfile.TemporaryDirectory() as temporary, patch.object(cache.urllib.request, "build_opener", return_value=Opener()): + self.assertEqual(api.download(7, Path(temporary) / "artifact.zip", hashlib.sha256(data).hexdigest()), len(data)) + self.assertEqual(observed[0].get_header("Authorization"), "Bearer private-token") + self.assertIsNone(observed[1].get_header("Authorization")) + self.assertNotIn("private-token", observed[1].full_url) + + def test_foreign_redirect_and_wrong_digest_are_refused(self): + api = cache.GitHub(REPO, "private-token") + + class Redirect: + def open(self, request, timeout): + raise urllib.error.HTTPError(request.full_url, 302, "redirect", {"Location": "https://attacker.example/artifact"}, None) + + with tempfile.TemporaryDirectory() as temporary, patch.object(cache.urllib.request, "build_opener", return_value=Redirect()): + with self.assertRaises(cache.Rejected): + api.download(7, Path(temporary) / "artifact.zip", "b" * 64) + with tempfile.TemporaryDirectory() as temporary, patch.object(cache.urllib.request, "build_opener") as opener: + opener.return_value.open.return_value = Response(b"wrong ZIP") + with self.assertRaises(cache.Rejected): + api.download(7, Path(temporary) / "artifact.zip", "b" * 64) + + def test_download_bytes_are_bounded_even_when_header_lies(self): + api = cache.GitHub(REPO, "private-token") + response = Response(b"too many streamed bytes") + response.headers["Content-Length"] = "1" + with tempfile.TemporaryDirectory() as temporary, patch.object(cache.urllib.request, "build_opener") as opener, \ + patch.object(cache, "MAX_ARCHIVE", 1), patch.object(cache, "MAX_MANIFEST", 1), patch.object(cache, "CHUNK", 1): + # Tighten the fixed transport-overhead margin by using an object + # whose stream lies about producing enormous chunks. + response.read = lambda size: b"x" * (1024**2 + 3) + opener.return_value.open.return_value = response + with self.assertRaises(cache.Rejected): + api.download(7, Path(temporary) / "artifact.zip", "b" * 64) +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_agent_master_selection.py b/scripts/ci/test_agent_master_selection.py index 214f85301..f02869d8d 100644 --- a/scripts/ci/test_agent_master_selection.py +++ b/scripts/ci/test_agent_master_selection.py @@ -1,4 +1,4 @@ -"""Master packaging must retain changes until all profiles are verified.""" +"""Master lanes retain cancelled, failed and skipped work across later pushes.""" from contextlib import redirect_stderr, redirect_stdout import io @@ -37,7 +37,7 @@ def verified_jobs(**conclusions): return {"total_count": len(jobs), "jobs": jobs} -class MasterSelectionTests(unittest.TestCase): +class GitFixture(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) @@ -59,6 +59,8 @@ def commit(self, path, content): self.git("commit", "--quiet", "-m", "fixture") return self.git("rev-parse", "HEAD") + +class MasterSelectionTests(GitFixture): def select(self, head, runs, jobsets=None, current_run=999): jobsets = jobsets or {} @@ -232,5 +234,81 @@ def test_manual_force_build_avoids_api(self): api.assert_not_called() +class MasterCISelectionTests(GitFixture): + def select(self, head, runs, jobsets=None): + jobsets = jobsets or {} + + def response(path): + if "/workflows/" in path: + self.assertIn("/agent-ci.yml/runs?", path) + return {"workflow_runs": runs} + run_id = int(path.split("/runs/")[1].split("/")[0]) + return jobsets.get(run_id, self.jobs()) + + with patch.object(selector, "api", side_effect=response): + return selector.select_master(head, 999, self.root, lane="ci") + + @staticmethod + def record(run_id, head, **overrides): + return run_record(run_id, head, path=selector.CI_WORKFLOW, **overrides) + + @staticmethod + def jobs(**conclusions): + jobs = [{"name": name, "status": "completed", + "conclusion": conclusions.get(name, "success")} + for name in sorted(selector.CI_VERIFIERS)] + return {"total_count": len(jobs), "jobs": jobs} + + def test_unrelated_push_retains_cancelled_or_failed_agent_changes(self): + changed = self.commit("apps/maple-agent/app/src/main.rs", "unverified change\n") + head = self.commit("services/opensecret/pcrDev.json", "unrelated later push\n") + for conclusion in ("cancelled", "failure"): + with self.subTest(conclusion=conclusion): + selected, reason = self.select(head, [ + self.record(2, changed, conclusion=conclusion), + self.record(1, self.base), + ]) + self.assertTrue(selected) + self.assertIn("master run 1", reason) + + def test_green_selector_only_does_not_consume_unverified_changes(self): + changed = self.commit("apps/maple-agent/app/src/main.rs", "cancelled change\n") + head = self.commit("README.md", "unrelated\n") + skipped = self.jobs(**{name: "skipped" for name in selector.CI_VERIFIERS}) + skipped["jobs"].append({"name": "Agent CI required jobs", "status": "completed", + "conclusion": "success"}) + skipped["total_count"] += 1 + selected, reason = self.select(head, [self.record(2, changed), self.record(1, self.base)], + {2: skipped}) + self.assertTrue(selected) + self.assertIn("master run 1", reason) + + def test_every_platform_provider_and_release_must_actually_pass(self): + changed = self.commit("apps/maple-agent/app/src/main.rs", "partly validated\n") + head = self.commit("README.md", "unrelated\n") + for name in selector.CI_VERIFIERS: + for conclusion in ("skipped", "failure", "cancelled"): + with self.subTest(job=name, conclusion=conclusion): + selected, reason = self.select( + head, [self.record(2, changed), self.record(1, self.base)], + {2: self.jobs(**{name: conclusion})}, + ) + self.assertTrue(selected) + self.assertIn("master run 1", reason) + + def test_completed_platform_validation_allows_later_unrelated_push_to_skip(self): + changed = self.commit("apps/maple-agent/app/src/main.rs", "fully validated\n") + head = self.commit("README.md", "unrelated\n") + selected, reason = self.select(head, [self.record(2, changed)]) + self.assertFalse(selected) + self.assertIn("master run 2", reason) + + def test_desktop_packaging_success_cannot_replace_ci_validation(self): + changed = self.commit("apps/maple-agent/app/src/main.rs", "only packaged\n") + head = self.commit("README.md", "unrelated\n") + selected, _ = self.select(head, [run_record(2, changed)]) + self.assertTrue(selected) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/ci/test_agent_workflows.py b/scripts/ci/test_agent_workflows.py index 1101a7f21..a0508af51 100644 --- a/scripts/ci/test_agent_workflows.py +++ b/scripts/ci/test_agent_workflows.py @@ -10,10 +10,17 @@ import tempfile import unittest +import agent_master_selection + ROOT = Path(__file__).resolve().parents[2] RUST_CACHE_COMMAND = 'python3 "${{ github.workspace }}/scripts/ci/agent_cache_command.py" {0}' -MASTER_CACHE_SAVE = "${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}" +TRUSTED_DEPENDENCY_WRITE = "${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' }}" +TRUSTED_DEPENDENCY_UPLOAD = "${{ success() && github.repository == 'MaplePrivacyLabs/Maple' && github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.dependency-seed.outputs.needs-upload == 'true' }}" +DEPENDENCY_BUILDERS = ( + ("agent-ci.yml", "desktop"), ("agent-ci.yml", "provider"), ("agent-ci.yml", "linux-release"), + ("agent-desktop-build.yml", "macos-build"), ("agent-desktop-build.yml", "linux"), +) @functools.cache @@ -160,14 +167,30 @@ def test_contributor_build_has_no_credentials_or_privileged_events(self): self.assertEqual(config["on"]["push"]["branches"], ["master"]) self.assertIsNone(config["on"]["pull_request"]) self.assertEqual(config["permissions"], {"contents": "read"}) - for value in strings(config): + for value in strings({key: value for key, value in config.items() if key != "jobs"}): self.assertNotRegex(value, r"\bsecrets\b|github\.token|\bGH_TOKEN\b") - for job in config["jobs"].values(): + for name, job in config["jobs"].items(): self.assertNotIn("environment", job) self.assertNotIn("uses", job) - self.assertIn(job.get("permissions"), (None, {"contents": "read"})) + if name in {"changes", "desktop", "provider", "linux-release"}: + self.assertEqual(job["permissions"], {"contents": "read", "actions": "read"}) + else: + self.assertIn(job.get("permissions"), (None, {"contents": "read"})) for step in job["steps"]: self.assertNotIn("${{", step.get("run", "")) + if name == "changes" and step.get("id") == "master": + self.assertEqual(step["env"], {"GH_TOKEN": "${{ github.token }}"}) + self.assertIn("agent_master_selection.py --lane ci", step["run"]) + self.assertEqual(step["if"], "${{ github.repository == 'MaplePrivacyLabs/Maple' && github.ref == 'refs/heads/master' && github.event_name == 'push' }}") + elif name in {"desktop", "provider", "linux-release"} and step.get("id") == "dependency-archive": + expected = {"GH_TOKEN": "${{ github.token }}"} + if name == "provider": + expected["PROVIDER_DEPENDENCY_FINGERPRINT"] = "${{ steps.provider.outputs.dependency-fingerprint }}" + self.assertEqual(step["env"], expected) + self.assertIn("agent_dependency_archive.py restore", step["run"]) + else: + for value in strings(step): + self.assertNotRegex(value, r"\bsecrets\b|github\.token|\bGH_TOKEN\b") action = step.get("uses", "") if action: self.assertRegex(action, r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)?@[0-9a-f]{40}$") @@ -183,15 +206,17 @@ def test_agent_cache_and_artifacts_do_not_share_research_publication(self): for cache in caches: self.assertEqual(cache["workspaces"], "apps/maple-agent -> target") self.assertTrue(cache["key"].startswith("maple-agent-")) - expected_save = MASTER_CACHE_SAVE if cache["add-rust-environment-hash-key"] == "true" else "false" - self.assertEqual(cache["save-if"], expected_save) + self.assertEqual(cache["save-if"], "false") for step in steps: action = step.get("uses", "") self.assertNotIn("release", action.lower()) self.assertNotIn("download-artifact", action) if "upload-artifact@" in action: - self.assertTrue(step["with"]["name"].startswith("maple-agent-")) - self.assertTrue(step["with"]["path"].startswith("apps/maple-agent/target/")) + if step.get("name") == "Upload durable dependency archive": + self.assertEqual(step["with"]["name"], "${{ steps.dependency-seed.outputs.artifact-name }}") + else: + self.assertTrue(step["with"]["name"].startswith("maple-agent-")) + self.assertTrue(step["with"]["path"].startswith("apps/maple-agent/target/")) self.assertFalse(any((ROOT / "apps/maple-agent/.github/workflows").glob("*.yml"))) def test_failed_or_missing_selection_cannot_skip_the_desktop_matrix(self): @@ -230,91 +255,172 @@ def test_parallel_jobs_retain_selection_platforms_and_required_gate(self): expected = result == ("skipped" if omitted else "success") self.assertEqual(command.returncode == 0, expected, env) + def test_master_selection_ci_verifiers_match_actual_platform_job_names(self): + jobs = workflow("agent-ci.yml")["jobs"] + actual = {jobs["linux-release"]["name"]} + for name in ("desktop", "provider"): + job = jobs[name] + self.assertEqual(set(job["strategy"]["matrix"]), {"os"}) + actual.update(job["name"].replace("${{ matrix.os }}", runner) + for runner in job["strategy"]["matrix"]["os"]) + self.assertEqual(agent_master_selection.CI_VERIFIERS, actual) + def test_dependency_cache_restore_prefix_uses_actual_environment_without_lock_hash(self): - for filename, names in (("agent-ci.yml", ("desktop", "linux-release")), - ("agent-desktop-build.yml", ("macos-build", "linux"))): - for name in names: - with self.subTest(workflow=filename, job=name): - steps = workflow(filename)["jobs"][name]["steps"] - fingerprint = next(i for i, step in enumerate(steps) if step.get("id") == "environment") - cache_index = next(i for i, step in enumerate(steps) if "rust-cache@" in step.get("uses", "")) - self.assertLess(fingerprint, cache_index) - cache = steps[cache_index]["with"] - self.assertEqual(cache["add-rust-environment-hash-key"], "true") - self.assertIn("steps.environment.outputs.compatibility", cache["key"]) - self.assertNotIn("hashFiles", cache["key"]) - self.assertEqual(cache["cmd-format"], RUST_CACHE_COMMAND) - self.assertEqual(cache["save-if"], "${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}") - self.assertIn("nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache.py", steps[fingerprint]["run"]) + for filename, name in DEPENDENCY_BUILDERS: + if name == "provider": + continue + with self.subTest(workflow=filename, job=name): + steps = workflow(filename)["jobs"][name]["steps"] + fingerprint = next(i for i, step in enumerate(steps) if step.get("id") == "environment") + artifact = next(i for i, step in enumerate(steps) if step.get("id") == "dependency-archive") + cache_index = next(i for i, step in enumerate(steps) if "rust-cache@" in step.get("uses", "")) + self.assertLess(fingerprint, artifact) + self.assertLess(artifact, cache_index) + cache = steps[cache_index]["with"] + self.assertEqual(steps[cache_index]["if"], "${{ steps.dependency-archive.outputs.restored != 'true' }}") + self.assertEqual(cache["add-rust-environment-hash-key"], "true") + self.assertIn("steps.environment.outputs.compatibility", cache["key"]) + self.assertNotIn("hashFiles", cache["key"]) + self.assertEqual(cache["cmd-format"], RUST_CACHE_COMMAND) + self.assertEqual(cache["save-if"], "false") + self.assertIn("nix develop --no-update-lock-file . -c python3 ../../scripts/ci/agent_cache_scope.py", steps[fingerprint]["run"]) def test_legacy_cache_restore_preserves_abi_scope_and_never_writes(self): - for filename, names in (("agent-ci.yml", ("desktop", "linux-release")), - ("agent-desktop-build.yml", ("macos-build", "linux"))): - for name in names: - with self.subTest(workflow=filename, job=name): - steps = workflow(filename)["jobs"][name]["steps"] - primary_index = next(i for i, step in enumerate(steps) if step.get("id") == "rust-cache") - marker_index = next(i for i, step in enumerate(steps) if step.get("id") == "restored-target") - caches = [(i, step) for i, step in enumerate(steps) if "rust-cache@" in step.get("uses", "")] - self.assertEqual(len(caches), 2) - fallback_index, fallback = caches[1] - primary = steps[primary_index] - self.assertLess(primary_index, marker_index) - self.assertLess(marker_index, fallback_index) - self.assertEqual(fallback["uses"], primary["uses"]) - self.assertEqual(fallback["if"], - "${{ steps.rust-cache.outputs.cache-hit != 'true' && steps.restored-target.outputs.present != 'true' }}") - # Disabling the upstream hash is permitted only on this - # read-only fallback, scoped to the very same actual ABI. - self.assertEqual(fallback["with"], {**primary["with"], - "add-rust-environment-hash-key": "false", "save-if": "false"}) - self.assertEqual(primary["with"]["add-rust-environment-hash-key"], "true") - self.assertEqual(primary["with"]["save-if"], MASTER_CACHE_SAVE) - self.assertNotIn("continue-on-error", fallback) + for filename, name in DEPENDENCY_BUILDERS: + if name == "provider": + continue + with self.subTest(workflow=filename, job=name): + steps = workflow(filename)["jobs"][name]["steps"] + primary_index = next(i for i, step in enumerate(steps) if step.get("id") == "rust-cache") + marker_index = next(i for i, step in enumerate(steps) if step.get("id") == "restored-target") + caches = [(i, step) for i, step in enumerate(steps) if "rust-cache@" in step.get("uses", "")] + self.assertEqual(len(caches), 2) + fallback_index, fallback = caches[1] + primary = steps[primary_index] + self.assertLess(primary_index, marker_index) + self.assertLess(marker_index, fallback_index) + self.assertEqual(fallback["uses"], primary["uses"]) + self.assertEqual(fallback["if"], "${{ steps.dependency-archive.outputs.restored != 'true' && steps.restored-target.outputs.present != 'true' }}") + self.assertEqual(fallback["with"], {**primary["with"], + "add-rust-environment-hash-key": "false", + "key": primary["with"]["key"].replace("outputs.compatibility", "outputs.legacy_compatibility")}) + self.assertEqual(primary["with"]["add-rust-environment-hash-key"], "true") + self.assertEqual(primary["with"]["save-if"], "false") + self.assertNotIn("continue-on-error", fallback) def test_primary_prefix_restore_does_not_download_a_second_archive(self): - # cache-hit=false also means a successful upstream prefix restore. - # Exercise the actual workflow's marker script against an empty target - # and a restored Cargo target before evaluating that fallback guard. - for filename, name in (("agent-ci.yml", "desktop"), ("agent-ci.yml", "linux-release"), - ("agent-desktop-build.yml", "macos-build"), ("agent-desktop-build.yml", "linux")): - steps = workflow(filename)["jobs"][name]["steps"] - marker = next(step for step in steps if step.get("id") == "restored-target") - with tempfile.TemporaryDirectory(prefix="cache restore ") as directory: - root = Path(directory) - target = root / "apps/maple-agent/target" - target.mkdir(parents=True) - for present in (False, True): - with self.subTest(workflow=filename, job=name, restored=present): - if present: - (target / "CACHEDIR.TAG").write_text("Signature: 8a477f597d28d172789f06886806bc55\n") - output = root / f"outputs-{present}" - subprocess.run(["bash", "-eu", "-c", marker["run"]], check=True, - env={**os.environ, "GITHUB_WORKSPACE": str(root), "GITHUB_OUTPUT": str(output)}, - capture_output=True) - self.assertEqual(output.read_text(), f"present={str(present).lower()}\n") + # Metadata and a restored CACHEDIR.TAG do not prove useful dependencies. + # Require real target fingerprints and a Rust library/metadata output. + for filename, name in DEPENDENCY_BUILDERS: + if name == "provider": + continue + marker = next(step for step in workflow(filename)["jobs"][name]["steps"] + if step.get("id") == "restored-target") + for case, expected in (("empty", False), ("tag_only", False), + ("fingerprint_only", False), ("library_only", False), ("empty_library", False), + ("rlib", True), ("rmeta", True), ("nested", True)): + with self.subTest(workflow=filename, job=name, case=case), tempfile.TemporaryDirectory(prefix="cache restore ") as directory: + root = Path(directory) + target = root / "apps/maple-agent/target" + profile = target / ("aarch64-apple-darwin/release" if case == "nested" else "debug") + profile.mkdir(parents=True) + if case != "empty": + (target / "CACHEDIR.TAG").write_text("Signature: 8a477f597d28d172789f06886806bc55\n") + if case in {"fingerprint_only", "empty_library", "rlib", "rmeta", "nested"}: + unit = profile / ".fingerprint/public-123" + unit.mkdir(parents=True) + (unit / "lib-public").write_text("fingerprint") + if case in {"library_only", "empty_library", "rlib", "rmeta", "nested"}: + deps = profile / "deps" + deps.mkdir() + extension = "rmeta" if case == "rmeta" else "rlib" + (deps / f"libpublic-123.{extension}").write_bytes(b"" if case == "empty_library" else b"actual dependency") + output = root / "outputs" + subprocess.run(["bash", "-eu", "-c", marker["run"]], check=True, + env={**os.environ, "GITHUB_WORKSPACE": str(root), "GITHUB_OUTPUT": str(output)}, + capture_output=True) + self.assertEqual(output.read_text(), f"present={str(expected).lower()}\n") def test_provider_restores_only_after_verified_graph_with_no_compatible_fallback(self): steps = workflow("agent-ci.yml")["jobs"]["provider"]["steps"] - prepare = next(i for i, step in enumerate(steps) if step.get("id") == "provider") - restore = next(i for i, step in enumerate(steps) if step.get("id") == "provider-cache") + indices = {step.get("id"): i for i, step in enumerate(steps) if step.get("id")} test = next(i for i, step in enumerate(steps) if step.get("name") == "Test released rusqlite against the static provider") - save = next(i for i, step in enumerate(steps) if "actions/cache/save@" in step.get("uses", "")) - environment = next(i for i, step in enumerate(steps) if step.get("id") == "environment") - self.assertLess(prepare, environment) - self.assertLess(environment, restore) - self.assertIn('--cargo-config-dir "$PROVIDER_WORK"', steps[environment]["run"]) - self.assertLess(restore, test) - self.assertLess(test, save) - cache = steps[restore]["with"] - self.assertNotIn("restore-keys", cache) - self.assertEqual(cache["path"], "${{ env.PROVIDER_TARGET }}") - self.assertIn("steps.environment.outputs.compatibility", cache["key"]) - self.assertIn("steps.provider.outputs.dependency-fingerprint", cache["key"]) - self.assertIn("--phase prepare", steps[prepare]["run"]) + self.assertLess(indices["provider"], indices["environment"]) + self.assertLess(indices["environment"], indices["dependency-archive"]) + self.assertLess(indices["dependency-archive"], indices["provider-cache"]) + self.assertLess(indices["provider-cache"], test) + self.assertLess(test, indices["dependency-seed"]) + self.assertIn('--cargo-config-dir "$PROVIDER_WORK"', steps[indices["environment"]]["run"]) + native = steps[indices["provider-cache"]] + self.assertEqual(native["if"], "${{ steps.dependency-archive.outputs.restored != 'true' }}") + self.assertNotIn("restore-keys", native["with"]) + self.assertEqual(native["with"]["path"], "${{ env.PROVIDER_TARGET }}") + self.assertIn("steps.environment.outputs.legacy_compatibility", native["with"]["key"]) + self.assertIn("steps.provider.outputs.dependency-fingerprint", native["with"]["key"]) + self.assertIn("--phase prepare", steps[indices["provider"]]["run"]) self.assertIn("--phase run", steps[test]["run"]) - self.assertIn("success() && github.event_name == 'push' && github.ref == 'refs/heads/master'", steps[save]["if"]) - self.assertEqual(steps[save]["with"]["key"], "${{ steps.provider-cache.outputs.cache-primary-key }}") + for ident in ("dependency-archive", "dependency-seed"): + self.assertIn('--dependency-fingerprint "$PROVIDER_DEPENDENCY_FINGERPRINT"', steps[indices[ident]]["run"]) + self.assertEqual(steps[indices[ident]]["env"]["PROVIDER_DEPENDENCY_FINGERPRINT"], "${{ steps.provider.outputs.dependency-fingerprint }}") + self.assertIn('--metadata "$PROVIDER_WORK/maple-provider-metadata.json"', steps[indices["dependency-seed"]]["run"]) + self.assertFalse(any("actions/cache/save@" in step.get("uses", "") for step in steps)) + + def test_durable_archive_restore_identity_and_trusted_fail_closed_upload(self): + for filename, name in DEPENDENCY_BUILDERS: + with self.subTest(workflow=filename, job=name): + job = workflow(filename)["jobs"][name] + self.assertEqual(job["permissions"], {"contents": "read", "actions": "read"}) + self.assertEqual(job["env"]["DEPENDENCY_CACHE_JOB"], job["name"]) + steps = job["steps"] + restore = next(step for step in steps if step.get("id") == "dependency-archive") + pack = next(step for step in steps if step.get("id") == "dependency-seed") + upload = next(step for step in steps if step.get("name") == "Upload durable dependency archive") + self.assertIn("agent_dependency_archive.py restore", restore["run"]) + self.assertIn("agent_dependency_archive.py pack", pack["run"]) + for step in (restore, pack): + self.assertIn(f"--workflow {filename}", step["run"]) + self.assertIn('--job-name "$DEPENDENCY_CACHE_JOB"', step["run"]) + self.assertIn('--receipt "$RUNNER_TEMP/agent-dependency-receipt.json"', step["run"]) + self.assertNotIn("continue-on-error", step) + self.assertEqual(restore["env"]["GH_TOKEN"], "${{ github.token }}") + self.assertNotIn("GH_TOKEN", pack.get("env", {})) + self.assertEqual(pack["if"], TRUSTED_DEPENDENCY_WRITE) + # A verified <24h receipt can suppress compression/upload. Only + # the helper's positive needs-upload result permits a write. + self.assertEqual(upload["if"], TRUSTED_DEPENDENCY_UPLOAD) + self.assertEqual(upload["uses"], "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02") + self.assertEqual(upload["with"], { + "name": "${{ steps.dependency-seed.outputs.artifact-name }}", + "path": "${{ runner.temp }}/agent-dependency-seed/manifest.json\n${{ runner.temp }}/agent-dependency-seed/dependencies.tar.gz\n", + "if-no-files-found": "error", "compression-level": 0, + "overwrite": True, "retention-days": 3, + }) + self.assertNotIn("continue-on-error", upload) + self.assertLess(steps.index(restore), steps.index(pack)) + self.assertLess(steps.index(pack), steps.index(upload)) + if name != "provider": + graph = next(step for step in steps if step.get("name") == "Verify the canonical SDK and proxy dependency graph") + self.assertIn('metadata="$RUNNER_TEMP/agent-workspace-metadata.json"', graph["run"]) + self.assertNotIn("rm -f", graph["run"]) + self.assertIn('--metadata "$RUNNER_TEMP/agent-workspace-metadata.json"', pack["run"]) + # Every earlier validation must succeed before pack/upload; + # diagnostics remain the only explicitly always-run steps. + for step in steps[:steps.index(pack)]: + self.assertNotIn("continue-on-error", step) + for step in steps[steps.index(pack) + 1:]: + if "run" in step: + self.assertEqual(step.get("if"), "always()") + self.assertIn("agent_measure.py --summarize", step["run"]) + self.assertNotIn("always()", pack["if"]) + self.assertNotIn("always()", upload["if"]) + + def test_dependency_archives_are_absent_from_protected_and_consumer_jobs(self): + for name, job in workflow("agent-desktop-build.yml")["jobs"].items(): + if name in {"macos-build", "linux"}: + continue + for step in job["steps"]: + self.assertNotIn("agent_dependency_archive.py", step.get("run", "")) + self.assertNotIn(step.get("id"), {"dependency-archive", "dependency-seed"}) def test_workspace_checks_and_plain_release_keep_original_coverage(self): jobs = workflow("agent-ci.yml")["jobs"] @@ -430,7 +536,7 @@ def test_only_trusted_master_can_receive_apple_signing_credentials(self): self.assertEqual(job["permissions"], { "contents": "read", "attestations": "read", }) - elif name == "changes": + elif name in ("changes", "macos-build", "linux"): self.assertEqual(job["permissions"], {"contents": "read", "actions": "read"}) else: self.assertIn(job.get("permissions"), (None, {"contents": "read"})) @@ -442,6 +548,9 @@ def test_only_trusted_master_can_receive_apple_signing_credentials(self): self.assertEqual(step["env"], {"GH_TOKEN": "${{ github.token }}"}) if name == "verify-linux": self.assertIn(self.MASTER_GUARD, " ".join(step["if"].split())) + elif name in ("macos-build", "linux") and step.get("id") == "dependency-archive": + self.assertEqual(step["env"], {"GH_TOKEN": "${{ github.token }}"}) + self.assertIn("agent_dependency_archive.py restore", step["run"]) elif name == "changes" and step.get("id") == "master": self.assertEqual(step["if"], "${{ github.repository == 'MaplePrivacyLabs/Maple' && " @@ -473,8 +582,7 @@ def test_actions_and_checkouts_do_not_grant_publication_or_persist_credentials(s self.assertEqual(cache["workspaces"], "apps/maple-agent -> target") self.assertTrue(cache["key"].startswith("maple-agent-release-")) self.assertIn("${{ matrix.variant }}", cache["key"]) - expected_save = MASTER_CACHE_SAVE if cache["add-rust-environment-hash-key"] == "true" else "false" - self.assertEqual(cache["save-if"], expected_save) + self.assertEqual(cache["save-if"], "false") def test_unsigned_pr_jobs_are_fail_safe_and_never_use_signed_macos_artifact_names(self): jobs = workflow(self.WORKFLOW)["jobs"] @@ -551,7 +659,8 @@ def test_profiles_and_artifacts_keep_platform_variant_and_run_identity(self): self.assertIn('"$MAPLE_RELEASE_PROFILE"', profile_step["run"]) uploads = [step for step in job["steps"] if step.get("uses", "").startswith("actions/upload-artifact@") - and step.get("with", {}).get("path") != "apps/maple-agent/target/ci-metrics/"] + and step.get("with", {}).get("path") != "apps/maple-agent/target/ci-metrics/" + and step.get("name") != "Upload durable dependency archive"] self.assertEqual(len(uploads), 1) upload = uploads[0] self.assertNotIn("if", upload) @@ -940,12 +1049,13 @@ def commit_file(self, path, text): self.git("commit", "-qm", "fixture change") return self.git("rev-parse", "HEAD") - def select(self, event, base, head): + def select(self, event, base, head, *, repository="fork/Maple", ref="refs/heads/topic", master=""): step = next(step for step in workflow("agent-ci.yml")["jobs"]["changes"]["steps"] if step.get("id") == "classify") output = self.root / "output" output.unlink(missing_ok=True) env = {**os.environ, "GITHUB_EVENT_NAME": event, "BASE_SHA": base, "HEAD_SHA": head, + "GITHUB_REPOSITORY": repository, "GITHUB_REF": ref, "MASTER_AGENT": master, "GITHUB_OUTPUT": str(output)} result = subprocess.run(["bash", "-c", step["run"]], cwd=self.root, env=env, capture_output=True, text=True) @@ -958,6 +1068,16 @@ def test_docs_only_push_and_agent_runtime_push(self): runtime = self.commit_file("apps/maple-agent/app/src/main.rs", "fn main() {}\n") self.assertEqual(self.select("push", docs, runtime), "agent=true\n") + def test_canonical_master_preserves_unvalidated_work_and_omits_only_proven_history(self): + docs = self.commit_file("apps/maple-agent/docs/design.md", "design\n") + for selected, expected in (("", "true"), ("true", "true"), + ("unknown", "true"), ("false", "false")): + with self.subTest(selected=selected): + self.assertEqual(self.select("push", self.base, docs, + repository="MaplePrivacyLabs/Maple", + ref="refs/heads/master", master=selected), + f"agent={expected}\n") + def test_pull_request_uses_merge_base_instead_of_unrelated_base_changes(self): master = self.commit_file("apps/maple-agent/app/src/main.rs", "fn main() {}\n") self.git("checkout", "-qb", "contributor", self.base) diff --git a/scripts/ci/test_opensecret_workflows.py b/scripts/ci/test_opensecret_workflows.py index 24c7634df..029369b46 100644 --- a/scripts/ci/test_opensecret_workflows.py +++ b/scripts/ci/test_opensecret_workflows.py @@ -301,9 +301,9 @@ def test_backend_retains_exact_rust_gates_and_disabled_stateful_shell_hooks(self commands = [step["run"] for step in config["jobs"]["rust"]["steps"] if "run" in step] self.assertEqual(commands, [ VERIFY_NIX, - "nix develop --no-update-lock-file '.?submodules=1' -c cargo fmt --all -- --check", - "nix develop --no-update-lock-file '.?submodules=1' -c cargo clippy --locked --all-targets --all-features -- -D warnings", - "nix develop --no-update-lock-file '.?submodules=1' -c cargo test --locked --all-features", + "nix develop --no-update-lock-file '.?submodules=1#ci' -c cargo fmt --all -- --check", + "nix develop --no-update-lock-file '.?submodules=1#ci' -c cargo clippy --locked --all-targets --all-features -- -D warnings", + "nix develop --no-update-lock-file '.?submodules=1#ci' -c cargo test --locked --all-features", ]) nix_commands = [step["run"] for step in config["jobs"]["nix"]["steps"] if "run" in step] self.assertEqual(nix_commands, [ @@ -378,12 +378,17 @@ def test_sdk_integration_uses_checked_out_backend_and_disposable_services(self): self.assertNotIn("opensecret-integration-revision", commands) self.assertFalse((ROOT / "sdk/opensecret-integration-revision").exists()) self.assertIn("cd services/opensecret", commands) - self.assertIn("'./services/opensecret?submodules=1'", commands) + self.assertEqual(commands.count("'./services/opensecret?submodules=1#ci'"), 4) + self.assertNotIn("'./services/opensecret?submodules=1'", commands) self.assertIn("services/opensecret/target/debug/opensecret", commands) self.assertIn("diesel migration run", commands) + self.assertIn("bash ../../.agents/skills/validate-opensecret/scripts/disposable_db_tests.sh", commands) self.assertIn("sdk/test/integration/bootstrap.sql", commands) self.assertIn("bun test src/lib/test/integration/api.test.ts", commands) self.assertIn("cargo test --locked --all-features --tests", commands) + self.assertIn("client::tests::native_handoff_full_stack_round_trip", commands) + for key in ("OPENSECRET_DEV_POSTGRES", "OPENSECRET_DEV_ENV", "OPENSECRET_DEV_CONTAINERS"): + self.assertEqual(config["env"][key], "0") self.assertEqual(config["env"]["APP_MODE"], "local") self.assertEqual(config["env"]["OPENAI_API_BASE"], "http://127.0.0.1:9") self.assertEqual(job["services"]["postgres"]["env"]["POSTGRES_DB"], "opensecret") diff --git a/services/opensecret/docs/dev-shell.md b/services/opensecret/docs/dev-shell.md index bffd5aee6..91152f51b 100644 --- a/services/opensecret/docs/dev-shell.md +++ b/services/opensecret/docs/dev-shell.md @@ -27,6 +27,14 @@ OPENSECRET_DEV_POSTGRES=0 OPENSECRET_DEV_ENV=0 OPENSECRET_DEV_CONTAINERS=0 \ nix develop --no-write-lock-file '.?submodules=1' -c cargo fmt --all -- --check ``` +Backend Rust CI and SDK integration use `'.?submodules=1#ci'`. This shell keeps +the same pinned Rust compiler and native build environment, PostgreSQL/Diesel +tools, and Python, without local container, credential, editor, or proxy tools. +It has no database startup, `.env` creation, container configuration, or secret +resolution hooks. SDK integration still owns its PostgreSQL service, migrations, +disposable OAuth/database suites, and encrypted SDK tests. The default shell +continues to provide the local development environment above. + For concurrent live checkouts, choose distinct `PGDATA`, `PGSOCKETS`, `PGPORT`, `DATABASE_URL`, and backend bind addresses. Verify the database identity before running migrations or destructive tests; never assume that a responding port diff --git a/services/opensecret/flake.nix b/services/opensecret/flake.nix index f9adfa97a..1b96b6646 100644 --- a/services/opensecret/flake.nix +++ b/services/opensecret/flake.nix @@ -131,6 +131,37 @@ ++ pkgs.lib.optionals pkgs.stdenv.isLinux linuxOnlyInputs ++ pkgs.lib.optionals pkgs.stdenv.isDarwin darwinOnlyInputs; + # CI builds the backend and runs disposable database tests. Local + # containers, credential providers, editors and proxy toolchains are + # not inputs to those gates and can have expensive Linux closures. + ciInputs = [ + rust + pkgs.bash + pkgs.coreutils + pkgs.git + pkgs.pkg-config + pkgs.openssl + pkgs.zlib + pkgs.gcc + pkgs.clang + pkgs.postgresql + pkgs.diesel-cli + pkgs.python3 + ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin darwinOnlyInputs; + + nativeBuildEnvironment = '' + export LIBCLANG_PATH=${pkgs.libclang.lib}/lib/ + export LD_LIBRARY_PATH=${pkgs.openssl}/lib:$LD_LIBRARY_PATH + export CC_wasm32_unknown_unknown=${pkgs.llvmPackages.clang-unwrapped}/bin/clang + export CFLAGS_wasm32_unknown_unknown="-I ${pkgs.llvmPackages.libclang.lib}/lib/clang/${pkgs.lib.versions.major pkgs.llvmPackages.libclang.version}/include/" + export PKG_CONFIG_PATH=${pkgs.openssl.dev}/lib/pkgconfig + + ${pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + export CC=clang + export CXX=clang++ + ''} + ''; + setupPostgresScript = pkgs.writeShellScript "setup-postgres" '' case "''${OPENSECRET_DEV_POSTGRES:-1}" in 0|false|False|FALSE|no|No|NO|skip|Skip|SKIP) @@ -644,6 +675,13 @@ ]; }; + # No database startup, .env creation, container configuration or + # credential resolution. CI owns its disposable database lifecycle. + devShells.ci = pkgs.mkShell { + packages = ciInputs; + shellHook = nativeBuildEnvironment; + }; + devShells.default = pkgs.mkShell { packages = inputs ++ [ secretsPkgs.secretspec secretsPkgs.bws ]; shellHook = '' @@ -652,16 +690,7 @@ export PGSOCKETS="''${PGSOCKETS:-$PGDATA/sockets}" export OPENSECRET_DEV_DATABASE_URL="''${OPENSECRET_DEV_DATABASE_URL:-postgres://opensecret_user:password@localhost:$PGPORT/opensecret}" - export LIBCLANG_PATH=${pkgs.libclang.lib}/lib/ - export LD_LIBRARY_PATH=${pkgs.openssl}/lib:$LD_LIBRARY_PATH - export CC_wasm32_unknown_unknown=${pkgs.llvmPackages.clang-unwrapped}/bin/clang - export CFLAGS_wasm32_unknown_unknown="-I ${pkgs.llvmPackages.libclang.lib}/lib/clang/${pkgs.lib.versions.major pkgs.llvmPackages.libclang.version}/include/" - export PKG_CONFIG_PATH=${pkgs.openssl.dev}/lib/pkgconfig - - ${pkgs.lib.optionalString pkgs.stdenv.isDarwin '' - export CC=clang - export CXX=clang++ - ''} + ${nativeBuildEnvironment} ${pkgs.lib.optionalString pkgs.stdenv.isLinux '' alias docker='podman'