From 029878bbb000d82e6e6d38d895c0865a4adbf43b Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:39:12 +0800 Subject: [PATCH 1/2] chore: retire the synthetic CPU KV slice tooling Remove the bounded synthetic KV experiment's tooling, tests, task fixtures, operator pages and manual pilot workflow, plus its ready-lane qualification job. The slice had its own consumers only: nothing in the general harness, the SkillEval modules, the runner or the evaluator imported it, so the dependency ran one way onto shared parts. Retained evidence is untouched. docs/kv-reclamation-qualification.md stays at its path, rewritten as a retirement record, because a hash-pinned evidence README links to it and must not be edited. Agent-assisted-by: Claude Code / claude-opus-5 (author) Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> --- .github/workflows/ci-ready.yml | 52 - .github/workflows/kv-edit-pilot.yml | 96 -- docs/kv-edit-integration.md | 119 --- docs/kv-live-launcher.md | 167 --- docs/kv-pilot-admission.md | 148 --- docs/kv-reclamation-qualification.md | 143 +-- docs/operations.md | 4 - src/inferrsi/harness/kv_edit_workbench.py | 206 ---- src/inferrsi/harness/kv_reclamation.py | 214 ---- tests/fixtures/kv_reclamation/candidate.py | 69 -- tests/fixtures/kv_reclamation/reference.py | 78 -- tests/fixtures/kv_reclamation/workload.py | 15 - tests/test_kv_actor_cleanup.py | 163 --- tests/test_kv_edit_inputs.py | 49 - tests/test_kv_edit_pilot.py | 182 ---- tests/test_kv_edit_workbench.py | 166 --- tests/test_kv_live_job.py | 848 ---------------- tests/test_kv_live_policy.py | 300 ------ tests/test_kv_live_worker.py | 166 --- tests/test_kv_pilot_admission.py | 572 ----------- tests/test_kv_reclamation.py | 254 ----- tests/test_kv_runtime_qualification.py | 292 ------ tests/test_kv_worker_supervisor.py | 135 --- tests/test_qualify_kv_edit_actor.py | 302 ------ tests/test_qualify_kv_reclamation.py | 148 --- tools/kv_edit_inputs.py | 91 -- tools/kv_edit_pilot.py | 257 ----- tools/kv_live_authorization.json | 9 - tools/kv_live_job.py | 1058 -------------------- tools/kv_live_policy.py | 346 ------- tools/kv_live_worker.py | 246 ----- tools/kv_pilot_admission.py | 376 ------- tools/kv_runtime_qualification.py | 256 ----- tools/kv_worker_supervisor.py | 139 --- tools/qualify_kv_actor_cleanup.py | 329 ------ tools/qualify_kv_edit_actor.py | 446 --------- tools/qualify_kv_reclamation.py | 295 ------ 37 files changed, 36 insertions(+), 8700 deletions(-) delete mode 100644 .github/workflows/kv-edit-pilot.yml delete mode 100644 docs/kv-edit-integration.md delete mode 100644 docs/kv-live-launcher.md delete mode 100644 docs/kv-pilot-admission.md delete mode 100644 src/inferrsi/harness/kv_edit_workbench.py delete mode 100644 src/inferrsi/harness/kv_reclamation.py delete mode 100644 tests/fixtures/kv_reclamation/candidate.py delete mode 100644 tests/fixtures/kv_reclamation/reference.py delete mode 100644 tests/fixtures/kv_reclamation/workload.py delete mode 100644 tests/test_kv_actor_cleanup.py delete mode 100644 tests/test_kv_edit_inputs.py delete mode 100644 tests/test_kv_edit_pilot.py delete mode 100644 tests/test_kv_edit_workbench.py delete mode 100644 tests/test_kv_live_job.py delete mode 100644 tests/test_kv_live_policy.py delete mode 100644 tests/test_kv_live_worker.py delete mode 100644 tests/test_kv_pilot_admission.py delete mode 100644 tests/test_kv_reclamation.py delete mode 100644 tests/test_kv_runtime_qualification.py delete mode 100644 tests/test_kv_worker_supervisor.py delete mode 100644 tests/test_qualify_kv_edit_actor.py delete mode 100644 tests/test_qualify_kv_reclamation.py delete mode 100644 tools/kv_edit_inputs.py delete mode 100644 tools/kv_edit_pilot.py delete mode 100644 tools/kv_live_authorization.json delete mode 100644 tools/kv_live_job.py delete mode 100644 tools/kv_live_policy.py delete mode 100644 tools/kv_live_worker.py delete mode 100644 tools/kv_pilot_admission.py delete mode 100644 tools/kv_runtime_qualification.py delete mode 100644 tools/kv_worker_supervisor.py delete mode 100644 tools/qualify_kv_actor_cleanup.py delete mode 100644 tools/qualify_kv_edit_actor.py delete mode 100644 tools/qualify_kv_reclamation.py diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 4f84c3dac..aeda080ff 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -67,55 +67,3 @@ jobs: env: PR_BODY: ${{ github.event.pull_request.body }} run: python3 tools/check_pr_references.py - - kv-task-qualification: - # Bounded software-control qualification; not a branch-protection or efficacy gate. - if: github.event.pull_request.draft == false - runs-on: ubuntu-22.04 - timeout-minutes: 20 - permissions: - contents: read - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Install distribution bubblewrap without changing security policy - run: | - sudo apt-get update - sudo apt-get install --yes --no-install-recommends bubblewrap - - name: Synthetic CPU controls (no model, secret or submitted artifact) - shell: bash - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - /usr/bin/python3 -m tools.qualify_kv_reclamation \ - --plan evidence/kv-reclamation-20261030/plan/qualification-plan.json \ - --revision "$GITHUB_SHA" --output "$RUNNER_TEMP/kv-qualification" - - name: Retain privacy-safe qualification receipt including failures - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: kv-qualification-${{ github.run_id }}-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/kv-qualification/qualification.json - ${{ runner.temp }}/kv-qualification/raw-evidence.zip - if-no-files-found: error - retention-days: 14 - - name: Scripted actor and descendant qualification (no provider or credentials) - if: always() - shell: bash - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - /usr/bin/python3 -m tools.qualify_kv_edit_actor \ - --plan evidence/kv-actor-qualification-20261031/plan/qualification-plan.json \ - --revision "$GITHUB_SHA" --output "$RUNNER_TEMP/kv-actor-qualification" - - name: Retain separate scripted actor receipt including failures - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: kv-actor-qualification-${{ github.run_id }}-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/kv-actor-qualification/qualification.json - ${{ runner.temp }}/kv-actor-qualification/raw-evidence.zip - if-no-files-found: error - retention-days: 14 diff --git a/.github/workflows/kv-edit-pilot.yml b/.github/workflows/kv-edit-pilot.yml deleted file mode 100644 index 2433c35ef..000000000 --- a/.github/workflows/kv-edit-pilot.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Bounded KV editing feasibility (manual, one pair) - -on: - workflow_dispatch: - -permissions: {} - -concurrency: - group: kv-reclamation-one-pair - cancel-in-progress: false - -jobs: - execute: - if: github.repository == 'InferMatrixLab/InferRSI' && github.ref == 'refs/heads/main' && github.run_attempt == 1 && github.actor == 'Dong1017' && github.triggering_actor == 'Dong1017' - runs-on: ubuntu-22.04 - timeout-minutes: 30 - permissions: - contents: write - actions: read - pull-requests: read - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - name: Same qualified system Python and distribution sandbox, no credentials - run: | - sudo apt-get update - sudo apt-get install --yes --no-install-recommends bubblewrap - - name: Exact freeze and same-host preflight, before credentials - env: - PYTHONPATH: src - run: /usr/bin/python3 -m tools.kv_live_job prepare --output "$RUNNER_TEMP/kv-live" - - name: Fixed bounded execution after exact public freeze and preflight - env: - PYTHONPATH: src - GITHUB_TOKEN: ${{ github.token }} - SKILL_EVAL_YIBUAPI_KEY: ${{ secrets.SKILL_EVAL_YIBUAPI_KEY }} - # Existing credentials enter only this trusted controller step. Admission - # still requires the exact public locator, qualification and both ledgers. - run: /usr/bin/python3 -m tools.kv_live_job execute --output "$RUNNER_TEMP/kv-live" - - name: Data-only public projection - if: always() - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - /usr/bin/python3 -m tools.kv_live_job package --output "$RUNNER_TEMP/kv-live" - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: kv-live-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/kv-live/public - if-no-files-found: error - retention-days: 30 - grade: - needs: execute - if: always() && github.repository == 'InferMatrixLab/InferRSI' && github.ref == 'refs/heads/main' && github.run_attempt == 1 && github.actor == 'Dong1017' && github.triggering_actor == 'Dong1017' - runs-on: ubuntu-22.04 - timeout-minutes: 10 - permissions: - contents: read - actions: read - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Install fixed secret-free grader dependencies - run: | - sudo apt-get update - sudo apt-get install --yes --no-install-recommends bubblewrap - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # verified v4 - with: - name: kv-live-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/kv-final-input - - name: Final grader of the exact captured public authorization - if: always() - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - GITHUB_SHA="$GITHUB_SHA" GITHUB_RUN_ID="$GITHUB_RUN_ID" \ - /usr/bin/python3 -m tools.kv_live_job grade --bundle "$RUNNER_TEMP/kv-final-input" \ - --output "$RUNNER_TEMP/kv-grade" - - name: Retain first grader outputs even after interruption, never rerun - if: always() - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - GITHUB_SHA="$GITHUB_SHA" GITHUB_RUN_ID="$GITHUB_RUN_ID" \ - /usr/bin/python3 -m tools.kv_live_job package-grade \ - --bundle "$RUNNER_TEMP/kv-final-input" --output "$RUNNER_TEMP/kv-grade" - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: kv-final-grade-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/kv-grade/public - if-no-files-found: error - retention-days: 30 diff --git a/docs/kv-edit-integration.md b/docs/kv-edit-integration.md deleted file mode 100644 index f16a3e896..000000000 --- a/docs/kv-edit-integration.md +++ /dev/null @@ -1,119 +0,0 @@ -# Bounded KV edit/check integration — model-free - -This slice connects the fixed edit protocol to the synthetic CPU task. It is not -a general coding agent, paid launcher, production KV implementation or efficacy -result. A2/A3 remain one prospective exploratory paired block, not a completed -model experiment. - -## Implemented boundaries - -- `kv_edit_workbench.VisibleWorkbench` accepts a frozen, extracted DEVELOPMENT - suite and public benchmark inputs/expected data. The full verifier collection, - optimized control and hidden grader inputs are not supplied to the worker. -- Candidate code runs only through `kv_reclamation.invoke` and its existing - network-free bubblewrap boundary. The visible benchmark composes the existing - `run_experiment`/`evaluate` AB/BA path with content-addressed source snapshots. - Host-observed whole-command elapsed time includes sandbox/interpreter startup; - full output must match trusted expected data before a metric is accepted. -- Every revision is retained outside the exposed workspace. Final hidden grading - takes the immutable returned source bytes and their exact digest, never a later - reopening of the mutable actor path. Its results do not feed another edit. -- `tools.kv_edit_pilot.prepare_scripted` freezes explicit actor files, prompt, - public suite, baseline, scripted responses and trusted dependency hashes. - `run_supervised` launches the exact fixed worker with `-I` through existing - `execute_command`, under the same inherited absolute deadline used by every - tool. A blocked worker is terminated, not granted a fresh callback deadline. -- `tools.kv_worker_supervisor.supervise` owns the shared private grant/completion - bridge. The scripted wrapper supplies only a fixed secret-free worker command - and environment. An injected live transport is not part of this slice. -- Existing `authorization_broker` is reused. Bounded private files outside the - actor workspace relay only the exact request grant and raw four-class usage. - The trusted parent reconciles completion before another grant. Missing, late, - invalid or unknown usage does not free capacity or permit another arm. -- An optional injected admission session exercises existing durable PR/day - claims and their live checks. The actual session and GitHub authority remain - in the parent; the worker receives only an ephemeral loopback gate capability. - See [pilot admission](kv-pilot-admission.md). - -The worker has **no provider transport, provider credential option, account -configuration or live-execution mode**. Its response file is JSON data, not an -executable script. Every result identifies zero real model calls and zero new -provider charges. Candidate source is never imported on the host. - -## Qualification and limits - -`tests/test_kv_edit_workbench.py` exercises edit/check/ABBA/rewrite/submission and -separate hidden grading with scripted sandbox outputs. `tests/test_kv_edit_pilot.py` -exercises the private request/usage bridge against fake durable stores, real -loopback authorization, missing usage, expired admission day, source/prompt drift -and a real hard timeout of a trusted blocking worker fixture. Scripted sandbox -outputs are **not** evidence that bubblewrap isolation worked. - -A real worker always probes isolation before its first simulated request. An -unavailable namespace or missing trusted boundary yields `INCONCLUSIVE`, without -host execution, a weaker sandbox, a different agent or a new paid attempt. -Qualified-host results must bind the exact implementation/plan/source hashes. -Artifact replay qualification alone does not certify the entire actor path or -arbitrary descendant cleanup. - -The actor/tool worker is trusted controller code, not an OS-sandboxed coding -agent. It interprets bounded response JSON as fixed tools. A2/A3 model-visible -files are constrained by the exact exposure manifest and read allowlist; -generated candidate execution alone is isolated by fresh workdirs, restricted -runtime mounts and unchanged candidate resource limits. - -`tools.qualify_kv_edit_actor` adds a separate, zero-provider software-control -receipt to the existing Ready Ubuntu 22.04 job. It requires all of: - -- Complete pinned K-owner exposure only in scripted A3, identical fixed tools, - two edit/check/measurement revisions, independent readback of retained source - bytes and receipts for revisions 0–2, and immutable final hidden grading. -- `tools.kv_edit_inputs` constructs identical common task/contract/starter, - workbench/development files and prompt for both arms; only A3 adds K. Public - development bytes contain IDs/events, not expected outputs or grader traces. - A per-arm common-contract digest binds exact files, protocol/client, limits, - check IDs, metric, suite and baseline independently of scripted responses. -- Production-profile denial of actual package/K/oracle/reference paths and - synthetic canaries. Runtime enumeration resolves bind roots but never follows - escaped descendant aliases; bounded size-indexed hashes detect renamed copies - including the complete hidden corpus. Escaped aliases must be unreadable in - the unchanged production sandbox; missing visibility is INCONCLUSIVE. -- A surviving observer around the actual outer-executor → trusted worker → - nested executor → sandbox chain. An authored fork/setsid probe binds its - namespace PID, starttime and PID-namespace identity to an already-owned host - descendant. Normal exit, outer-worker hard timeout and cancellation require - this observed chain and zero live survivors. Recovery kills never convert a - failed cleanup into PASS. Candidate execution still forbids fork. - -The separate `kv-actor-qualification--` artifact binds the exact -clean revision, dependency closure, plan, exposure, per-arm receipts and raw -archive digest. Prior task-only qualification is not this actor qualification. -Scripted actor identities include scripted responses; they do not identify a -future live actor or establish exact live exposure/transport qualification. -The final submitted tree still needs its actual hosted result; local fake -fixtures, inventory-only checks and green repository tests cannot substitute. - -## Future paid launcher — design only - -After independent integration/security review and exact-head qualification: - -1. Freeze the one legitimate execution PR, immutable campaign/attempt, actor/K - manifests, provider/model/group/rate evidence and all limits. Refresh expired - pricing and reread existing PR/day obligations; do not reuse consumed approval. -2. Reserve and claim the complete conservative block before disclosure. Preserve - the fixed eight requests per arm, sixteen total and no retry policy. Every send - must consume its preallocated grant; all uncertain outcomes retain their bound. -3. Supply the existing provider key only to trusted provider transport, never to - candidate sandboxes, exposed files, CLI arguments, artifacts or ordinary PR CI. - Replace scripted transport only through reviewed, exact-revision wiring; do - not treat this document or a response artifact as execution authority. -4. Keep the parent-owned admission/usage boundary and hard worker deadline. If - actual transport/worker failure leaves usage unresolved, stop subsequent calls - across both arms and retain the full reservation. -5. Freeze both final artifacts and finish independent hidden correctness/performance - evaluation under the prospectively frozen gates. Retain failures and missing - outcomes. One pair demonstrates execution feasibility only, not K efficacy or - production/service performance. - -This document does not supply credentials, choose a new execution environment, -alter repository settings, add a CI lane or authorize a paid launch. diff --git a/docs/kv-live-launcher.md b/docs/kv-live-launcher.md deleted file mode 100644 index 1cc1750de..000000000 --- a/docs/kv-live-launcher.md +++ /dev/null @@ -1,167 +0,0 @@ -# Bounded KV live launcher (one-pair gated execution) - -This is an Issue-scoped synthetic editing feasibility controller, not a default -CI lane, model benchmark, efficacy result or service qualification. The manual -workflow binds the existing provider key and GitHub token **only in the trusted -execute step**, after secret-free preparation. A disabled or incomplete public -locator still prevents paid admission. Ordinary tests use fake providers, fake -ledgers and synthetic keys only; no key values belong in the freeze or checkout. - -## Trust and execution boundaries - -1. `kv_live_policy` validates an exact owner-authorized public freeze, immutable - artifact/source digests, existing ledger anchors, pricing expiry and exact - main/run identity. No missing-ledger initialization or retry is supported. -2. `prepare` stages the fixed A2/A3 allowlists. It requires prior hosted actor - qualification plus same-host isolation, descendant cleanup and a scripted - read/edit/check/benchmark/submit replay of the exact staged inputs. Preparation - rejects a provider credential. Scripted performance failure is retained, not - relabelled as an optimization success. -3. `execute` reserves and claims the entire pair before disclosing either arm. - GitHub authority stays in the controller. The trusted child receives only the - provider key, ephemeral broker capability and explicit minimal environment. -4. `GrantedSender` consumes one outer authorization for one canonical request, - burns the attempt before sending, and invokes the existing transport once. - Unknown/lost/late usage, cancellation and ambiguous ledger writes stop the - pair without refund or relaunch. Each arm retains one absolute deadline. -5. Candidate code executes only in the existing network-free bubblewrap profile; - unavailable isolation has no host fallback. The trusted worker is not itself - claimed to be another OS sandbox. -6. `package` copies bounded final/revision sources and selected structured - feedback/usage records, never raw provider bodies, worker logs or environment - dumps. Unknown attempts and not-started/missing arms remain explicit. -7. A separate read-only, secret-free job downloads only this run's fixed artifact - name using a pinned action. It verifies actual source bytes and run/plan/actor - identities and current-revision feedback links before final hidden semantics - and fixed performance evaluation. The public authorization snapshot is checked - against the immutable locator offline; later revocation still stops spending - without deleting already retained no-spend grading evidence. - Missing results remain INCONCLUSIVE and never become actor feedback or retries. - -## First-run evidence, including interruptions - -The execution artifact contains the exact verified public authorization, a bounded -projection of preparation progress, staged workspace hashes and the actual -isolation/runtime/cleanup/replay receipts. Each preparation stage records its -start before running. Missing, failed or interrupted stages are explicit; no -exception text, process command, environment or arbitrary log is published. - -The execution bundle also projects closed worker diagnostics from fixed trusted -result paths: worker status, an allowlisted reason/error code, `provider_attempts` -and outer executor status/return code/elapsed seconds. `provider_attempts` counts -invocations of the transport function, **not confirmed HTTP delivery or charges**. -A present zero is distinct from a missing/malformed/interrupted record (`null`). -Only fixed reason enums or `http_100` through `http_599` are retained; unknown -exceptions become `unknown`, never arbitrary exception text. The transport code -is captured separately before the edit agent generalizes its failure. These -diagnostics do not settle costs, authorize retry or upgrade a verdict. Packaging -only reads existing first-run files, including when controller writing stopped. -This future retention correction cannot recover a reason omitted from an earlier -artifact; that earlier cause and billing remain unknown without separate evidence. - -The grader uploads only its dedicated `public/` projection, not its working -directory. `evidence.json` enumerates every assigned hidden trace and the fixed -alternating AB/BA timing trials plus quality validation. It binds the original -candidate, frozen trace/workload hashes, actual raw task-JSON hashes, measured -values, execution status/exit code and before/after source hashes where available. -The shared executor now records only status, return code and elapsed seconds -immediately after a candidate process returns, before any verifier early return -or JSON parsing. The projection accepts that exact three-field receipt. A process -killed before receipt persistence still has explicitly unavailable status; it is -never inferred from stdout or a PASS. Activation requires fresh model-free -qualification of this changed shared dependency, not the earlier receipt. - -Only exact raw JSON conforming to the fixed KV result schema is copied. Its -strings must be known task request IDs or the fixed `ValueError` marker. Each -output is at most 4 MiB, the whole two-arm projection at most 32 MiB, with at most -256 semantic traces per arm. Arbitrary/malformed candidate-only stdout is withheld -with observed byte count, content hash, parsing/schema failure and truncation -status; oversized/unsafe files remain unhashed/unavailable. These diagnostics are -not a sanitizer for private inputs or provider logs. No claim is made to preserve -all arbitrary raw bytes. Failed and missing trials are never upgraded to PASS. -Projection completion is not full-evidence acceptance: every arm's -`raw_outputs_complete` and unavailable execution fields must be inspected even -when a retained grading summary says PASS. - -`package-grade --bundle --output ` runs in -a separate `always()` environment-cleared workflow step. It validates the same -captured authorization and run identity, then packages **only already generated -first-run evidence**; it cannot execute candidates, call a model or retry grading. -This covers a grader process interrupted before its final cleanup. It also leaves -`projection.json` with started/completed/failed-or-interrupted state and final -manifest hashes. A workflow/host loss that prevents even the always-step or upload -cannot guarantee artifact retention; missing evidence remains INCONCLUSIVE, not -a reason to rerun the paid trial. Both Actions artifacts have 30-day retention; -long-term public archive publication remains a separately approved action. - -## Hosted qualification provenance - -The frozen `qualification` record contains `run_id`, `run_attempt`, `head` -(GitHub run head), `revision` (actual qualified checkout), `artifact_id`, the -bounded downloaded `archive`, exact `receipt` and `dependencies_sha256`. -`head` and `revision` are separate because PR workflow run identity and checkout -identity need not be the same commit. - -The archive must contain exactly `qualification.json` and `raw-evidence.zip`. -The launcher compares the receipt bytes and raw-evidence digest, verifies the -common actor dependency closure against this checkout, checks successful -isolation/cleanup/two-revision tool evidence and requires exact live exposure and -prompt hashes. A common-actor contract digest also binds client settings, input -ceiling, visible suite, baseline, edit limits, checks, metrics and exposed files; -only scripted callback/response-specific identity is excluded. Hosted and current -mount inventories need not have identical hashes: selective sensitive-file -hashing can differ. The canonical dependency closure binds the sandbox profile, -and a fresh current-host inventory/probe must independently PASS; both inventory -digests are retained. The live actor identity is deliberately distinct from the -scripted identity that also binds response fixtures. - -Immediately before admission and each request, live metadata must identify the -successful pinned Ready run and the exact nonexpired -`kv-actor-qualification--` artifact with matching archive -SHA-256. Generic green CI, a numeric task-only artifact or a locally authored -`PASS` JSON file cannot replace this receipt. - -## Required one-pair operational freeze - -- Accepted final common task/workbench/development bytes, limits and arm order; - exact A3 knowledge export audit and independent final-verifier inventory. -- Hosted actor qualification of the actual integrated dependency closure and - exact exposure/prompt; same-host probes must also pass at execution time. -- One legitimate charge PR/head and audited existing PR/day ledger histories. - Existing reservations/carryover remain obligations; a launcher PR does not - create another allowance. -- Refreshed price evidence, supported authorization expiry and enough full-pair - runway before expiry or Shanghai midnight. An expired plan is not renewed by - an offline test or by the larger task ceiling. -- Reviewed public freeze/locator with actual hashes and provenance, plus explicit - owner authorization for this existing-key binding and single manual execution. - -No deployment, credential or paid action is authorized merely by this template. -An operator must not enable the locator, add a secret binding or dispatch merely -because repository tests passed. These tests establish software behavior only; -real hosted isolation, provider delivery, measured model output and efficacy -remain separate evidence. - -## Explicit one-additional-attempt exception - -The default PR lifetime cap is unchanged. A v2 public plan may carry the reviewed -`budget_exception` object: exact repository/charge PR, trusted PR cap, fixed new -campaign/execution IDs, exact prior claimed binding and PR/day checkpoints. The -root validates it under the existing owner-comment/locator identity, then passes -an immutable key-bound limit only to the anchored PR reader. It is never a ledger -JSON field, environment/CLI setting, worker input or candidate-selected allowance. -Historical ledger bytes remain unchanged; an ordinary default reader still -rejects a cumulative balance exceeding its default cap. - -The additional admission requires the exact sole prior KV claim and retains it -in full. It rejects any other KV reservation/claim, including a partial uncertain -write. Once the new pair is claimed, every request requires exactly those two -claims. A third campaign, changed ID, new PR, different day or smaller requested -budget cannot reset this guard. Both caps are checked before the existing -daily-first CAS sequence; races, ambiguous writes and unknown usage still stop -without retries or refunds. The pilot total ceiling remains unchanged. - -Issue-scoped proposed values and remaining activation prerequisites live in -[`SECOND-ATTEMPT.md`](../evidence/kv-live-pilot-20261031/plan/SECOND-ATTEMPT.md). -The original v1 plan/binding remains historical; this software support alone -neither renews its expiry nor authorizes a model request. diff --git a/docs/kv-pilot-admission.md b/docs/kv-pilot-admission.md deleted file mode 100644 index 08d72a104..000000000 --- a/docs/kv-pilot-admission.md +++ /dev/null @@ -1,148 +0,0 @@ -# One-pair KV pilot admission seam (offline) - -`tools/kv_pilot_admission.py` is a narrow controller interface for the accepted -exploratory KV editing feasibility envelope: **one A2/A3 paired block**, at most -**8 provider attempts per arm / 16 total**, with a full-context worst-case -**USD 10.17 per request / USD 162.72 total**. These are spending bounds, not -scientific acceptance thresholds. One pair can establish implementation/execution -feasibility only; it cannot establish efficacy. No experiment is run by this module. - -The module introduces no CLI, default transport, credential read, workflow launch, -ledger initializer, new ledger schema or automatic retry. Tests use in-memory -stores and injected clocks/launch callbacks only. A real launcher remains a -separate reviewed integration, not authority implied by this offline seam. - -## Trust and immutable identity - -Only reviewed controller/root configuration can create `PilotPolicy`. Never load -it, its stores/anchors, or `check_current` from candidate code, PR artifacts, -author-written requests, mutable labels or model output. `check_current(policy)` -must return exactly `True` only after independently verifying owner authorization, -repository/legitimate execution PR and exact head, frozen source revision, visible -input/K/verifier hashes, model/provider/pricing identity, and qualified isolation. -The plan SHA-256 must bind those accepted inputs and the single-pair sample plan; -the per-arm actor hashes bind the complete editing protocol/manifests. The source -author model/effort provenance remains distinct from the evaluated provider model. - -`provider_json` is immutable canonical JSON and uses the separate editing protocol, -not the pre-existing read/submit agent. Existing `reservation()` recomputes the -full 1,000,000-token ceiling and 4,096-output bound; USD amounts are converted -without floating-point arithmetic. Plan and pricing expiration are enforced both -before admission and before every send. A frozen root-policy change must cause -`check_current` to deny; this API is not a way for candidate inputs to authorize -such a change. - -The **existing global daily ledger entry is the durable campaign-binding record**: - -- `Binding.run_id`: fixed `kv-reclamation-one-pair-v1:` identity. -- `Binding.plan_digest`: SHA-256 of the complete immutable policy, including - execution ID, repository/PR, exact head/source/plan, actors and pricing. -- `Binding.worst_case`: the entire USD 162.72 envelope. - -Admission rejects *any* existing entry in this one-pilot namespace, not only an -identical digest. Thus changing the campaign ID, digest, run/attempt, PR, date or -expiry cannot create a new allowance. The unchanged global CAS arbitrates racing -first admissions. A later legitimate experiment would need separately reviewed -scope and code; this seam deliberately offers no reset, takeover or resume path. - -## Durable ordering and accounting - -`admit(...)` reads both existing verified histories, checks their identities and -requires the selected PR's retained global obligations to equal its cumulative PR -charge. An incomplete old PR tip cannot erase global imported charges. It computes -both cap checks before writing, then performs exactly: - -```text -full daily reservation CAS → full PR reservation CAS - → daily one-shot claim → PR one-shot claim - → fresh verification of BOTH claims, live policy, price and Shanghai day - → process-local supervisor session -``` - -All earlier audited charges, reservations, retries and unresolved obligations must -already be present in verified history. Absent/corrupt/unknown history blocks; -there is no missing-file-zero fallback. One immutable campaign on one PR subject -to the USD 200 lifetime cap is already strictly below the USD 1,000 pilot-task -ceiling; this does not introduce a global lifetime cap. Unrelated settled past-day -campaigns are not new pilot spending. The USD 1,000 Shanghai-day cap still includes -all charges for that day and unresolved historical carryover. No accounting -meaning is inferred from a successful test or claimed actual usage. - -The synthetic regression fixture preserves USD 491.35 of unresolved historical -obligations, producing USD 654.07 retained after the complete reservation, USD -345.93 remaining that day, and USD 37.28 remaining on a fresh execution PR. The -new pilot itself moves from USD 0 reserved to USD 162.72 reserved, leaving USD -837.28 of its USD 1,000 task ceiling; the earlier USD 491.35 is global daily -carryover, not new pilot spend. These are fixture/audit amounts, **not a current -live ledger readback or invoice**. -Current verified refs/checkpoints and refreshed pricing remain launch prerequisites. - -A stale CAS, partial reservation, lost response or crash never grants execution. -Unknown claim outcomes cannot be reconciled into a new grant. A caller receiving -an uncertain first daily write must stop rather than blindly retry; an absent -observation is not proof that the old write cannot arrive. The durable no-relaunch -guarantee applies once a campaign entry persists, with CAS preventing competing -first admissions from both winning. Reservations and -claims stay in the existing ledgers after success, failure, cancellation or process -death; there is no refund or terminal-settlement writer here. Shanghai rollover -stops new requests, even if admitted earlier. Outstanding entries carry over; -midnight does not refund or relaunch the campaign. - -## Trusted supervisor interface - -`run_admitted(..., launch=callback)` invokes a trusted callback only after both -claims and closes the session in `finally`. For finer integration, `admit(...)` -returns the same process-local, non-pickleable session. Do not expose this object, -its stores, policy checker or completion method to model/candidate code. - -1. `start_arm(arm, absolute_deadline)` immediately **before disclosure/launch**. - The deadline is at most 600 seconds away, cannot be renewed, and arms cannot - overlap. The qualified external supervisor must enforce that same absolute - deadline; in-process checks alone cannot terminate blocked code. -2. `authorize(grant, absolute_deadline) -> True` immediately before **one send**. - The exact grant shape matches `anthropic_edit_agent.RequestSlots`: plan digest, - execution ID, arm, slot, payload SHA-256, actor SHA-256 and per-request bound. - Admission has independent counters, burns before live checks, verifies both - durable claims and day/price after reads, and allows no ninth slot or borrowing - from the other arm. The worker's local counters are an additional guard only. -3. `complete(same_grant, raw_usage)` after the trusted transport/worker has observed - the response for that exact request. Raw usage requires all four integer fields: - `input_tokens`, `output_tokens`, `cache_read_input_tokens`, - `cache_creation_input_tokens`. All input classes count toward the 1M bound. - Candidate-authored JSON, a narrative or unverified trace is not usage evidence. -4. `finish_arm(arm)` only when no request remains unresolved. Preserve failures - and missing artifacts in the experiment evidence rather than discarding them. -5. `close()` irreversibly stops this process-local session. No new process can - reacquire the already consumed durable claims. - -There may be only one pending request across both arms. A lost grant/reply, -transport failure, unknown/malformed/over-bound usage, mismatched completion, -revoked authorization or inconsistent claims stops both arms; failed/ambiguous -work cannot replenish slots. The full cost stays reserved. Report and reconcile -any observed over-bound exposure through the existing trusted quarantine/accounting -process before other paid work; this module does not claim to settle provider bills. - -`context(arm)` supplies an exact frozen context for the unchanged existing -`request_guard.authorization_broker`. No new network protocol is introduced. -The existing broker is a **trusted-client gate, not a candidate sandbox**. Real -wiring must keep provider credentials and GitHub authority outside candidate/test -execution, independently enforce exactly one send per grant/no transport retries, -and relay usage from trusted observation before the next authorization. Neither -this seam nor a green fake test proves that real isolation or billing is qualified. - -## Verification - -Run offline: - -```sh -pytest tests/test_kv_pilot_admission.py tests/test_skill_eval_budget.py \ - tests/test_skill_eval_daily_budget.py tests/test_skill_eval_budget_git.py \ - tests/test_request_guard.py tests/test_anthropic_edit_agent.py tests/test_anthropic_agent.py -ruff check tools/kv_pilot_admission.py tests/test_kv_pilot_admission.py -``` - -Coverage includes stale CAS, partial/unknown writes, death after every durable -step, duplicate claims and relaunch, new digest/campaign/PR/attempt, cap overflow, -Shanghai carryover/rollover, pricing expiry, final exact-claim verification, -revocation, unknown/mismatched usage across arms, frozen deadlines and the hard -8-per-arm/16-total limit. These are controller regressions, not paid-model results. diff --git a/docs/kv-reclamation-qualification.md b/docs/kv-reclamation-qualification.md index 6b8c3e1e7..50bab84b9 100644 --- a/docs/kv-reclamation-qualification.md +++ b/docs/kv-reclamation-qualification.md @@ -1,118 +1,47 @@ -# Synthetic CPU KV-block reclamation qualification +# Synthetic CPU KV-block reclamation qualification (retired) -This is a bounded **model-free mechanism-transfer task**, not vLLM conformance, -a natural historical-repository holdout, an accelerator result or agent efficacy. -The accepted slice is recorded in [the scoped execution record][scope]. It does -not replace the broader task-efficacy proposal, the serving-parser task or the -Indexer acceptance boundaries. +The tooling this page documented has been removed. It described a bounded, +model-free mechanism-transfer task built for the task-efficacy pilot in +[issue 248](https://github.com/InferMatrixLab/InferRSI/issues/248): a synthetic +CPU reclamation contract with an independently authored frozen oracle, an +isolated input-only replay and trusted event-by-event comparison through the +existing AB/BA runner. -This synthetic task was deliberately constructed around the nominated knowledge's -dual-readiness mechanism. Independent oracle authorship is **not independent task -selection**. Both arms' visible contract and correct starter already state the -readiness rules; a later comparison tests utility while preserving those rules, -not discovery of a hidden requirement. It is a selected synthetic near-transfer -probe, not an independently sampled case. +This page is kept because the retained evidence links to it, and that evidence +is hash-pinned and must not be edited. -The visible starter is a correct but deliberately repeated linear scan. Its -contract includes request finish, asynchronous save completion, a processed-step -write fence, deterministic reclamation order, generation identity, idempotent -callbacks and atomic invalid-input handling. Only `candidate.py` may change. -Shared blocks, refcounts, prefix matching, transport and real concurrency are -outside this task. +## What was removed -## Trust and evidence boundaries +`tools/qualify_kv_reclamation.py`, `tools/qualify_kv_edit_actor.py`, +`tools/qualify_kv_actor_cleanup.py`, the `tools/kv_*` live-launcher and +admission modules, `src/inferrsi/harness/kv_reclamation.py`, +`src/inferrsi/harness/kv_edit_workbench.py`, their tests and task fixtures, +the `kv-edit-pilot` workflow and the `kv-task-qualification` job in the ready +lane. None of the commands named on this page can be run from the repository +any more. -```mermaid -flowchart LR - Contract[Visible neutral contract] --> Starter[Editable candidate.py] - Contract --> Oracle[Independent frozen oracle] - Starter --> Sandbox[Isolated input-only replay] - Sandbox --> Compare[Trusted event-by-event comparison] - Oracle --> Compare - Compare --> Runner[Existing AB/BA runner and evaluator] -``` +## What was kept -- The independent verifier author freezes the semantic oracle and disjoint - development/grader traces without reading the nominated knowledge or target - implementation. Static trace observations are checked against that oracle. -- The candidate process receives its source, a trusted driver and input events, - never oracle code, expected answers, other trials, knowledge or reference code. - Return types, event counts, every reclamation list and pending state are checked. - Raw inputs necessarily reach the candidate during replay; expected results and - the grader fixture store do not. Every output byte remains untrusted, even if - the candidate tampers with its Python driver or output serializer. - In-process tuple/list assertions are ordinary compatibility checks, **not** - anti-tamper acceptance: malicious same-interpreter code can bypass them. The - externally enforced acceptance is frozen observable trace-envelope equality - and host whole-command timing, not immutable per-call driver/type behavior. -- `kv_reclamation.invoke` reuses the existing bubblewrap command and host - `execute_command` deadline. Network/PID namespaces, clear environment, - read-only mounts and hard process/memory/CPU/output limits are probed. There is - **no unisolated fallback**. This boundary confines artifacts, not a future actor. - Group-cleanup probes cover the tested host process group; the hard no-fork - boundary is checked separately. This is not arbitrary detached-tree or future - actor cleanup certification. -- Missing output, changed protected paths and incorrect artifacts are rejected. - Infrastructure failure, unavailable isolation and timeout remain unverified - (`INCONCLUSIVE`), distinct from a completed comparison refuting a claim. -- Workbench timing reuses `run_experiment` and `evaluate`. It measures host elapsed - time of the whole CPU task command, including sandbox/interpreter startup. The - candidate cannot supply a claimed timing metric. Inputs, repetition count and - numeric headroom gates live only in the frozen qualification plan. -- Correctness precedes timing. An optimized software control must pass the full - verifier and the headroom gate; premature save/fence release, generation errors, - cached answers/event skipping, missing output and protected edits are negative - controls. A/A and no-op comparisons must not meet the improvement gate. -- All runs, including failures, are retained. There is no retry-until-pass rule. - The CI receipt retains hashes, statuses, raw timing samples and control coverage. - Its `raw-evidence.zip` retains actual control outputs and AB/BA evidence with - labelled path redactions, without hostnames, credentials or full environment dumps. +The retained evidence stays in place and unmodified: -## Meaning of a successful qualification +- [`evidence/kv-reclamation-20261030/`](../evidence/kv-reclamation-20261030/README.md) +- `evidence/kv-actor-qualification-20261030/` +- `evidence/kv-actor-qualification-20261031/` +- `evidence/kv-live-pilot-20261031/` -Qualification can establish only that this exact host/task/verifier combination -has discriminating controls and measurable software headroom. CI success is not -an engine campaign `PASS`, a knowledge benefit, an actor isolation certificate or -permission to infer broad transfer. The original tests and historical outcomes -remain unchanged. +Their entries in `tools/evidence_inventory.json` are unchanged, so +`python -m tools.validate_evidence` still verifies them. -The prospective A2/A3 comparison holds W constant and changes only a complete, -pinned `audit-vllm-kv-lifetimes` owner bundle. Before any paid step it still needs a -frozen executable actor/tool protocol, demonstrated read isolation (including W -package data and host files), full K-to-target contamination audit, known charges, -conservative reservations, authorized plan identity and a final-artifact grader. -A provider `read_file`/`submit_answer` path is not an iterative editing agent. -No model calls or paid adapter are part of this delivery. +## What the evidence does and does not show -## Reproduce +The qualification receipts record **software and scripted-control execution** +only. Both live A2/A3 attempts on 2026-10-31 ended `INCONCLUSIVE`, the second +stopping at its first transport call on HTTP 403, so no completed arm, per-arm +outcome or A/A variance exists. The recorded dollar figures are reserved upper +bounds, not observed spend. Nothing here is a model-effect, knowledge-effect or +hardware result. -From a checkout with Linux system Python, Git and distribution bubblewrap: - -```sh -PYTHONPATH=src /usr/bin/python3 -m tools.qualify_kv_reclamation \ - --plan evidence/kv-reclamation-20261030/plan/qualification-plan.json \ - --revision "$(git rev-parse HEAD)" --output -``` - -The command never accepts arbitrary source or grader paths. The existing Ready -workflow runs a non-required Ubuntu job with no secrets and retains the sanitized -receipt. Its exit code gates software controls, **not** numerical performance: -always read `qualification_verdict` in the receipt. A green job with insufficient -headroom does not qualify the task. The original manual parser qualification, -test matrix, branch protection and default serving-parser adapter are unchanged. - -The initial local run is retained as `INCONCLUSIVE`: the cloud host refused the -required network namespace setup. It did not run any candidate without isolation. -Trusted direct-import controls are labelled separately and do not upgrade that -result. See the [frozen plan and retained evidence](../evidence/kv-reclamation-20261030/README.md). - -The independently frozen benchmark tests retain their original standalone layout. -In this integrated checkout, use the checked wrapper test or set the documented -oracle location without modifying frozen files: - -```sh -KV_FROZEN_ORACLE="$PWD/evidence/kv-reclamation-20261030/validator/independent/oracle.py" \ - pytest evidence/kv-reclamation-20261030/validator/test_benchmark.py -``` - -[scope]: https://github.com/InferMatrixLab/InferRSI/issues/248#issuecomment-6526000390 +The general SkillEval capability is unaffected: nothing in +`src/inferrsi/harness/skill_eval*.py`, the runner, the evaluator or +`tools/skill_eval_*.py` referenced the removed modules. The dependency ran one +way only, from this slice onto those shared parts. diff --git a/docs/operations.md b/docs/operations.md index 9614db4f2..bfafe399d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -32,10 +32,6 @@ Effect classes used below: | Export a cloud-generated parser pair | [Generation-only protocol](skill-eval-generation-only.md), `python -m tools.skill_eval_cloud_export` | Local evidence-writing; no execution or publication | Validates frozen inputs and retained sources; writes one bounded inert JSON bundle, never executes parser source | | Run the fixed historical parser generation pilot | [Main-only historical job](skill-eval-historical-job.md), `python -m tools.skill_eval_historical_job` | Authorized paid generation / ledger-writing / evidence-writing | No dispatch inputs; reviewed main policy, pre-secret freeze, existing credentials, two-request cap; no generated-source execution or automatic retry | | Qualify an artifact verifier host | [Manual reviewer qualification](skill-eval-reviewer-qualification.md), `python -m tools.skill_eval_qualify` | Trusted-control execution / report-generating | Manual main-only workflow; fixed software controls, no model or external artifact input; unavailable isolation fails rather than skips | -| Qualify the synthetic CPU reclamation task | [Task qualification](kv-reclamation-qualification.md), `python -m tools.qualify_kv_reclamation` | Trusted-control execution / report-generating | Explicit frozen plan; independent verifier, isolated controls and existing AB/BA runner; no paid actor or efficacy claim | -| Qualify fixed KV editing mechanics | [Actor integration](kv-edit-integration.md), `python -m tools.qualify_kv_edit_actor` | Scripted trusted-control execution / report-generating | Existing Ready Ubuntu host; real isolated artifact replay, fixed edit/check/ABBA tools, immutable final grading and descendant cleanup; zero provider calls, no paid launcher or efficacy claim | -| Exercise pilot admission offline | [Admission seam](kv-pilot-admission.md), Python `tools.kv_pilot_admission.run_admitted` | Explicit injected-controller effect | Reuses audited PR/day ledger contracts; fake stores for tests, no credential discovery, ledger initialization, CLI or automatic paid launch | -| Run the one-pair gated KV launcher | [Live launcher boundaries](kv-live-launcher.md), `python -m tools.kv_live_job` (`prepare`, `execute`, `package`, `grade`, `package-grade`) | Manual workflow-only gated provider/ledger execution; separate grading and interruption-safe, data-only `package-grade` projection | Existing credentials bind only to the trusted execute step after secret-free preparation; exact public freeze, hosted/current-host qualification and durable full-pair admission required; `package-grade` never reruns candidates or models; no default CI spending | | Replay a cloud-generated parser pair | [Isolated cloud replay](skill-eval-cloud-replay.md), `python -m tools.skill_eval_cloud_replay` | Untrusted-artifact execution / report-generating | Manual pinned data ingress; same-host controls before bubblewrap-only replay; no models or credentials | | Validate one harness-improvement step | [Offline improvement](harness-improvement.md), Python `tools.skill_eval_improve.prepare` / `run_offline` | Snapshot-writing / offline experiment-executing | Supplied declarative INV/DOM edits, shared total limits, rerun and one holdout gate; trusted fixed-output clients only; no real-model or Store execution | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | diff --git a/src/inferrsi/harness/kv_edit_workbench.py b/src/inferrsi/harness/kv_edit_workbench.py deleted file mode 100644 index c887ea9f6..000000000 --- a/src/inferrsi/harness/kv_edit_workbench.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Fixed visible KV workbench; submitted Python executes only through kv.invoke. - -The visible suite contains extracted DEVELOPMENT cases and a public benchmark, -never the full verifier collection. Candidate sandboxes receive events only. -Hidden grading is a separate operation on the final immutable source snapshot. -""" - -import hashlib -import json -import math -import os -import time -from dataclasses import dataclass -from pathlib import Path - -from ..infra.artifacts import atomic_write_json, digest -from ..model import anthropic_agent as provider -from ..model.anthropic_edit_agent import EditPlan -from ..runspec import RunSpec -from . import kv_reclamation as kv -from .runner import run_experiment - - -def _copy(value): - return json.loads(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)) - - -def _compare(trace, actual): - # Projection of frozen expected DATA, not a second state-machine oracle. - expected = {"observations": trace["expected"], - "pending": trace["expected"][-1]["pending"] if trace["expected"] else []} - return digest(expected) == digest(actual) - - -def _traces(traces, split): - if type(traces) is not list or not traces: - raise ValueError("Nonempty frozen trace set required") - ids = [] - for trace in traces: - if (type(trace) is not dict or set(trace) != {"id", "split", "events", "expected"} - or trace["split"] != split or type(trace["id"]) is not str - or type(trace["events"]) is not list or type(trace["expected"]) is not list - or len(trace["events"]) != len(trace["expected"])): - raise ValueError("Wrong trace split or frozen trace shape") - ids.append(trace["id"]) - if len(set(ids)) != len(ids): - raise ValueError("Duplicate trace identity") - return _copy(traces) - - -@dataclass(frozen=True) -class VisibleSuite: - """The whole byte string is hashed in the actor's callback/dependency freeze.""" - - serialized: bytes - - def __post_init__(self): - if type(self.serialized) is not bytes or not 0 < len(self.serialized) <= 16 * 1024 * 1024: - raise ValueError("Visible suite must be bounded immutable JSON bytes") - value = json.loads(self.serialized, object_pairs_hook=provider.unique_object) - if (type(value) is not dict or set(value) != {"schema", "development", "benchmark_events", - "benchmark_expected", "min_improvement_pct"} - or value["schema"] != "kv-visible-suite-v1" - or type(value["benchmark_events"]) is not list - or type(value["benchmark_expected"]) is not dict - or type(value["min_improvement_pct"]) not in (int, float) - or not math.isfinite(value["min_improvement_pct"]) - or value["min_improvement_pct"] < 0): - raise ValueError("Wrong visible suite shape") - _traces(value["development"], "development") - _copy(value) - - def read(self): - return json.loads(self.serialized) - - -class VisibleWorkbench: - """Fixed callbacks; no model-supplied paths, commands, traces or metric gates.""" - - def __init__(self, plan: EditPlan, suite: VisibleSuite, baseline: bytes, output: Path): - value = suite.read() - if tuple(t["id"] for t in value["development"]) != plan.check_ids: - raise ValueError("Actor check identities differ from the frozen visible suite") - if (type(baseline) is not bytes or not 0 < len(baseline) <= plan.limits.max_source_bytes - or plan.metric_key != "elapsed_seconds" or plan.metric_unit != "seconds" - or plan.direction != "minimize"): - raise ValueError("Wrong fixed KV source/metric contract") - baseline.decode("utf-8") - self.plan, self.suite, self.baseline = plan, suite, baseline - self.output = Path(output) - self.output.mkdir(parents=True, exist_ok=False) - self.check_count = self.benchmark_count = 0 - - def retain_revision(self, source, metadata, deadline): - if time.monotonic() >= deadline: - return False - if hashlib.sha256(source).hexdigest() != metadata["sha256"]: - raise ValueError("Revision digest does not match retained bytes") - path = self.output / "revisions" / f"revision-{metadata['revision']:04d}" - path.mkdir(parents=True, exist_ok=False) - with os.fdopen(os.open(path / "candidate.py", os.O_WRONLY | os.O_CREAT | os.O_EXCL, - 0o600), "wb") as stream: - stream.write(source) - stream.flush() - os.fsync(stream.fileno()) - atomic_write_json(path / "receipt.json", metadata) - return kv.source_bytes(path / "candidate.py") == source - - def visible_checks(self, source, deadline): - self.check_count += 1 - if self.check_count > self.plan.limits.max_checks: - raise ValueError("Frozen visible-check count exhausted") - traces = self.suite.read()["development"] - try: - result = kv.verify_source(source, traces, self.output / f"check-{self.check_count:03d}", - deadline, _compare) - except kv.SandboxUnavailable: - return {"verdict": "INCONCLUSIVE", "checks": {}, "reason": "sandbox_unavailable"} - reason = "none" if result["verdict"] == "PASS" else ( - "check_failed" if result["verdict"] == "FAIL" else - "deadline" if result.get("reason") in {"timeout", "deadline"} else "candidate_error") - return {"verdict": result["verdict"], "checks": result["checks"], "reason": reason} - - def visible_benchmark(self, source, deadline): - self.benchmark_count += 1 - if self.benchmark_count > self.plan.limits.max_benchmarks: - raise ValueError("Frozen visible-benchmark count exhausted") - empty = {"verdict": "INCONCLUSIVE", "baseline_values": [], "candidate_values": [], - "quality_verdict": "INCONCLUSIVE", "improvement_pct": None, - "reason": "deadline"} - left = deadline - time.monotonic() - if left <= 0: - return empty - root = self.output / f"benchmark-{self.benchmark_count:03d}" - sources = {"baseline": self.baseline, "candidate": bytes(source)} - for label, data in sources.items(): - directory = root / "sources" / label - directory.mkdir(parents=True) - (directory / "candidate.py").write_bytes(data) - suite = self.suite.read() - expected = digest(suite["benchmark_expected"]) - quality = [] - - def snapshot(repo, excluded): - data = kv.source_bytes(repo / "candidate.py") - return {"available": True, "snapshot_hash": hashlib.sha256(data).hexdigest(), - "kind": "immutable-candidate-bytes"} - - def execute(command, cwd, trial_dir, runner_deadline, metric_key): - if command not in (("fixed-kv", "baseline"), ("fixed-kv", "candidate")): - raise ValueError("Not a fixed KV workbench command") - label = command[1] - record, actual = kv.invoke(sources[label], {"mode": "benchmark", - "events": _copy(suite["benchmark_events"])}, trial_dir, - min(deadline, runner_deadline)) - valid = record["status"] == "completed" and digest(actual) == expected - elapsed = record.get("elapsed_seconds") - finite = type(elapsed) in (float, int) and math.isfinite(elapsed) and elapsed > 0 - record["metric"] = elapsed if valid and finite else None - record["metric_error"] = None if valid and finite else "unverified complete trace" - if record["status"] == "completed" and not valid: - record["status"] = "failed" - if metric_key is None: - quality.append("PASS" if valid else "FAIL" if record["status"] == "failed" - else "INCONCLUSIVE") - atomic_write_json(trial_dir / "execution.json", record) - return record - - spec = RunSpec( - repo=str(root / "sources/baseline"), candidate_repo=str(root / "sources/candidate"), - repo_type="synthetic-cpu-kv", backend="cpu", mode="code", - scenario="kv-visible-development", metric_key=self.plan.metric_key, - metric_unit=self.plan.metric_unit, direction=self.plan.direction, - comparison_scope="whole CPU task including sandbox and interpreter startup", - baseline_command=("fixed-kv", "baseline"), candidate_command=("fixed-kv", "candidate"), - validation_command=("fixed-kv", "candidate"), - repetitions=self.plan.limits.benchmark_repetitions, - max_minutes=left / 60, min_improvement_pct=suite["min_improvement_pct"], - output_dir=str(root / "runs"), - agent="bounded-edit-check-protocol") - try: - run = run_experiment(spec, {"scope": "synthetic visible development only"}, - execute=execute, snapshot=snapshot) - except kv.SandboxUnavailable: - return {**empty, "reason": "sandbox_unavailable"} - evaluation = json.loads((run / "evaluation.json").read_text()) - return {"verdict": evaluation["verdict"], - "baseline_values": (evaluation["baseline"] or {}).get("values", []), - "candidate_values": (evaluation["candidate"] or {}).get("values", []), - "quality_verdict": quality[-1] if quality else "INCONCLUSIVE", - "improvement_pct": evaluation["improvement_pct"], - "reason": "none" if evaluation["verdict"] == "PASS" else - "check_failed" if evaluation["verdict"] == "FAIL" else "measurement_incomplete"} - - -def grade_final(source, expected_sha256, hidden_traces, output, deadline): - """Separate post-submission grader; never return this data to the actor.""" - if type(source) is not bytes or hashlib.sha256(source).hexdigest() != expected_sha256: - raise ValueError("Final grader requires the exact immutable submitted source") - traces = _traces(hidden_traces, "grader") - try: - result = kv.verify_source(source, traces, output, deadline, _compare) - except kv.SandboxUnavailable: - result = {"verdict": "INCONCLUSIVE", "checks": {}, "reason": "sandbox_unavailable"} - return {**result, "artifact_sha256": expected_sha256, - "scope": "post-submission hidden correctness only; no search feedback"} diff --git a/src/inferrsi/harness/kv_reclamation.py b/src/inferrsi/harness/kv_reclamation.py deleted file mode 100644 index 261f5285c..000000000 --- a/src/inferrsi/harness/kv_reclamation.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Bounded synthetic CPU artifact replay; no agent or model execution adapter. - -The oracle stays in the controller. Submitted code sees only inputs inside the -existing Linux bubblewrap boundary. Unavailable isolation has no host fallback. -""" - -import hashlib -import json -import os -import socket -import tempfile -import time -from pathlib import Path - -from ..infra.artifacts import atomic_write_json, write_json -from ..infra.executors.local import execute_command -from .skill_eval_engineering import SandboxUnavailable, read_regular, sandbox_command - -MAX_SOURCE_BYTES = 65536 - -# This driver contains no expected answers, verifier, knowledge or timing API. -# Timing is the existing host executor's elapsed time, not an artifact assertion. -DRIVER = '''import json, os, resource, socket, sys -resource.setrlimit(resource.RLIMIT_CPU, (16, 16)) -resource.setrlimit(resource.RLIMIT_AS, (268435456, 268435456)) -resource.setrlimit(resource.RLIMIT_FSIZE, (4194304, 4194304)) -resource.setrlimit(resource.RLIMIT_NOFILE, (32, 32)) -resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) -resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) -print("INFERRSI_KV_READY", file=sys.stderr, flush=True) -with open("/work/inputs.json", encoding="utf-8") as stream: - payload = json.load(stream) -if payload["mode"] == "probe": - inaccessible = [] - for path in payload["inaccessible"]: - try: - with open(path, "rb") as stream: - stream.read(1) - except OSError: - inaccessible.append(True) - else: - inaccessible.append(False) - try: - open("/work/write-probe", "w").close() - readonly = False - except OSError: - readonly = True - connection = socket.socket() - connection.settimeout(0.25) - try: - network_blocked = connection.connect_ex(("127.0.0.1", payload["host_port"])) != 0 - except OSError: - network_blocked = True - finally: - connection.close() - try: - child = os.fork() - except OSError: - fork_denied = True - else: - if child == 0: - os._exit(0) - os.waitpid(child, 0) - fork_denied = False - print(json.dumps({"inaccessible": inaccessible, "readonly": readonly, - "clean_env": "KV_QUALIFICATION_SECRET" not in os.environ, - "network_blocked": network_blocked, "fork_denied": fork_denied, - "limits": {name: list(resource.getrlimit(k)) for name, k in - (("memory", resource.RLIMIT_AS), ("processes", resource.RLIMIT_NPROC), - ("cpu", resource.RLIMIT_CPU), ("output", resource.RLIMIT_FSIZE), - ("files", resource.RLIMIT_NOFILE), ("core", resource.RLIMIT_CORE))}})) -else: - namespace = {"__name__": "submitted_candidate"} - with open("/work/candidate.py", encoding="utf-8") as stream: - exec(compile(stream.read(), "candidate.py", "exec"), namespace) - target = namespace["Reclaimer"]() - observations = [] - def pending(): - value = target.pending() - if (type(value) is not list or any(type(item) is not tuple or len(item) != 3 - or type(item[0]) is not str or type(item[1]) is not int - or type(item[2]) is not tuple - or any(type(block) is not int for block in item[2]) for item in value)): - raise TypeError("pending shape") - return list(value) - for event in payload["events"]: - try: - result = target.apply(event) - except ValueError: - observation = {"error": "ValueError"} - else: - if type(result) is not list or any(type(block) is not int for block in result): - raise TypeError("reclaimed shape") - observation = {"reclaimed": list(result)} - if payload["mode"] == "verify": - observation["pending"] = pending() - observations.append(observation) - print(json.dumps({"observations": observations, "pending": pending()}, allow_nan=False)) -''' - - -def source_bytes(path: Path) -> bytes: - source = read_regular(path, MAX_SOURCE_BYTES) - if not source: - raise ValueError("Candidate source is empty") - source.decode("utf-8") - return source - - -def workspace_hashes(workspace: Path) -> dict: - """Only candidate.py may differ; reject symlinks, special and extra nodes.""" - hashes = {} - for path in sorted(workspace.rglob("*")): - if path.is_symlink() or not (path.is_dir() or path.is_file()): - raise ValueError("Task workspace contains unsafe nodes") - if path.is_file(): - hashes[path.relative_to(workspace).as_posix()] = hashlib.sha256( - read_regular(path, 16 * 1024 * 1024)).hexdigest() - return hashes - - -def capture_artifact(workspace: Path, frozen: dict) -> bytes: - current = workspace_hashes(workspace) - if set(current) != set(frozen) or any(current[name] != value - for name, value in frozen.items() - if name != "candidate.py"): - raise ValueError("Protected paths changed or artifact missing") - return source_bytes(workspace / "candidate.py") - - -def invoke(source: bytes, payload: dict, output: Path, deadline: float) -> tuple[dict, dict | None]: - """One resource-bounded process; output/answer checking belongs to the parent.""" - if type(source) is not bytes or not 0 < len(source) <= MAX_SOURCE_BYTES: - raise ValueError("Candidate source must be bounded nonempty bytes") - source.decode("utf-8") - with tempfile.TemporaryDirectory(prefix="kv-replay-") as temporary: - work = Path(temporary) - (work / "candidate.py").write_bytes(source) - (work / "runner.py").write_text(DRIVER, encoding="utf-8", newline="\n") - write_json(work / "inputs.json", payload) - environment = {"PATH": "/usr/bin:/bin", "LANG": "C"} - if payload.get("mode") == "probe": - environment["KV_QUALIFICATION_SECRET"] = "synthetic-not-a-secret" - record = execute_command( - tuple(sandbox_command(work)), work, output, deadline, - environment=environment, - ) - # Retain the original outcome before readiness/output checks can return or raise. - # Closed projection only: no host paths, arbitrary error text or environment. - atomic_write_json(output / "execution.json", { - name: record.get(name) for name in ("status", "returncode", "elapsed_seconds") - }) - if record["status"] in {"timeout", "error", "cancelled"}: - return record, None - marker = Path(record["stderr_path"]).read_bytes()[:128] - if not marker.startswith(b"INFERRSI_KV_READY\n"): - raise SandboxUnavailable("KV replay did not establish its sandbox boundary") - if record["status"] != "completed": - return record, None - try: - data = read_regular(Path(record["stdout_path"]), 4 * 1024 * 1024) - return record, json.loads(data) - except (ValueError, UnicodeError, RecursionError): - return record, None - - -def process_group_gone(record: dict) -> bool: - group = record.get("process_group_id") - if os.name == "nt" or group is None: - return False - try: - os.killpg(group, 0) - except ProcessLookupError: - return True - return False - - -def probe_isolation(output: Path, deadline: float, forbidden: list[str]) -> dict: - forbidden = [*forbidden, f"/proc/{os.getpid()}/cmdline"] - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - record, value = invoke(b"# no submitted code\n", { - "mode": "probe", "inaccessible": forbidden, - "host_port": listener.getsockname()[1], - }, output, deadline) - if (record["status"] != "completed" or not isinstance(value, dict) - or value.get("inaccessible") != [True] * len(forbidden) - or value.get("readonly") is not True or value.get("clean_env") is not True - or value.get("network_blocked") is not True or value.get("fork_denied") is not True - or value.get("limits") != {"memory": [268435456, 268435456], - "processes": [0, 0], "cpu": [16, 16], - "output": [4194304, 4194304], "files": [32, 32], - "core": [0, 0]} - or not process_group_gone(record)): - raise SandboxUnavailable("KV isolation probes failed; no host fallback") - value["normal_exit_process_group_gone"] = True - return value - - -def verify_source(source: bytes, traces: list[dict], output: Path, deadline: float, - compare) -> dict: - """Compare every event and pending state using an independently frozen oracle.""" - checks = {} - for index, trace in enumerate(traces): - if time.monotonic() >= deadline: - return {"verdict": "INCONCLUSIVE", "checks": checks, "reason": "deadline"} - record, value = invoke(source, {"mode": "verify", "events": trace["events"]}, - output / f"trace-{index:03d}", deadline) - if record["status"] in {"error", "timeout", "cancelled"}: - return {"verdict": "INCONCLUSIVE", "checks": checks, - "reason": record["status"]} - checks[trace["id"]] = compare(trace, value) - return {"verdict": "PASS" if all(checks.values()) else "FAIL", "checks": checks} diff --git a/tests/fixtures/kv_reclamation/candidate.py b/tests/fixtures/kv_reclamation/candidate.py deleted file mode 100644 index 467a8425b..000000000 --- a/tests/fixtures/kv_reclamation/candidate.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Synthetic CPU task starter: a deliberately repeated linear scan, not vLLM.""" - - -class Reclaimer: - def __init__(self): - self._frontier = -1 - self._leases = [] - self._generations = {} - self._blocks = set() - - def apply(self, event): - if type(event) is not dict or type(event.get("op")) is not str: - raise ValueError("event must have an operation") - op = event["op"] - fields = { - "allocate": {"op", "request_id", "generation", "blocks", "write_step", - "save_required"}, - "finish": {"op", "request_id", "generation"}, - "save_complete": {"op", "request_id", "generation"}, - "advance": {"op", "processed_step"}, - } - if op not in fields or set(event) != fields[op]: - raise ValueError("unexpected operation or fields") - integers = ("processed_step",) if op == "advance" else ("generation",) - if op == "allocate": - integers += ("write_step",) - if any(type(event[key]) is not int or event[key] < 0 for key in integers): - raise ValueError("integer fields must be nonnegative integers, excluding bool") - if op != "advance" and type(event["request_id"]) is not str: - raise ValueError("request_id must be a string") - if op == "allocate": - request, generation, blocks = event["request_id"], event["generation"], event["blocks"] - if (type(blocks) is not list or not blocks - or any(type(block) is not int or block < 0 for block in blocks) - or len(set(blocks)) != len(blocks) - or type(event["save_required"]) is not bool): - raise ValueError("invalid allocation fields") - if (generation <= self._generations.get(request, -1) - or any(lease["request_id"] == request for lease in self._leases) - or self._blocks.intersection(blocks)): - raise ValueError("allocation conflicts with lifetime history") - self._leases.append({**event, "blocks": tuple(blocks), "finished": False, - "saved": False}) - self._generations[request] = generation - self._blocks.update(blocks) - elif op == "advance": - if event["processed_step"] < self._frontier: - raise ValueError("processed frontier cannot decrease") - self._frontier = event["processed_step"] - else: - for lease in self._leases: - if (lease["request_id"], lease["generation"]) == ( - event["request_id"], event["generation"]): - lease["finished" if op == "finish" else "saved"] = True - break - reclaimed, remaining = [], [] - for lease in self._leases: - if (lease["finished"] and (not lease["save_required"] or lease["saved"]) - and lease["write_step"] <= self._frontier): - reclaimed.extend(lease["blocks"]) - self._blocks.difference_update(lease["blocks"]) - else: - remaining.append(lease) - self._leases = remaining - return reclaimed - - def pending(self): - return [(lease["request_id"], lease["generation"], lease["blocks"]) - for lease in self._leases] diff --git a/tests/fixtures/kv_reclamation/reference.py b/tests/fixtures/kv_reclamation/reference.py deleted file mode 100644 index e529ff9e6..000000000 --- a/tests/fixtures/kv_reclamation/reference.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Trusted authored headroom control, created after the independent oracle freeze. - -Never expose this implementation to a trial actor. It is not a model result. -""" - -import heapq - - -class Reclaimer: - def __init__(self): - self._frontier = -1 - self._leases = {} - self._generations = {} - self._blocks = set() - self._ready = [] - self._sequence = 0 - - def apply(self, event): - if type(event) is not dict or type(event.get("op")) is not str: - raise ValueError("event must have an operation") - op = event["op"] - fields = { - "allocate": {"op", "request_id", "generation", "blocks", "write_step", - "save_required"}, - "finish": {"op", "request_id", "generation"}, - "save_complete": {"op", "request_id", "generation"}, - "advance": {"op", "processed_step"}, - } - if op not in fields or set(event) != fields[op]: - raise ValueError("unexpected operation or fields") - integers = ("processed_step",) if op == "advance" else ("generation",) - if op == "allocate": - integers += ("write_step",) - if any(type(event[key]) is not int or event[key] < 0 for key in integers): - raise ValueError("integer fields must be nonnegative integers, excluding bool") - if op != "advance" and type(event["request_id"]) is not str: - raise ValueError("request_id must be a string") - if op == "allocate": - request, generation, blocks = event["request_id"], event["generation"], event["blocks"] - if (type(blocks) is not list or not blocks - or any(type(block) is not int or block < 0 for block in blocks) - or len(set(blocks)) != len(blocks) - or type(event["save_required"]) is not bool): - raise ValueError("invalid allocation fields") - if (generation <= self._generations.get(request, -1) - or request in self._leases or self._blocks.intersection(blocks)): - raise ValueError("allocation conflicts with lifetime history") - self._leases[request] = {**event, "blocks": tuple(blocks), "finished": False, - "saved": False, "queued": False, "sequence": self._sequence} - self._sequence += 1 - self._generations[request] = generation - self._blocks.update(blocks) - elif op == "advance": - if event["processed_step"] < self._frontier: - raise ValueError("processed frontier cannot decrease") - self._frontier = event["processed_step"] - else: - lease = self._leases.get(event["request_id"]) - if lease is not None and lease["generation"] == event["generation"]: - lease["finished" if op == "finish" else "saved"] = True - if (lease["finished"] and (not lease["save_required"] or lease["saved"]) - and not lease["queued"]): - heapq.heappush(self._ready, (lease["write_step"], lease["sequence"], - lease["request_id"])) - lease["queued"] = True - reclaimable = [] - while self._ready and self._ready[0][0] <= self._frontier: - _, sequence, request = heapq.heappop(self._ready) - reclaimable.append((sequence, self._leases.pop(request))) - reclaimed = [] - for _, lease in sorted(reclaimable): - reclaimed.extend(lease["blocks"]) - self._blocks.difference_update(lease["blocks"]) - return reclaimed - - def pending(self): - return [(lease["request_id"], lease["generation"], lease["blocks"]) - for lease in self._leases.values()] diff --git a/tests/fixtures/kv_reclamation/workload.py b/tests/fixtures/kv_reclamation/workload.py deleted file mode 100644 index fce20ce86..000000000 --- a/tests/fixtures/kv_reclamation/workload.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Protected deterministic input construction, independent of submitted code.""" - - -def workload(count): - if type(count) is not int or count < 1: - raise ValueError("count must be a positive integer") - events = [{"op": "allocate", "request_id": str(index), "generation": 0, - "blocks": [2 * index, 2 * index + 1], "write_step": index % 257, - "save_required": index % 3 != 0} for index in range(count)] - events += [{"op": "finish", "request_id": str(index), "generation": 0} - for index in reversed(range(count))] - events += [{"op": "save_complete", "request_id": str(index), "generation": 0} - for index in range(count)] - events += [{"op": "advance", "processed_step": frontier} for frontier in range(257)] - return events diff --git a/tests/test_kv_actor_cleanup.py b/tests/test_kv_actor_cleanup.py deleted file mode 100644 index 192d379e9..000000000 --- a/tests/test_kv_actor_cleanup.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Observer and fail-closed controls; no submitted artifact executes on host.""" - -import json -import sys -import time - -import pytest - -from tools import qualify_kv_actor_cleanup as cleanup - - -def stat_file(root, pid, parent, start, state="S"): - path = root / str(pid) - path.mkdir() - fields = [state, str(parent)] + ["0"] * 17 + [str(start)] - (path / "stat").write_text(f"{pid} (trusted fixture) " + " ".join(fields)) - - -def test_observer_tracks_only_owned_tree_and_starttime_not_reused_pid(tmp_path): - stat_file(tmp_path, 10, 1, 100) - stat_file(tmp_path, 11, 10, 101) - stat_file(tmp_path, 12, 11, 102) - stat_file(tmp_path, 99, 1, 999) - owned, visible = cleanup._owned_snapshot(10, {}, proc=tmp_path) - assert visible and set(owned) == {11, 12} - # A formerly owned PID reused by an unrelated process is not a survivor. - owned, visible = cleanup._owned_snapshot(50, {99: 998}, proc=tmp_path) - assert visible and owned == {} - # An owned detached/reparented descendant remains tracked by PID+starttime. - owned, visible = cleanup._owned_snapshot(50, {12: 102}, proc=tmp_path) - assert visible and set(owned) == {12} - - -def test_unreadable_proc_is_not_empty_success(tmp_path): - assert cleanup._owned_snapshot(1, {}, proc=tmp_path / "missing") == ({}, False) - (tmp_path / "12").mkdir() - (tmp_path / "12/stat").write_text("malformed") - assert cleanup._owned_snapshot(1, {}, proc=tmp_path)[1] is False - - -def test_no_sandbox_never_runs_probe_on_host(tmp_path, monkeypatch): - monkeypatch.setattr(cleanup, "sandbox_command", lambda *_: (_ for _ in ()).throw( - cleanup.SandboxUnavailable("blocked"))) - monkeypatch.setattr(cleanup, "execute_command", - lambda *_args, **_kwargs: pytest.fail("no fallback")) - result = cleanup.probe_worker("normal", tmp_path / "out", time.monotonic() + 1) - assert result["verdict"] == "INCONCLUSIVE" - assert result["reason"] == "sandbox_unavailable" - - -def test_qualification_requires_all_three_modes_and_readable_worker_evidence(tmp_path, monkeypatch): - calls = [] - def execute(command, cwd, output, deadline, **kwargs): - calls.append(command[4]) - assert command[1] == "-I" - assert kwargs["environment"] == {"PATH": "/usr/bin:/bin", "LANG": "C"} - destination = tmp_path / "out" / command[4] - destination.mkdir(parents=True) - (destination / "result.json").write_text(json.dumps({"verdict": "PASS"})) - return {"status": "completed"} - monkeypatch.setattr(cleanup, "execute_command", execute) - result = cleanup.qualify(tmp_path / "out", time.monotonic() + 10, 1) - assert calls == ["normal", "timeout", "cancel"] and result["verdict"] == "PASS" - - -def test_probe_is_authored_fork_setsid_only_not_candidate_or_provider(): - assert "os.fork()" in cleanup.PROBE and "os.setsid()" in cleanup.PROBE - assert "candidate" not in cleanup.PROBE and "exec(" not in cleanup.PROBE - assert "SKILL_EVAL" not in cleanup.PROBE - - -def test_nested_worker_rejects_other_source_before_execution(tmp_path, monkeypatch): - (tmp_path / "runner.py").write_text("raise RuntimeError('not the trusted probe')\n") - monkeypatch.setattr(cleanup, "execute_command", lambda *a, **k: pytest.fail("must not run")) - with pytest.raises(ValueError, match="exact authored"): - cleanup.nested_worker("normal", tmp_path, tmp_path / "out", time.monotonic() + 1) - - -def test_detached_marker_requires_exact_owned_host_identity(tmp_path): - stat_file(tmp_path, 10, 1, 100) - stat_file(tmp_path, 11, 10, 101) - stat_file(tmp_path, 12, 11, 102) - marker = {"role": "detached", "own_session": True, "namespace_pid": 3, - "start": 102, "pid_namespace": "pid:[1234]"} - owned, _ = cleanup._owned_snapshot(1, {}, proc=tmp_path) - # A marker plus two ancestors never substitutes for observing the detached PID. - assert cleanup._bind_detached([marker], {p: owned[p] for p in (10, 11)}, proc=tmp_path) is None - (tmp_path / "12/status").write_text("Name:\ttrusted\nNSpid:\t12\t3\n") - (tmp_path / "12/ns").mkdir() - (tmp_path / "12/ns/pid").symlink_to("pid:[1234]") - detached = cleanup._bind_detached([marker], owned, proc=tmp_path) - assert detached == {"pid": 12, "start": 102} - assert cleanup._bind_chain([{"role": "worker", "pid": 10, "start": 100}], - detached, owned)["chain_length"] == 3 - for altered in ({"start": 999}, {"namespace_pid": 4}, {"pid_namespace": "pid:[4321]"}): - assert cleanup._bind_detached([{**marker, **altered}], owned, proc=tmp_path) is None - assert cleanup._bind_chain([{"role": "worker", "pid": 11, "start": 101}], - detached, owned) is None - - -@pytest.mark.skipif(sys.platform != "linux", reason="requires Linux proc identity observation") -@pytest.mark.parametrize("mode", ["timeout", "cancel"]) -def test_outer_worker_loss_detects_and_recovers_independent_session_survivors( - tmp_path, monkeypatch, mode -): - # Authored host-only fixture intentionally lacks bubblewrap die-with-parent. - # It exercises negative observer/supervisor behavior, never candidate code or - # qualification success. The detached descendants must be detected and killed. - output = tmp_path / "out" - events = output / "sandbox/stdout.log" - events.parent.mkdir(parents=True) - detached = '''import json, os, time -with open("/proc/self/stat") as stream: - start = int(stream.read().rsplit(")", 1)[1].split()[19]) -print(json.dumps({"role": "detached", "own_session": os.getsid(0) == os.getpid(), - "namespace_pid": os.getpid(), "start": start, - "pid_namespace": os.readlink("/proc/self/ns/pid")}), flush=True) -print(json.dumps({"role": "parent"}), flush=True) -time.sleep(30) -''' - middle = f'''import subprocess, sys, time -with open({str(events)!r}, "w") as stream: - subprocess.Popen([sys.executable, "-c", {detached!r}], stdout=stream, start_new_session=True) - time.sleep(30) -''' - worker = f'''import json, os, subprocess, sys, time -with open("/proc/self/stat") as stream: - start = int(stream.read().rsplit(")", 1)[1].split()[19]) -print(json.dumps({{"role": "worker", "pid": os.getpid(), "start": start}}), flush=True) -subprocess.Popen([sys.executable, "-c", {middle!r}], start_new_session=True) -time.sleep(30) -''' - execute = cleanup.execute_command - - def fixture_execute(command, cwd, trial, deadline, **kwargs): - assert "--nested-worker" in command - return execute((sys.executable, "-c", worker), cwd, trial, deadline, **kwargs) - - monkeypatch.setattr(cleanup, "sandbox_command", lambda _: ("never-run",)) - monkeypatch.setattr(cleanup, "execute_command", fixture_execute) - # probe_worker owns creation, not the fixture's precreated output parent. - events.parent.rmdir() - output.rmdir() - original_events = cleanup._events - - def read_events(path): - events.parent.mkdir(parents=True, exist_ok=True) - return original_events(path) - - monkeypatch.setattr(cleanup, "_events", read_events) - result = cleanup.probe_worker(mode, output, time.monotonic() + 2) - assert result["verdict"] == "FAIL" - assert result["outer_worker_supervised"] and result["bound_chain"]["chain_length"] >= 3 - assert result["surviving_descendants"] > 0 and result["recovery_kills"] > 0 - assert result["recovery_survivors"] == 0 and result["recovery_visibility_complete"] - assert result["explicit_cancel_sent"] == (mode == "cancel") - for role in ("worker", "detached"): - identity = result["bound_chain"][role] - try: - current = cleanup._stat(cleanup.Path("/proc") / str(identity["pid"]) / "stat") - except FileNotFoundError: - continue - assert current["start"] != identity["start"] or current["state"] == "Z" diff --git a/tests/test_kv_edit_inputs.py b/tests/test_kv_edit_inputs.py deleted file mode 100644 index b595e3f37..000000000 --- a/tests/test_kv_edit_inputs.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -from copy import deepcopy - -import pytest - -from tools import kv_edit_inputs as inputs - - -def test_common_inputs_and_prompt_are_shared_without_expected_or_hidden_data(): - development = [{"id": "visible", "split": "development", "events": [{}], - "expected": [{"trusted_expected_canary": True}]}] - public, prompt, suite = inputs.build_inputs( - b"# starter\n", b"# neutral contract\n", development, [], {"benchmark_canary": True}, 0 - ) - assert set(public) == inputs.COMMON - assert prompt == inputs.PROMPT - assert json.loads(public["development.json"])["traces"] == [{"id": "visible", "events": [{}]}] - assert all(b"trusted_expected_canary" not in data and b"benchmark_canary" not in data - for data in public.values()) - assert suite.read()["development"] == development - assert inputs.build_inputs(b"# starter\n", b"# neutral contract\n", development, - [], {"benchmark_canary": True}, 0)[:2] == (public, prompt) - - -def test_hidden_split_is_never_accepted_as_public_input(): - with pytest.raises(ValueError, match="split"): - inputs.build_inputs(b"# starter\n", b"# contract\n", - [{"id": "secret", "split": "grader", "events": [], "expected": []}], - [], {}, 0) - - -@pytest.mark.parametrize("field", ["arm", "suite_sha256", "baseline_sha256", "client", - "model", "input_token_ceiling", "prompt", "files", "limits", - "check_ids", "metric_key", "metric_unit", "direction"]) -def test_common_contract_binds_all_non_transport_actor_inputs(field): - config = {k: k for k in ("arm", "model", "client", "input_token_ceiling", "prompt", - "suite_sha256", "baseline_sha256")} - config["edit_plan"] = {k: k for k in ("files", "limits", "check_ids", "metric_key", - "metric_unit", "direction", "callbacks_sha256")} - original = inputs.common_actor_contract_sha256(config) - changed = deepcopy(config) - (changed["edit_plan"] if field in config["edit_plan"] else changed)[field] = "changed" - assert inputs.common_actor_contract_sha256(changed) != original - scripted = deepcopy(config) - scripted["edit_plan"]["callbacks_sha256"] = "different-response-fixture-binding" - scripted["responses_sha256"] = "scripted-only" - scripted["plan_sha256"] = "separate-campaign" - scripted["actor_sha256"] = "different-transport-identity" - assert inputs.common_actor_contract_sha256(scripted) == original diff --git a/tests/test_kv_edit_pilot.py b/tests/test_kv_edit_pilot.py deleted file mode 100644 index ec7d82169..000000000 --- a/tests/test_kv_edit_pilot.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Supervised scripted worker and fake-store admission: no model/provider calls.""" - -import json -import sys -import time -from datetime import timedelta -from pathlib import Path - -import pytest - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.executors.local import execute_command -from inferrsi.model import anthropic_agent as provider -from tests.test_anthropic_edit_agent import read_task, submit, work -from tests.test_kv_edit_workbench import BASELINE, CANDIDATE, fake_sandbox, stage, suite -from tests.test_kv_pilot_admission import World, policy -from tools import kv_edit_pilot as pilot -from tools import kv_pilot_admission as admission - - -def prepare(tmp_path, arm="A2", scripts=None): - parent = tmp_path / arm - parent.mkdir() - root, plan = stage(parent) - spec = {"kind": provider.EDIT_KIND, "provider": "yibuapi", "endpoint": provider.ENDPOINT, - "api_version": "2023-06-01", "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 262144, "max_read_bytes": 32768, "max_tool_calls": 32} - config = pilot.prepare_scripted(parent / "inputs", root, plan, suite(), BASELINE, - scripts or [read_task(), work(CANDIDATE), submit()], plan_sha256="1" * 64, - execution_id="example-run-1.1", arm=arm, model="claude-opus-5-5", spec=spec, - prompt="Optimize only the public task.", input_token_ceiling=1000000) - return parent / "inputs", root, config - - -def in_process_worker(command, cwd, trial_dir, deadline, *, environment): - """Test-only scripted executor: does not certify the subprocess/isolation boundary.""" - assert provider.KEY_ENV not in environment and "GITHUB_TOKEN" not in environment - assert "-I" in command and command[0] == sys.executable - assert "--offline-worker" in command - args = dict(zip(command[4::2], command[5::2])) - assert float(args["--deadline"]) == deadline - trial_dir.mkdir(parents=True) - with (trial_dir / "stdout.log").open("w") as stream: - code = pilot.scripted_worker(Path(args["--inputs"]), Path(args["--workspace"]), - Path(args["--output"]), deadline, environment=environment, - emit=lambda e: stream.write(provider.canonical(e) + "\n")) - (trial_dir / "stderr.log").write_text("") - return {"status": "completed" if code == 0 else "failed", "returncode": code, - "stdout_path": str(trial_dir / "stdout.log"), - "stderr_path": str(trial_dir / "stderr.log"), "elapsed_seconds": 0.01} - - -def fake_isolation(output, deadline, forbidden): - assert deadline > time.monotonic() and len(forbidden) == 4 - return {"scope": "scripted unit-test output; not isolation qualification"} - - -def test_supervised_scripted_pair_with_existing_claims_and_usage_bridge(tmp_path, monkeypatch): - a, b = prepare(tmp_path, "A2"), prepare(tmp_path, "A3") - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - world = World() - config = policy(actor_sha256=(a[2]["actor_sha256"], b[2]["actor_sha256"])) - session = admission.admit(config, world.daily, world.pr, check_current=lambda _: True, - wall_clock=lambda: world.now, monotonic_clock=time.monotonic) - assert len(world.events) == 4 # Whole plan reserved/claimed before either disclosure. - for inputs, root, _ in (a, b): - result = pilot.run_supervised(inputs, root, inputs.parent / "run", - time.monotonic() + 600, admission=session, execute=in_process_worker) - assert result["worker"]["status"] == "completed" - assert result["simulated_slots_consumed"] == 3 - assert result["request_completion_known"] - assert result["model_calls"] == 0 and result["provider_charges_usd"] == 0 - final = inputs.parent / "run/worker/final/candidate.py" - assert final.read_text() == CANDIDATE - assert world.pr.ledger.charged == 162720000 # No refund from simulated usage. - assert world.daily.ledger.charged_on("2026-10-30") == 654070000 - with pytest.raises(Exception): - session.start_arm("A2", time.monotonic() + 600) - session.close() - - -def test_unknown_usage_blocks_other_arm_and_retains_full_reservation(tmp_path, monkeypatch): - a = prepare(tmp_path, scripts=[{**read_task(), "usage": None}]) - b = prepare(tmp_path, "A3") - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - world = World() - session = admission.admit(policy(actor_sha256=(a[2]["actor_sha256"], b[2]["actor_sha256"])), - world.daily, world.pr, check_current=lambda _: True, wall_clock=lambda: world.now, - monotonic_clock=time.monotonic) - result = pilot.run_supervised(a[0], a[1], a[0].parent / "run", time.monotonic() + 600, - admission=session, execute=in_process_worker) - assert result["worker"]["status"] == "failed" - assert not result["request_completion_known"] and result["simulated_slots_consumed"] == 1 - with pytest.raises(Exception): - pilot.run_supervised(b[0], b[1], b[0].parent / "run", time.monotonic() + 600, - admission=session, execute=in_process_worker) - assert world.pr.ledger.charged == 162720000 - - -def test_missing_completion_after_grant_is_not_recovered_or_refunded(tmp_path, monkeypatch): - a = prepare(tmp_path) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - original = pilot.atomic_write_json - def lose_completion(path, value): - if Path(path).name == "complete.json": - raise OSError("simulated loss") - return original(path, value) - monkeypatch.setattr(pilot, "atomic_write_json", lose_completion) - result = pilot.run_supervised(a[0], a[1], a[0].parent / "run", time.monotonic() + 600, - execute=in_process_worker) - assert result["simulated_slots_consumed"] == 1 - assert not result["request_completion_known"] - assert result["worker"]["reason"] == "provider_transport_error" - - -@pytest.mark.parametrize("name", ["suite.json", "baseline.py", "responses.json"]) -def test_changed_private_frozen_inputs_fail_before_worker_or_slots(tmp_path, name): - inputs, root, _ = prepare(tmp_path) - (inputs / name).write_text("changed") - with pytest.raises(ValueError): - pilot.run_supervised(inputs, root, inputs.parent / "run", time.monotonic() + 600, - execute=lambda *_args, **_kwargs: pytest.fail("must not launch")) - - -def test_changed_prompt_or_dependencies_fail_before_launch(tmp_path): - inputs, root, _ = prepare(tmp_path) - path = inputs / "worker.json" - config = json.loads(path.read_text()) - config["prompt"] = "changed prompt" - path.write_text(json.dumps(config)) - with pytest.raises(ValueError, match="binding changed"): - pilot.run_supervised(inputs, root, inputs.parent / "run", time.monotonic() + 600) - - -def test_sandbox_unavailable_precedes_every_simulated_request(tmp_path, monkeypatch): - inputs, root, _ = prepare(tmp_path) - def unavailable(*_): - raise kv.SandboxUnavailable("sandbox unavailable") - monkeypatch.setattr(kv, "probe_isolation", unavailable) - result = pilot.run_supervised(inputs, root, inputs.parent / "run", time.monotonic() + 30, - execute=in_process_worker) - assert result["worker"]["status"] == "inconclusive" - assert result["worker"]["reason"] == "sandbox_unavailable" - assert result["simulated_slots_consumed"] == 0 - - -def test_existing_executor_really_kills_blocked_trusted_worker_at_inherited_deadline(tmp_path): - inputs, root, _ = prepare(tmp_path) - def blocking_executor(command, cwd, output, deadline, *, environment): - # Trusted sleep fixture, NOT submitted Python or a candidate host fallback. - return execute_command((sys.executable, "-I", "-c", "import time; time.sleep(30)"), - cwd, output, deadline, environment=environment) - started = time.monotonic() - result = pilot.run_supervised(inputs, root, inputs.parent / "run", started + 0.2, - execute=blocking_executor) - assert result["execution"]["status"] == "timeout" - assert result["execution"]["elapsed_seconds"] < 2 - assert kv.process_group_gone(result["execution"]) - assert result["worker"]["status"] == "missing" - - -def test_shanghai_day_change_blocks_bridge_next_grant(tmp_path, monkeypatch): - a = prepare(tmp_path) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - world = World() - session = admission.admit(policy(actor_sha256=(a[2]["actor_sha256"], "5" * 64)), - world.daily, world.pr, check_current=lambda _: True, wall_clock=lambda: world.now, - monotonic_clock=time.monotonic) - original = session.complete - def midnight(grant, usage): - original(grant, usage) - world.now += timedelta(hours=1) - session.complete = midnight - result = pilot.run_supervised(a[0], a[1], a[0].parent / "run", time.monotonic() + 600, - admission=session, execute=in_process_worker) - assert result["worker"]["status"] == "failed" - assert result["worker"]["reason"] == "request_authorization_denied" - assert world.pr.ledger.charged == 162720000 diff --git a/tests/test_kv_edit_workbench.py b/tests/test_kv_edit_workbench.py deleted file mode 100644 index 3017cfbbf..000000000 --- a/tests/test_kv_edit_workbench.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Fixed workbench integration with scripted sandbox outputs; never host candidate exec.""" - -import copy -import json -import time -from pathlib import Path - -import pytest - -from inferrsi.harness import kv_edit_workbench as workbench -from inferrsi.harness import kv_reclamation as kv -from inferrsi.model import anthropic_edit_agent as edit -from tests.test_anthropic_edit_agent import call, read_task, response, submit, work - -BASELINE = b"# visible linear starter\n" -CANDIDATE = "raise AssertionError('must never run on host')\n" -TRACE = {"id": "dev_empty", "split": "development", "events": [], "expected": []} -EXPECTED = {"observations": [], "pending": []} - - -def suite(): - return workbench.VisibleSuite(json.dumps({ - "schema": "kv-visible-suite-v1", "development": [TRACE], "benchmark_events": [], - "benchmark_expected": EXPECTED, "min_improvement_pct": 0, - }).encode()) - - -def stage(tmp_path): - root = tmp_path / "workspace" - root.mkdir() - (root / "candidate.py").write_bytes(BASELINE) - (root / "task.json").write_text('{"request":"optimize"}') - plan = edit.EditPlan(edit.freeze_files(root, ["task.json", "candidate.py"]), - edit.EditLimits(65536, 4096, 8, 8, 4, 2, 600), ("dev_empty",), - "elapsed_seconds", "seconds", "minimize", "b" * 64) - return root, plan - - -def fake_sandbox(source, payload, output, deadline): - assert type(source) is bytes and deadline > time.monotonic() - assert set(payload) == {"mode", "events"} # Never expected answers or hidden data. - output.mkdir(parents=True) - return {"status": "completed", "elapsed_seconds": 2.0 if source == BASELINE else 1.0, - "metric": None, "metric_error": None}, copy.deepcopy(EXPECTED) - - -def test_scripted_edit_check_abba_rewrite_submit_hidden_grade(tmp_path, monkeypatch): - root, plan = stage(tmp_path) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - scripts = [read_task(), work(CANDIDATE, "1"), work(CANDIDATE + "# final\n", "2"), submit()] - payloads, traces = [], [] - def send(payload, spec, remaining): - payloads.append(copy.deepcopy(payload)) - return scripts[len(payloads) - 1] - spec = {"kind": edit.KIND, "provider": "yibuapi", "endpoint": edit.provider.ENDPOINT, - "api_version": "2023-06-01", "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 1048576, "max_read_bytes": 65536, "max_tool_calls": 24} - answer = edit.run_edit_agent(root, plan, "Optimize", "claude-opus-5-5", spec, - time.monotonic() + 600, traces.append, arm="A2", - slots=edit.RequestSlots("c" * 64, {"A2": 8, "A3": 8}, 10170000, 162720000, - execution_id="offline-1"), - authorize=lambda *_: True, send=send, visible_checks=bench.visible_checks, - visible_benchmark=bench.visible_benchmark, retain_revision=bench.retain_revision, - input_token_ceiling=1000000) - assert answer["artifact_source"] == CANDIDATE + "# final\n" - assert len(list((tmp_path / "private/revisions").glob("*/candidate.py"))) == 3 - runs = sorted((tmp_path / "private").glob("benchmark-*/runs/*")) - assert len(runs) == 2 - for run in runs: - events = [json.loads(line) for line in (run / "events.jsonl").read_text().splitlines()] - assert [e["side"] for e in events if e["phase"] == "measurement"] == [ - "baseline", "candidate", "candidate", "baseline"] - evaluation = json.loads((run / "evaluation.json").read_text()) - assert evaluation["source_integrity"]["verified"] - assert evaluation["improvement_pct"] == 50 - # Freeze actual returned bytes before the separate hidden grader; changing the - # live actor path afterward must neither change grading input nor cause a retry. - (root / "candidate.py").write_text("# later unrelated mutation\n") - hidden = [{**TRACE, "split": "grader", "id": "hidden-id-not-for-actor"}] - result = workbench.grade_final(answer["artifact_source"].encode(), answer["sha256"], hidden, - tmp_path / "grading", time.monotonic() + 30) - assert result["verdict"] == "PASS" - assert "hidden-id-not-for-actor" not in edit.provider.canonical(payloads) - assert "private" not in edit.provider.canonical(payloads) - - -def test_benchmark_validates_entire_output_before_accepting_host_metric(tmp_path, monkeypatch): - _, plan = stage(tmp_path) - def bad(source, payload, output, deadline): - record, _ = fake_sandbox(source, payload, output, deadline) - return record, {"observations": [], "pending": [], "fake_latency": 0.00001} - monkeypatch.setattr(kv, "invoke", bad) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - result = bench.visible_benchmark(CANDIDATE.encode(), time.monotonic() + 30) - assert result["verdict"] == "INCONCLUSIVE" - assert result["baseline_values"] == [] and result["candidate_values"] == [] - - -def test_same_absolute_deadline_reaches_every_sandbox_call(tmp_path, monkeypatch): - _, plan = stage(tmp_path) - seen = [] - def invoke(source, payload, output, deadline): - seen.append(deadline) - return fake_sandbox(source, payload, output, deadline) - monkeypatch.setattr(kv, "invoke", invoke) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - deadline = time.monotonic() + 30 - bench.visible_checks(CANDIDATE.encode(), deadline) - bench.visible_benchmark(CANDIDATE.encode(), deadline) - assert len(seen) == 6 and all(value <= deadline for value in seen) - - -def test_unavailable_sandbox_is_inconclusive_without_fallback(tmp_path, monkeypatch): - _, plan = stage(tmp_path) - def unavailable(*_): - raise kv.SandboxUnavailable("local permission denied") - monkeypatch.setattr(kv, "invoke", unavailable) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - for callback in (bench.visible_checks, bench.visible_benchmark): - result = callback(CANDIDATE.encode(), time.monotonic() + 30) - assert result["verdict"] == "INCONCLUSIVE" - assert result["reason"] == "sandbox_unavailable" - assert "permission" not in json.dumps(result) - - -def test_hidden_data_never_accepted_as_visible_suite(): - value = suite().read() - value["development"][0]["split"] = "grader" - with pytest.raises(ValueError): - workbench.VisibleSuite(json.dumps(value).encode()) - - -def test_hidden_grade_rejects_wrong_final_hash_before_execution(tmp_path, monkeypatch): - monkeypatch.setattr(kv, "invoke", lambda *_: pytest.fail("must not execute")) - with pytest.raises(ValueError, match="exact immutable"): - workbench.grade_final(BASELINE, "a" * 64, [], tmp_path, time.monotonic() + 1) - - -def test_retention_is_exclusive_and_exact(tmp_path): - _, plan = stage(tmp_path) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - metadata = {"revision": 0, "sha256": edit.digest(BASELINE)} - assert bench.retain_revision(BASELINE, metadata, time.monotonic() + 30) - with pytest.raises(FileExistsError): - bench.retain_revision(BASELINE, metadata, time.monotonic() + 30) - with pytest.raises(ValueError): - bench.retain_revision(BASELINE, {"revision": 1, "sha256": "a" * 64}, - time.monotonic() + 30) - - -def test_does_not_call_candidate_import_or_trusted_observation_helpers(): - source = Path(workbench.__file__).read_text() - assert "exec(" not in source and "importlib" not in source and "observe_trusted" not in source - - -def test_malformed_tool_measurement_arguments_fail_before_callback(tmp_path, monkeypatch): - # Existing protocol strict tool validation is reused by the integration. - from tests.test_anthropic_edit_agent import run - root, plan = stage(tmp_path) - spec = {"kind": edit.KIND, "provider": "yibuapi", "endpoint": edit.provider.ENDPOINT, - "api_version": "2023-06-01", "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 1048576, "max_read_bytes": 65536, "max_tool_calls": 24} - with pytest.raises(edit.provider.AgentError, match="unsupported_tool"): - run((root, plan, spec), [read_task(), response(call("run_visible_benchmark", { - "command": "untrusted command", "workload": "hidden"}))]) diff --git a/tests/test_kv_live_job.py b/tests/test_kv_live_job.py deleted file mode 100644 index 778d5712e..000000000 --- a/tests/test_kv_live_job.py +++ /dev/null @@ -1,848 +0,0 @@ -"""Offline controller/workflow tests: synthetic identities, keys, stores and HTTP only.""" - -import copy -import json -import time -from datetime import datetime, timezone -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest -import yaml - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.model import anthropic_agent as provider -from inferrsi.model import anthropic_edit_agent as edit -from tests.test_anthropic_edit_agent import read_task, submit, work -from tests.test_kv_edit_workbench import BASELINE, CANDIDATE, TRACE, fake_sandbox, suite -from tests.test_kv_pilot_admission import World -from tests.test_kv_pilot_admission import policy as sample_policy -from tools import kv_live_job as job -from tools import kv_live_policy as policy -from tools import kv_live_worker as worker - - -def write(root, name, data): - if not isinstance(data, bytes): - data = provider.canonical(data).encode() - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return {"path": name, "bytes": len(data), "sha256": policy.sha(data)} - - -@pytest.fixture -def staged(tmp_path, monkeypatch): - root, output = tmp_path / "checkout", tmp_path / "output" - root.mkdir() - output.mkdir() - monkeypatch.setattr(job, "ROOT", root) - monkeypatch.setattr(policy, "ROOT", root) - visible = suite().read() - common, prompt, _ = job.inputs.build_inputs(BASELINE, b"contract", visible["development"], - visible["benchmark_events"], visible["benchmark_expected"], visible["min_improvement_pct"]) - paths = {"candidate.py": "tests/fixtures/kv_reclamation/candidate.py", - "CONTRACT.md": "evidence/kv-reclamation-20261030/validator/independent/CONTRACT.md"} - sources = {name: (paths.get(name, "fixture/" + name), data) for name, data in common.items()} - sources.update({name: (job.KNOWLEDGE_ROOT + name[len("knowledge/"):], b"offline knowledge") - for name in job.KNOWLEDGE}) - records = {name: write(root, path, data) for name, (path, data) in sources.items()} - provider_config = json.loads(sample_policy().provider_json) - plan = {"actors": {}, "provider": provider_config, "execution_id": "offline-1", - "expires_at": int(time.time()) + 7200, "arm_order": ["A2", "A3"], - "suite": write(root, "fixture/suite.json", suite().serialized), - "final_suite": write(root, "fixture/final.json", suite().serialized), - "hidden_traces": write(root, "fixture/hidden.json", [{**TRACE, "split": "grader"}]), - "charge_pr": 999, "charge_head": "2" * 40, - "final_seconds": 180, "final_repetitions": 8} - configs = {} - for arm in ("A2", "A3"): - names = job.COMMON | (job.KNOWLEDGE if arm == "A3" else set()) - workspace = tmp_path / f"freeze-{arm}" - workspace.mkdir() - for name in names: - write(workspace, name, sources[name][1]) - ep = edit.EditPlan(edit.freeze_files(workspace, names), - edit.EditLimits(65536, 8192, 8, 8, 4, 2, 600), (TRACE["id"],), - "elapsed_seconds", "seconds", "minimize", "0" * 64) - configs[arm] = worker.configuration(workspace, ep, suite(), BASELINE, arm=arm, - model="claude-opus-5-5", spec=provider_config["client"], prompt=prompt) - plan["actors"][arm] = {"config": write(root, f"fixture/{arm}.json", configs[arm]), - "workspace": [{"path": n, "source": records[n]} - for n in sorted(names)]} - job._stage(plan, output) - atomic_write_json(output / "authorization.json", {"body": "offline frozen public fixture"}) - authorization = {"plan": plan, "plan_sha256": policy.fingerprint(plan)} - locator = {"offline": True} - prepared = {"schema": "kv-live-prepared-v1", "implementation_sha": "a" * 40, - "run_id": "123", "plan_sha256": authorization["plan_sha256"], - "locator_sha256": policy.fingerprint(locator), "cleanup": {"verdict": "PASS"}, - "runtime_profile": {"verdict": "PASS"}, - "replay": {a: {"verdict": "PASS", "model_calls": 0} for a in ("A2", "A3")}, - "inputs": kv.workspace_hashes(output)} - atomic_write_json(output / "prepared.json", prepared) - monkeypatch.setattr(policy, "frozen_authorization", lambda **_: (authorization, locator)) - monkeypatch.setattr(policy, "runtime", lambda: ("a" * 40, "123")) - monkeypatch.setattr(policy, "verify_live", lambda *_: None) - monkeypatch.setattr(job, "runway", lambda *_: None) - monkeypatch.setenv(provider.KEY_ENV, "OFFLINE-FAKE-PROVIDER-KEY") - monkeypatch.setenv("GITHUB_RUN_ID", "123") - monkeypatch.setenv("GITHUB_SHA", "a" * 40) - monkeypatch.delenv("GITHUB_TOKEN", raising=False) - world = World() - root_policy = sample_policy( - actor_sha256=tuple(configs[a]["actor_sha256"] for a in ("A2", "A3")), - plan_sha256=authorization["plan_sha256"], execution_id=plan["execution_id"]) - monkeypatch.setattr(policy, "policy", lambda *_: root_policy) - original_admit = job.admission.admit - monkeypatch.setattr(job.admission, "admit", lambda *args, **kwargs: - original_admit(*args, **({"wall_clock": lambda: world.now} | kwargs))) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", lambda *_: {"offline": True}) - return SimpleNamespace(root=root, output=output, plan=plan, world=world, - configs=configs, authorization=authorization) - - -def launcher(scripts, seen, *, transport=None): - def execute(command, cwd, output, deadline, *, environment): - assert environment[provider.KEY_ENV] == "OFFLINE-FAKE-PROVIDER-KEY" - forbidden = {"GITHUB_TOKEN", "ACTIONS_RUNTIME_TOKEN", "HOME", "PYTHONPATH"} - assert not forbidden & set(environment) - args = dict(zip(command[3::2], command[4::2])) - assert float(args["--deadline"]) == deadline - arm = Path(args["--inputs"]).parent.name - seen.append(arm) - responses = iter(copy.deepcopy(scripts)) - code = worker.live_worker(args["--inputs"], args["--workspace"], args["--output"], - deadline, environment=environment, emit=lambda _: None, - transport=transport or (lambda *a, **kw: next(responses))) - return {"status": "completed" if code == 0 else "failed", "returncode": code} - - return lambda *args, **kwargs: worker.run_supervised(*args, **kwargs, execute=execute) - - -def test_fake_pair_reserves_before_disclosure_and_grades_immutable_bytes(staged, monkeypatch): - seen = [] - scripts = [read_task(), work(CANDIDATE), work(CANDIDATE + "# final\n", "2"), submit()] - launch = launcher(scripts, seen) - def checked_launch(*args, **kwargs): - assert len(staged.world.events) == 4 - return launch(*args, **kwargs) - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), launch=checked_launch) - assert seen == ["A2", "A3"] - assert all(a["status"] == "completed" for a in result["arms"].values()) - assert staged.world.pr.ledger.charged == 162720000 - assert staged.world.daily.ledger.charged_on("2026-10-30") == 654070000 - summary = job.package(staged.output) - assert all(len(r) == 4 for r in summary["requests"].values()) - assert all(a["status"] == "submitted" for a in summary["artifacts"].values()) - assert all(len(v["revisions"]) == 3 and len(v["feedback"]) == 4 - for v in summary["evidence"].values()) - for arm in seen: - (staged.output / arm / "workspace/candidate.py").write_text("changed after submission") - monkeypatch.delenv(provider.KEY_ENV) - graded = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["semantic"]["verdict"] == "PASS" for a in graded["arms"].values()) - public = b"".join(p.read_bytes() for p in (staged.output / "public").rglob("*") if p.is_file()) - assert b"OFFLINE-FAKE-PROVIDER-KEY" not in public - - -def test_unknown_usage_retains_attempt_and_does_not_disclose_other_arm(staged): - seen = [] - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), - launch=launcher([{**read_task(), "usage": None}], seen)) - assert seen == ["A2"] - assert result["arms"]["A3"]["status"] == "not_started" - summary = job.package(staged.output) - assert summary["requests"]["A2"][0]["status"] == "unresolved" - assert summary["requests"]["A2"][0]["usage"] is None - assert staged.world.pr.ledger.charged == 162720000 - assert summary["diagnostics"]["A2"]["reason"] == "unknown_usage" - assert summary["diagnostics"]["A2"]["provider_attempts"] == 1 - assert summary["diagnostics"]["A3"]["provider_attempts"] is None - - -def test_closed_worker_failure_is_packaged_without_retry_or_arbitrary_text(staged): - seen = [] - transport = Mock(side_effect=provider.AgentError("http_429")) - job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), - launch=launcher([], seen, transport=transport)) - assert seen == ["A2"] and transport.call_count == 1 - assert staged.world.pr.ledger.charged == 162720000 - summary = job.package(staged.output) - diagnostic = summary["diagnostics"]["A2"] - assert diagnostic == {"worker_status": "failed", "reason": "provider_transport_error", - "provider_attempts": 1, "transport_error_code": "http_429", - "execution": {"status": "failed", "returncode": 1, "elapsed_seconds": None}} - assert summary["requests"]["A2"][0]["usage"] is None - assert summary["diagnostics"]["A3"]["worker_status"] == "unavailable" - public = b"".join(p.read_bytes() for p in (staged.output / "public").rglob("*") if p.is_file()) - assert b"OFFLINE-FAKE-PROVIDER-KEY" not in public - - -def test_failed_request_gate_consumes_slot_but_retains_zero_transport_invocations( - staged, monkeypatch): - monkeypatch.setattr(job.admission._Admission, "authorize", lambda *_: False) - seen, transport = [], Mock() - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), - launch=launcher([], seen, transport=transport)) - assert seen == ["A2"] and transport.call_count == 0 - assert result["arms"]["A2"]["slots_consumed"] == 1 - assert result["arms"]["A2"]["request_completion_known"] is False - summary = job.package(staged.output) - assert summary["diagnostics"]["A2"]["provider_attempts"] == 0 - assert summary["diagnostics"]["A2"]["reason"] == "request_authorization_denied" - assert summary["diagnostics"]["A3"]["provider_attempts"] is None - assert staged.world.pr.ledger.charged == 162720000 - - -@pytest.mark.parametrize("attempts", [0, 1, 8, True, -1, 9, "1", None]) -def test_closed_outcome_recovers_fixed_files_even_without_execution_summary( - staged, attempts): - root = staged.output / "A2/run" - (root / "worker").mkdir(parents=True) - atomic_write_json(root / "worker/result.json", {"schema": worker.SCHEMA, - "status": "failed", "reason": "PRIVATE-CANARY arbitrary text", - "provider_attempts": attempts, "transport_error_code": "PRIVATE-CANARY", - "environment": {"secret": "PRIVATE-CANARY"}}) - atomic_write_json(root / "supervision.json", {"schema": "kv-worker-supervision-v1", - "execution": {"status": "failed", "returncode": -15, "elapsed_seconds": 1.5, - "metric": 99, "argv": ["PRIVATE-CANARY"], "error": "PRIVATE-CANARY"}}) - assert not (staged.output / "execution.json").exists() - summary = job.package(staged.output) - diagnostic = summary["diagnostics"]["A2"] - assert diagnostic["reason"] == diagnostic["transport_error_code"] == "unknown" - assert diagnostic["provider_attempts"] == ( - attempts if type(attempts) is int and 0 <= attempts <= 8 else None) - assert diagnostic["execution"] == { - "status": "failed", "returncode": -15, "elapsed_seconds": 1.5} - assert summary["diagnostics"]["A3"]["provider_attempts"] is None - assert "PRIVATE-CANARY" not in provider.canonical(summary) - assert summary["artifacts"]["A2"]["status"] == "missing_or_invalid" - - -@pytest.mark.parametrize("data", [b"not-json private", b"[]", b'{"schema":"wrong"}']) -def test_missing_or_malformed_worker_result_does_not_infer_zero(staged, data): - root = staged.output / "A2/run" - write(root, "worker/result.json", data) - diagnostic = job.package(staged.output)["diagnostics"]["A2"] - assert diagnostic == {"worker_status": "unavailable", "reason": None, - "provider_attempts": None, "transport_error_code": None, - "execution": {"status": "unavailable", "returncode": None, "elapsed_seconds": None}} - - -def test_prepared_input_mutation_blocks_before_reservation(staged): - (staged.output / "A2/workspace/task.json").write_text("changed") - with pytest.raises(ValueError, match="inputs changed"): - job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), launch=Mock()) - assert staged.world.events == [] - - -def test_stage_rejects_sibling_leak_or_nonidentical_common_bytes(staged, tmp_path): - value = copy.deepcopy(staged.plan) - value["actors"]["A2"]["workspace"].append({"path": "knowledge/SKILL.md"}) - with pytest.raises(ValueError, match="allowlist"): - job._stage(value, tmp_path / "bad-stage") - - -@pytest.mark.parametrize("credential", [provider.KEY_ENV, "GITHUB_TOKEN"]) -def test_final_grade_rejects_injected_authority(tmp_path, monkeypatch, credential): - monkeypatch.setenv(credential, "OFFLINE-FAKE") - with pytest.raises(ValueError, match="without injected"): - job.grade(tmp_path / "input", tmp_path / "output") - - -def test_immutable_final_bundle_rejects_wrong_run_before_grading(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setenv("GITHUB_RUN_ID", "124") - result = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["reason"] == "invalid_execution_identity" for a in result["arms"].values()) - - -@pytest.mark.parametrize("now,seconds", [("2026-10-30T15:59:00", 7200), - ("2026-10-30T12:00:00", 1319)]) -def test_no_partial_pair_before_expiry_or_shanghai_midnight(now, seconds): - now = datetime.fromisoformat(now).replace(tzinfo=timezone.utc) - with pytest.raises(ValueError, match="runway"): - job.runway(now.timestamp() + seconds, now=now) - - -@pytest.mark.parametrize("method,path,data", [("POST", "/git/refs", {}), - ("PATCH", "/git/refs/heads/main", {"sha": "a" * 40, "force": False}), - ("PATCH", "/git/refs/heads/skill-eval-budget/1409534281/999", - {"sha": "a" * 40, "force": True}), ("GET", "/pulls/1?x=1", None)]) -def test_controller_api_rejects_scope_before_http(method, path, data): - opener = Mock() - api = job.DeadlineGitHub("OFFLINE-FAKE", time.monotonic() + 30, opener=opener) - with pytest.raises(ValueError, match="scope"): - api.request(method, path, data) - opener.open.assert_not_called() - - -def test_workflow_manual_no_inputs_execute_only_credentials_and_separate_pinned_grade(): - path = Path(__file__).parents[1] / ".github/workflows/kv-edit-pilot.yml" - text = path.read_text() - value = yaml.load(text, Loader=yaml.BaseLoader) - assert value["on"] == {"workflow_dispatch": ""} - assert value["permissions"] == {} - assert value["concurrency"]["cancel-in-progress"] == "false" - assert text.count("secrets.") == 1 - assert "env" not in value - execution = value["jobs"]["execute"] - bound = [s for s in execution["steps"] if " execute --output " in s.get("run", "")] - assert len(bound) == 1 - assert bound[0]["env"] == { - "PYTHONPATH": "src", "GITHUB_TOKEN": "${{ github.token }}", - "SKILL_EVAL_YIBUAPI_KEY": "${{ secrets.SKILL_EVAL_YIBUAPI_KEY }}", - } - prepare = next(s for s in execution["steps"] if " prepare --output " in s.get("run", "")) - assert execution["steps"].index(prepare) < execution["steps"].index(bound[0]) - for name, job_value in value["jobs"].items(): - assert "env" not in job_value - condition = job_value["if"] - for guard in ("github.repository", "github.ref", "github.run_attempt", "github.actor", - "github.triggering_actor"): - assert guard in condition - for step in job_value["steps"]: - if step is not bound[0]: - assert not {"GITHUB_TOKEN", provider.KEY_ENV} & set(step.get("env", {})) - assert "secrets." not in provider.canonical(step) - if "uses" in step: - assert policy._hex(step["uses"].split("@")[1], 40) - if step.get("uses", "").startswith("actions/checkout@"): - assert step["with"]["persist-credentials"] == "false" - assert step["with"]["ref"] == "${{ github.sha }}" - grade = value["jobs"]["grade"] - assert grade["permissions"]["contents"] == "read" - download = next(s for s in grade["steps"] if s.get("uses", "").startswith("actions/download")) - assert set(download["with"]) == {"name", "path"} # Current-run only; no URL/run-id input. - assert download["with"]["name"] == "kv-live-${{ github.run_id }}-${{ github.run_attempt }}" - projection = next(s for s in grade["steps"] if "package-grade" in s.get("run", "")) - assert projection["if"] == "always()" and "env -i" in projection["run"] - upload = next(s for s in grade["steps"] if s.get("uses", "").startswith("actions/upload")) - assert upload["with"]["path"] == "${{ runner.temp }}/kv-grade/public" - operations = (path.parents[2] / "docs/operations.md").read_text() - assert "`package-grade` never reruns candidates or models" in operations - - -def test_same_host_preflight_uses_real_scoped_scripted_loop_with_fake_sandbox(staged, monkeypatch): - from tests.test_kv_edit_pilot import in_process_worker - original = worker.scripted.run_supervised - monkeypatch.setattr(worker.scripted, "run_supervised", lambda *a, **kw: - original(*a, **kw, execute=in_process_worker)) - monkeypatch.delenv(provider.KEY_ENV) - result = job.replay_preflight(staged.output, time.monotonic() + 120) - assert set(result) == {"A2", "A3"} - assert all(v["model_calls"] == 0 and v["verdict"] == "PASS" for v in result.values()) - - -def test_grade_rejects_symlink_source_as_inconclusive(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - candidate = staged.output / "public/A2/candidate.py" - candidate.unlink() - candidate.symlink_to(staged.output / "A2/run/worker/final/candidate.py") - monkeypatch.delenv(provider.KEY_ENV) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["verdict"] == "INCONCLUSIVE" - assert result["arms"]["A3"]["semantic"]["verdict"] == "PASS" - - -def test_missing_entire_artifact_bundle_retains_both_arms_as_inconclusive(staged, monkeypatch): - monkeypatch.delenv(provider.KEY_ENV) - result = job.grade(staged.output / "missing", staged.output / "grade") - assert set(result["arms"]) == {"A2", "A3"} - assert all(v["verdict"] == "INCONCLUSIVE" for v in result["arms"].values()) - assert (staged.output / "grade/grade.json").is_file() - - -def test_uncertain_admission_retains_identity_without_zero_cost_claim(staged, monkeypatch): - staged.world.daily.outcomes = ["unknown_commit"] - with pytest.raises(job.policy.budget.BudgetError): - job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), launch=Mock()) - assert staged.world.daily.ledger.charged_on("2026-10-30") == 654070000 - assert staged.world.pr.ledger.charged == 0 - result = job.package(staged.output) - assert result["execution"]["admission_outcome"].endswith("unknown_no_retry") - assert result["execution"]["reserved_usd"] is None - assert result["execution"]["actual_cost_usd"] is None - monkeypatch.delenv(provider.KEY_ENV) - grade = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["verdict"] == "INCONCLUSIVE" for a in grade["arms"].values()) - - -@pytest.mark.parametrize("field,value", [("execution_id", "wrong-execution"), ("revision", 999), - ("artifact", "wrong.py"), ("check_receipt", "0" * 64), ("benchmark_receipt", "0" * 64)]) -def test_corrupt_final_receipt_cannot_be_packaged_or_graded(staged, monkeypatch, field, value): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - path = staged.output / "A2/run/worker/final/receipt.json" - receipt = policy.read_json(path) - receipt[field] = value - atomic_write_json(path, receipt) - summary = job.package(staged.output) - assert summary["artifacts"]["A2"]["status"] == "missing_or_invalid" - monkeypatch.delenv(provider.KEY_ENV) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["verdict"] == "INCONCLUSIVE" - - -def test_corrupted_feedback_bytes_are_not_current_revision_evidence(staged): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - path = staged.output / "A2/run/worker/feedback/0001.json" - receipt = policy.read_json(path) - receipt["feedback"]["reason"] = "check_failed" - atomic_write_json(path, receipt) - summary = job.package(staged.output) - assert summary["artifacts"]["A2"]["status"] == "missing_or_invalid" - - -def test_final_grade_uses_captured_public_freeze_not_revoked_live_comment(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - def offline_only(*, fetch, now): - assert now == 0 - assert fetch(123)["body"] == "offline frozen public fixture" - return staged.authorization, {"offline": True} - monkeypatch.setattr(policy, "frozen_authorization", offline_only) - monkeypatch.setattr(policy, "fetch_comment", Mock(side_effect=AssertionError("no live fetch"))) - result = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["semantic"]["verdict"] == "PASS" for a in result["arms"].values()) - - -@pytest.mark.parametrize("failure", ["exception", "duplicate_keys", "oversize", "late"]) -def test_controller_http_response_failure_is_bounded_redacted_and_never_retried(failure): - class Response: - def __enter__(self): - return self - def __exit__(self, *_): - return None - def read1(self, size): - if failure == "duplicate_keys": - raw, self.raw = getattr(self, "raw", b'{"x":1,"x":2}'), b"" - return raw - if failure == "late": - api.deadline = time.monotonic() - 1 - return b"{}" - return b"x" * size - opener = Mock() - opener.open.side_effect = (OSError("OFFLINE-FAKE never retain") - if failure == "exception" else None) - opener.open.return_value = Response() - api = job.DeadlineGitHub("OFFLINE-FAKE", time.monotonic() + 30, opener=opener) - with pytest.raises(ValueError, match="response unavailable") as error: - api.request("GET", "") - assert "OFFLINE-FAKE" not in str(error.value) - assert opener.open.call_count == 1 - - -def test_pre_execution_rejection_retains_inconclusive_grade(staged, monkeypatch): - summary = job.package(staged.output) # Successful prepare but execute was never admitted. - assert summary["execution"] == {} - monkeypatch.delenv(provider.KEY_ENV) - grader = Mock(side_effect=AssertionError("unbound candidates must not be graded")) - monkeypatch.setattr(job.workbench, "grade_final", grader) - result = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["reason"] == "invalid_execution_identity" for a in result["arms"].values()) - assert (staged.output / "grade/grade.json").is_file() - grader.assert_not_called() - - -@pytest.mark.parametrize("change", ["none", "inventory", "mounts", "inconclusive"]) -def test_same_host_runtime_must_match_actual_qualified_profile(tmp_path, monkeypatch, change): - import sys - - import tools - expected = {"verdict": "PASS", "inventory_sha256": "a" * 64, "mount_count": 4} - actual = dict(expected) - if change == "inventory": - actual["inventory_sha256"] = "b" * 64 - elif change == "mounts": - actual["mount_count"] = 5 - elif change == "inconclusive": - actual["verdict"] = "INCONCLUSIVE" - runtime = SimpleNamespace(qualify=lambda *a, **kw: actual) - monkeypatch.setitem(sys.modules, "tools.kv_runtime_qualification", runtime) - monkeypatch.setattr(tools, "kv_runtime_qualification", runtime, raising=False) - monkeypatch.setattr(policy, "validate_qualification", lambda _: {"runtime_isolation": expected, - "dependencies": {"src/inferrsi/harness/skill_eval_engineering.py": "c" * 64}}) - if change in {"none", "inventory"}: - result = job.runtime_preflight(tmp_path, time.monotonic() + 30, {}) - assert result["verdict"] == "PASS" - assert result["inventory_sha256"] == actual["inventory_sha256"] - assert result["qualified_inventory_sha256"] == expected["inventory_sha256"] - assert result["profile_source_sha256"] == "c" * 64 - else: - with pytest.raises(ValueError, match="runtime/profile"): - job.runtime_preflight(tmp_path, time.monotonic() + 30, {}) - - -def retained_fake_sandbox(source, payload, output, deadline): - record, actual = fake_sandbox(source, payload, output, deadline) - # Mimic the real executor's fixed filenames, including deliberately excluded text. - (output / "stdout.log").write_bytes(provider.canonical(actual).encode()) - (output / "stderr.log").write_text("OFFLINE-PRIVATE-CANARY") - (output / "unrequested.json").write_text('"OFFLINE-PRIVATE-CANARY"') - atomic_write_json(output / "execution.json", {"status": record["status"], - "returncode": 0, "elapsed_seconds": record["elapsed_seconds"]}) - return {**record, "returncode": 0, "argv": ["OFFLINE-PRIVATE-CANARY"], - "cwd": "OFFLINE-PRIVATE-CANARY", "error": "OFFLINE-PRIVATE-CANARY"}, actual - - -def test_final_artifact_reconstructs_original_semantics_and_abba_samples(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(kv, "invoke", retained_fake_sandbox) - result = job.grade(staged.output / "public", staged.output / "grade") - public = staged.output / "grade/public" - manifest = policy.read_json(public / "evidence.json") - assert manifest["execution"]["run_id"] == "123" - assert policy.read_json(public / "authorization.json")["body"].startswith("offline") - assert policy.read_json(public / "grade.json") == result - for arm in ("A2", "A3"): - evidence = manifest["arms"][arm] - assert evidence["raw_outputs_complete"] - assert len(evidence["semantics"]) == 1 and len(evidence["performance"]) == 17 - assert evidence["semantics"][0]["check"] is True - assert evidence["semantics"][0]["execution"] == { - "status": "completed", "returncode": 0, "elapsed_seconds": 1.0} - for row in evidence["semantics"] + evidence["performance"]: - raw = (public / row["output"]["path"]).read_bytes() - assert policy.sha(raw) == row["output"]["sha256"] - assert len(raw) == row["output"]["bytes"] - assert policy.fingerprint(json.loads(raw)) == row["output"]["data_sha256"] - assert [r["side"] for r in evidence["performance"][:4]] == [ - "baseline", "candidate", "candidate", "baseline"] - for side in ("baseline", "candidate"): - observed = [r["execution"]["metric"] for r in evidence["performance"] - if r["side"] == side and r["phase"] == "measurement"] - assert observed == result["arms"][arm]["performance"][side + "_values"] - assert all(r["source_verified"] and r["execution"]["returncode"] == 0 - for r in evidence["performance"]) - assert all(r["output"]["data_sha256"] == r["expected_sha256"] - for r in evidence["performance"]) - published = b"".join(p.read_bytes() for p in public.rglob("*") if p.is_file()) - assert b"OFFLINE-PRIVATE-CANARY" not in published - assert b"OFFLINE-FAKE-PROVIDER-KEY" not in published - assert str(staged.output).encode() not in published - assert not list(public.rglob("*.log")) - - -@pytest.mark.parametrize("bad", [b'not JSON OFFLINE-PRIVATE-CANARY', - b'{"observations":[],"pending":[],"extra":"OFFLINE-PRIVATE-CANARY"}', - b'{"observations":[],"pending":[["OFFLINE-PRIVATE-CANARY",0,[]]]}', - b'{"observations":[],"pending":[],"pending":[]}', - b'{"observations":[],"pending":[[[],0,[]]]}']) -def test_raw_projection_never_uploads_arbitrary_or_malformed_output(tmp_path, bad): - source, target = tmp_path / "raw", tmp_path / "published.json" - source.write_bytes(bad) - result = job._raw_kv_output(source, target, [], verify=True, remaining=[1024]) - assert result["status"] == "redacted_invalid_output" - assert not target.exists() - assert result["sha256"] == policy.sha(bad) and result["bytes"] == len(bad) - assert result["truncated"] is False - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(result) - - -@pytest.mark.parametrize("unsafe", ["symlink", "oversize", "whole_budget"]) -def test_raw_projection_file_and_total_limits(tmp_path, unsafe): - source, target = tmp_path / "raw", tmp_path / "published.json" - good = b'{"observations":[],"pending":[]}' - if unsafe == "symlink": - (tmp_path / "other").write_bytes(good) - source.symlink_to(tmp_path / "other") - else: - source.write_bytes(good if unsafe == "whole_budget" else b" " * (4 * policy.MAX_JSON + 1)) - result = job._raw_kv_output(source, target, [], verify=True, remaining=[1]) - assert result["status"] != "retained" and not target.exists() - - -def test_all_frozen_hidden_expected_outputs_fit_public_projection(tmp_path): - # Real frozen DATA only; this does not execute a candidate or re-author the oracle. - root = Path(__file__).parents[1] - frozen = policy.read_json(root / ( - "evidence/kv-reclamation-20261030/validator/independent/fixtures.json")) - traces = [t for t in frozen["traces"] if t["split"] == "grader"] - assert len(traces) == 77 - remaining = [32 * policy.MAX_JSON] - for index, trace in enumerate(traces): - expected = {"observations": trace["expected"], - "pending": trace["expected"][-1]["pending"] if trace["expected"] else []} - raw = provider.canonical(expected).encode() - source, target = tmp_path / "source", tmp_path / f"trace-{index:03d}.json" - source.write_bytes(raw) - result = job._raw_kv_output(source, target, trace["events"], - verify=True, remaining=remaining) - assert result["status"] == "retained", trace["id"] - assert target.read_bytes() == raw - - -def test_partial_preflight_retains_failed_stage_without_exception_text(staged, monkeypatch): - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(policy, "fetch_comment", lambda _: {"id": 123, "body": "offline", - "issue_url": "https://example.invalid/public", "user": {"login": "owner", "type": "User"}}) - def frozen(*, fetch=None, **kwargs): - if fetch is not None: - fetch(123) - return staged.authorization, {"offline": True} - monkeypatch.setattr(policy, "frozen_authorization", frozen) - monkeypatch.setattr(kv, "probe_isolation", - Mock(side_effect=ValueError("OFFLINE-PRIVATE-CANARY"))) - fresh = staged.output.parent / "failed-prepare" - with pytest.raises(ValueError): - job.prepare(fresh) - job.package(fresh) - proof = policy.read_json(fresh / "public/preflight.json") - assert proof["status"] == "partial" and proof["stages"]["stage"] == "completed" - assert proof["stages"]["isolation"] == "failed_or_interrupted" - assert proof["stages"]["cleanup"] == "not_started" - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(proof) - - -def test_preflight_projection_excludes_arbitrary_nested_text(staged): - path = staged.output / "prepared.json" - value = policy.read_json(path) - value["runtime_profile"].update(inventory_sha256="a" * 64, - error="OFFLINE-PRIVATE-CANARY", environment={"KEY": "OFFLINE-PRIVATE-CANARY"}, - probe={"network_blocked": True, "inaccessible": [True, False], - "unexpected": "OFFLINE-PRIVATE-CANARY"}) - value["cleanup"]["cases"] = {mode: {"verdict": "FAIL", "surviving_descendants": 1, - "error": "OFFLINE-PRIVATE-CANARY"} for mode in ("normal", "timeout", "cancel")} - atomic_write_json(path, value) - job.package(staged.output) - proof = policy.read_json(staged.output / "public/preflight.json") - assert proof["proofs"]["runtime_profile"]["probe"]["inaccessible"] == [True, False] - assert proof["proofs"]["cleanup"]["cases"]["normal"]["surviving_descendants"] == 1 - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(proof) - - -def test_interrupted_grader_keeps_first_outputs_and_explicit_unstarted_arm(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(kv, "invoke", retained_fake_sandbox) - original = job.workbench.grade_final - def interrupted(*args): - original(*args) - raise KeyboardInterrupt - monkeypatch.setattr(job.workbench, "grade_final", interrupted) - with pytest.raises(KeyboardInterrupt): - job.grade(staged.output / "public", staged.output / "grade") - public = staged.output / "grade/public" - result = policy.read_json(public / "grade.json") - assert result["arms"]["A2"]["reason"] == "grading_interrupted" - assert result["arms"]["A3"]["reason"] == "not_evaluated" - evidence = policy.read_json(public / "evidence.json")["arms"] - assert evidence["A2"]["semantics"][0]["output"]["status"] == "retained" - assert not evidence["A3"]["raw_outputs_complete"] - - -def test_always_projection_recovers_failed_projection_without_any_reexecution(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(kv, "invoke", retained_fake_sandbox) - original = job._raw_kv_output - monkeypatch.setattr(job, "_raw_kv_output", - Mock(side_effect=RuntimeError("projection interrupted"))) - with pytest.raises(RuntimeError, match="projection interrupted"): - job.grade(staged.output / "public", staged.output / "grade") - public = staged.output / "grade/public" - assert policy.read_json(public / "projection.json")["status"] == "failed_or_interrupted" - first = (staged.output / "grade/A2/semantics/trace-000/stdout.log").read_bytes() - monkeypatch.setattr(job, "_raw_kv_output", original) - monkeypatch.setattr(kv, "invoke", Mock(side_effect=AssertionError("no candidate rerun"))) - result = job.package_grade(staged.output / "public", staged.output / "grade") - assert all(arm["raw_outputs_complete"] for arm in result["arms"].values()) - assert (public / "A2/semantics/trace-000.json").read_bytes() == first - assert policy.read_json(public / "projection.json")["status"] == "completed" - grade = policy.read_json(public / "grade.json") - assert grade["arms"]["A2"]["semantic"]["verdict"] == "PASS" - assert "verdict" not in grade["arms"]["A2"] # No invented aggregate verdict. - - -def test_always_projection_keeps_raw_partial_output_without_finally(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - grade = staged.output / "grade" - retained_fake_sandbox(BASELINE, {"mode": "verify", "events": []}, - grade / "A2/semantics/trace-000", time.monotonic() + 10) - monkeypatch.setattr(kv, "invoke", Mock(side_effect=AssertionError("no candidate rerun"))) - manifest = job.package_grade(staged.output / "public", grade) - assert manifest["arms"]["A2"]["semantics"][0]["output"]["status"] == "retained" - assert manifest["arms"]["A2"]["semantics"][0]["check"] is None - assert not manifest["arms"]["A2"]["raw_outputs_complete"] - assert all(arm["verdict"] == "INCONCLUSIVE" - for arm in policy.read_json(grade / "public/grade.json")["arms"].values()) - - -def test_grade_projection_retains_failed_and_skipped_timing_rows(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - def failed(source, payload, output, deadline): - record, actual = retained_fake_sandbox(source, payload, output, deadline) - if payload["mode"] == "benchmark": - record.update(status="timeout", returncode=-9) - return record, actual - monkeypatch.setattr(kv, "invoke", failed) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["performance"]["verdict"] == "INCONCLUSIVE" - rows = policy.read_json(staged.output / "grade/public/evidence.json")["arms"]["A2"] - assert len(rows["performance"]) == 17 - assert rows["performance"][0]["execution"]["status"] == "timeout" - assert rows["performance"][0]["execution"]["returncode"] == -9 - assert all(row["execution"]["status"] == "unavailable" for row in rows["performance"][1:]) - assert not rows["raw_outputs_complete"] - - -@pytest.mark.parametrize("corruption", ["run", "locator", "workspace"]) -def test_preflight_projection_rejects_unbound_or_private_prepared_fields(staged, corruption): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - path = staged.output / "prepared.json" - value = policy.read_json(path) - if corruption == "run": - value["run_id"] = "9999" - elif corruption == "locator": - value["locator_sha256"] = "0" * 64 - else: - value["inputs"]["A2/workspace/task.json"] = "OFFLINE-PRIVATE-CANARY" - atomic_write_json(path, value) - job.package(staged.output) - projected = policy.read_json(staged.output / "public/preflight.json") - assert projected["status"] != "prepared" - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(projected) - - -def test_always_projection_rejects_corrupt_grade_text_and_false_pass(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - output = staged.output / "grade" - output.mkdir() - atomic_write_json(output / "grade.json", {"schema": "kv-live-final-grade-v1", - "plan_sha256": staged.authorization["plan_sha256"], "scope": "OFFLINE-PRIVATE-CANARY", - "arms": {a: {"reason": "OFFLINE-PRIVATE-CANARY", "semantic": { - "verdict": "PASS", "checks": {TRACE["id"]: False}, "artifact_sha256": "a" * 64, - "environment": "OFFLINE-PRIVATE-CANARY"}} for a in ("A2", "A3")}}) - monkeypatch.setattr(kv, "invoke", Mock(side_effect=AssertionError("no candidate rerun"))) - job.package_grade(staged.output / "public", output) - raw = (output / "public/grade.json").read_bytes() - assert b"OFFLINE-PRIVATE-CANARY" not in raw - assert all(a["verdict"] == "INCONCLUSIVE" for a in json.loads(raw)["arms"].values()) - - -@pytest.mark.parametrize("credential", [provider.KEY_ENV, "GITHUB_TOKEN"]) -def test_always_projection_rejects_injected_secrets(tmp_path, monkeypatch, credential): - monkeypatch.setenv(credential, "OFFLINE-FAKE") - with pytest.raises(ValueError, match="without injected"): - job.package_grade(tmp_path / "bundle", tmp_path / "output") - - -@pytest.mark.parametrize("receipt", ["failed", "absent", "unexpected_fields"]) -def test_trace_exit_failure_cannot_be_upgraded_by_correct_stdout(staged, monkeypatch, receipt): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - def failure(source, payload, output, deadline): - record, actual = retained_fake_sandbox(source, payload, output, deadline) - if payload["mode"] != "verify": - return record, actual - record.update(status="failed", returncode=1) - path = output / "execution.json" - if receipt == "failed": - atomic_write_json(path, {k: record[k] for k in - ("status", "returncode", "elapsed_seconds")}) - elif receipt == "absent": - path.unlink() # Process loss before first receipt cannot imply successful exit. - else: - atomic_write_json(path, {**record, "secret": "OFFLINE-PRIVATE-CANARY"}) - return record, None # Real invoke rejects a nonzero exit even with valid JSON. - monkeypatch.setattr(kv, "invoke", failure) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["semantic"]["verdict"] == "FAIL" - row = policy.read_json(staged.output / "grade/public/evidence.json")[ - "arms"]["A2"]["semantics"][0] - assert row["output"]["status"] == "retained" and row["check"] is False - assert row["execution"]["status"] == ("failed" if receipt == "failed" else "unavailable") - if receipt == "failed": - assert row["execution"]["returncode"] == 1 - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(row) - - -def test_fake_second_pair_preserves_first_claim_and_cannot_launch_third(staged, monkeypatch): - from dataclasses import replace - - from tests.test_kv_pilot_admission import additional_world - world, first, second = additional_world() - staged.world = world - extra = replace(second.additional_attempt, execution_id=staged.plan['execution_id']) - second = replace(second, execution_id=extra.execution_id, additional_attempt=extra, - plan_sha256=staged.authorization['plan_sha256'], - actor_sha256=tuple(staged.configs[a]['actor_sha256'] for a in ('A2', 'A3'))) - monkeypatch.setattr(policy, 'policy', lambda *_: second) - seen = [] - scripts = [read_task(), work(CANDIDATE), work(CANDIDATE + '# second\n', '2'), submit()] - launch = launcher(scripts, seen) - def after_both_claims(*args, **kwargs): - assert len(world.events) == 8 - assert world.pr.ledger.charged == 325_440_000 - assert world.pr.ledger.entries[0].binding == first.binding - return launch(*args, **kwargs) - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(world.daily, world.pr), launch=after_both_claims) - assert seen == ['A2', 'A3'] - assert all(a['status'] == 'completed' for a in result['arms'].values()) - assert world.daily.ledger.charged_on('2026-10-30') == 816_790_000 - summary = job.package(staged.output) - assert all(len(r) == 4 for r in summary['requests'].values()) - assert all(a['status'] == 'submitted' for a in summary['artifacts'].values()) - with pytest.raises(policy.budget.BudgetError): - world.admit(second) - assert seen == ['A2', 'A3'] and len(world.events) == 8 - - -def test_controller_supplies_only_root_bound_limit_to_anchored_pr_store(staged, monkeypatch): - from tests.test_kv_pilot_admission import additional_world - world, _, second = additional_world() - monkeypatch.setattr(policy, 'policy', lambda *_: second) - staged.plan['ledger'] = {'ref': 'refs/heads/skill-eval-budget/123/999', - 'genesis_commit': 'a' * 40, 'checkpoint_commit': 'b' * 40} - staged.plan['daily_ledger'] = {'ref': 'refs/heads/skill-eval-budget/1409534281/daily', - 'genesis_commit': 'c' * 40, 'checkpoint_commit': 'd' * 40} - captured = [] - monkeypatch.setattr(job, 'GitHubDailyStore', lambda api, anchor: world.daily) - def pr_store(api, anchor, **kwargs): - captured.append((anchor, kwargs)) - raise ValueError('offline stop before admission') - monkeypatch.setattr(job, 'GitHubGitStore', pr_store) - with pytest.raises(ValueError, match='offline stop'): - job.execute(staged.output, api=SimpleNamespace()) - assert captured[0][0].key == second.key - assert captured[0][1] == {'trusted_limit': second.trusted_limit} - assert len(world.events) == 4 # Only the synthetic previous pair, no new write. diff --git a/tests/test_kv_live_policy.py b/tests/test_kv_live_policy.py deleted file mode 100644 index 00da1f98b..000000000 --- a/tests/test_kv_live_policy.py +++ /dev/null @@ -1,300 +0,0 @@ -"""All identities, files, API reads and clock values below are offline fixtures.""" - -import copy -import json -import zipfile -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from tests.test_kv_pilot_admission import NOW -from tests.test_kv_pilot_admission import policy as sample_policy -from tools import kv_live_policy as live - - -def write(root, name, value): - data = json.dumps(value).encode() - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return {"path": name, "bytes": len(data), "sha256": live.sha(data)} - - -@pytest.fixture -def case(tmp_path, monkeypatch): - monkeypatch.setattr(live, "ROOT", tmp_path) - monkeypatch.setattr(live, "LOCATOR", tmp_path / "locator.json") - monkeypatch.setattr(live, "DEPENDENCIES", ("code.py",)) - monkeypatch.setattr(live.scripted, "dependencies", lambda: {}) - monkeypatch.setattr(live, "QUALIFICATION_DEPENDENCIES", {"code.py"}) - (tmp_path / "code.py").write_text("# offline code fixture\n") - sources = {"code.py": live.sha((tmp_path / "code.py").read_bytes())} - monkeypatch.setattr(live, "qualification_dependencies", lambda: sources) - actor_config = {"actor_sha256": "4" * 64, "edit_plan": {"files": []}, - "prompt": "offline prompt", "arm": "A2", "model": "claude-opus-5-5", - "client": {}, "input_token_ceiling": 1000000, - "suite_sha256": "7" * 64, "baseline_sha256": "8" * 64} - actor = write(tmp_path, "actor.json", actor_config) - evidence = b"offline data-only archive fixture" - qualified = write(tmp_path, "qualified.json", { - "schema": "kv-scripted-actor-qualification-v1", "revision": "6" * 40, - "verdict": "PASS", "model_calls": 0, "provider_charges_usd": 0, - "dependencies": sources, "dependencies_sha256": live.fingerprint(sources), - "software_checks_passed": True, "source_unchanged": True, - "cleanup": {"verdict": "PASS"}, - "runtime_isolation": {"verdict": "PASS", "inventory_sha256": "9" * 64, - "mount_count": 4}, - "raw_evidence": {"sha256": live.sha(evidence)}, - "arms": {arm: {"executor_status": "completed", "worker_status": "completed", - "request_completion_known": True, - "simulated_slots_consumed": 4, "visible_workbench_verified": True, - "two_revision_receipts_retained": True, "hidden_correctness": {"verdict": "PASS"}, - "exposure": [], "prompt_sha256": live.sha(b"offline prompt"), - "common_actor_contract_sha256": live.inputs.common_actor_contract_sha256(actor_config)} - for arm in ("A2", "A3")}}) - with zipfile.ZipFile(tmp_path / "qualified.zip", "w") as archive: - archive.writestr("qualification.json", (tmp_path / "qualified.json").read_bytes()) - archive.writestr("raw-evidence.zip", evidence) - raw_archive = (tmp_path / "qualified.zip").read_bytes() - archive_record = {"path": "qualified.zip", "bytes": len(raw_archive), - "sha256": live.sha(raw_archive)} - suite = write(tmp_path, "suite.json", {}) - plan = {"schema": "kv-live-plan-v1", "repository_id": live.REPOSITORY_ID, "issue": 248, - "campaign_id": "offline-campaign", "execution_id": "offline-execution", - "charge_pr": 999, "charge_head": "2" * 40, "expires_at": int(NOW.timestamp()) + 7200, - "provider": json.loads(sample_policy().provider_json), "dependencies": sources, - "actors": {arm: {"config": actor, "workspace": [{"path": "candidate.py"}]} - for arm in ("A2", "A3")}, - "ledger": {"ref": f"refs/heads/skill-eval-budget/{live.REPOSITORY_ID}/999", - "genesis_commit": "3" * 40, "checkpoint_commit": "3" * 40}, - "daily_ledger": {"ref": f"refs/heads/skill-eval-budget/{live.REPOSITORY_ID}/daily", - "genesis_commit": "5" * 40, "checkpoint_commit": "5" * 40}, - "qualification": {"run_id": 123, "run_attempt": 1, "head": "6" * 40, - "revision": "6" * 40, "artifact_id": 456, - "archive": archive_record, "receipt": qualified, - "dependencies_sha256": live.fingerprint(sources)}, - "suite": suite, "final_suite": suite, "hidden_traces": suite, - "arm_order": ["A2", "A3"], "scope": "synthetic feasibility only", - "price_evidence": "offline synthetic price fixture", - "prior_owner_group_statement": "offline fixture, not live attestation", - "final_seconds": 180, "final_repetitions": 8} - authorization = {"schema": "kv-live-authorization-v1", "plan_sha256": live.fingerprint(plan), - "plan": plan} - body = live.PREFIX + json.dumps(authorization) + live.SUFFIX - locator = {"schema": "kv-live-authorization-locator-v1", "enabled": True, - "repository_id": live.REPOSITORY_ID, "issue": 248, "comment_id": 12345, - "author_login": live.OPERATOR, "body_sha256": live.sha(body.encode())} - write(tmp_path, "locator.json", locator) - comment = {"id": 12345, "body": body, "user": {"login": live.OPERATOR, "type": "User"}, - "issue_url": f"https://api.github.com/repos/{live.REPOSITORY}/issues/248"} - return SimpleNamespace(root=tmp_path, plan=plan, locator=locator, comment=comment, - authorization=authorization) - - -def test_exact_frozen_public_authorization_has_no_spending_side_effect(case): - actual = live.frozen_authorization(fetch=lambda n: case.comment, now=NOW.timestamp()) - assert actual == (case.authorization, case.locator) - assert live.policy(case.plan, "7" * 40).binding.worst_case == 162720000 - - -@pytest.mark.parametrize("field,value", [ - ("id", 12346), ("body", "changed"), ("issue_url", "https://example.invalid/other"), - ("user", {"login": "other", "type": "User"}), - ("user", {"login": live.OPERATOR, "type": "Bot"}), -]) -def test_comment_identity_hash_and_author_are_not_interchangeable(case, field, value): - case.comment[field] = value - with pytest.raises(ValueError, match="identity"): - live.frozen_authorization(fetch=lambda n: case.comment, now=NOW.timestamp()) - - -@pytest.mark.parametrize("change", ["expired", "source", "qualifier", "missing_dependency", - "amount", "requests", "charge_ref", "unknown_field"]) -def test_bad_frozen_plan_fails_before_any_request_or_ledger(case, change): - plan = copy.deepcopy(case.plan) - if change == "expired": - plan["expires_at"] = 1 - elif change == "source": - (case.root / "code.py").write_text("changed") - elif change == "qualifier": - plan["qualification"]["dependencies_sha256"] = "0" * 64 - elif change == "missing_dependency": - plan["dependencies"].clear() - elif change == "amount": - plan["provider"]["spend_limit"]["amount"] = "201" - elif change == "requests": - plan["provider"]["client"]["max_requests"] = 9 - elif change == "charge_ref": - plan["ledger"]["ref"] += "1" - else: - plan["secret"] = "offline-not-allowed" - with pytest.raises((ValueError, live.budget.BudgetError)): - live.validate_plan(plan, now=NOW.timestamp()) - - -def test_disabled_default_does_not_fetch_comment(tmp_path, monkeypatch): - monkeypatch.setattr(live, "LOCATOR", tmp_path / "locator.json") - write(tmp_path, "locator.json", {"schema": "kv-live-authorization-locator-v1", - "enabled": False}) - with pytest.raises(ValueError, match="disabled"): - live.frozen_authorization(fetch=lambda _: pytest.fail("must not fetch")) - - -@pytest.mark.parametrize("name,value", [("GITHUB_EVENT_NAME", "pull_request"), - ("GITHUB_REF", "refs/heads/other"), ("GITHUB_RUN_ATTEMPT", "2"), - ("GITHUB_REPOSITORY", "other/repo"), ("GITHUB_ACTOR", "other"), - ("GITHUB_TRIGGERING_ACTOR", "other")]) -def test_runtime_rejects_other_entry_points(name, value, monkeypatch): - env = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", - "GITHUB_REPOSITORY": live.REPOSITORY, "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_ACTOR": live.OPERATOR, "GITHUB_TRIGGERING_ACTOR": live.OPERATOR, - "GITHUB_SHA": "a" * 40, "GITHUB_RUN_ID": "123"} - env[name] = value - with pytest.raises(ValueError): - live.runtime(env) - - -def test_live_current_main_charge_head_run_and_qualification(case): - repo = {"id": live.REPOSITORY_ID} - calls = {"": repo, "/git/ref/heads/main": {"ref": "refs/heads/main", - "object": {"type": "commit", "sha": "a" * 40}}, - "/pulls/999": {"number": 999, "head": {"sha": "2" * 40}, "draft": False, - "base": {"ref": "main", "repo": repo}, "merged": True}, - "/actions/runs/1": {"event": "workflow_dispatch", "head_branch": "main", - "head_sha": "a" * 40, "run_attempt": 1, "status": "in_progress", - "path": live.WORKFLOW, "repository": repo, - "actor": {"login": live.OPERATOR}, "triggering_actor": {"login": live.OPERATOR}}, - "/actions/runs/123": {"head_sha": "6" * 40, "path": ".github/workflows/ci-ready.yml", - "status": "completed", "conclusion": "success", "repository": repo, - "run_attempt": 1}, - "/actions/artifacts/456": {"id": 456, "name": "kv-actor-qualification-123-1", - "expired": False, "digest": "sha256:" + case.plan["qualification"]["archive"]["sha256"], - "workflow_run": {"id": 123, "head_sha": "6" * 40, - "head_repository_id": live.REPOSITORY_ID}}} - api = SimpleNamespace(request=lambda method, path: calls[path]) - live.verify_live(api, case.plan, "a" * 40, "1") - for path, field, value in [("/pulls/999", "merged", False), - ("/actions/runs/1", "run_attempt", 2), - ("/actions/runs/123", "conclusion", "failure"), - ("/actions/artifacts/456", "digest", "sha256:" + "0" * 64), - ("/actions/artifacts/456", "name", "kv-qualification-123-1")]: - previous = calls[path][field] - calls[path][field] = value - with pytest.raises(ValueError): - live.verify_live(api, case.plan, "a" * 40, "1") - calls[path][field] = previous - - -@pytest.mark.parametrize("change", ["exposure", "prompt", "suite", "client", "limits", - "receipt", "archive", "dependency"]) -def test_hosted_qualification_cannot_be_swapped_for_generic_ci_or_other_actor(case, change): - plan = copy.deepcopy(case.plan) - if change in {"exposure", "prompt", "suite", "client", "limits"}: - actor = live.read_json(case.root / "actor.json") - if change == "exposure": - actor["edit_plan"]["files"] = [{"path": "other-file"}] - elif change == "suite": - actor["suite_sha256"] = "9" * 64 - elif change == "client": - actor["client"] = {"max_requests": 9} - elif change == "limits": - actor["edit_plan"]["limits"] = {"max_checks": 9} - else: - actor["prompt"] = "different prompt" - for arm in ("A2", "A3"): - plan["actors"][arm]["config"] = write(case.root, "actor.json", actor) - elif change == "receipt": - plan["qualification"]["receipt"] = write(case.root, "other.json", {"verdict": "PASS"}) - elif change == "archive": - plan["qualification"]["archive"]["sha256"] = "0" * 64 - else: - (case.root / "code.py").write_text("changed dependency") - with pytest.raises(ValueError): - live.validate_qualification(plan) - - -def add_exception(case): - from tools import kv_pilot_admission as admission - value = {"schema": "kv-one-additional-attempt-v1", - "key": {"repository_id": live.REPOSITORY_ID, "pr_number": case.plan["charge_pr"]}, - "pr_lifetime_cap_microusd": 350_000_000, - "pilot_lifetime_cap_microusd": 1_000_000_000, "additional_attempts": 1, - "prior_binding": {"plan_digest": "8" * 64, - "run_id": admission.CAMPAIGN_NAMESPACE + "original-campaign", - "worst_case": admission.FULL_RESERVATION_MICROUSD}, - "pr_checkpoint": case.plan["ledger"]["checkpoint_commit"], - "daily_checkpoint": case.plan["daily_ledger"]["checkpoint_commit"], - "campaign_id": case.plan["campaign_id"], "execution_id": case.plan["execution_id"]} - case.plan.update(schema="kv-live-plan-v2", budget_exception=value) - return value - - -def test_only_exact_public_v2_freeze_can_resolve_scoped_cap(case): - value = add_exception(case) - # Changing local plan/exception cannot change the verified public bytes. - assert live.frozen_authorization(fetch=lambda _: case.comment, now=NOW.timestamp())[0][ - "plan"]["schema"] == "kv-live-plan-v1" - case.authorization.update(plan=case.plan, plan_sha256=live.fingerprint(case.plan)) - case.comment["body"] = live.PREFIX + json.dumps(case.authorization) + live.SUFFIX - with pytest.raises(ValueError, match="exact bytes"): - live.frozen_authorization(fetch=lambda _: case.comment, now=NOW.timestamp()) - case.locator["body_sha256"] = live.sha(case.comment["body"].encode()) - write(case.root, "locator.json", case.locator) # Simulated separately reviewed root binding. - authorization, _ = live.frozen_authorization(fetch=lambda _: case.comment, now=NOW.timestamp()) - resolved = live.policy(authorization["plan"], "7" * 40) - assert resolved.trusted_limit.key == live.budget.PRKey(live.REPOSITORY_ID, 999) - assert resolved.trusted_limit.limit_microusd == 350_000_000 - assert resolved.additional_attempt.prior_binding.run_id == value["prior_binding"]["run_id"] - assert resolved.additional_attempt.pr_checkpoint == case.plan["ledger"]["checkpoint_commit"] - assert resolved.reservation(now=NOW.timestamp())["amount"] == "162.72" - - -@pytest.mark.parametrize("field,value", [ - ("key", {"repository_id": 1409534281, "pr_number": 275}), - ("key", {"repository_id": 1, "pr_number": 999}), - ("pr_lifetime_cap_microusd", True), ("pr_lifetime_cap_microusd", 200_000_000), - ("pr_lifetime_cap_microusd", 1_000_000_001), - ("pilot_lifetime_cap_microusd", 1_000_000_001), - ("additional_attempts", True), ("additional_attempts", 2), - ("pr_checkpoint", "9" * 40), ("daily_checkpoint", "9" * 40), - ("campaign_id", "different"), ("execution_id", "different"), - ("prior_binding", None), ("prior_binding", {}), ("unexpected", 350_000_000), -]) -def test_v2_exception_cannot_change_key_identity_count_or_anchored_history(case, field, value): - add_exception(case)[field] = value - with pytest.raises(ValueError): - live.validate_plan(case.plan, now=NOW.timestamp()) - - -def test_legacy_plan_rejects_exception_without_versioned_freeze(case): - add_exception(case) - case.plan["schema"] = "kv-live-plan-v1" - with pytest.raises(ValueError): - live.validate_plan(case.plan, now=NOW.timestamp()) - with pytest.raises(ValueError): - live.policy(case.plan, "7" * 40) - - -def test_checked_in_second_attempt_is_exactly_owner_scoped_and_preserves_old_binding(): - root = Path(__file__).resolve().parents[1] - folder = root / 'evidence/kv-live-pilot-20261031/plan' - old = live.read_json(folder / 'plan.json') - old_binding = live.policy(old, '3ece458ceaa92c5dad8d8e0b1e782a6961400b96').binding - extra = live.read_json(folder / 'second-attempt-budget.json') - assert extra['key'] == {'repository_id': live.REPOSITORY_ID, 'pr_number': 275} - assert extra['pr_lifetime_cap_microusd'] == 350_000_000 - assert extra['pilot_lifetime_cap_microusd'] == 1_000_000_000 - assert extra['additional_attempts'] == 1 - assert extra['prior_binding'] == {'plan_digest': old_binding.plan_digest, - 'run_id': old_binding.run_id, 'worst_case': old_binding.worst_case} - new = {**old, 'schema': 'kv-live-plan-v2', 'budget_exception': extra, - 'campaign_id': extra['campaign_id'], 'execution_id': extra['execution_id'], - 'ledger': {**old['ledger'], 'checkpoint_commit': extra['pr_checkpoint']}, - 'daily_ledger': {**old['daily_ledger'], 'checkpoint_commit': extra['daily_checkpoint']}} - resolved = live.additional_attempt(new) - assert resolved.limit.key == live.budget.PRKey(live.REPOSITORY_ID, 275) - assert resolved.limit.limit_microusd == 350_000_000 - assert live.additional_attempt(old) is None - assert live.budget.PR_LIMIT_MICROUSD == 200_000_000 - assert live.budget.DAILY_LIMIT_MICROUSD == 1_000_000_000 diff --git a/tests/test_kv_live_worker.py b/tests/test_kv_live_worker.py deleted file mode 100644 index 802290b8b..000000000 --- a/tests/test_kv_live_worker.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Synthetic-key/fake-HTTP controls only; never provider calls or live ledger access.""" - -import copy -import io -import json -import time -from unittest.mock import Mock -from urllib.error import HTTPError - -import pytest - -from inferrsi.model import anthropic_agent as provider -from tests.test_anthropic_edit_agent import read_task -from tests.test_kv_pilot_admission import policy as sample_policy -from tools import kv_live_worker as worker - -GATE = {"SKILL_EVAL_GATE_PORT": "1234", "SKILL_EVAL_GATE_TOKEN": "a" * 64, - "SKILL_EVAL_GATE_CONTEXT": "b" * 64} - - -def sender(tmp_path, *, transport=None): - config = {"model": "claude-opus-5-5", "input_token_ceiling": 1000000, - "client": {"max_requests": 8, "max_output_tokens": 4096}} - http = transport or Mock(return_value=read_task()) - ipc = Mock() - value = worker.GrantedSender(config, tmp_path, time.monotonic() + 600, - key="OFFLINE-FAKE-KEY", gate=GATE, transport=http, authorize_ipc=ipc) - return value, http, ipc - - -def grant(payload, slot=1): - return {"request_sha256": worker.edit.digest(provider.canonical(payload).encode()), - "slot": slot} - - -def test_exactly_one_gate_and_http_per_slot_with_no_implicit_second_gate(tmp_path): - value, http, ipc = sender(tmp_path) - payload = {"model": "claude-opus-5-5"} - for slot in range(1, 9): - assert value.authorize(grant(payload, slot), value.deadline) - value.send(payload, value.config["client"], 100) - assert value.attempts == ipc.call_count == http.call_count == 8 - assert http.call_args.kwargs == {"environment": {}, "require_authorization": False} - assert http.call_args.args[2] == "OFFLINE-FAKE-KEY" - with pytest.raises(provider.AgentError): - value.send(payload, value.config["client"], 100) - assert http.call_count == 8 - raw = worker.scripted._read_json(tmp_path / "complete.json") - assert raw["usage"] == read_task()["usage"] # RAW cache classes, not normalized totals. - assert "OFFLINE-FAKE-KEY" not in (tmp_path / "complete.json").read_text() - - -@pytest.mark.parametrize("failure", ["transport", "lost_gate", "hash", "spec", "deadline"]) -def test_no_retry_or_http_without_one_exact_valid_grant(tmp_path, failure): - value, http, ipc = sender(tmp_path) - payload = {"model": "claude-opus-5-5"} - if failure == "lost_gate": - ipc.side_effect = OSError("offline lost reply") - with pytest.raises(OSError): - value.authorize(grant(payload), value.deadline) - else: - value.authorize(grant(payload), value.deadline) - spec = value.config["client"] - if failure == "transport": - http.side_effect = OSError("OFFLINE-FAKE-KEY must not enter error") - elif failure == "hash": - payload["changed"] = True - elif failure == "spec": - spec = {**spec, "max_requests": 9} - elif failure == "deadline": - value.deadline = time.monotonic() - 1 - with pytest.raises(provider.AgentError) as error: - value.send(payload, spec, 100) - assert "OFFLINE-FAKE-KEY" not in str(error.value) - with pytest.raises(provider.AgentError): - value.send(payload, spec, 100) - assert http.call_count == (1 if failure == "transport" else 0) - - -@pytest.mark.parametrize("change", ["missing", "late", "wrong_model", "over_bound"]) -def test_unknown_response_usage_is_relayed_as_unresolved_not_refunded(tmp_path, change): - response = copy.deepcopy(read_task()) - if change == "missing": - response["usage"].pop("cache_creation_input_tokens") - elif change == "wrong_model": - response["model"] = "other" - elif change == "over_bound": - response["usage"]["output_tokens"] = 4097 - value, _, _ = sender(tmp_path, transport=Mock(return_value=response)) - if change == "late": - def late(*args, **kwargs): - value.deadline = time.monotonic() - 1 - return response - value.transport = late - payload = {"model": "claude-opus-5-5"} - value.authorize(grant(payload), value.deadline) - value.send(payload, value.config["client"], 100) - assert worker.scripted._read_json(tmp_path / "complete.json")["usage"] is None - - - -def test_ninth_authorized_send_cannot_make_a_ninth_http_attempt(tmp_path): - value, http, _ = sender(tmp_path) - payload = {"model": "claude-opus-5-5"} - for slot in range(1, 9): - value.authorize(grant(payload, slot), value.deadline) - value.send(payload, value.config["client"], 100) - value.authorize(grant(payload, 9), value.deadline) - with pytest.raises(provider.AgentError): - value.send(payload, value.config["client"], 100) - assert http.call_count == 8 - - -@pytest.mark.parametrize("failure,expected", [ - (401, "http_401"), (429, "http_429"), (500, "http_500"), (503, "http_503"), - ("bad_json", "invalid_response_json"), ("echo", "credential_echo"), - ("network", "transport_error"), -]) -def test_real_transport_closed_failure_code_survives_generic_failure( - tmp_path, monkeypatch, failure, expected): - value, _, _ = sender(tmp_path, transport=provider.send_message) - value.config["client"] = json.loads(sample_policy().provider_json)["client"] - opener = Mock() - monkeypatch.setattr(provider.urllib.request, "build_opener", Mock(return_value=opener)) - if type(failure) is int: - opener.open.side_effect = HTTPError("https://unused.invalid", failure, - "OFFLINE-FAKE-KEY private error", {}, io.BytesIO(b"private response body")) - elif failure == "network": - opener.open.side_effect = OSError("OFFLINE-FAKE-KEY private network detail") - else: - raw = b"not JSON: private response" if failure == "bad_json" else ( - provider.canonical({"text": "OFFLINE-FAKE-KEY"}).encode()) - opener.open.return_value = io.BytesIO(raw) - payload = {"model": "claude-opus-5-5"} - value.authorize(grant(payload), value.deadline) - with pytest.raises(provider.AgentError, match="^provider_transport_error$"): - value.send(payload, value.config["client"], 100) - assert value.transport_error_code == expected - assert value.attempts == opener.open.call_count == 1 - assert worker.scripted._read_json(tmp_path / "complete.json")["usage"] is None - with pytest.raises(provider.AgentError): - value.send(payload, value.config["client"], 100) - assert opener.open.call_count == 1 - - -def test_arbitrary_exception_and_nonclosed_codes_are_never_retained(tmp_path): - value, http, _ = sender(tmp_path, transport=Mock( - side_effect=provider.AgentError("OFFLINE-FAKE-KEY http_401 private payload"))) - payload = {"model": "claude-opus-5-5"} - value.authorize(grant(payload), value.deadline) - with pytest.raises(provider.AgentError, match="^provider_transport_error$"): - value.send(payload, value.config["client"], 100) - assert value.transport_error_code == "unknown" - assert value.attempts == http.call_count == 1 - for invalid in ("http_099", "http_600", "http_４０１", "http_401\n", {}, None, True): - assert worker.closed_reason(invalid) == "unknown" - - -def test_failed_presend_gate_has_known_zero_transport_invocations(tmp_path): - value, http, ipc = sender(tmp_path) - ipc.side_effect = OSError("OFFLINE-FAKE-KEY private gate detail") - with pytest.raises(OSError): - value.authorize(grant({"model": "claude-opus-5-5"}), value.deadline) - assert value.attempts == http.call_count == 0 - assert value.transport_error_code is None - assert not (tmp_path / "complete.json").exists() diff --git a/tests/test_kv_pilot_admission.py b/tests/test_kv_pilot_admission.py deleted file mode 100644 index 226f164d1..000000000 --- a/tests/test_kv_pilot_admission.py +++ /dev/null @@ -1,572 +0,0 @@ -"""Fake stores/clock/launch only: no credentials, network or provider calls.""" - -import json -import pickle -from dataclasses import replace -from datetime import datetime, timedelta, timezone - -import pytest - -from inferrsi.model import anthropic_agent as provider -from tools import kv_pilot_admission as pilot -from tools import skill_eval_budget as b - -NOW = datetime(2026, 10, 30, 15, tzinfo=timezone.utc) -KEY = b.PRKey(123, 999) -DAY = b.shanghai_day(NOW) -BOUND = pilot.FULL_RESERVATION_MICROUSD -USAGE = {"input_tokens": 10, "output_tokens": 4, - "cache_read_input_tokens": 5, "cache_creation_input_tokens": 6} - - -def policy(**changes): - config = { - "model": {"name": "claude-opus-5-5", "effort": "not_applicable"}, - "client": {"kind": provider.EDIT_KIND, "provider": "yibuapi", - "endpoint": provider.ENDPOINT, "api_version": "2023-06-01", - "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 262144, "max_read_bytes": 32768, - "max_tool_calls": 32}, - "pricing": {"currency": "USD", "unit": "per_million_tokens", - "model": "claude-opus-5-5", "group": "audited-example-group", - "rates": {"input": "8", "output": "40", "cache_read": "0.4", - "cache_write": "10"}, "request_fee": "0", - "max_input_tokens": 1_000_000, - "rate_evidence": "https://github.com/example/repo/issues/1#issuecomment-1", - "expires_at": int(NOW.timestamp()) + 7200}, - "spend_limit": {"currency": "USD", "amount": "200"}, - } - values = {"campaign_id": "example-feasibility", "key": KEY, - "execution_id": "example-run-1.1", "plan_sha256": "1" * 64, - "pr_head": "2" * 40, "source_revision": "3" * 40, - "actor_sha256": ("4" * 64, "5" * 64), - "provider_json": provider.canonical(config), - "expires_at": int(NOW.timestamp()) + 7200} - return pilot.PilotPolicy(**(values | changes)) - - -class FakeStore: - """Exercise the existing store transition/CAS contract, including ambiguity.""" - - def __init__(self, ledger, events): - self.ledger, self.events = ledger, events - self.daily = type(ledger) is b.DailyLedger - self.name = "daily" if self.daily else "pr" - self.token = 0 - self.outcomes = [] - self.before_cas = None - self.on_read = None - - def read_verified(self, key): - if self.on_read: - self.on_read(self) - assert key == self.ledger.key - cls = b.DailySnapshot if self.daily else b.Snapshot - return cls(str(self.token), self.ledger) - - def compare_and_swap(self, expected, successor): - validate = b.validate_daily_successor if self.daily else b.validate_successor - validate(expected.ledger, successor) - if self.before_cas: - callback, self.before_cas = self.before_cas, None - callback(self) - if expected != self.read_verified(self.ledger.key): - return b.WriteResult.CONFLICT - outcome = self.outcomes.pop(0) if self.outcomes else "commit" - if outcome == "conflict": - return b.WriteResult.CONFLICT - if outcome == "unknown": - return b.WriteResult.UNKNOWN - self.ledger = successor - self.token += 1 - self.events.append((self.name, successor.entries[-1].phase)) - if outcome == "crash": - raise KeyboardInterrupt("simulated process death after durable write") - return b.WriteResult.UNKNOWN if outcome == "unknown_commit" else b.WriteResult.COMMITTED - - -class World: - def __init__(self, *, carryover=True, legacy=0): - self.now, self.monotonic = NOW, 100.0 - self.events, self.live = [], True - entries = [] - if carryover: - # Synthetic ledger identities reproduce the audited $491.35 retained - # obligation, not invoices or a claim that old reservations settled. - for index, amount in enumerate((162_720_000, 122_720_000, 185_570_000, - 20_340_000), 1): - entries.append(b.DailyEntry(b.PRKey(KEY.repository_id, index), - b.Binding(f"{index:064x}", f"old-{index}.1", amount), - "2026-10-29", "claimed")) - if legacy: - entries.append(b.DailyEntry(KEY, b.Binding("9" * 64, "legacy.1", legacy), - "2026-10-29", "claimed")) - self.daily = FakeStore(b.DailyLedger(b.DailyKey(KEY.repository_id), "audited-import", - tuple(entries)), self.events) - self.pr = FakeStore(b.Ledger(KEY, b.Genesis("audited-pr-history", legacy)), self.events) - - def kwargs(self): - return {"check_current": lambda value: self.live, - "wall_clock": lambda: self.now, "monotonic_clock": lambda: self.monotonic} - - def admit(self, config=None): - return pilot.admit(config or policy(), self.daily, self.pr, **self.kwargs()) - - -def grant(session, arm="A2", slot=1, **changes): - return {"plan_sha256": session.policy.plan_sha256, - "execution_id": session.policy.execution_id, "arm": arm, "slot": slot, - "request_sha256": f"{slot:064x}", - "actor_sha256": session.policy.actor_sha256[pilot.ARMS.index(arm)], - "reserved_microusd": pilot.PER_REQUEST_MICROUSD} | changes - - -def started(world=None): - world = world or World() - session = world.admit() - session.start_arm("A2", 700.0) - return world, session - - -def test_full_reservation_and_both_claims_precede_launch_and_every_send(): - world = World() - launched = [] - - def launch(session): - assert world.events == [("daily", "reserved"), ("pr", "reserved"), - ("daily", "claimed"), ("pr", "claimed")] - assert world.daily.ledger.charged_on(DAY) == 654_070_000 - assert world.pr.ledger.remaining == 37_280_000 - session.start_arm("A2", 700.0) - request = grant(session) - assert session.authorize(request, 700.0) is True - session.complete(request, USAGE) - launched.append(session) - return "offline-result" - - assert pilot.run_admitted(policy(), world.daily, world.pr, launch=launch, - **world.kwargs()) == "offline-result" - with pytest.raises(b.BudgetError): - launched[0].authorize(grant(launched[0], slot=2), 700.0) - assert world.pr.ledger.charged == BOUND # No release after success either. - - -@pytest.mark.parametrize("changes", [{}, {"plan_sha256": "6" * 64}, - {"campaign_id": "new-id"}, - {"execution_id": "example-run-2.1"}, - {"expires_at": int(NOW.timestamp()) + 7000}]) -def test_duplicate_new_digest_new_id_replan_relaunch_cannot_reset(changes): - world = World() - world.admit() - with pytest.raises(b.BudgetError): - world.admit(policy(**changes)) - assert len(world.events) == 4 - assert world.pr.ledger.charged == BOUND - - -def test_moving_campaign_to_another_pr_cannot_reset(): - world = World() - world.admit() - other_key = b.PRKey(KEY.repository_id, KEY.pr_number + 1) - other_pr = FakeStore(b.Ledger(other_key, b.Genesis("new-pr-audit", 0)), world.events) - with pytest.raises(b.BudgetError, match="already bound"): - pilot.admit(policy(key=other_key, campaign_id="also-new-id"), world.daily, - other_pr, **world.kwargs()) - assert other_pr.ledger.charged == 0 - - -@pytest.mark.parametrize("store_name, outcomes", [ - ("daily", ["conflict"]), ("daily", ["unknown"]), ("daily", ["unknown_commit"]), - ("pr", ["conflict"]), ("pr", ["unknown"]), ("pr", ["unknown_commit"]), - ("daily", ["commit", "unknown_commit"]), ("pr", ["commit", "unknown_commit"]), -]) -def test_conflict_unknown_and_partial_reservation_never_launch(store_name, outcomes): - world = World() - getattr(world, store_name).outcomes = outcomes - launches = [] - with pytest.raises(b.BudgetError): - pilot.run_admitted(policy(), world.daily, world.pr, - launch=launches.append, **world.kwargs()) - assert not launches - charged = world.daily.ledger.charged_on(DAY) - if charged > 491_350_000: - with pytest.raises(b.BudgetError): - world.admit() - assert world.daily.ledger.charged_on(DAY) == charged - - -@pytest.mark.parametrize("store_name, outcomes", [ - ("daily", ["crash"]), ("pr", ["crash"]), - ("daily", ["commit", "crash"]), ("pr", ["commit", "crash"]), -]) -def test_crash_after_any_durable_step_has_no_takeover(store_name, outcomes): - world = World() - getattr(world, store_name).outcomes = outcomes - with pytest.raises(KeyboardInterrupt): - world.admit() - with pytest.raises(b.BudgetError): - world.admit() - assert world.daily.ledger.charged_on(DAY) == 654_070_000 - - -def test_stale_pr_cas_keeps_daily_reservation_no_launch(): - world = World() - world.pr.before_cas = lambda store: setattr(store, "token", store.token + 1) - with pytest.raises(b.BudgetError, match="conflicted"): - world.admit() - assert world.events == [("daily", "reserved")] - assert world.daily.ledger.charged_on(DAY) == 654_070_000 - assert world.pr.ledger.charged == 0 - - -def test_pr_cap_includes_legacy_and_prechecks_before_any_write(): - world = World(legacy=37_280_001) - with pytest.raises(b.BudgetError, match="PR cumulative"): - world.admit() - assert not world.events - assert World(legacy=37_280_000).admit() - - -def test_unrelated_settled_history_is_not_new_pilot_spend_or_current_day_charge(): - world = World(carryover=False) - entries = tuple(b.DailyEntry(b.PRKey(123, index), - b.Binding(f"{index:064x}", f"past-{index}.1", 200_000_000), - f"2026-10-{index:02}", "finished", f"2026-10-{index:02}", 1) - for index in range(1, 6)) - world.daily.ledger = b.DailyLedger(b.DailyKey(123), "audited-import", entries) - assert world.daily.ledger.charged_on(DAY) == 0 - world.admit() - assert world.pr.ledger.charged == BOUND - assert world.daily.ledger.charged_on(DAY) == BOUND - assert sum(e.binding.worst_case for e in world.daily.ledger.entries) > 1_000_000_000 - - -def test_unsettled_historical_charges_still_carry_and_enforce_daily_cap(): - world = World(carryover=False) - entries = tuple(b.DailyEntry(b.PRKey(123, index), - b.Binding(f"{index:064x}", f"past-{index}.1", 200_000_000), - f"2026-10-{index:02}", "claimed") for index in range(1, 6)) - world.daily.ledger = b.DailyLedger(b.DailyKey(123), "audited-import", entries) - assert world.daily.ledger.charged_on(DAY) == 1_000_000_000 - with pytest.raises(b.BudgetError, match="Global Shanghai daily budget"): - world.admit() - assert not world.events - - -def test_inconsistent_pr_tip_cannot_erase_imported_daily_obligations(): - world = World(legacy=1) - world.pr.ledger = b.Ledger(KEY, b.Genesis("incorrect-tip-subtotal", 0)) - with pytest.raises(b.BudgetError, match="Inconsistent"): - world.admit() - assert not world.events - - -def test_rollover_during_claim_burns_reservations_without_launch(): - world = World() - - def rollover(store): - world.now = NOW.replace(hour=16) - - world.pr.before_cas = rollover - with pytest.raises(b.BudgetError, match="admission day"): - world.admit() - assert world.pr.ledger.charged == BOUND - assert world.daily.ledger.charged_on("2026-10-31") == 654_070_000 - assert world.events == [("daily", "reserved"), ("pr", "reserved")] - - -def test_rollover_and_pricing_expiry_block_new_grants_without_refund(): - for change in (lambda w: setattr(w, "now", NOW.replace(hour=16)), - lambda w: setattr(w, "now", NOW + timedelta(hours=2))): - world, session = started() - change(world) - with pytest.raises(b.BudgetError): - session.authorize(grant(session), 700.0) - assert session.attempts() == {"A2": 1, "A3": 0} - assert world.pr.ledger.charged == BOUND - - -@pytest.mark.parametrize("which", ["daily", "pr"]) -def test_each_grant_rechecks_both_exact_durable_claims(which): - world, session = started() - store = getattr(world, which) - old = store.ledger.entries[-1] - store.ledger = replace(store.ledger, entries=store.ledger.entries[:-1] + ( - replace(old, phase="reserved"),), revision=store.ledger.revision - 1) - with pytest.raises(b.BudgetError, match="Both exact durable"): - session.authorize(grant(session), 700.0) - assert session.attempts()["A2"] == 1 - - -def test_revoked_authorization_burns_attempt_and_stops_both_arms(): - world, session = started() - world.live = False - with pytest.raises(b.BudgetError): - session.authorize(grant(session), 700.0) - world.live = True - with pytest.raises(b.BudgetError): - session.start_arm("A3", 700.0) - assert session.attempts()["A2"] == 1 - - -def test_known_usage_allows_exactly_eight_per_arm_sixteen_total(): - world = World() - session = world.admit() - for arm in pilot.ARMS: - session.start_arm(arm, 700.0) - for slot in range(1, 9): - request = grant(session, arm, slot) - assert session.authorize(request, 700.0) - session.complete(request, USAGE) - session.finish_arm(arm) - assert session.attempts() == {"A2": 8, "A3": 8} - with pytest.raises(b.BudgetError): - session.authorize(grant(session, "A3", 9), 700.0) - assert session.attempts() == {"A2": 8, "A3": 8} - assert world.pr.ledger.charged == BOUND - - -def test_ninth_slot_cannot_borrow_from_other_arm(): - _, session = started() - for slot in range(1, 9): - request = grant(session, slot=slot) - session.authorize(request, 700.0) - session.complete(request, USAGE) - with pytest.raises(b.BudgetError): - session.authorize(grant(session, slot=9), 700.0) - assert session.attempts() == {"A2": 8, "A3": 0} - - -@pytest.mark.parametrize("usage", [None, {}, {**USAGE, "output_tokens": True}, - {**USAGE, "output_tokens": 4097}, - {**USAGE, "cache_creation_input_tokens": 1_000_001}]) -def test_unknown_or_overbound_usage_stops_both_arms(usage): - world, session = started() - request = grant(session) - session.authorize(request, 700.0) - with pytest.raises((b.BudgetError, provider.AgentError)): - session.complete(request, usage) - for action in (lambda: session.authorize(grant(session, slot=2), 700.0), - lambda: session.finish_arm("A2"), - lambda: session.start_arm("A3", 700.0)): - with pytest.raises(b.BudgetError): - action() - assert world.pr.ledger.charged == BOUND - - -def test_missing_usage_lost_reply_blocks_next_request_and_restart(): - world, session = started() - session.authorize(grant(session), 700.0) - with pytest.raises(b.BudgetError): - session.authorize(grant(session, slot=2), 700.0) - with pytest.raises(b.BudgetError): - world.admit() - assert session.attempts() == {"A2": 1, "A3": 0} - - -@pytest.mark.parametrize("changes", [{"plan_sha256": "a" * 64}, {"slot": True}, - {"slot": 2}, {"execution_id": "other"}, - {"actor_sha256": "a" * 64}, - {"reserved_microusd": 1}]) -def test_candidate_grant_cannot_choose_plan_actor_slot_or_budget(changes): - _, session = started() - with pytest.raises(b.BudgetError): - session.authorize(grant(session, **changes), 700.0) - assert session.attempts()["A2"] == 1 - - -def test_repeated_completion_and_wrong_request_never_reopen_latch(): - for wrong in (False, True): - _, session = started() - request = grant(session) - session.authorize(request, 700.0) - if not wrong: - session.complete(request, USAGE) - with pytest.raises(b.BudgetError): - session.complete(grant(session, slot=2) if wrong else request, USAGE) - with pytest.raises(b.BudgetError): - session.authorize(grant(session, slot=2), 700.0) - - -def test_deadline_cannot_be_extended_and_session_cannot_be_serialized(): - for deadline in (701.0, 699.0): - _, session = started() - with pytest.raises(b.BudgetError): - session.authorize(grant(session), deadline) - world, session = started() - world.monotonic = 700 - with pytest.raises(b.BudgetError, match="deadline"): - session.authorize(grant(session), 700) - with pytest.raises(TypeError): - pickle.dumps(session) - - -def test_existing_full_context_read_submit_kind_is_not_editing_authority(): - config = json.loads(policy().provider_json) - config["client"].update(kind=provider.FULL_CONTEXT_KIND, max_requests=1) - with pytest.raises(b.BudgetError): - policy(provider_json=provider.canonical(config)) - - -def additional_world(): - world = World() - first = policy() - world.admit(first) - original_pr = b.encode(world.pr.ledger) - limit = b.TrustedPRLimit(KEY, 350_000_000) - world.pr.ledger = b.decode(original_pr, trusted_limit=limit) - extra = pilot.AdditionalAttempt(limit, first.binding, "one-fixed-additional", - "one-fixed-additional.1", "a" * 40, "b" * 40) - second = policy(campaign_id=extra.campaign_id, execution_id=extra.execution_id, - plan_sha256="6" * 64, additional_attempt=extra) - assert b.encode(world.pr.ledger) == original_pr - return world, first, second - - -def test_one_additional_pair_keeps_both_full_claims_and_every_ceiling(): - world, first, second = additional_world() - prior_pr, prior_daily = world.pr.ledger.entries, world.daily.ledger.entries - session = world.admit(second) - assert world.pr.ledger.entries[:1] == prior_pr - assert world.daily.ledger.entries[:-1] == prior_daily - assert world.pr.ledger.charged == 325_440_000 - assert world.pr.ledger.remaining == 24_560_000 - assert world.daily.ledger.charged_on(DAY) == 816_790_000 - assert b.DAILY_LIMIT_MICROUSD - world.daily.ledger.charged_on(DAY) == 183_210_000 - assert pilot.PILOT_LIMIT_MICROUSD - 2 * BOUND == 674_560_000 - assert all(e.phase == "claimed" and e.actual_cost is None for e in world.pr.ledger.entries) - session.start_arm("A2", 700.0) - session.authorize(grant(session), 700.0) - with pytest.raises(provider.AgentError, match="unknown_usage"): - session.complete(grant(session), None) - with pytest.raises(b.BudgetError): - session.start_arm("A3", 700.0) - assert world.pr.ledger.entries[0].binding == first.binding - assert len(world.events) == 8 - - -def test_historical_v1_binding_digest_is_unchanged(): - from dataclasses import asdict - old = policy() - fields = asdict(old) - fields.pop("additional_attempt") - assert old.binding.plan_digest == pilot._fingerprint(fields) - - -@pytest.mark.parametrize("change", ["same", "digest", "third", "day", "smaller"]) -def test_additional_attempt_cannot_be_reused_or_turned_into_third(change): - world, _, second = additional_world() - world.admit(second) - before = list(world.events) - with pytest.raises(b.BudgetError): - if change == "digest": - second = replace(second, plan_sha256="7" * 64) - elif change == "third": - extra = replace(second.additional_attempt, prior_binding=second.binding, - campaign_id="third", execution_id="third.1") - second = replace(second, campaign_id=extra.campaign_id, - execution_id=extra.execution_id, additional_attempt=extra) - elif change == "day": - world.now += timedelta(days=1) - elif change == "smaller": - config = json.loads(second.provider_json) - config["client"]["max_requests"] = 2 - second = replace(second, provider_json=provider.canonical(config)) - world.admit(second) - assert world.events == before - assert world.pr.ledger.charged == 325_440_000 - - -def test_other_pr_or_repository_cannot_borrow_exception_context(): - world, _, second = additional_world() - for key in (b.PRKey(KEY.repository_id, KEY.pr_number + 1), b.PRKey(9999, KEY.pr_number)): - with pytest.raises(b.BudgetError, match="reviewed scope"): - replace(second, key=key) - # Even a separately constructed context on another PR cannot move the old KV claim. - key = b.PRKey(KEY.repository_id, KEY.pr_number + 1) - extra = replace(second.additional_attempt, limit=b.TrustedPRLimit(key, 350_000_000)) - other = replace(second, key=key, additional_attempt=extra) - store = FakeStore(b.Ledger(key, b.Genesis("other audit", 0), - trusted_limit=extra.limit), world.events) - with pytest.raises(b.BudgetError, match="prior claim"): - pilot.admit(other, world.daily, store, **world.kwargs()) - assert store.ledger.charged == 0 and len(world.events) == 4 - - -@pytest.mark.parametrize("change", ["prior_binding", "prior_phase", "missing", "extra_pr"]) -def test_exact_original_claim_is_required_before_any_new_write(change): - world, _, second = additional_world() - if change == "prior_binding": - second = replace(second, additional_attempt=replace(second.additional_attempt, - prior_binding=replace(second.additional_attempt.prior_binding, plan_digest="8" * 64))) - elif change == "prior_phase": - old = world.pr.ledger.entries[0].binding - world.pr.ledger = b.advance(world.pr.ledger, old, "finished", actual_cost=0) - world.daily.ledger = b.daily_advance(world.daily.ledger, KEY, old, - "finished", DAY, actual_cost=0) - elif change == "missing": - world = World() - world.pr.ledger = replace(world.pr.ledger, trusted_limit=second.trusted_limit) - else: - other = b.Binding("8" * 64, "unrelated-pr-history", 1) - world.pr.ledger = b.reserve(world.pr.ledger, other) - world.daily.ledger = b.daily_reserve(world.daily.ledger, KEY, other, DAY) - before = list(world.events) - with pytest.raises(b.BudgetError): - world.admit(second) - assert world.events == before - - -@pytest.mark.parametrize("store_name,outcomes", [ - ("daily", ["conflict"]), ("daily", ["unknown"]), ("daily", ["unknown_commit"]), - ("pr", ["conflict"]), ("pr", ["unknown"]), ("pr", ["unknown_commit"]), - ("daily", ["commit", "unknown_commit"]), ("pr", ["commit", "unknown_commit"]), - ("daily", ["crash"]), ("pr", ["crash"]), -]) -def test_additional_partial_or_uncertain_cas_never_grants_or_refunds(store_name, outcomes): - world, _, second = additional_world() - getattr(world, store_name).outcomes = outcomes - launched = [] - with pytest.raises((b.BudgetError, KeyboardInterrupt)): - pilot.run_admitted(second, world.daily, world.pr, launch=launched.append, **world.kwargs()) - assert launched == [] - assert world.pr.ledger.charged >= BOUND - assert world.daily.ledger.charged_on(DAY) >= 654_070_000 - assert world.pr.ledger.entries[0].phase == "claimed" - if world.daily.ledger.charged_on(DAY) > 654_070_000: - with pytest.raises(b.BudgetError): - world.admit(second) - - -def test_two_racing_additional_admissions_have_only_one_durable_winner(): - world, _, second = additional_world() - winners = [] - world.daily.before_cas = lambda _: winners.append(world.admit(second)) - with pytest.raises(b.BudgetError): - world.admit(second) - assert len(winners) == 1 and len(world.events) == 8 - assert world.pr.ledger.charged == 325_440_000 - assert world.daily.ledger.charged_on(DAY) == 816_790_000 - - -def test_additional_pair_checks_full_daily_cap_before_any_write(): - world, _, second = additional_world() - extra = b.Binding('9' * 64, 'unrelated-new-obligation', 200_000_000) - world.daily.ledger = b.daily_reserve(world.daily.ledger, b.PRKey(KEY.repository_id, 50), - extra, DAY) - before = list(world.events) - with pytest.raises(b.BudgetError, match='daily'): - world.admit(second) - assert world.events == before - assert world.pr.ledger.charged == BOUND - - -def test_namespace_history_is_rechecked_before_each_additional_request(): - world, _, second = additional_world() - session = world.admit(second) - session.start_arm('A2', 700.0) - unrelated = b.Binding('9' * 64, pilot.CAMPAIGN_NAMESPACE + 'unapproved-third', 1) - world.daily.ledger = b.daily_reserve(world.daily.ledger, b.PRKey(KEY.repository_id, 50), - unrelated, DAY) - with pytest.raises(b.BudgetError, match='already bound'): - session.authorize(grant(session), 700.0) diff --git a/tests/test_kv_reclamation.py b/tests/test_kv_reclamation.py deleted file mode 100644 index edf0ee164..000000000 --- a/tests/test_kv_reclamation.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Software contract tests; mocks do not establish artifact isolation or efficacy.""" - -import json -import subprocess -import sys -import time -from pathlib import Path -from unittest.mock import patch - -import pytest - -from inferrsi.harness import kv_reclamation as kv - - -def test_capture_only_allows_candidate_bytes(tmp_path): - (tmp_path / "candidate.py").write_text("# starter\n") - (tmp_path / "contract.md").write_text("contract\n") - frozen = kv.workspace_hashes(tmp_path) - (tmp_path / "candidate.py").write_text("# changed\n") - assert kv.capture_artifact(tmp_path, frozen) == b"# changed\n" - (tmp_path / "contract.md").write_text("tampered\n") - with pytest.raises(ValueError, match="Protected"): - kv.capture_artifact(tmp_path, frozen) - - -@pytest.mark.parametrize("change", ["missing", "extra", "symlink", "empty", "oversized"]) -def test_bad_artifact_and_workspace_fail_before_execution(tmp_path, change): - target = tmp_path / "candidate.py" - target.write_text("# starter\n") - frozen = kv.workspace_hashes(tmp_path) - if change == "missing": - target.unlink() - elif change == "extra": - (tmp_path / "other.py").write_text("# extra\n") - elif change == "symlink": - target.unlink() - target.symlink_to("other.py") - else: - target.write_bytes(b"" if change == "empty" else b"x" * 65537) - with pytest.raises(ValueError): - kv.capture_artifact(tmp_path, frozen) - - -def fake_executor(command, cwd, output, deadline, *, environment): - assert environment == {"PATH": "/usr/bin:/bin", "LANG": "C"} - assert set(p.name for p in cwd.iterdir()) == {"candidate.py", "runner.py", "inputs.json"} - assert "expected" not in (cwd / "inputs.json").read_text() - assert "oracle" not in (cwd / "inputs.json").read_text() - output.mkdir(parents=True) - (output / "stdout.log").write_text(json.dumps({"observations": [], "pending": []})) - (output / "stderr.log").write_text("INFERRSI_KV_READY\n") - return {"status": "completed", "returncode": 0, "elapsed_seconds": 0.5, - "stdout_path": str(output / "stdout.log"), - "stderr_path": str(output / "stderr.log")} - - -def test_candidate_receives_only_inputs_not_answers_or_knowledge(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=fake_executor): - record, value = kv.invoke(b"# source", {"mode": "verify", "events": []}, - tmp_path / "output", time.monotonic() + 1) - assert record["elapsed_seconds"] == 0.5 - assert value == {"observations": [], "pending": []} - assert json.loads((tmp_path / "output/execution.json").read_text()) == { - "status": "completed", "returncode": 0, "elapsed_seconds": 0.5} - - -@pytest.mark.parametrize("status,returncode,output_kind,expected", [ - ("completed", 0, "valid", "value"), - ("completed", 0, "invalid_json", None), - ("completed", 0, "empty", None), - ("failed", 1, "valid", None), - ("failed", -9, "valid", None), - ("timeout", -9, "no_logs", None), - ("cancelled", -15, "no_logs", None), - ("error", None, "no_logs", None), - ("completed", 0, "no_marker", kv.SandboxUnavailable), - ("completed", 0, "missing_stdout", None), - ("completed", 0, "missing_stderr", FileNotFoundError), -]) -def test_invoke_retains_closed_original_execution_before_all_output_paths( - tmp_path, status, returncode, output_kind, expected -): - output = tmp_path / "run" - actual = {"observations": [], "pending": []} - original = {"status": status, "returncode": returncode, "elapsed_seconds": 0.125, - "stdout_path": str(output / "stdout.log"), - "stderr_path": str(output / "stderr.log"), "metric": 123, - "argv": ["private-canary"], "cwd": "private-canary", - "error": "private-canary", "environment": {"KEY": "private-canary"}, - "pid": 12345, "extra": "private-canary"} - - def execute(*args, **kwargs): - output.mkdir() - if output_kind not in {"no_logs", "missing_stdout"}: - data = "invalid JSON" if output_kind == "invalid_json" else ( - "" if output_kind == "empty" else json.dumps(actual)) - (output / "stdout.log").write_text(data) - if output_kind not in {"no_logs", "missing_stderr"}: - (output / "stderr.log").write_text( - "" if output_kind == "no_marker" else "INFERRSI_KV_READY\n") - return original - - before = dict(original) - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=execute) as launch: - if isinstance(expected, type): - with pytest.raises(expected): - kv.invoke(b"# source", {}, output, time.monotonic() + 1) - else: - record, value = kv.invoke(b"# source", {}, output, time.monotonic() + 1) - assert record is original - assert value == (actual if expected == "value" else None) - launch.assert_called_once() - assert original == before # Preserve elapsed/metric and all existing return semantics. - raw = (output / "execution.json").read_text() - assert json.loads(raw) == {"status": status, "returncode": returncode, - "elapsed_seconds": 0.125} - assert "private-canary" not in raw and str(tmp_path) not in raw - - -def test_execution_retention_failure_stops_before_output_check_without_retry(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=fake_executor) as launch, \ - patch.object(kv, "atomic_write_json", side_effect=OSError("storage unavailable")), \ - patch.object(kv, "read_regular") as read: - with pytest.raises(OSError, match="storage unavailable"): - kv.invoke(b"# source", {}, tmp_path / "run", time.monotonic() + 1) - launch.assert_called_once() - read.assert_not_called() - assert not (tmp_path / "run/execution.json").exists() - - -def test_executor_did_not_return_has_no_invented_execution_record(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=KeyboardInterrupt) as launch: - with pytest.raises(KeyboardInterrupt): - kv.invoke(b"# source", {}, tmp_path / "run", time.monotonic() + 1) - launch.assert_called_once() - assert not (tmp_path / "run/execution.json").exists() - - -def test_no_sandbox_has_no_host_fallback(tmp_path): - with patch.object(kv, "sandbox_command", side_effect=kv.SandboxUnavailable("blocked")), \ - patch.object(kv, "execute_command") as execute: - with pytest.raises(kv.SandboxUnavailable): - kv.invoke(b"# source", {}, tmp_path, time.monotonic() + 1) - execute.assert_not_called() - - -def test_missing_worker_marker_is_infrastructure_failure(tmp_path): - def execute(*args, **kwargs): - record = fake_executor(*args, **kwargs) - Path(record["stderr_path"]).write_bytes(b"") - return record - - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=execute): - with pytest.raises(kv.SandboxUnavailable, match="boundary"): - kv.invoke(b"# source", {}, tmp_path / "run", time.monotonic() + 1) - - -def test_each_event_compared_by_trusted_oracle_and_missing_outputs_rejected(tmp_path): - traces = [{"id": "one", "events": []}, {"id": "two", "events": []}] - def compare(trace, value): - return value == trace["id"] - with patch.object(kv, "invoke", side_effect=[ - ({"status": "completed"}, "one"), ({"status": "completed"}, None)]): - result = kv.verify_source(b"# source", traces, tmp_path, time.monotonic() + 1, compare) - assert result == {"verdict": "FAIL", "checks": {"one": True, "two": False}} - - -def test_grader_failure_is_unverified_not_refuted(tmp_path): - with patch.object(kv, "invoke", return_value=({"status": "timeout"}, None)): - result = kv.verify_source(b"# source", [{"id": "one", "events": []}], tmp_path, - time.monotonic() + 1, lambda *_: True) - assert result["verdict"] == "INCONCLUSIVE" - assert result["checks"] == {} - - -def test_expired_deadline_never_launches(tmp_path): - with patch.object(kv, "invoke") as invoke: - result = kv.verify_source(b"# source", [{"id": "one", "events": []}], tmp_path, - time.monotonic() - 1, lambda *_: True) - invoke.assert_not_called() - assert result["verdict"] == "INCONCLUSIVE" - - -def test_real_executor_expired_before_launch_needs_no_output_files(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["not-launched"]): - record, value = kv.invoke(b"# source", {"mode": "verify", "events": []}, - tmp_path / "run", time.monotonic() - 1) - assert record["status"] == "timeout" and value is None - assert not (tmp_path / "run/stderr.log").exists() - assert json.loads((tmp_path / "run/execution.json").read_text()) == { - "status": "timeout", "returncode": None, - "elapsed_seconds": record["elapsed_seconds"]} - - -def test_driver_snapshots_mutable_return_buffers(): - assert 'observation = {"reclaimed": list(result)}' in kv.DRIVER - assert "return list(value)" in kv.DRIVER - - -@pytest.mark.skipif(sys.platform != "linux", reason="Trusted Linux resource-limit driver") -def test_authored_driver_control_can_reuse_return_buffers(tmp_path): - # Trusted local software test, not a submitted artifact or sandbox fallback. - source = '''class Reclaimer: - def __init__(self): - self.result = [] - self.state = [] - def apply(self, event): - self.result[:] = [event["value"]] - return self.result - def pending(self): - self.state[:] = [(str(self.result[0]), 0, (self.result[0],))] - return self.state -''' - (tmp_path / "candidate.py").write_text(source) - (tmp_path / "inputs.json").write_text(json.dumps( - {"mode": "verify", "events": [{"value": 1}, {"value": 2}]})) - (tmp_path / "runner.py").write_text(kv.DRIVER.replace("/work/", str(tmp_path) + "/")) - run = subprocess.run([sys.executable, "-I", "-S", "-B", str(tmp_path / "runner.py")], - capture_output=True, text=True, timeout=5, check=False) - assert run.returncode == 0, run.stderr - value = json.loads(run.stdout) - assert value["observations"][0] == {"reclaimed": [1], "pending": [["1", 0, [1]]]} - assert value["observations"][1] == {"reclaimed": [2], "pending": [["2", 0, [2]]]} - - -@pytest.mark.parametrize("broken", [None, "inaccessible", "readonly", "clean_env", - "network_blocked", "fork_denied", "limits", "cleanup"]) -def test_preflight_requires_each_actual_probe(tmp_path, broken): - value = {"inaccessible": [True, True], "readonly": True, "clean_env": True, - "network_blocked": True, "fork_denied": True, - "limits": {"memory": [268435456, 268435456], "processes": [0, 0], "cpu": [16, 16], - "output": [4194304, 4194304], "files": [32, 32], "core": [0, 0]}} - if broken and broken != "cleanup": - value[broken] = False - with patch.object(kv, "invoke", return_value=({"status": "completed"}, value)), \ - patch.object(kv, "process_group_gone", return_value=broken != "cleanup"): - if broken: - with pytest.raises(kv.SandboxUnavailable): - kv.probe_isolation(tmp_path, time.monotonic() + 1, ["/synthetic-hidden"]) - else: - assert kv.probe_isolation(tmp_path, time.monotonic() + 1, - ["/synthetic-hidden"])["normal_exit_process_group_gone"] - - -def test_driver_has_no_expected_answers_or_candidate_timing_authority(): - assert "perf_counter" not in kv.DRIVER and "process_time" not in kv.DRIVER - assert "expected" not in kv.DRIVER and "oracle" not in kv.DRIVER - assert "resource.RLIMIT_NPROC, (0, 0)" in kv.DRIVER - assert "resource.RLIMIT_AS, (268435456, 268435456)" in kv.DRIVER diff --git a/tests/test_kv_runtime_qualification.py b/tests/test_kv_runtime_qualification.py deleted file mode 100644 index 7d4844f8c..000000000 --- a/tests/test_kv_runtime_qualification.py +++ /dev/null @@ -1,292 +0,0 @@ -"""Metadata/runtime probes only: no submitted artifact is run by these unit tests.""" - -import importlib.metadata -import json -import time -from pathlib import Path - -import pytest - -from tools import kv_runtime_qualification as runtime - - -def fixture(tmp_path): - mount = tmp_path / "runtime" - mount.mkdir() - (mount / "stdlib.py").write_text("# trusted standard runtime\n") - sensitive = tmp_path / "private/skill_library/SKILL.md" - sensitive.parent.mkdir(parents=True) - sensitive.write_text("Known sensitive knowledge bytes. " * 10) - return mount, sensitive - - -def test_runtime_inventory_keeps_only_digest_counts_and_no_absolute_paths(tmp_path): - mount, sensitive = fixture(tmp_path) - result, escapes = runtime.inventory( - [(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5 - ) - assert result["verdict"] == "PASS" and result["nodes"] == 2 and escapes == [] - assert str(tmp_path) not in json.dumps(result) - - -@pytest.mark.parametrize("variant", ["named_package", "renamed_copy", "symlink_sensitive"]) -def test_runtime_inventory_rejects_sensitive_data_inside_mounted_tree(tmp_path, variant): - mount, sensitive = fixture(tmp_path) - if variant == "named_package": - (mount / "inferrsi").mkdir() - elif variant == "renamed_copy": - (mount / "unremarkable.txt").write_bytes(sensitive.read_bytes()) - else: - (mount / "alias").symlink_to(sensitive) - result, _ = runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - assert result["verdict"] == "FAIL" - - -def test_escaped_symlink_requires_same_profile_probe_not_assumed_safe(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - unrelated = tmp_path / "external.txt" - unrelated.write_text("outside\n") - (mount / "escape").symlink_to(unrelated) - monkeypatch.setattr(runtime, "sensitive_files", lambda *a: [sensitive]) - monkeypatch.setattr(runtime, "runtime_mounts", lambda *a: [(mount, Path("/runtime"))]) - forbidden = [] - - def probe(output, deadline, paths): - forbidden.extend(paths) - raise runtime.kv.SandboxUnavailable("unavailable") - - monkeypatch.setattr(runtime.kv, "probe_isolation", probe) - result = runtime.qualify( - tmp_path / "out", - time.monotonic() + 5, - root=tmp_path, - independent=tmp_path, - fixtures=tmp_path, - knowledge_root=tmp_path, - ) - assert "/runtime/escape" in forbidden - assert str(sensitive) in forbidden - assert result["verdict"] == "INCONCLUSIVE" - assert len([p for p in forbidden if p.endswith("-canary.txt")]) == 2 - - -def test_bind_source_symlink_inventories_actual_mounted_contents(tmp_path): - mount, sensitive = fixture(tmp_path) - alias = tmp_path / "bind-source" - alias.symlink_to(mount, target_is_directory=True) - (mount / "renamed.txt").write_bytes(sensitive.read_bytes()) - result, escapes = runtime.inventory( - [(alias, Path("/runtime"))], [sensitive], time.monotonic() + 5 - ) - assert result["verdict"] == "FAIL" and result["nodes"] == 3 - assert escapes == [] - - -def test_complete_renamed_large_hidden_corpus_is_rejected(tmp_path): - from tools import qualify_kv_reclamation as task - - mount, _ = fixture(tmp_path) - sensitive = task.INDEPENDENT / "fixtures.json" - assert sensitive.stat().st_size > 65536 - (mount / "large-data.bin").write_bytes(sensitive.read_bytes()) - result, _ = runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - assert result["verdict"] == "FAIL" - - -def test_hash_byte_limit_fails_closed(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - monkeypatch.setattr(runtime, "MAX_HASH_BYTES", 1) - with pytest.raises(ValueError, match="hash byte limit"): - runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - - -def test_failed_inventory_does_not_probe_or_claim_success(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - (mount / "inferrsi").mkdir() - monkeypatch.setattr(runtime, "sensitive_files", lambda *a: [sensitive]) - monkeypatch.setattr(runtime, "runtime_mounts", lambda *a: [(mount, Path("/runtime"))]) - monkeypatch.setattr(runtime.kv, "probe_isolation", lambda *a: pytest.fail("inventory rejected")) - result = runtime.qualify( - tmp_path / "out", - time.monotonic() + 5, - root=tmp_path, - independent=tmp_path, - fixtures=tmp_path, - knowledge_root=tmp_path, - ) - assert result["verdict"] == "FAIL" - - -def test_unreadable_enumeration_is_inconclusive(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - monkeypatch.setattr(runtime, "sensitive_files", lambda *a: [sensitive]) - monkeypatch.setattr(runtime, "runtime_mounts", lambda *a: [(mount, Path("/runtime"))]) - - def denied(*a): - raise PermissionError("private-path-canary") - - monkeypatch.setattr(runtime.os, "scandir", denied) - result = runtime.qualify( - tmp_path / "out", - time.monotonic() + 5, - root=tmp_path, - independent=tmp_path, - fixtures=tmp_path, - knowledge_root=tmp_path, - ) - assert result["verdict"] == "INCONCLUSIVE" - assert "private-path-canary" not in json.dumps(result) - - -def test_actual_package_paths_and_oracle_are_regular_forbidden_files(): - from tools import qualify_kv_reclamation as task - - files = runtime.sensitive_files( - task.ROOT, - task.INDEPENDENT, - task.FIXTURES, - task.ROOT / "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes", - time.monotonic() + 10, - ) - assert task.INDEPENDENT / "oracle.py" in files - assert task.INDEPENDENT / "fixtures.json" in files - assert task.FIXTURES / "reference.py" in files - assert all(p.is_file() for p in files) - assert any("skill_library" in p.parts for p in files) - - -def test_missing_required_sensitive_anchor_is_not_silently_dropped(tmp_path): - with pytest.raises(ValueError, match="anchor missing"): - runtime.sensitive_files(tmp_path, tmp_path, tmp_path, tmp_path, time.monotonic() + 5) - - -def test_inventory_deadline_and_node_limit_fail_closed(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - with pytest.raises(ValueError, match="inventory limit"): - runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() - 1) - monkeypatch.setattr(runtime, "MAX_NODES", 1) - with pytest.raises(ValueError, match="inventory limit"): - runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - - -def package(directory): - directory.mkdir(parents=True) - (directory / "__init__.py").write_text("# synthetic package\n") - (directory / "data").mkdir() - (directory / "data/private.json").write_text('{"synthetic_sensitive_copy": true}\n') - return directory - - -def installed_distribution(metadata, *, record=None, source_list=None, direct_url=None): - metadata.mkdir(parents=True) - (metadata / ("PKG-INFO" if metadata.suffix == ".egg-info" else "METADATA")).write_text( - "Metadata-Version: 2.1\nName: inferrsi\nVersion: 0.1.0\n" - ) - for name, value in (("RECORD", record), ("SOURCES.txt", source_list), - ("direct_url.json", direct_url)): - if value is not None: - (metadata / name).write_text(value) - return importlib.metadata.PathDistribution(metadata) - - -def installed_files(distribution, monkeypatch): - from tools import qualify_kv_reclamation as task - - monkeypatch.setattr(runtime.importlib.metadata, "distributions", lambda: [distribution]) - return runtime.sensitive_files( - task.ROOT, task.INDEPENDENT, task.FIXTURES, - task.ROOT / "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes", - time.monotonic() + 10, - ) - - -def test_source_egg_info_inventories_actual_tree_not_source_archive_paths(tmp_path, monkeypatch): - actual = package(tmp_path / "project/src/inferrsi") - distribution = installed_distribution( - tmp_path / "project/src/inferrsi.egg-info", - source_list="src/inferrsi/__init__.py\nsrc/inferrsi/data/private.json\n", - ) - # This is the real PathDistribution mismatch seen on Python 3.10/3.11 CI. - assert not distribution.locate_file("src/inferrsi/__init__.py").exists() - files = installed_files(distribution, monkeypatch) - assert actual / "__init__.py" in files and actual / "data/private.json" in files - - -def test_installed_wheel_copy_includes_all_actual_package_nodes(tmp_path, monkeypatch): - actual = package(tmp_path / "site/inferrsi") - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", record="inferrsi/__init__.py,,\n" - ) - files = installed_files(distribution, monkeypatch) - assert actual / "__init__.py" in files and actual / "data/private.json" in files - - -@pytest.mark.parametrize("variant", ["missing", "symlink", "absolute", "traversal"]) -def test_raw_record_does_not_filter_missing_or_unsafe_installed_nodes( - tmp_path, monkeypatch, variant -): - actual = package(tmp_path / "site/inferrsi") - entry = "inferrsi/missing.py" - if variant == "symlink": - (actual / "alias.py").symlink_to(actual / "__init__.py") - entry = "inferrsi/alias.py" - elif variant == "absolute": - entry = str(actual / "missing.py") - elif variant == "traversal": - entry = "inferrsi/../missing.py" - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", record=f"inferrsi/__init__.py,,\n{entry},,\n" - ) - - class FilteredFilesDistribution: - # Newer importlib.metadata filters missing paths from .files; never use - # that lossy view to decide completeness of the authoritative RECORD. - files = [] - metadata = distribution.metadata - read_text = distribution.read_text - locate_file = distribution.locate_file - - with pytest.raises(ValueError, match="package (inventory incomplete|RECORD)"): - installed_files(FilteredFilesDistribution(), monkeypatch) - - -@pytest.mark.parametrize("layout", ["src", "flat"]) -def test_editable_record_maps_independent_source_copy(tmp_path, monkeypatch, layout): - source = tmp_path / "project" - actual = package(source / ("src/inferrsi" if layout == "src" else "inferrsi")) - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", - record="__editable__.inferrsi.pth,,\n", - direct_url=json.dumps({"url": source.as_uri(), "dir_info": {"editable": True}}), - ) - files = installed_files(distribution, monkeypatch) - assert actual / "__init__.py" in files and actual / "data/private.json" in files - - -@pytest.mark.parametrize("variant", ["missing", "ambiguous", "symlink", "remote"]) -def test_unknown_editable_layout_never_silently_drops_installed_copy( - tmp_path, monkeypatch, variant -): - source = tmp_path / "project" - source.mkdir() - if variant in {"ambiguous", "remote"}: - package(source / "src/inferrsi") - if variant == "ambiguous": - package(source / "inferrsi") - if variant == "symlink": - actual = package(tmp_path / "other/inferrsi") - (source / "inferrsi").symlink_to(actual, target_is_directory=True) - url = "https://example.invalid/source" if variant == "remote" else source.as_uri() - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", record="__editable__.inferrsi.pth,,\n", - direct_url=json.dumps({"url": url, "dir_info": {"editable": True}}), - ) - with pytest.raises(ValueError): - installed_files(distribution, monkeypatch) - - -def test_missing_installed_record_is_not_treated_as_source_egg_info(tmp_path, monkeypatch): - package(tmp_path / "site/inferrsi") - distribution = installed_distribution(tmp_path / "site/inferrsi-0.1.0.dist-info") - with pytest.raises(ValueError, match="inventory metadata"): - installed_files(distribution, monkeypatch) diff --git a/tests/test_kv_worker_supervisor.py b/tests/test_kv_worker_supervisor.py deleted file mode 100644 index ed2adc40e..000000000 --- a/tests/test_kv_worker_supervisor.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Fake supervisor controls only; no providers or live ledger writes.""" - -import time -from contextlib import contextmanager -from unittest.mock import Mock - -import pytest - -from inferrsi.infra.artifacts import atomic_write_json -from tests.test_kv_edit_pilot import prepare -from tools import kv_edit_pilot as pilot -from tools import kv_worker_supervisor as supervisor - - -@pytest.mark.parametrize("extra", ["GITHUB_TOKEN", "ACTIONS_RUNTIME_TOKEN", "PYTHONPATH", "HOME"]) -def test_shared_supervisor_rejects_authority_and_path_environment_injection(tmp_path, extra): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - with pytest.raises(ValueError, match="environment"): - supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=Mock(), environment_factory=lambda gate: {**gate, extra: "offline"}, - execute=Mock(side_effect=AssertionError("must not execute"))) - admission.close.assert_called_once() - - -def test_shared_supervisor_closes_admission_on_outer_cancellation(tmp_path): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - with pytest.raises(KeyboardInterrupt): - supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=Mock(side_effect=KeyboardInterrupt)) - admission.close.assert_called_once() - - -@pytest.mark.parametrize("status", ["cancelled", "timeout", "error"]) -def test_returned_executor_failure_stops_session_without_finishing_arm(tmp_path, status): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - result = supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=Mock(return_value={"status": status})) - assert not result["request_completion_known"] - admission.close.assert_called_once() - admission.finish_arm.assert_not_called() - - - -@pytest.mark.parametrize("returncode,has_result", [(-15, True), (-9, True), (0, False), - (1, False), (1, True)]) -def test_signal_death_or_missing_result_blocks_following_arm(tmp_path, returncode, has_result): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - def execute(command, checkout, output, deadline, *, environment): - if has_result: - (output.parent / "worker").mkdir() - atomic_write_json(output.parent / "worker/result.json", {"status": "completed"}) - return {"status": "failed" if returncode else "completed", "returncode": returncode} - result = supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=execute) - assert not result["request_completion_known"] - admission.close.assert_called_once() - admission.finish_arm.assert_not_called() - - -@pytest.mark.parametrize("record,accepted", [ - ({"status": "completed", "returncode": 0}, True), - ({"status": "failed", "returncode": 1}, False), - ({"status": "failed", "returncode": 0}, False), - ({"status": "completed", "returncode": 1}, False), - ({"status": "completed", "returncode": False}, False), - ({"status": "completed", "returncode": True}, False), - ({"status": "completed", "returncode": 0.0}, False), - ({"status": "completed", "returncode": "0"}, False), - ({"status": "completed", "returncode": None}, False), - ({"status": "completed"}, False), - ({"returncode": 0}, False), - ({"status": "unknown", "returncode": 0}, False), - ({"status": "cancelled", "returncode": 0}, False), - ({"status": "timeout", "returncode": 0}, False), - ({"status": "error", "returncode": 0}, False), -]) -def test_known_request_and_worker_result_cannot_upgrade_executor_outcome( - tmp_path, monkeypatch, record, accepted -): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - admission.authorize.return_value = True - broker_check = None - - @contextmanager - def broker(check, *args, **kwargs): - nonlocal broker_check - broker_check = check - yield {} - - monkeypatch.setattr(supervisor, "authorization_broker", broker) - - def execute(command, checkout, output, deadline, *, environment): - grant = {k: config[k] for k in ("plan_sha256", "execution_id", "arm", "actor_sha256")} - grant.update(slot=1, request_sha256="a" * 64, reserved_microusd=10170000) - bridge = output.parent / "bridge" - atomic_write_json(bridge / "pending.json", grant) - assert broker_check() is True - atomic_write_json(bridge / "complete.json", {"grant": grant, "usage": { - "input_tokens": 1, "output_tokens": 1, - "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}}) - (output.parent / "worker").mkdir() - atomic_write_json(output.parent / "worker/result.json", {"status": "completed"}) - return record - - result = supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=execute) - assert result["execution"] == record and result["worker"]["status"] == "completed" - assert result["slots_consumed"] == 1 and result["completions"][0]["status"] == "known" - assert result["request_completion_known"] is accepted - admission.complete.assert_called_once() - if accepted: - admission.finish_arm.assert_called_once_with(config["arm"]) - admission.close.assert_not_called() - else: - admission.close.assert_called_once() - admission.finish_arm.assert_not_called() diff --git a/tests/test_qualify_kv_edit_actor.py b/tests/test_qualify_kv_edit_actor.py deleted file mode 100644 index 5d77e5f31..000000000 --- a/tests/test_qualify_kv_edit_actor.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Fixed actor qualifier controls; fake worker outputs are not host qualification.""" - -import json -import zipfile - -import pytest -import yaml - -from inferrsi.model import anthropic_edit_agent as edit -from tools import qualify_kv_edit_actor as qualify - -PLAN = qualify.ROOT / "evidence/kv-actor-qualification-20261031/plan/qualification-plan.json" -HISTORICAL = qualify.ROOT / "evidence/kv-actor-qualification-20261030" - - -def test_historical_qualification_bundle_is_retained_byte_for_byte(): - integrity = HISTORICAL / "plan/integrity.json" - assert qualify.task.file_hash(integrity) == ( - "d6c018da4cf1cde265d9dce36cb4368d75953d2291acb1ef48ddb67d09d89336" - ) - for name, digest in json.loads(integrity.read_text()).items(): - assert qualify.task.file_hash(HISTORICAL / name) == digest - - -def test_new_payload_plan_preserves_the_software_contract(): - current = qualify.checked_plan(PLAN) - historical = json.loads((HISTORICAL / "plan/qualification-plan.json").read_text()) - identity_fields = {"base_revision", "knowledge_revision", "knowledge_files_sha256"} - assert {k: v for k, v in current.items() if k not in identity_fields} == { - k: v for k, v in historical.items() if k not in identity_fields - } - assert current["knowledge_revision"] == "a04c599c49465b67ea40a1f0de622706ab3da955" - changed = { - name for name, digest in current["knowledge_files_sha256"].items() - if historical["knowledge_files_sha256"][name] != digest - } - assert changed == {"metadata.json"} - - -def test_current_plan_rejects_metadata_drift(tmp_path, monkeypatch): - plan = qualify.checked_plan(PLAN) - owner = tmp_path / plan["knowledge_root"] - for name in plan["knowledge_files_sha256"]: - source = qualify.ROOT / plan["knowledge_root"] / name - target = owner / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(source.read_bytes()) - monkeypatch.setattr(qualify, "ROOT", tmp_path) - qualify.checked_plan(PLAN) - metadata = owner / "metadata.json" - metadata.write_bytes(metadata.read_bytes() + b"\n") - with pytest.raises(ValueError, match="Pinned K bytes changed"): - qualify.checked_plan(PLAN) - - -def identity(): - return {"available": True, "head": "a" * 40, "snapshot_hash": "exact", "dirty": False} - - -def fake_worker(inputs, workspace, output, deadline): - config = json.loads((inputs / "worker.json").read_text()) - scripts = json.loads((inputs / "responses.json").read_text()) - source = scripts[-2]["content"][0]["input"]["source"].encode() - final = output / "worker/final" - final.mkdir(parents=True) - (final / "candidate.py").write_bytes(source) - # Parent final grader must use final bytes, not this later live-path mutation. - (workspace / "candidate.py").write_bytes(b"# later live actor path\n") - for i in (0, 1, 2): - directory = output / f"worker/workbench/revisions/revision-{i:04d}" - directory.mkdir(parents=True) - (directory / "candidate.py").write_bytes(source) - return { - "execution": {"status": "completed"}, - "worker": { - "status": "completed", - "artifact": {"sha256": edit.digest(source), "revision": 2}, - }, - "simulated_slots_consumed": 4, - "request_completion_known": True, - "config": config, - } - - -def test_fixed_actual_owner_exposure_and_separate_final_grade(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.pilot, "run_supervised", fake_worker) - monkeypatch.setattr(qualify, "visible_receipts", lambda *_: True) - seen = [] - - def grade(source, digest, hidden, output, deadline): - assert edit.digest(source) == digest - assert b"later live actor path" not in source - assert all(t["split"] == "grader" for t in hidden) - seen.append(source) - return {"verdict": "PASS"} - - monkeypatch.setattr(qualify.workbench, "grade_final", grade) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "PASS" and result["software_checks_passed"] - assert result["model_calls"] == 0 and len(seen) == 2 - assert not any(f["path"].startswith("knowledge/") for f in result["arms"]["A2"]["exposure"]) - names = {f["path"] for f in result["arms"]["A3"]["exposure"]} - assert names == { - "task.json", - "candidate.py", - "CONTRACT.md", - "workbench.md", - "development.json", - "knowledge/SKILL.md", - "knowledge/metadata.json", - "knowledge/references/source-notes.md", - } - common = qualify.shared_inputs.COMMON - a2 = {f["path"]: f for f in result["arms"]["A2"]["exposure"]} - a3 = {f["path"]: f for f in result["arms"]["A3"]["exposure"]} - assert {p: a2[p] for p in common} == {p: a3[p] for p in common} - assert result["arms"]["A2"]["prompt_sha256"] == result["arms"]["A3"]["prompt_sha256"] - assert all(f["bytes"] <= qualify.checked_plan(PLAN)["client"]["max_read_bytes"] - for f in a3.values()) - with zipfile.ZipFile(tmp_path / "out/raw-evidence.zip") as archive: - raw = "\n".join(archive.read(name).decode() for name in archive.namelist()) - assert str(tmp_path) not in raw - assert "" in raw or "scripted" in raw - assert any(name.endswith("candidate.py") for name in archive.namelist()) - - -def test_unavailable_worker_is_not_pass_and_does_not_grade(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "INCONCLUSIVE"}) - monkeypatch.setattr( - qualify.pilot, - "run_supervised", - lambda *_: { - "execution": {"status": "failed"}, - "worker": {"status": "inconclusive", "reason": "sandbox_unavailable"}, - "simulated_slots_consumed": 0, - "request_completion_known": True, - }, - ) - monkeypatch.setattr(qualify.workbench, "grade_final", lambda *_: pytest.fail("not submitted")) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "INCONCLUSIVE" and not result["software_checks_passed"] - assert all(a["simulated_slots_consumed"] == 0 for a in result["arms"].values()) - - -def test_worker_receipt_cannot_override_failed_outer_execution(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "PASS"}) - monkeypatch.setattr(qualify, "visible_receipts", lambda *_: True) - monkeypatch.setattr(qualify.workbench, "grade_final", lambda *_: {"verdict": "PASS"}) - - def failed_worker(*args): - return {**fake_worker(*args), "execution": {"status": "failed"}} - - monkeypatch.setattr(qualify.pilot, "run_supervised", failed_worker) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "INCONCLUSIVE" and not result["software_checks_passed"] - - -def test_current_clean_exact_revision_is_required(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: {**identity(), "dirty": True}) - with pytest.raises(ValueError, match="clean Git revision"): - qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert not (tmp_path / "out").exists() - - -def test_outbound_owner_links_cannot_expand_allowlist(tmp_path): - plan = json.loads(PLAN.read_text()) - plan["knowledge_files_sha256"]["../SKILL.md"] = "a" * 64 - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - with pytest.raises(ValueError, match="complete pinned K owner"): - qualify.checked_plan(path) - - -def test_qualification_reuses_1800_workload_not_a_post_outcome_choice(): - plan = qualify.checked_plan(PLAN) - task_plan = qualify.task.checked_plan(qualify.TASK_PLAN) - assert plan["visible_workload_count"] == task_plan["timing"]["lease_count"] == 1800 - assert plan["actor_limits"]["max_seconds"] == 600 - - -def test_existing_ready_job_only_adds_secret_free_separate_receipt(): - workflow = yaml.safe_load((qualify.ROOT / ".github/workflows/ci-ready.yml").read_text()) - job = workflow["jobs"]["kv-task-qualification"] - assert job["runs-on"] == "ubuntu-22.04" and job["permissions"] == {"contents": "read"} - commands = "\n".join(step.get("run", "") for step in job["steps"]) - assert "tools.qualify_kv_edit_actor" in commands and "env -i" in commands - assert f"--plan {PLAN.relative_to(qualify.ROOT).as_posix()}" in commands - assert "secrets." not in commands and "send_message" not in commands - assert "--privileged" not in commands and "sysctl" not in commands - uploads = [s["with"] for s in job["steps"] if "upload-artifact" in s.get("uses", "")] - assert len(uploads) == 2 and uploads[0]["name"].startswith("kv-qualification-") - assert uploads[1]["name"].startswith("kv-actor-qualification-") - - -def receipt_fixture(tmp_path): - actor = "a" * 64 - revisions = tmp_path / "revisions" - sources = {r: f"# Scripted revision {r}\n".encode() for r in (0, 1, 2)} - for revision, source in sources.items(): - directory = revisions / f"revision-{revision:04d}" - directory.mkdir(parents=True) - (directory / "candidate.py").write_bytes(source) - artifact = { - "revision": 2, - "sha256": edit.digest(sources[2]), - "plan_sha256": "c" * 64, - "execution_id": "fixture", - "arm": "A2", - } - events = [] - for revision, source in sources.items(): - metadata = {"revision": revision, "sha256": edit.digest(source), - "actor_sha256": actor, "artifact": "candidate.py", - **{k: artifact[k] for k in ("plan_sha256", "execution_id", "arm")}} - (revisions / f"revision-{revision:04d}/receipt.json").write_text(json.dumps(metadata)) - events.append({"type": "edit.artifact", "bytes": len(source), **metadata}) - for revision in (1, 2): - for kind in ("checks", "benchmark"): - feedback = ( - {"verdict": "PASS"} - if kind == "checks" - else { - "verdict": "FAIL", - "quality_verdict": "PASS", - "baseline_values": [1, 1], - "candidate_values": [1, 1], - "improvement_pct": 0, - } - ) - receipt = { - "revision": revision, - "kind": kind, - "feedback": feedback, - "artifact_sha256": edit.digest(sources[revision]), - **{k: artifact[k] for k in ("plan_sha256", "execution_id", "arm")}, - } - receipt["receipt_sha256"] = edit.fingerprint({**receipt, "actor_sha256": actor}) - events.append({"type": "agent.tool", "receipt": receipt}) - if revision == 2: - artifact["check_receipt" if kind == "checks" else "benchmark_receipt"] = receipt[ - "receipt_sha256" - ] - path = tmp_path / "trace.jsonl" - path.write_text("\n".join(json.dumps(e) for e in events)) - return path, artifact, actor, revisions, events - - -def test_numeric_gain_is_not_required_by_receipt_readback(tmp_path): - path, artifact, actor, revisions, events = receipt_fixture(tmp_path) - assert qualify.visible_receipts(path, artifact, actor, 2, revisions) - events[-1]["receipt"]["feedback"]["quality_verdict"] = "FAIL" - path.write_text("\n".join(json.dumps(e) for e in events)) - assert not qualify.visible_receipts(path, artifact, actor, 2, revisions) - - -@pytest.mark.parametrize("variant", ["duplicate_final", "wrong_binding", "retained_drift", - "retained_receipt_drift", "initial_source_drift"]) -def test_receipt_readback_requires_both_bound_revisions_and_retained_bytes(tmp_path, variant): - path, artifact, actor, revisions, events = receipt_fixture(tmp_path) - if variant == "duplicate_final": - events[3:5] = events[5:] - elif variant == "wrong_binding": - receipt = events[3]["receipt"] - receipt["arm"] = "A3" - body = {k: v for k, v in receipt.items() if k != "receipt_sha256"} - receipt["receipt_sha256"] = edit.fingerprint({**body, "actor_sha256": actor}) - elif variant == "retained_drift": - (revisions / "revision-0001/candidate.py").write_text("# changed retained bytes\n") - elif variant == "retained_receipt_drift": - (revisions / "revision-0001/receipt.json").write_text("{}") - else: - (revisions / "revision-0000/candidate.py").write_text("# changed initial source\n") - path.write_text("\n".join(json.dumps(e) for e in events)) - assert not qualify.visible_receipts(path, artifact, actor, 2, revisions) - - -def test_runtime_visibility_failure_cannot_be_hidden_by_good_scripted_arms(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "FAIL"}) - monkeypatch.setattr(qualify.pilot, "run_supervised", fake_worker) - monkeypatch.setattr(qualify, "visible_receipts", lambda *_: True) - monkeypatch.setattr(qualify.workbench, "grade_final", lambda *_: {"verdict": "PASS"}) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "FAIL" and not result["software_checks_passed"] - assert result["dependencies_sha256"] == edit.fingerprint(result["dependencies"]) - - -def test_dependency_receipt_binds_transitive_identity_metrics_budget_and_initializers(): - assert { - "src/inferrsi/infra/targets/git.py", "src/inferrsi/infra/targets/__init__.py", - "src/inferrsi/model/protocols/metrics.py", "src/inferrsi/harness/scenarios/external.py", - "tools/skill_eval_budget.py", "tools/kv_edit_inputs.py", "tools/kv_worker_supervisor.py", - "src/inferrsi/__init__.py", - } <= qualify.dependencies().keys() diff --git a/tests/test_qualify_kv_reclamation.py b/tests/test_qualify_kv_reclamation.py deleted file mode 100644 index 2ebac1c2e..000000000 --- a/tests/test_qualify_kv_reclamation.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Model-free controller tests; mocked timing values are not performance evidence.""" - -import json -import os -import subprocess -import sys -import zipfile -from types import SimpleNamespace -from unittest.mock import patch - -import pytest -import yaml - -from tools import qualify_kv_reclamation as qualify - -PLAN = qualify.EVIDENCE / "plan/qualification-plan.json" - - -def control_results(): - plan = qualify.checked_plan(PLAN) - return ([{"verdict": "PASS", "checks": {"fixture": True}} - for _ in plan["required_good"]] - + [{"verdict": "FAIL", "checks": {"fixture": False}} - for _ in plan["required_bad"]]) - - -def module(name, path): - if name == "kv_independent_verifier": - return SimpleNamespace(load_traces=lambda: [{"split": "development"}], compare=lambda: True) - if name == "kv_workload": - return SimpleNamespace(workload=lambda count: []) - return SimpleNamespace(benchmark_expected=lambda count: {"observations": [], "pending": []}) - - -def measurement(name, *args): - gain = 50 if name == "reference" else 0 - return {"evaluation": {"verdict": "PASS" if gain else "FAIL", "improvement_pct": gain}, - "records": []} - - -def test_full_qualification_requires_good_bad_integrity_deadline_and_headroom(tmp_path): - identity = {"available": True, "head": "a" * 40, "snapshot_hash": "frozen", "dirty": False} - with patch.object(qualify, "repository_identity", return_value=identity), \ - patch.object(qualify.kv, "probe_isolation", return_value={"readonly": True}), \ - patch.object(qualify.kv, "verify_source", side_effect=control_results()), \ - patch.object(qualify.kv, "invoke", return_value=( - {"status": "timeout", "elapsed_seconds": 0.3}, None)), \ - patch.object(qualify.kv, "process_group_gone", return_value=True), \ - patch.object(qualify, "load_module", side_effect=module), \ - patch.object(qualify, "file_hash", wraps=qualify.file_hash) as hashes, \ - patch.object(qualify, "measure", side_effect=measurement): - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "output") - assert result["verdict"] == "PASS" - assert result["software_checks_passed"] - assert len(result["control_source_sha256"]) == 9 - assert hashes.call_count > 5 - - -def test_unavailable_is_retained_inconclusive_and_no_controls_run(tmp_path): - identity = {"available": True, "head": "a" * 40, "snapshot_hash": "frozen", "dirty": False} - with patch.object(qualify, "repository_identity", return_value=identity), \ - patch.object(qualify.kv, "probe_isolation", side_effect=qualify.kv.SandboxUnavailable( - "unavailable")), patch.object(qualify.kv, "verify_source") as verify: - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "output") - verify.assert_not_called() - assert result["verdict"] == "INCONCLUSIVE" - assert not result["software_checks_passed"] - assert json.loads((tmp_path / "output/qualification.json").read_text()) == result - assert result["model_calls"] == 0 and result["provider_charges_usd"] == 0 - - -def test_wrong_revision_is_rejected_before_execution(tmp_path): - with pytest.raises(ValueError, match="exact"): - qualify.qualify(PLAN, "b" * 40, tmp_path / "output") - assert not (tmp_path / "output").exists() - - -def test_frozen_plan_changed_input_fails(tmp_path): - plan = json.loads(PLAN.read_text()) - plan["starter_sha256"] = "0" * 64 - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - with pytest.raises(ValueError, match="changed"): - qualify.checked_plan(path) - - -def test_cli_does_not_turn_numeric_gate_into_general_ci_gate(tmp_path): - result = {"verdict": "FAIL", "software_checks_passed": True} - with patch("sys.argv", ["qualify", "--plan", str(PLAN), "--revision", "a" * 40, - "--output", str(tmp_path / "output")]), \ - patch.object(qualify, "qualify", return_value=result): - assert qualify.main() == 0 - result["software_checks_passed"] = False - assert qualify.main() == 1 - - -def test_existing_ready_job_security_and_parser_default_preserved(): - root = qualify.ROOT - workflow = yaml.safe_load((root / ".github/workflows/ci-ready.yml").read_text()) - job = workflow["jobs"]["kv-task-qualification"] - assert job["runs-on"] == "ubuntu-22.04" - assert job["permissions"] == {"contents": "read"} - assert job["if"] == "github.event.pull_request.draft == false" - assert job["steps"][0]["with"]["persist-credentials"] is False - commands = "\n".join(step.get("run", "") for step in job["steps"]) - assert "env -i" in commands and "secrets." not in commands - assert "sysctl" not in commands and "--privileged" not in commands - assert "--plan evidence/kv-reclamation-20261030/plan/qualification-plan.json" in commands - assert "/qualification.json" in job["steps"][-1]["with"]["path"] - assert "/raw-evidence.zip" in job["steps"][-1]["with"]["path"] - parser = (root / ".github/workflows/skill-eval-qualify.yml").read_text() - assert "tools.skill_eval_qualify" in parser - assert "kv_reclamation" not in parser - - -def test_independently_frozen_verifier_suite(): - result = subprocess.run([sys.executable, "-m", "pytest", "-q", - str(qualify.INDEPENDENT / "test_verifier.py")], - capture_output=True, text=True, timeout=60, check=False) - assert result.returncode == 0, result.stdout + result.stderr - - -def test_independently_frozen_benchmark_suite(): - result = subprocess.run([sys.executable, "-m", "pytest", "-q", - str(qualify.EVIDENCE / "validator/test_benchmark.py")], - env={**os.environ, - "KV_FROZEN_ORACLE": str(qualify.INDEPENDENT / "oracle.py")}, - capture_output=True, text=True, timeout=60, check=False) - assert result.returncode == 0, result.stdout + result.stderr - - -def test_retained_actual_outputs_have_labelled_path_redactions(tmp_path): - private, output = tmp_path / "private", tmp_path / "output" - private.mkdir() - output.mkdir() - original = {"source": str(qualify.ROOT / "candidate.py"), "output": str(private / "trial"), - "cwd": "/tmp/kv-replay-example", "observations": [{"reclaimed": [1, 2]}]} - (private / "actual.json").write_text(json.dumps(original)) - (private / "not-evidence.txt").write_text("not retained") - receipt = qualify.retain_raw_evidence(private, output) - with zipfile.ZipFile(output / receipt["path"]) as archive: - value = json.loads(archive.read("actual.json")) - assert value["observations"] == original["observations"] - assert value["source"] == "/candidate.py" - assert value["output"] == "/trial" - assert value["cwd"] == "" - assert "not-evidence.txt" not in archive.namelist() - assert receipt["sha256"] == qualify.file_hash(output / receipt["path"]) diff --git a/tools/kv_edit_inputs.py b/tools/kv_edit_inputs.py deleted file mode 100644 index 98e4c548f..000000000 --- a/tools/kv_edit_inputs.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Pure public input construction shared by scripted qualification and live freeze. - -No discovery, credentials, provider calls, ledger actions or hidden-trace input. -Expected development/benchmark data stays in the trusted workbench suite only. -""" - -import hashlib - -from inferrsi.harness.kv_edit_workbench import VisibleSuite -from inferrsi.model.anthropic_agent import canonical - -COMMON = {"candidate.py", "task.json", "CONTRACT.md", "workbench.md", "development.json"} -KNOWLEDGE = {"knowledge/SKILL.md", "knowledge/metadata.json", - "knowledge/references/source-notes.md"} -KNOWLEDGE_ROOT = "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/" -PROMPT = ( - "Use only the exposed task, contract, starter and fixed visible workbench. " - "Only candidate.py may change. External links are not readable files. " - "Finish after current-revision checks and measurement. " - "Any supplied knowledge is fallible guidance." -) -WORKBENCH = """# Fixed synthetic CPU workbench - -Read task.json, CONTRACT.md, candidate.py and development.json. Only candidate.py -may change. The exposed development file contains public case IDs and events; -trusted expected outputs remain outside the read allowlist. - -Use write_candidate to retain a revision, run_visible_checks for bounded public -correctness feedback, and run_visible_benchmark for fixed whole-command AB/BA -timing against the starter. Measurement includes sandbox/interpreter startup and -requires correct complete output. Rewriting invalidates previous receipts. -Submit only after checks and measurement on the current revision. Hidden final -grading occurs after immutable submission and never feeds another edit. - -This synthetic feasibility task does not establish production performance or -knowledge efficacy. Any knowledge files explicitly exposed in the manifest are -fallible guidance; their outbound links do not expand the read allowlist. -""" - - -def common_actor_contract_sha256(config): - """Bind exact shared exposure/protocol/workload despite transport-specific IDs. - - Scripted callback identities include response fixture bytes and therefore - cannot equal live callback identities. No other edit-plan field is omitted. - Outer campaign IDs are separately frozen by each supervising controller. - """ - value = {k: config[k] for k in ( - "arm", "model", "client", "input_token_ceiling", "prompt", "suite_sha256", - "baseline_sha256", - )} - value["schema"] = "kv-common-actor-contract-v1" - value["edit_plan"] = {k: v for k, v in config["edit_plan"].items() - if k != "callbacks_sha256"} - return hashlib.sha256(canonical(value).encode()).hexdigest() - - -def build_inputs(baseline, contract, development, benchmark_events, benchmark_expected, - min_improvement_pct): - """Return (public byte map, common prompt, private visible-workbench suite). - - All workload/gate inputs must come from the prospectively frozen task. No - grader split is accepted, and no expected outputs enter the public byte map. - Both arms use this same result; callers add only the allowlisted K to A3. - """ - if (type(baseline) is not bytes or not 0 < len(baseline) <= 65536 - or type(contract) is not bytes or not contract): - raise ValueError("Explicit bounded starter and contract bytes required") - baseline.decode("utf-8") - contract.decode("utf-8") - suite = VisibleSuite(canonical({ - "schema": "kv-visible-suite-v1", "development": development, - "benchmark_events": benchmark_events, "benchmark_expected": benchmark_expected, - "min_improvement_pct": min_improvement_pct, - }).encode()) - public = { - "candidate.py": baseline, - "CONTRACT.md": contract, - "task.json": canonical({ - "request": "Optimize Reclaimer; preserve its complete contract.", - "workbench": "fixed public checks and whole-command AB/BA timing", - "scope": "synthetic CPU feasibility only; no production or knowledge-efficacy claim", - }).encode(), - "workbench.md": WORKBENCH.encode(), - "development.json": canonical({ - "schema": "kv-public-development-v1", - "traces": [{k: t[k] for k in ("id", "events")} - for t in suite.read()["development"]], - }).encode(), - } - return public, PROMPT, suite diff --git a/tools/kv_edit_pilot.py b/tools/kv_edit_pilot.py deleted file mode 100644 index c850cd00b..000000000 --- a/tools/kv_edit_pilot.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Secret-free, SCRIPTED KV edit/check worker and existing-executor supervisor. - -This module cannot call a model provider. It exercises the real fixed-tool and -sandbox path with supplied response DATA. A future paid launcher is a separately -reviewed integration, not a switch or credential/config option offered here. -""" - -import argparse -import hashlib -import json -import math -import os -import sys -import time -from dataclasses import asdict, replace -from pathlib import Path - -# -I worker bootstrap: this exact trusted checkout, never cwd/PYTHONPATH/home. -ROOT = Path(__file__).resolve().parents[1] -if __package__ in {None, ""}: - sys.path.insert(0, str(ROOT / "src")) - -from inferrsi.harness import kv_edit_workbench as workbench # noqa: E402 -from inferrsi.harness import kv_reclamation as kv # noqa: E402 -from inferrsi.infra.artifacts import atomic_write_json # noqa: E402 -from inferrsi.infra.executors.local import execute_command # noqa: E402 -from inferrsi.model import anthropic_agent as provider # noqa: E402 -from inferrsi.model import anthropic_edit_agent as edit # noqa: E402 -from inferrsi.model.request_guard import ( # noqa: E402 - ENV_KEYS, - authorize_request, - gate_environment, -) - -SCHEMA = "kv-scripted-worker-v1" -FILES = ("tools/kv_edit_pilot.py", "tools/kv_worker_supervisor.py", - "src/inferrsi/harness/kv_edit_workbench.py", - "src/inferrsi/harness/kv_reclamation.py", - "src/inferrsi/harness/skill_eval_engineering.py", "src/inferrsi/harness/runner.py", - "src/inferrsi/harness/evaluation.py", "src/inferrsi/harness/reporting.py", - "src/inferrsi/infra/executors/local.py", "src/inferrsi/infra/processes.py", - "src/inferrsi/infra/artifacts.py", "src/inferrsi/runspec.py", - "src/inferrsi/claim.py", "src/inferrsi/summary_chart.py") -FIELDS = {"schema", "plan_sha256", "execution_id", "arm", "model", "client", "edit_plan", - "input_token_ceiling", "suite_sha256", "baseline_sha256", "responses_sha256", - "actor_sha256", "dependencies", "prompt"} - - -def _read_json(path, limit=16 * 1024 * 1024): - return json.loads(kv.read_regular(Path(path), limit), object_pairs_hook=provider.unique_object, - parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON"))) - - -def dependencies(): - return {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in FILES} - - -def _plan(value): - if type(value) is not dict or set(value) != { - "files", "limits", "check_ids", "metric_key", "metric_unit", "direction", - "callbacks_sha256"}: - raise ValueError("Wrong frozen edit-plan shape") - return edit.EditPlan(files=tuple(edit.ExposedFile(**f) for f in value["files"]), - limits=edit.EditLimits(**value["limits"]), check_ids=tuple(value["check_ids"]), - **{k: value[k] for k in ("metric_key", "metric_unit", "direction", "callbacks_sha256")}) - - -def prepare_scripted(inputs, workspace, plan, suite, baseline, responses, *, - plan_sha256, execution_id, arm, model, spec, prompt, input_token_ceiling): - """Freeze offline inputs. No discovery, credentials, provider calls or ledger writes.""" - inputs = Path(inputs) - if (not edit._sha(plan_sha256) or arm not in {"A2", "A3"} - or type(responses) is not list or not 0 < len(responses) <= spec["max_requests"] - or type(prompt) is not str): - raise ValueError("Explicit frozen scripted inputs required") - actual = edit.freeze_files(workspace, [f.path for f in plan.files]) - if actual != plan.files or kv.source_bytes(Path(workspace) / "candidate.py") != baseline: - raise ValueError("Staged inputs differ from frozen actor manifest/baseline") - scripts = provider.canonical(responses).encode() - if len(scripts) > spec["max_requests"] * provider.MAX_RESPONSE: - raise ValueError("Scripted response file exceeds fixed response bound") - deps = dependencies() - callback_binding = {"dependencies": deps, "suite_sha256": edit.digest(suite.serialized), - "baseline_sha256": edit.digest(baseline), - "responses_sha256": edit.digest(scripts), "prompt": prompt} - plan = replace(plan, callbacks_sha256=edit.fingerprint(callback_binding)) - identity = edit.frozen_identity(plan, model, spec, input_token_ceiling) - config = {"schema": SCHEMA, "plan_sha256": plan_sha256, "execution_id": execution_id, - "arm": arm, "model": model, "client": spec, "edit_plan": asdict(plan), - "input_token_ceiling": input_token_ceiling, - "suite_sha256": edit.digest(suite.serialized), - "baseline_sha256": edit.digest(baseline), "responses_sha256": edit.digest(scripts), - "actor_sha256": edit.fingerprint(identity), "dependencies": deps, "prompt": prompt} - inputs.mkdir(parents=True, exist_ok=False) - (inputs / "suite.json").write_bytes(suite.serialized) - (inputs / "baseline.py").write_bytes(baseline) - (inputs / "responses.json").write_bytes(scripts) - atomic_write_json(inputs / "worker.json", config) - return config - - -def checked_inputs(inputs): - inputs = Path(inputs) - config = _read_json(inputs / "worker.json", 2 * 1024 * 1024) - if (type(config) is not dict or set(config) != FIELDS or config["schema"] != SCHEMA - or config["arm"] not in {"A2", "A3"} or config["dependencies"] != dependencies()): - raise ValueError("Wrong scripted worker identity/dependencies") - plan = _plan(config["edit_plan"]) - binding = {k: config[k] for k in ("dependencies", "suite_sha256", "baseline_sha256", - "responses_sha256", "prompt")} - if plan.callbacks_sha256 != edit.fingerprint(binding): - raise ValueError("Visible callback/prompt/script binding changed") - identity = edit.frozen_identity(plan, config["model"], config["client"], - config["input_token_ceiling"]) - if edit.fingerprint(identity) != config["actor_sha256"]: - raise ValueError("Frozen actor identity changed") - raw = {} - for name, field, limit in (("suite.json", "suite_sha256", 16 * 1024 * 1024), - ("baseline.py", "baseline_sha256", plan.limits.max_source_bytes), - ("responses.json", "responses_sha256", - config["client"]["max_requests"] * provider.MAX_RESPONSE)): - raw[name] = kv.read_regular(inputs / name, limit) - if edit.digest(raw[name]) != config[field]: - raise ValueError("Frozen scripted input changed") - suite = workbench.VisibleSuite(raw["suite.json"]) - responses = _read_json(inputs / "responses.json", len(raw["responses.json"])) - if type(responses) is not list or not 0 < len(responses) <= config["client"]["max_requests"]: - raise ValueError("Invalid bounded scripted response collection") - return config, plan, suite, raw["baseline.py"], responses - - -def _known_usage(response, config, deadline): - """Only known compatible raw usage is relayed; never response text or errors.""" - try: - if (time.monotonic() >= deadline or type(response) is not dict - or response.get("type") != "message" or response.get("role") != "assistant" - or response.get("model") != config["model"] - or response.get("stop_reason") != "tool_use" - or type(response.get("content")) is not list - or any(not provider.valid_content_block(c) for c in response["content"]) - or len(provider.canonical(response).encode()) > provider.MAX_RESPONSE): - return None - usage = provider.token_usage(response.get("usage")) - if (usage["input_tokens"] > config["input_token_ceiling"] - or usage["output_tokens"] > config["client"]["max_output_tokens"]): - return None - return {k: response["usage"][k] for k in ("input_tokens", "output_tokens", - "cache_read_input_tokens", "cache_creation_input_tokens")} - except Exception: - return None - - -def scripted_worker(inputs, workspace, output, deadline, *, environment, emit): - """Fixed worker body. Test injection is via module mocks, never CLI commands.""" - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - bridge = output.parent / "bridge" - result = {"schema": SCHEMA, "status": "failed", "model_calls": 0, - "provider_charges_usd": 0, "scope": "scripted adapter qualification only"} - try: - config, plan, suite, baseline, responses = checked_inputs(inputs) - gate = gate_environment(environment) - if (not math.isfinite(deadline) - or not 0 < deadline - time.monotonic() <= plan.limits.max_seconds): - raise ValueError("Invalid inherited absolute deadline") - # No actor/provider request occurs if the real sandbox cannot be established. - result["isolation"] = kv.probe_isolation(output / "isolation", deadline, - [str(Path(inputs).resolve()), str(bridge.resolve()), str(ROOT), - f"/proc/{os.getppid()}/environ"]) - bench = workbench.VisibleWorkbench(plan, suite, baseline, output / "workbench") - grant, sent = None, 0 - - def authorize(value, inherited): - nonlocal grant - if inherited != deadline: - return False - grant = dict(value) - atomic_write_json(bridge / "pending.json", grant) - authorize_request(provider.canonical(grant).encode(), environment=gate, - timeout=max(0.001, deadline - time.monotonic())) - return True - - def send(payload, spec, remaining): - nonlocal sent - if (grant is None or edit.digest(provider.canonical(payload).encode()) - != grant["request_sha256"] or sent >= len(responses)): - raise ValueError("No exact scripted request grant/response") - value = responses[sent] - sent += 1 - atomic_write_json(bridge / "complete.json", { - "grant": grant, "usage": _known_usage(value, config, deadline)}) - return value - - # These values describe simulated request slots, not a fresh paid claim. - requests = config["client"]["max_requests"] - slots = edit.RequestSlots(config["plan_sha256"], {config["arm"]: requests}, - 10170000, requests * 10170000, execution_id=config["execution_id"]) - answer = edit.run_edit_agent(workspace, plan, config["prompt"], config["model"], - config["client"], deadline, emit, arm=config["arm"], slots=slots, - authorize=authorize, send=send, visible_checks=bench.visible_checks, - visible_benchmark=bench.visible_benchmark, retain_revision=bench.retain_revision, - input_token_ceiling=config["input_token_ceiling"]) - source = answer.pop("artifact_source").encode("utf-8") - final = output / "final" - final.mkdir() - (final / "candidate.py").write_bytes(source) - atomic_write_json(final / "receipt.json", answer) - result.update(status="completed", artifact=answer, simulated_requests=sent) - except kv.SandboxUnavailable: - result.update(status="inconclusive", reason="sandbox_unavailable") - except provider.AgentError as error: - result["reason"] = str(error) - except Exception: - result["reason"] = "scripted_worker_error" - atomic_write_json(output / "result.json", result) - return 0 if result["status"] == "completed" else 1 - - -def run_supervised(inputs, workspace, output, deadline, *, admission=None, execute=execute_command): - """SCRIPTED ONLY; shared private supervisor retains all deadline/admission semantics.""" - from tools.kv_worker_supervisor import supervise - - config, plan, _, _, _ = checked_inputs(inputs) - - def command(inputs, workspace, worker_output, inherited): - return (sys.executable, "-I", str(Path(__file__).resolve()), "--offline-worker", - "--inputs", str(inputs.resolve()), "--workspace", str(workspace.resolve()), - "--output", str(worker_output), "--deadline", repr(inherited)) - - result = supervise(config, plan, inputs, workspace, output, deadline, checkout=ROOT, - command_factory=command, - environment_factory=lambda gate: {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "LANG": "C", **gate}, - admission=admission, execute=execute) - result.update(schema=SCHEMA, simulated_slots_consumed=result.pop("slots_consumed"), - model_calls=0, provider_charges_usd=0) - atomic_write_json(Path(output) / "supervision.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--offline-worker", action="store_true", required=True) - parser.add_argument("--inputs", type=Path, required=True) - parser.add_argument("--workspace", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--deadline", type=float, required=True) - args = parser.parse_args() - # Only the inherited ephemeral request-gate capability is read. No provider - # credential discovery/configuration or live-transport path exists here. - environment = {key: os.environ.get(key, "") for key in ENV_KEYS} - return scripted_worker(args.inputs, args.workspace, args.output, args.deadline, - environment=environment, emit=lambda event: print(provider.canonical(event), flush=True)) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/kv_live_authorization.json b/tools/kv_live_authorization.json deleted file mode 100644 index dbab9c3af..000000000 --- a/tools/kv_live_authorization.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "author_login": "Dong1017", - "body_sha256": "085c799c5cf907254802d07edef5f58587a957b837fa36a70ceb1cc0a15c3549", - "comment_id": 6542223785, - "enabled": true, - "issue": 248, - "repository_id": 1409534281, - "schema": "kv-live-authorization-locator-v1" -} diff --git a/tools/kv_live_job.py b/tools/kv_live_job.py deleted file mode 100644 index 86e5f7496..000000000 --- a/tools/kv_live_job.py +++ /dev/null @@ -1,1058 +0,0 @@ -"""Fixed main-only KV launcher; disabled until a reviewed authorization is installed. - -No dispatch inputs, PR-code checkout, ledger initializer, refund or retry. Ordinary -CI tests use fake transports/stores only. Candidate source never executes on host. -""" - -import argparse -import json -import math -import os -import re -import shutil -import stat -import time -import urllib.request -from dataclasses import replace -from datetime import datetime, timedelta, timezone -from itertools import islice -from pathlib import Path - -from inferrsi.harness import kv_edit_workbench as workbench -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.model import anthropic_agent as provider -from tools import kv_edit_inputs as inputs -from tools import kv_live_policy as policy -from tools import kv_live_worker as worker -from tools import kv_pilot_admission as admission -from tools import qualify_kv_actor_cleanup as cleanup -from tools.skill_eval_budget_git import GitHubDailyStore, GitHubGitStore - -ROOT = policy.ROOT -COMMON, KNOWLEDGE, KNOWLEDGE_ROOT = inputs.COMMON, inputs.KNOWLEDGE, inputs.KNOWLEDGE_ROOT - - -class DeadlineGitHub: - """Existing Git ledger API with bounded single-attempt IO, no new storage protocol.""" - - def __init__(self, token, deadline, *, opener=None): - self.token, self.deadline = token, deadline - self.opener = opener or urllib.request.build_opener( - urllib.request.ProxyHandler({}), policy.NoRedirect()) - - def request(self, method, path, data=None): - allowed = ((method == "GET" and re.fullmatch( - r"(?:|/git/ref/heads/[A-Za-z0-9/_-]+|/git/(?:commits|trees|blobs)/[0-9a-f]{40}" - r"|/pulls/[1-9][0-9]*|/actions/(?:runs|artifacts)/[1-9][0-9]*" - r"|/issues/comments/[1-9][0-9]*)", path)) - or (method == "POST" and path in {"/git/trees", "/git/commits"}) - or (method == "PATCH" and re.fullmatch( - r"/git/refs/heads/skill-eval-budget/1409534281/(?:[1-9][0-9]*|daily)", path))) - left = self.deadline - time.monotonic() - if method == "PATCH" and (type(data) is not dict - or set(data) != {"sha", "force"} or data["force"] is not False - or not policy._hex(data["sha"], 40)): - allowed = False - if not allowed or left <= 0: - raise ValueError("Controller API scope/deadline denied") - request = urllib.request.Request( - "https://api.github.com/repos/" + policy.REPOSITORY + path, method=method, - headers={"Authorization": "Bearer " + self.token, - "Accept": "application/vnd.github+json", "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28"}, - data=None if data is None else provider.canonical(data).encode()) - try: - chunks, size = [], 0 - with self.opener.open(request, timeout=min(left, 10)) as response: - while True: - if time.monotonic() >= self.deadline: - raise ValueError("deadline") - chunk = response.read1(min(65536, 4 * 1024 * 1024 + 1 - size)) - if not chunk: - break - chunks.append(chunk) - size += len(chunk) - if size > 4 * 1024 * 1024: - raise ValueError("size") - if time.monotonic() >= self.deadline: - raise ValueError("deadline") - return json.loads(b"".join(chunks), object_pairs_hook=provider.unique_object) - except Exception: - # An uncertain write remains uncertain in the existing CAS adapter. - # No response body, URL, credential or urllib error text is retained. - raise ValueError("Controller API response unavailable") from None - - -def runway(expires_at, now=None): - now = datetime.now(timezone.utc) if now is None else now - local = now.astimezone(timezone(timedelta(hours=8))) - midnight = (local + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) - # Two 600s arms, plus bounded admission/launch margin. Not an extra model allowance. - if min(expires_at - now.timestamp(), (midnight - local).total_seconds()) < 1320: - raise ValueError("Insufficient expiry/Shanghai-day runway for the complete pair") - - -def _stage(plan, output): - suite = policy.artifact(plan["suite"], limit=16 * policy.MAX_JSON) - common_bytes, common_protocol = None, None - for arm in ("A2", "A3"): - value = plan["actors"][arm] - config = policy.read_json(ROOT / value["config"]["path"]) - folder, workspace = output / arm / "inputs", output / arm / "workspace" - folder.mkdir(parents=True) - workspace.mkdir() - names = [e["path"] for e in value["workspace"]] - expected = COMMON | (KNOWLEDGE if arm == "A3" else set()) - if set(names) != expected or len(names) != len(expected): - raise ValueError("Actor exposure differs from the fixed A2/A3 allowlist") - staged = {} - for entry in value["workspace"]: - if type(entry) is not dict or set(entry) != {"path", "source"}: - raise ValueError("Wrong exposure record") - name = policy.relative(entry["path"]) - source = entry["source"] - if name in KNOWLEDGE and source["path"] != KNOWLEDGE_ROOT + name[len("knowledge/"):]: - raise ValueError("Knowledge export is not the complete nominated owner") - if name == "candidate.py" and source["path"] != ( - "tests/fixtures/kv_reclamation/candidate.py"): - raise ValueError("Candidate starter differs from qualified task") - if name == "CONTRACT.md" and source["path"] != ( - "evidence/kv-reclamation-20261030/validator/independent/CONTRACT.md"): - raise ValueError("Neutral contract changed") - data = policy.artifact(source) - path = workspace / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - staged[name] = data - common = {name: staged[name] for name in COMMON} - if common_bytes is not None and common != common_bytes: - raise ValueError("A2/A3 common workbench bytes differ") - common_bytes = common - suite_data = workbench.VisibleSuite(suite).read() - expected_common, expected_prompt, _ = inputs.build_inputs( - staged["candidate.py"], staged["CONTRACT.md"], suite_data["development"], - suite_data["benchmark_events"], suite_data["benchmark_expected"], - suite_data["min_improvement_pct"]) - if common != expected_common or config["prompt"] != expected_prompt: - raise ValueError("Actor inputs differ from the shared qualified public workbench") - protocol = {k: config[k] for k in ("model", "client", "prompt", "input_token_ceiling")} - protocol["edit_plan"] = {k: v for k, v in config["edit_plan"].items() - if k not in {"files", "callbacks_sha256"}} - if common_protocol is not None and protocol != common_protocol: - raise ValueError("A2/A3 tool/metric/prompt limits differ") - common_protocol = protocol - edit_plan = worker.scripted._plan(config["edit_plan"]) - expected_config = worker.configuration(workspace, edit_plan, workbench.VisibleSuite(suite), - staged["candidate.py"], arm=arm, model=plan["provider"]["model"]["name"], - spec=plan["provider"]["client"], prompt=config["prompt"]) - if provider.canonical(config) != provider.canonical(expected_config): - raise ValueError("Frozen worker config differs from staged actor identity") - atomic_write_json(folder / "worker.json", config) - (folder / "suite.json").write_bytes(suite) - (folder / "baseline.py").write_bytes(staged["candidate.py"]) - atomic_write_json(folder / "binding.json", {"plan_sha256": policy.fingerprint(plan), - "execution_id": plan["execution_id"]}) - worker.checked_inputs(folder) - - -def replay_preflight(output, deadline): - """Same-host fixed tool replay, with response DATA and no provider transport. - - Rewrites the starter itself (not a hidden solution), checks/benchmarks that - exact revision and submits it. A failed speed gate is valid feedback, not a - reason to discard or rerun this mechanical qualification. - """ - results = {} - for arm in ("A2", "A3"): - config, plan, suite, baseline = worker.checked_inputs(output / arm / "inputs") - root = output / "preflight/replay" / arm - root.mkdir(parents=True) - workspace = root / "workspace" - shutil.copytree(output / arm / "workspace", workspace) - - def call(name, inputs, number): - return {"type": "tool_use", "id": f"replay-{number}", "name": name, "input": inputs} - - def response(content): - return {"type": "message", "role": "assistant", "model": config["model"], - "stop_reason": "tool_use", "content": content, - "usage": {"input_tokens": 1, "output_tokens": 1, - "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}} - - scripts = [response([call("read_file", {"path": "task.json"}, 1)]), - response([call("write_candidate", {"source": baseline.decode()}, 2), - call("run_visible_checks", {}, 3), call("run_visible_benchmark", {}, 4)]), - response([call("submit_candidate", {}, 5)])] - worker.scripted.prepare_scripted(root / "inputs", workspace, plan, suite, baseline, - scripts, plan_sha256=config["plan_sha256"], execution_id=config["execution_id"], - arm=arm, model=config["model"], spec=config["client"], prompt=config["prompt"], - input_token_ceiling=config["input_token_ceiling"]) - result = worker.scripted.run_supervised(root / "inputs", workspace, root / "run", deadline) - if (result["worker"].get("status") != "completed" - or result["execution"].get("status") != "completed" - or not result["request_completion_known"] - or result["simulated_slots_consumed"] != 3 - or kv.source_bytes(root / "run/worker/final/candidate.py") != baseline): - raise ValueError("Same-host scripted tool replay incomplete") - results[arm] = {"verdict": "PASS", "model_calls": 0, - "artifact_sha256": policy.sha(baseline), "scripted_requests": 3} - atomic_write_json(root / "receipt.json", results[arm]) - return results - - -def runtime_preflight(output, deadline, plan): - """Recheck actual mounted runtime/K/oracle isolation; hosted evidence is not a bypass.""" - from tools import kv_runtime_qualification - receipt = policy.validate_qualification(plan) - result = kv_runtime_qualification.qualify(output, deadline, root=ROOT, - independent=ROOT / "evidence/kv-reclamation-20261030/validator/independent", - fixtures=ROOT / "tests/fixtures/kv_reclamation", knowledge_root=ROOT / KNOWLEDGE_ROOT) - qualified = receipt["runtime_isolation"] - profile_source = receipt["dependencies"]["src/inferrsi/harness/skill_eval_engineering.py"] - result = {**result, "qualified_inventory_sha256": qualified["inventory_sha256"], - "profile_source_sha256": profile_source} - atomic_write_json(output / "receipt.json", result) - if (result.get("verdict") != "PASS" - or not policy._hex(result.get("inventory_sha256")) - or not policy._hex(qualified.get("inventory_sha256")) - or not policy._hex(profile_source) - or result.get("mount_count") != qualified.get("mount_count")): - raise ValueError("Same-host runtime/profile differs from hosted actor qualification") - # Inventory hashing is sensitive-set dependent, not a portable host fingerprint. - # The exact sandbox source/profile is bound by the canonical qualification - # closure; independently require this host's full inventory/probe PASS. - return result - - -def prepare(output): - if os.environ.get(provider.KEY_ENV): - raise ValueError("Preparation must precede provider-secret injection") - captured = {} - def capture(number): - comment = policy.fetch_comment(number) - captured.update({"id": comment["id"], "body": comment["body"], - "issue_url": comment["issue_url"], - "user": {k: comment["user"][k] for k in ("login", "type")}}) - return comment - authorization, locator = policy.frozen_authorization(fetch=capture) - plan = authorization["plan"] - revision, run_id = policy.runtime() - runway(plan["expires_at"]) - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - atomic_write_json(output / "authorization.json", captured) - progress = {"schema": "kv-live-preflight-progress-v1", "implementation_sha": revision, - "run_id": run_id, "plan_sha256": authorization["plan_sha256"], - "stages": dict.fromkeys(("stage", "isolation", "cleanup", "runtime", "replay"), - "not_started")} - def step(name, callback): - progress["stages"][name] = "started_outcome_unknown" - atomic_write_json(output / "preflight.json", progress) - try: - result = callback() - except BaseException: - progress["stages"][name] = "failed_or_interrupted" - atomic_write_json(output / "preflight.json", progress) - raise - progress["stages"][name] = "completed" - atomic_write_json(output / "preflight.json", progress) - return result - step("stage", lambda: _stage(plan, output)) - deadline = time.monotonic() + 120 - isolation = step("isolation", lambda: kv.probe_isolation( - output / "preflight/isolation", deadline, - [str(ROOT), str(output / "A2/inputs"), str(output / "A3/workspace")])) - descendants = step("cleanup", lambda: cleanup.qualify( - output / "preflight/cleanup", deadline, 3)) - if descendants["verdict"] != "PASS": - raise ValueError("Same-host actor cleanup qualification incomplete") - runtime_profile = step("runtime", lambda: runtime_preflight( - output / "preflight/runtime", deadline, plan)) - replay = step("replay", lambda: replay_preflight(output, deadline)) - receipt = {"schema": "kv-live-prepared-v1", "implementation_sha": revision, - "run_id": run_id, "plan_sha256": authorization["plan_sha256"], - "locator_sha256": policy.fingerprint(locator), - "isolation": isolation, "cleanup": descendants, "replay": replay, - "runtime_profile": runtime_profile, - "inputs": kv.workspace_hashes(output)} - atomic_write_json(output / "prepared.json", receipt) - return receipt - - -def execute(output, *, api=None, stores=None, launch=worker.run_supervised): - output = Path(output) - authorization, locator = policy.frozen_authorization() - plan = authorization["plan"] - revision, run_id = policy.runtime() - prepared = policy.read_json(output / "prepared.json") - if (prepared["schema"] != "kv-live-prepared-v1" - or prepared["implementation_sha"] != revision or prepared["run_id"] != run_id - or prepared["plan_sha256"] != authorization["plan_sha256"] - or prepared["locator_sha256"] != policy.fingerprint(locator) - or prepared["cleanup"]["verdict"] != "PASS" - or prepared["runtime_profile"]["verdict"] != "PASS" - or set(prepared["replay"]) != {"A2", "A3"} - or any(v.get("verdict") != "PASS" or v.get("model_calls") != 0 - for v in prepared["replay"].values())): - raise ValueError("Prepared exact source/authorization/host identity changed") - current = kv.workspace_hashes(output) - current.pop("prepared.json", None) - if current != prepared["inputs"]: - raise ValueError("Prepared inputs changed before admission") - runway(plan["expires_at"]) - # Only the existing named injected key is used. Never inspect secret files or env dumps. - safe = provider.execution_environment({provider.KEY_ENV: os.environ.get(provider.KEY_ENV)}) - api = api or DeadlineGitHub(os.environ["GITHUB_TOKEN"], time.monotonic() + 120) - root_policy = policy.policy(plan, revision) - - def check_current(value): - if value != root_policy: - return False - fresh = policy.frozen_authorization(fetch=lambda n: api.request( - "GET", f"/issues/comments/{n}")) - if fresh != (authorization, locator): - return False - policy.verify_live(api, plan, revision, run_id) - return True - - if stores is None: - stores = (GitHubDailyStore(api, policy.DailyLedgerAnchor( - key=policy.budget.DailyKey(policy.REPOSITORY_ID), **plan["daily_ledger"])), - GitHubGitStore(api, policy.LedgerAnchor(key=root_policy.key, **plan["ledger"]), - trusted_limit=root_policy.trusted_limit)) - summary = {"schema": "kv-live-execution-v1", "implementation_sha": revision, - "run_id": run_id, "plan_sha256": authorization["plan_sha256"], - "execution_id": plan["execution_id"], "charge_pr": plan["charge_pr"], - "charge_head": plan["charge_head"], - "reservation_upper_bound_usd": "162.72", "reserved_usd": None, - "admission_outcome": "pending_durable_state_unknown_no_retry", - "actual_cost_usd": None, - "arms": {arm: {"status": "not_started", "request_completion_known": False, - "slots_consumed": 0} for arm in plan["arm_order"]}} - atomic_write_json(output / "execution.json", summary) - try: - session = admission.admit(root_policy, *stores, check_current=check_current) - except BaseException: - # Failed admission can have written a partial/uncertain durable claim. - # Preserve identity for no-secret grading, but never assert zero spend. - summary["admission_outcome"] = "not_granted_durable_state_unknown_no_retry" - atomic_write_json(output / "execution.json", summary) - raise - summary.update(admission_outcome="granted", reserved_usd="162.72") - atomic_write_json(output / "execution.json", summary) - try: - for arm in plan["arm_order"]: - summary["arms"][arm] = {"status": "launching_or_interrupted", - "request_completion_known": False, "slots_consumed": None} - atomic_write_json(output / "execution.json", summary) - deadline = time.monotonic() + 600 - api.deadline = deadline - result = launch(output / arm / "inputs", output / arm / "workspace", - output / arm / "run", deadline, admission=session, key=safe[provider.KEY_ENV]) - summary["arms"][arm] = {"status": result["worker"].get("status", "missing"), - "request_completion_known": result["request_completion_known"], - "slots_consumed": result["slots_consumed"]} - atomic_write_json(output / "execution.json", summary) - if not result["request_completion_known"]: - break - finally: - session.close() - return summary - - -def _validate_feedback(receipt, config, arm, execution): - fields = {"plan_sha256", "execution_id", "arm", "artifact_sha256", "revision", - "kind", "feedback", "receipt_sha256"} - if (type(receipt) is not dict or set(receipt) != fields or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"] - or receipt["execution_id"] != execution["execution_id"] - or receipt["kind"] not in {"checks", "benchmark"} - or type(receipt["revision"]) is not int or not 0 <= receipt["revision"] <= 8 - or not policy._hex(receipt["artifact_sha256"])): - raise ValueError("Wrong visible receipt identity") - value = {k: v for k, v in receipt.items() if k != "receipt_sha256"} - if receipt["receipt_sha256"] != policy.fingerprint({**value, - "actor_sha256": config["actor_sha256"]}): - raise ValueError("Visible feedback digest changed") - worker.edit._feedback(receipt["kind"], receipt["feedback"], - worker.scripted._plan(config["edit_plan"])) - - -def _validate_submission(root, receipt, source, config, arm, execution): - fields = {"plan_sha256", "execution_id", "arm", "artifact", "revision", "sha256", - "actor_sha256", "check_receipt", "benchmark_receipt"} - if (type(receipt) is not dict or set(receipt) != fields or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"] - or receipt["execution_id"] != execution["execution_id"] - or receipt["artifact"] != "candidate.py" or type(receipt["revision"]) is not int - or not 0 <= receipt["revision"] <= 8 - or receipt["actor_sha256"] != config["actor_sha256"] - or receipt["sha256"] != policy.sha(source)): - raise ValueError("Wrong final submission identity") - revision = root / "revisions" / f"revision-{receipt['revision']:04d}" - expected = {k: v for k, v in receipt.items() if k not in {"check_receipt", "benchmark_receipt"}} - if (policy.read_json(revision / "receipt.json", 16384) != expected - or kv.source_bytes(revision / "candidate.py") != source): - raise ValueError("Final source differs from retained revision") - found = set() - for number in range(1, 13): - path = root / "feedback" / f"{number:04d}.json" - if not path.exists(): - continue - feedback = policy.read_json(path, 8192) - _validate_feedback(feedback, config, arm, execution) - field = "check_receipt" if feedback["kind"] == "checks" else "benchmark_receipt" - if (feedback["revision"] == receipt["revision"] - and feedback["artifact_sha256"] == receipt["sha256"] - and feedback["receipt_sha256"] == receipt[field]): - found.add(field) - if found != {"check_receipt", "benchmark_receipt"}: - raise ValueError("Missing exact current-revision visible receipts") - - -def _public_evidence(root, destination, arm, execution): - """Bounded structured revision/feedback only; raw logs are never publication inputs.""" - result = {"revisions": [], "feedback": [], "status": "incomplete"} - try: - config = policy.read_json(root.parents[1] / "inputs/worker.json") - except (OSError, ValueError): - return result - for revision in range(9): - source = root / "workbench/revisions" / f"revision-{revision:04d}" - try: - receipt = policy.read_json(source / "receipt.json", 16384) - data = kv.source_bytes(source / "candidate.py") - fields = {"plan_sha256", "execution_id", "arm", "artifact", "revision", - "sha256", "actor_sha256"} - if (set(receipt) != fields or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"] - or receipt["execution_id"] != execution["execution_id"] - or receipt["actor_sha256"] != config["actor_sha256"] - or receipt["artifact"] != "candidate.py" or receipt["revision"] != revision - or receipt["sha256"] != policy.sha(data) - or not policy._hex(receipt["actor_sha256"])): - raise ValueError("Wrong revision identity") - target = destination / "revisions" / f"revision-{revision:04d}" - target.mkdir(parents=True) - (target / "candidate.py").write_bytes(data) - atomic_write_json(target / "receipt.json", receipt) - result["revisions"].append({"revision": revision, "sha256": policy.sha(data)}) - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - continue - for number in range(1, 13): - try: - receipt = policy.read_json(root / "feedback" / f"{number:04d}.json", 8192) - _validate_feedback(receipt, config, arm, execution) - (destination / "feedback").mkdir(parents=True, exist_ok=True) - atomic_write_json(destination / "feedback" / f"{number:04d}.json", receipt) - result["feedback"].append({"number": number, "sha256": policy.fingerprint(receipt)}) - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - continue - if result["revisions"] and result["feedback"]: - result["status"] = "retained" - return result - - -def _proof_projection(value): - """Exact public probe fields only: never scope/error/path/log/environment text.""" - if type(value) is not dict: - return {} - result = {} - for key in ("verdict", "executor_status", "worker_status"): - if value.get(key) in {"PASS", "FAIL", "INCONCLUSIVE", "completed", "failed", - "timeout", "cancelled", "error"}: - result[key] = value[key] - for key in ("readonly", "clean_env", "network_blocked", "fork_denied", - "normal_exit_process_group_gone", "detached_session_observed", - "visibility_complete", "explicit_cancel_sent", "candidate_profile_unchanged", - "outer_worker_supervised", "source_unchanged", "dirty", - "recovery_visibility_complete"): - if type(value.get(key)) is bool: - result[key] = value[key] - for key in ("model_calls", "observed_descendants", "surviving_descendants", "recovery_kills", - "recovery_survivors", "nodes", "mount_count", "escaped_symlinks", - "sensitive_files", "scripted_requests"): - if type(value.get(key)) is int and 0 <= value[key] <= 200000: - result[key] = value[key] - for key in ("probe_sha256", "qualifier_sha256", "snapshot_hash", "inventory_sha256", - "qualified_inventory_sha256", "profile_source_sha256", "artifact_sha256"): - if policy._hex(value.get(key)): - result[key] = value[key] - if policy._hex(value.get("revision"), 40): - result["revision"] = value["revision"] - denied = value.get("inaccessible") - if type(denied) is list and len(denied) <= 30000 and all(type(v) is bool for v in denied): - result["inaccessible"] = denied - limits = value.get("limits") - if (type(limits) is dict - and set(limits) == {"memory", "processes", "cpu", "output", "files", "core"}): - if all(type(v) is list and len(v) == 2 and all(type(n) is int and -1 <= n <= 2**40 - for n in v) for v in limits.values()): - result["limits"] = limits - for key in ("probe",): - if key in value: - result[key] = _proof_projection(value[key]) - if type(value.get("cases")) is dict: - result["cases"] = {mode: _proof_projection(value["cases"].get(mode)) - for mode in ("normal", "timeout", "cancel")} - chain = value.get("bound_chain") - if (type(chain) is dict and type(chain.get("chain_length")) is int - and 3 <= chain["chain_length"] <= 128): - result["bound_chain"] = {"chain_length": chain["chain_length"]} - for role in ("worker", "detached"): - identity = chain.get(role) - if (type(identity) is dict and set(identity) == {"pid", "start"} - and all(type(v) is int and 0 < v < 2**63 for v in identity.values())): - result["bound_chain"][role] = identity - return result - - -def _public_preflight(output, authorization, locator): - """Project fixed receipts, including partial preparation; no directory walk.""" - result = {"schema": "kv-live-public-preflight-v1", "status": "missing_or_invalid", - "plan_sha256": authorization["plan_sha256"], "stages": {}, "proofs": {}} - binding = None - try: - execution = policy.read_json(output / "execution.json") - binding = tuple(execution[k] for k in ("implementation_sha", "run_id", "plan_sha256")) - except (OSError, ValueError, KeyError, TypeError): - pass - for filename in ("preflight.json", "prepared.json"): - try: - value = policy.read_json(output / filename) - schema = ("kv-live-preflight-progress-v1" if filename == "preflight.json" - else "kv-live-prepared-v1") - if (value["schema"] != schema or value["plan_sha256"] != authorization["plan_sha256"] - or not policy._hex(value["implementation_sha"], 40) - or not re.fullmatch(r"[1-9][0-9]*", value["run_id"])): - raise ValueError("Unbound preparation") - identity = tuple(value[k] for k in ("implementation_sha", "run_id", "plan_sha256")) - if binding is not None and identity != binding: - raise ValueError("Preparation execution identity changed") - binding = identity - result.update(implementation_sha=value["implementation_sha"], run_id=value["run_id"]) - if filename == "preflight.json": - result["stages"] = {name: state if state in { - "not_started", "started_outcome_unknown", - "failed_or_interrupted", "completed"} else "missing_or_invalid" - for name in ("stage", "isolation", "cleanup", "runtime", "replay") - for state in [value.get("stages", {}).get(name)]} - result["status"] = "partial" - else: - if value.get("locator_sha256") != policy.fingerprint(locator): - raise ValueError("Unbound prepared locator") - result["locator_sha256"] = value["locator_sha256"] - for name in ("isolation", "cleanup", "runtime_profile"): - result["proofs"][name] = _proof_projection(value.get(name)) - result["proofs"]["replay"] = {a: _proof_projection(value.get("replay", {}).get(a)) - for a in ("A2", "A3")} - workspace = {arm: {entry["path"]: { - "expected_sha256": entry["source"]["sha256"], - "observed_sha256": value.get("inputs", {}).get( - f"{arm}/workspace/{entry['path']}")} - for entry in authorization["plan"]["actors"][arm]["workspace"] - if entry["path"] in COMMON | (KNOWLEDGE if arm == "A3" else set())} - for arm in ("A2", "A3")} - if any(not policy._hex(entry["observed_sha256"]) - or entry["observed_sha256"] != entry["expected_sha256"] - for paths in workspace.values() for entry in paths.values()): - raise ValueError("Missing staged workspace hashes") - result["workspace"] = workspace - result["status"] = "prepared" - except (OSError, ValueError, KeyError, TypeError, RecursionError): - continue - for name, path in (("isolation", "preflight/isolation/stdout.log"), - ("cleanup", "preflight/cleanup/qualification.json"), - ("runtime_profile", "preflight/runtime/receipt.json")): - if not result["proofs"].get(name): - try: - result["proofs"][name] = _proof_projection(policy.read_json(output / path)) - except (OSError, ValueError, TypeError, RecursionError): - result["proofs"][name] = {"verdict": "INCONCLUSIVE"} - for kind, names, pattern in (("replay", ("A2", "A3"), "preflight/replay/{}/receipt.json"), - ("cleanup", ("normal", "timeout", "cancel"), "preflight/cleanup/{}/result.json")): - target = result["proofs"].setdefault(kind, {}) - if kind == "cleanup": - target = target.setdefault("cases", {}) - for name in names: - if not target.get(name): - try: - target[name] = _proof_projection(policy.read_json( - output / pattern.format(name))) - except (OSError, ValueError, TypeError, RecursionError): - target[name] = {"verdict": "INCONCLUSIVE"} - return result - - -def _raw_kv_output(path, destination, events, *, verify, remaining): - """Retain exact raw JSON only after a closed KV-data schema/privacy check. - - Malformed candidate-only output is withheld, with bounded byte/hash diagnostics; - this is not a sanitizer for provider logs or private inputs. Frozen task inputs - identify allowed strings. Oversize/unsafe files are not opened or fully hashed. - """ - result = {"status": "missing_or_unreadable", "reason": "unavailable", - "bytes": None, "sha256": None, "truncated": None} - try: - if any(p.is_symlink() for p in (path, *path.parents)): - return {**result, "reason": "unsafe_path"} - info = path.lstat() - if not stat.S_ISREG(info.st_mode): - return {**result, "reason": "nonregular_file"} - if info.st_size > 4 * policy.MAX_JSON: - return {**result, "reason": "file_byte_limit", "bytes": info.st_size, - "truncated": True} - raw = kv.read_regular(path, 4 * policy.MAX_JSON) - result = {"status": "redacted_invalid_output", "reason": "json_parse_failed", - "bytes": len(raw), "sha256": policy.sha(raw), "truncated": False} - value = json.loads(raw, object_pairs_hook=provider.unique_object) - result["reason"] = "not_allowlisted_kv_data" - ids = {e["request_id"] for e in events if type(e) is dict - and type(e.get("request_id")) is str} - def integers(items): - return type(items) is list and len(items) <= 20000 and all( - type(n) is int for n in items) - def pending(items): - return type(items) is list and len(items) <= 10000 and all( - type(v) is list and len(v) == 3 and v[0] in ids - and type(v[1]) is int - and integers(v[2]) for v in items) - if (type(value) is not dict or set(value) != {"observations", "pending"} - or not pending(value["pending"]) or type(value["observations"]) is not list - or len(value["observations"]) > 10000): - raise ValueError("Not public KV output") - for observation in value["observations"]: - fields = {"pending"} if verify else set() - if (type(observation) is not dict - or set(observation) not in (fields | {"error"}, fields | {"reclaimed"}) - or (verify and not pending(observation["pending"])) - or ("error" in observation and observation["error"] != "ValueError") - or ("reclaimed" in observation and not integers(observation["reclaimed"]))): - raise ValueError("Not public observation") - if len(raw) > remaining[0]: - return {**result, "status": "total_evidence_limit", - "reason": "whole_evidence_budget_exhausted"} - result.update(status="projection_write_failed", reason="projection_write_failed") - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(raw) - remaining[0] -= len(raw) - result = {"status": "retained", "bytes": len(raw), "sha256": policy.sha(raw), - "data_sha256": policy.fingerprint(value), "truncated": False} - except OSError: - return result - except (ValueError, TypeError, RecursionError): - pass - return result - - -def _execution_projection(value): - result = {"status": "unavailable"} - if type(value) is dict: - if value.get("status") in {"completed", "failed", "timeout", "cancelled", "error"}: - result["status"] = value["status"] - for name in ("elapsed_seconds", "metric"): - number = value.get(name) - if type(number) in (int, float) and math.isfinite(number) and 0 <= number <= 3600: - result[name] = number - elif name in value and number is None: - result[name] = None - code = value.get("returncode") - if type(code) is int and -255 <= code <= 255: - result["returncode"] = code - elif "returncode" in value and code is None: - result["returncode"] = None - return result - - -def _grade_evidence(bundle, output, result, plan=None): - """Only known grader paths; all assigned cases/trials survive as explicit rows.""" - public = output / "public" - public.mkdir(exist_ok=True) - state = {"schema": "kv-live-grade-projection-v1", "status": "started_outcome_unknown"} - atomic_write_json(public / "projection.json", state) - try: - remaining = [32 * policy.MAX_JSON] # Whole two-arm raw evidence ceiling, not per file. - manifest = {"schema": "kv-live-grade-evidence-v1", "plan_sha256": result["plan_sha256"], - "arms": {a: {"status": "not_evaluated"} for a in ("A2", "A3")}} - if plan is not None: - captured = policy.read_json(bundle / "authorization.json") - atomic_write_json(public / "authorization.json", captured) - metadata = policy.read_json(bundle / "bundle.json") - manifest["execution"] = {k: metadata["execution"][k] for k in - ("implementation_sha", "run_id", "execution_id", "charge_pr", "charge_head")} - hidden = json.loads(policy.artifact(plan["hidden_traces"], limit=16 * policy.MAX_JSON)) - if type(hidden) is not list or not 0 < len(hidden) <= 256: - raise ValueError("Frozen grader trace-count evidence bound exceeded") - suite = workbench.VisibleSuite(policy.artifact( - plan["final_suite"], limit=16 * policy.MAX_JSON)).read() - for arm in ("A2", "A3"): - records = {"status": "recorded", "semantics": [], "performance": []} - manifest["arms"][arm] = records - arm_result = result["arms"].get(arm, {}) - source = arm_result.get("artifact_sha256", - arm_result.get("semantic", {}).get("artifact_sha256")) - records["artifact_sha256"] = source if policy._hex(source) else None - baseline_hash = next(e["source"]["sha256"] for e in plan["actors"][arm]["workspace"] - if e["path"] == "candidate.py") - checks = arm_result.get("semantic", {}).get("checks", {}) - for index, trace in enumerate(hidden): - name = f"{arm}/semantics/trace-{index:03d}.json" - trace_root = output / arm / "semantics" / f"trace-{index:03d}" - raw = _raw_kv_output( - trace_root / "stdout.log", - public / name, trace["events"], verify=True, remaining=remaining) - execution = {"status": "unavailable"} - try: - record = policy.read_json(trace_root / "execution.json", 16384) - if (type(record) is not dict or set(record) != { - "status", "returncode", "elapsed_seconds"}): - raise ValueError("Wrong trusted executor receipt") - execution = _execution_projection(record) - except (OSError, ValueError, TypeError): - pass - records["semantics"].append({"trace_id": trace["id"], - "trace_sha256": policy.fingerprint(trace), "output": {"path": name, **raw}, - "check": checks.get(trace["id"]), - "execution": execution}) - runs = output / arm / "benchmark/benchmark-001/runs" - run = None - try: - entries = list(islice(runs.iterdir(), 2)) - if (len(entries) == 1 - and re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{10}", entries[0].name) - and entries[0].is_dir() and not entries[0].is_symlink()): - run = entries[0] - except OSError: - pass - schedule = [(side, index + 1, f"{side}/trial-{index + 1:03d}") - for index in range(plan["final_repetitions"]) - for side in (("baseline", "candidate") if index % 2 == 0 - else ("candidate", "baseline"))] - schedule.append(("candidate", None, "validation")) - for side, trial, relative in schedule: - name = f"{arm}/performance/{relative}/output.json" - row = {"side": side, "trial": trial, - "phase": "quality" if trial is None else "measurement", - "source_sha256": baseline_hash if side == "baseline" else source, - "input_sha256": policy.fingerprint(suite["benchmark_events"]), - "expected_sha256": policy.fingerprint(suite["benchmark_expected"]), - "execution": {"status": "unavailable"}, - "source_verified": False, "output": {"status": "not_observed"}} - if run is not None: - row["output"] = {"path": name, **_raw_kv_output( - run / relative / "stdout.log", - public / name, suite["benchmark_events"], verify=False, - remaining=remaining)} - try: - row["execution"] = _execution_projection(policy.read_json( - run / relative / "execution.json")) - identity = policy.read_json(run / relative / "source.json") - row["source_snapshots"] = {k: identity.get(k, {}).get("snapshot_hash") - if policy._hex(identity.get(k, {}).get("snapshot_hash")) else None - for k in ("before", "after")} - row["source_verified"] = all( - identity.get(k, {}).get("available") is True - and identity[k].get("snapshot_hash") == row["source_sha256"] - for k in ("before", "after")) - except (OSError, ValueError, TypeError): - pass - records["performance"].append(row) - records["raw_outputs_complete"] = all(row["output"]["status"] == "retained" - for name in ("semantics", "performance") for row in records[name]) - atomic_write_json(public / "grade.json", result) - atomic_write_json(public / "evidence.json", manifest) # Completion marker written last. - except BaseException: - state["status"] = "failed_or_interrupted" - atomic_write_json(public / "projection.json", state) - raise - state.update(status="completed", evidence_sha256=policy.fingerprint(manifest), - grade_sha256=policy.fingerprint(result)) - atomic_write_json(public / "projection.json", state) - return manifest - - -def _worker_outcome(root): - """Recover only closed trusted outcome fields, even if controller writing stopped.""" - outcome = {"worker_status": "unavailable", "reason": None, "provider_attempts": None, - "transport_error_code": None, - "execution": {"status": "unavailable", "returncode": None, - "elapsed_seconds": None}} - try: - value = policy.read_json(root / "worker/result.json") - if type(value) is not dict or value.get("schema") != worker.SCHEMA: - raise ValueError("Wrong worker result") - if value.get("status") in {"completed", "failed", "inconclusive"}: - outcome["worker_status"] = value["status"] - if "reason" in value: - outcome["reason"] = worker.closed_reason(value["reason"]) - attempts = value.get("provider_attempts") - if type(attempts) is int and 0 <= attempts <= 8: - outcome["provider_attempts"] = attempts - if value.get("transport_error_code") is not None: - outcome["transport_error_code"] = worker.closed_reason(value["transport_error_code"]) - except (OSError, ValueError, KeyError, TypeError): - pass - try: - value = policy.read_json(root / "supervision.json") - if type(value) is not dict or value.get("schema") != "kv-worker-supervision-v1": - raise ValueError("Wrong supervision result") - projected = _execution_projection(value.get("execution")) - outcome["execution"].update({k: v for k, v in projected.items() - if k in outcome["execution"]}) - except (OSError, ValueError, KeyError, TypeError): - pass - return outcome - - -def package(output): - """Fixed bounded public projection. Never copy raw worker/provider logs or private paths.""" - output = Path(output) - public = output / "public" - public.mkdir(parents=True, exist_ok=False) - preflight_status = "missing_or_invalid_authorization" - try: - # Public bytes already verified before preparation; never credentials. - captured = policy.read_json(output / "authorization.json") - authorization, locator = policy.frozen_authorization(fetch=lambda _: captured, now=0) - atomic_write_json(public / "authorization.json", captured) - proof = _public_preflight(output, authorization, locator) - atomic_write_json(public / "preflight.json", proof) - preflight_status = proof["status"] - except (OSError, ValueError, KeyError, TypeError): - pass # Missing authorization is explicit INCONCLUSIVE in the no-spend grader. - execution = (policy.read_json(output / "execution.json") - if (output / "execution.json").is_file() else {}) - summary = {"schema": "kv-live-bundle-v1", "execution": execution, "artifacts": {}, - "requests": {}, "evidence": {}, "preflight_status": preflight_status} - summary["diagnostics"] = {arm: _worker_outcome(output / arm / "run") - for arm in ("A2", "A3")} - for arm in ("A2", "A3"): - try: - supervision = policy.read_json(output / arm / "run/supervision.json") - requests = supervision["completions"] - if type(requests) is not list or len(requests) > 8: - raise ValueError("Wrong request count") - safe = [] - for entry in requests: - raw = entry["usage"] - if raw is not None: - provider.token_usage(raw) - grant = entry["grant"] - if (grant["arm"] != arm or grant["plan_sha256"] != execution["plan_sha256"] - or not policy._hex(grant["request_sha256"])): - raise ValueError("Wrong completion identity") - safe.append({"slot": grant["slot"], "request_sha256": grant["request_sha256"], - "status": "known" if raw is not None else "unresolved", - "usage": None if raw is None else {k: raw[k] for k in ("input_tokens", - "output_tokens", "cache_read_input_tokens", - "cache_creation_input_tokens")}}) - summary["requests"][arm] = safe - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - summary["requests"][arm] = [] - for arm in ("A2", "A3"): - root = output / arm / "run/worker" - summary["evidence"][arm] = _public_evidence(root, public / arm, arm, execution) - try: - result = policy.read_json(root / "result.json") - if type(result) is not dict or result.get("status") != "completed": - raise ValueError("not submitted") - receipt = policy.read_json(root / "final/receipt.json", 16384) - data = kv.source_bytes(root / "final/candidate.py") - if (receipt["sha256"] != policy.sha(data) or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"]): - raise ValueError("wrong final artifact") - config = policy.read_json(output / arm / "inputs/worker.json") - _validate_submission(public / arm, receipt, data, config, arm, execution) - (public / arm).mkdir(exist_ok=True) - (public / arm / "candidate.py").write_bytes(data) - atomic_write_json(public / arm / "receipt.json", receipt) - summary["artifacts"][arm] = {"status": "submitted", "sha256": policy.sha(data), - "receipt_sha256": policy.fingerprint(receipt)} - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - summary["artifacts"][arm] = {"status": "missing_or_invalid"} - atomic_write_json(public / "bundle.json", summary) - return summary - - -def _grade_rejected(output, reason, plan_sha256=None): - output.mkdir(parents=True, exist_ok=True) - result = {"schema": "kv-live-final-grade-v1", "plan_sha256": plan_sha256, - "scope": "final input identity unavailable; no candidate evaluation or spending", - "arms": {arm: {"verdict": "INCONCLUSIVE", "reason": reason} - for arm in ("A2", "A3")}} - atomic_write_json(output / "grade.json", result) - _grade_evidence(None, output, result) - return result - - -def _grade_context(bundle): - """Shared no-network identity check for grading and interruption-only packaging.""" - try: - captured = policy.read_json(bundle / "authorization.json") - authorization, _ = policy.frozen_authorization(fetch=lambda _: captured, now=0) - except (OSError, ValueError, KeyError, TypeError): - raise ValueError("missing_or_invalid_authorization") from None - plan = authorization["plan"] - try: - metadata = policy.read_json(bundle / "bundle.json") - except (OSError, ValueError): - raise ValueError("missing_or_invalid_bundle") from None - if (type(metadata) is not dict or metadata.get("schema") != "kv-live-bundle-v1" - or type(metadata.get("execution")) is not dict - or metadata.get("execution", {}).get("plan_sha256") != authorization["plan_sha256"] - or metadata["execution"].get("run_id") != os.environ.get("GITHUB_RUN_ID") - or metadata["execution"].get("implementation_sha") != os.environ.get("GITHUB_SHA") - or metadata["execution"].get("execution_id") != plan["execution_id"] - or metadata["execution"].get("charge_pr") != plan["charge_pr"] - or metadata["execution"].get("charge_head") != plan["charge_head"]): - raise ValueError("invalid_execution_identity") - return authorization, metadata - - -def package_grade(bundle, output): - """Always-step recovery of already generated evidence; never invoke a candidate.""" - if os.environ.get(provider.KEY_ENV) or os.environ.get("GITHUB_TOKEN"): - raise ValueError("Final packaging must run without injected credentials") - bundle, output = Path(bundle), Path(output) - try: - authorization, _ = _grade_context(bundle) - except ValueError as exc: - return _grade_rejected(output, str(exc)) - plan = authorization["plan"] - try: - result = policy.read_json(output / "grade.json") - if (result.get("schema") != "kv-live-final-grade-v1" - or result.get("plan_sha256") != authorization["plan_sha256"] - or set(result.get("arms", {})) != {"A2", "A3"}): - raise ValueError("Unbound grade") - # Reuse the existing closed feedback contract; never publish arbitrary fields. - safe = {"schema": result["schema"], "plan_sha256": result["plan_sha256"], - "scope": "one synthetic pair; hidden semantics and nonheldout task timing", - "arms": {}} - hidden = json.loads(policy.artifact(plan["hidden_traces"], limit=16 * policy.MAX_JSON)) - ids = {t["id"] for t in hidden} - for arm, value in result["arms"].items(): - entry = {"verdict": "INCONCLUSIVE", "reason": "grading_interrupted"} - if value.get("reason") in {"not_evaluated", "grading_interrupted", - "missing_or_invalid_artifact"}: - entry["reason"] = value["reason"] - if policy._hex(value.get("artifact_sha256")): - entry["artifact_sha256"] = value["artifact_sha256"] - semantic = value.get("semantic", {}) - if (semantic.get("verdict") in {"PASS", "FAIL", "INCONCLUSIVE"} - and type(semantic.get("checks")) is dict and set(semantic["checks"]) <= ids - and all(type(v) is bool for v in semantic["checks"].values()) - and (semantic["verdict"] == "INCONCLUSIVE" or ( - set(semantic["checks"]) == ids - and (semantic["verdict"] == "PASS") == all(semantic["checks"].values()))) - and policy._hex(semantic.get("artifact_sha256"))): - entry["semantic"] = {k: semantic[k] for k in - ("verdict", "checks", "artifact_sha256")} - if semantic.get("reason") in {"deadline", "error", "timeout", "cancelled", - "sandbox_unavailable"}: - entry["semantic"]["reason"] = semantic["reason"] - feedback = value.get("performance") - if feedback is not None: - config = policy.read_json(ROOT / plan["actors"][arm]["config"]["path"]) - ep = worker.scripted._plan(config["edit_plan"]) - ep = replace(ep, limits=replace(ep.limits, - benchmark_repetitions=plan["final_repetitions"])) - try: - worker.edit._feedback("benchmark", feedback, ep) - entry["performance"] = feedback - except provider.AgentError: - pass - if "verdict" not in value and {"semantic", "performance"} <= set(entry): - entry.pop("verdict") - entry.pop("reason") - safe["arms"][arm] = entry - result = safe - except (OSError, ValueError, KeyError, TypeError, AttributeError): - result = {"schema": "kv-live-final-grade-v1", "plan_sha256": authorization["plan_sha256"], - "scope": "grade process incomplete; first outputs only, no re-execution", - "arms": {a: {"verdict": "INCONCLUSIVE", "reason": "grading_interrupted"} - for a in ("A2", "A3")}} - output.mkdir(parents=True, exist_ok=True) - return _grade_evidence(bundle, output, result, plan) - - -def grade(bundle, output): - """Separate no-secret hidden grade of immutable final bytes; never model feedback.""" - if os.environ.get(provider.KEY_ENV) or os.environ.get("GITHUB_TOKEN"): - raise ValueError("Final grading must run without injected provider/GitHub credentials") - # Revocation/expiry stops spending, not offline grading of already captured public bytes. - bundle, output = Path(bundle), Path(output) - try: - authorization, metadata = _grade_context(bundle) - except ValueError as exc: - return _grade_rejected(output, str(exc)) - plan = authorization["plan"] - output.mkdir(parents=True, exist_ok=False) - hidden = json.loads(policy.artifact(plan["hidden_traces"], limit=16 * policy.MAX_JSON)) - suite = workbench.VisibleSuite(policy.artifact(plan["final_suite"], limit=16 * policy.MAX_JSON)) - result = {"schema": "kv-live-final-grade-v1", "plan_sha256": authorization["plan_sha256"], - "scope": "one synthetic pair; hidden semantics and nonheldout task timing", - "arms": {arm: {"verdict": "INCONCLUSIVE", "reason": "not_evaluated"} - for arm in ("A2", "A3")}} - atomic_write_json(output / "grade.json", result) - try: - for arm in plan["arm_order"]: - try: - entry = metadata["artifacts"][arm] - source = kv.source_bytes(bundle / arm / "candidate.py") - receipt = policy.read_json(bundle / arm / "receipt.json", 16384) - config = policy.read_json(ROOT / plan["actors"][arm]["config"]["path"]) - if (entry["status"] != "submitted" or entry["sha256"] != policy.sha(source) - or entry["receipt_sha256"] != policy.fingerprint(receipt) - or receipt["sha256"] != policy.sha(source) or receipt["arm"] != arm - or receipt["actor_sha256"] != config["actor_sha256"] - or receipt["plan_sha256"] != authorization["plan_sha256"]): - raise ValueError("Final source/receipt identity changed") - _validate_submission(bundle / arm, receipt, source, config, arm, - metadata["execution"]) - deadline = time.monotonic() + plan["final_seconds"] - result["arms"][arm] = {"verdict": "INCONCLUSIVE", "reason": "grading_interrupted", - "artifact_sha256": policy.sha(source)} - atomic_write_json(output / "grade.json", result) - semantic = workbench.grade_final(source, policy.sha(source), hidden, - output / arm / "semantics", deadline) - ep = worker.scripted._plan(config["edit_plan"]) - ep = replace(ep, limits=replace(ep.limits, - benchmark_repetitions=plan["final_repetitions"])) - baseline = policy.artifact(next(e["source"] - for e in plan["actors"][arm]["workspace"] if e["path"] == "candidate.py")) - bench = workbench.VisibleWorkbench(ep, suite, baseline, output / arm / "benchmark") - result["arms"][arm] = {"semantic": semantic, "performance": { - "verdict": "INCONCLUSIVE", "reason": "grading_interrupted"}} - atomic_write_json(output / "grade.json", result) - result["arms"][arm]["performance"] = bench.visible_benchmark(source, deadline) - atomic_write_json(output / "grade.json", result) - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - result["arms"][arm].update(verdict="INCONCLUSIVE", - reason="missing_or_invalid_artifact") - finally: - atomic_write_json(output / "grade.json", result) - _grade_evidence(bundle, output, result, plan) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("prepare", "execute", "package", "grade", - "package-grade")) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--bundle", type=Path) - args = parser.parse_args() - if args.action in {"grade", "package-grade"}: - if args.bundle is None: - parser.error("grade requires the fixed current-run bundle") - {"grade": grade, "package-grade": package_grade}[args.action](args.bundle, args.output) - elif args.bundle is not None: - parser.error("bundle input is only for final grading") - else: - {"prepare": prepare, "execute": execute, "package": package}[args.action](args.output) - - -if __name__ == "__main__": - main() diff --git a/tools/kv_live_policy.py b/tools/kv_live_policy.py deleted file mode 100644 index c00532b78..000000000 --- a/tools/kv_live_policy.py +++ /dev/null @@ -1,346 +0,0 @@ -"""Fixed public freeze/current-main validation. No secrets, writes or fallback policy.""" - -import hashlib -import io -import json -import os -import re -import subprocess -import time -import urllib.request -import zipfile -from pathlib import Path, PurePosixPath - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.model import anthropic_agent as provider -from tools import kv_edit_inputs as inputs -from tools import kv_edit_pilot as scripted -from tools import kv_pilot_admission as admission -from tools import skill_eval_budget as budget -from tools.skill_eval_budget_git import DailyLedgerAnchor, LedgerAnchor - -ROOT = Path(__file__).resolve().parents[1] -REPOSITORY = "InferMatrixLab/InferRSI" -REPOSITORY_ID = 1409534281 -OPERATOR = "Dong1017" -WORKFLOW = ".github/workflows/kv-edit-pilot.yml" -LOCATOR = ROOT / "tools/kv_live_authorization.json" -PREFIX = "\n```json\n" -SUFFIX = "\n```" -MAX_JSON = 1024 * 1024 -PLAN_FIELDS = { - "schema", "repository_id", "issue", "campaign_id", "execution_id", "charge_pr", - "charge_head", "expires_at", "provider", "actors", "dependencies", "ledger", - "daily_ledger", "qualification", "suite", "final_suite", "hidden_traces", - "arm_order", "scope", "price_evidence", "prior_owner_group_statement", - "final_seconds", "final_repetitions", -} -DEPENDENCIES = ( - "tools/kv_edit_inputs.py", "tools/kv_live_policy.py", "tools/kv_live_job.py", - "tools/kv_live_worker.py", - "tools/kv_pilot_admission.py", "tools/kv_edit_pilot.py", - "src/inferrsi/model/anthropic_agent.py", "src/inferrsi/model/anthropic_edit_agent.py", - "src/inferrsi/model/request_guard.py", "tools/skill_eval_budget.py", - "tools/skill_eval_budget_git.py", "tools/skill_eval_pr.py", - "tools/qualify_kv_actor_cleanup.py", "src/inferrsi/infra/targets/__init__.py", - "src/inferrsi/infra/targets/git.py", - ".github/workflows/kv-edit-pilot.yml", "pyproject.toml", -) - -QUALIFICATION_DEPENDENCIES = { - "tools/kv_edit_inputs.py", - "tools/qualify_kv_edit_actor.py", "tools/qualify_kv_actor_cleanup.py", - "tools/kv_runtime_qualification.py", "tools/qualify_kv_reclamation.py", - "tools/kv_pilot_admission.py", "src/inferrsi/model/anthropic_edit_agent.py", - "src/inferrsi/model/anthropic_agent.py", "src/inferrsi/model/request_guard.py", -} - - -def read_json(path, limit=MAX_JSON): - return json.loads(kv.read_regular(Path(path), limit), object_pairs_hook=provider.unique_object, - parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON"))) - - -def sha(data): - return hashlib.sha256(data).hexdigest() - - -def fingerprint(value): - return sha(provider.canonical(value).encode()) - - -def relative(name): - if (type(name) is not str or not name or "\\" in name - or PurePosixPath(name).is_absolute() - or any(p in {"", ".", ".."} for p in name.split("/"))): - raise ValueError("Invalid frozen relative path") - return name - - -def _hex(value, size=64): - return type(value) is str and re.fullmatch(r"[0-9a-f]{" + str(size) + "}", value) is not None - - -def _positive(value): - return type(value) is int and value > 0 - - -def artifact(record, *, limit=MAX_JSON): - if (type(record) is not dict or set(record) != {"path", "bytes", "sha256"} - or not _positive(record["bytes"]) or record["bytes"] > limit - or not _hex(record["sha256"])): - raise ValueError("Invalid frozen artifact record") - path = ROOT / relative(record["path"]) - data = kv.read_regular(path, limit) - if len(data) != record["bytes"] or sha(data) != record["sha256"]: - raise ValueError("Frozen artifact changed") - return data - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise ValueError("Authorization redirect forbidden") - - -def fetch_comment(number): - request = urllib.request.Request( - f"https://api.github.com/repos/{REPOSITORY}/issues/comments/{number}", - headers={"Accept": "application/vnd.github+json", "User-Agent": "InferRSI-KV-controller"}) - with urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()).open( - request, timeout=20) as response: - raw = response.read(MAX_JSON + 1) - if len(raw) > MAX_JSON: - raise ValueError("Authorization response too large") - return json.loads(raw, object_pairs_hook=provider.unique_object) - - -def frozen_authorization(*, fetch=fetch_comment, now=None): - locator = read_json(LOCATOR, 16384) - fields = {"schema", "enabled", "repository_id", "issue", "comment_id", "author_login", - "body_sha256"} - if (type(locator) is not dict or set(locator) != fields or locator.get("enabled") is not True - or locator["schema"] != "kv-live-authorization-locator-v1" - or locator["repository_id"] != REPOSITORY_ID or locator["author_login"] != OPERATOR - or not all(_positive(locator[k]) for k in ("issue", "comment_id")) - or not _hex(locator["body_sha256"])): - raise ValueError("Live KV authorization is disabled or incomplete") - comment = fetch(locator["comment_id"]) - body = comment.get("body") - if (comment.get("id") != locator["comment_id"] - or comment.get("issue_url") != - f"https://api.github.com/repos/{REPOSITORY}/issues/{locator['issue']}" - or comment.get("user", {}).get("login") != OPERATOR - or comment.get("user", {}).get("type") != "User" - or type(body) is not str or sha(body.encode()) != locator["body_sha256"] - or not body.startswith(PREFIX) or not body.endswith(SUFFIX)): - raise ValueError("Fixed authorization identity or exact bytes changed") - value = json.loads(body[len(PREFIX):-len(SUFFIX)], object_pairs_hook=provider.unique_object) - if (type(value) is not dict or set(value) != {"schema", "plan_sha256", "plan"} - or value["schema"] != "kv-live-authorization-v1" - or value["plan_sha256"] != fingerprint(value["plan"])): - raise ValueError("Authorization does not bind the exact plan") - validate_plan(value["plan"], now=time.time() if now is None else now) - if value["plan"]["issue"] != locator["issue"]: - raise ValueError("Authorization Issue changed") - return value, locator - - -def validate_plan(plan, *, now): - fields = PLAN_FIELDS | ({"budget_exception"} if type(plan) is dict - and plan.get("schema") == "kv-live-plan-v2" else set()) - if (type(plan) is not dict or set(plan) != fields - or plan["schema"] not in {"kv-live-plan-v1", "kv-live-plan-v2"} - or plan["repository_id"] != REPOSITORY_ID - or not all(_positive(plan[k]) for k in ("issue", "charge_pr", "expires_at")) - or not _hex(plan["charge_head"], 40) or plan["expires_at"] <= now - or plan["arm_order"] not in (["A2", "A3"], ["A3", "A2"]) - or plan["final_seconds"] != 180 or plan["final_repetitions"] != 8 - or type(plan["actors"]) is not dict or set(plan["actors"]) != {"A2", "A3"} - or any(type(plan[k]) is not str or not plan[k].strip() - for k in ("scope", "price_evidence", "prior_owner_group_statement"))): - raise ValueError("Wrong, expired or incomplete fixed KV scope") - for actor in plan["actors"].values(): - if (type(actor) is not dict or set(actor) != {"config", "workspace"} - or type(actor["workspace"]) is not list or not actor["workspace"]): - raise ValueError("Wrong frozen actor inputs") - artifact(actor["config"]) - key = budget.PRKey(REPOSITORY_ID, plan["charge_pr"]) - LedgerAnchor(key=key, **plan["ledger"]) - DailyLedgerAnchor(key=budget.DailyKey(REPOSITORY_ID), **plan["daily_ledger"]) - additional_attempt(plan) - sources = plan["dependencies"] - required = set(DEPENDENCIES) | set(scripted.dependencies()) - if (type(sources) is not dict or not required <= set(sources) - or any(not _hex(value) or sha(kv.read_regular(ROOT / relative(name), MAX_JSON)) != value - for name, value in sources.items())): - raise ValueError("Frozen executable dependencies changed") - for name in ("suite", "final_suite", "hidden_traces"): - artifact(plan[name], limit=16 * MAX_JSON) - validate_qualification(plan) - # Reuse strict cost identity validation; no new rates or token arithmetic here. - policy(plan, "0" * 40).reservation(now=now) - - -def qualification_dependencies(): - # Canonical closure belongs to the actor qualifier, including package - # initializers and transitive runtime/metric/scenario helpers. A checkout - # without that reviewed prerequisite cannot activate this launcher. - from tools import qualify_kv_edit_actor - return qualify_kv_edit_actor.dependencies() - - -def validate_qualification(plan): - q = plan["qualification"] - fields = {"run_id", "run_attempt", "head", "revision", "artifact_id", "archive", "receipt", - "dependencies_sha256"} - if (type(q) is not dict or set(q) != fields - or not all(_positive(q[k]) for k in ("run_id", "run_attempt", "artifact_id")) - or not _hex(q["head"], 40) or not _hex(q["revision"], 40)): - raise ValueError("Exact hosted actor qualification provenance required") - raw = artifact(q["receipt"]) - receipt = json.loads(raw, object_pairs_hook=provider.unique_object) - sources = receipt.get("dependencies", {}) - required = QUALIFICATION_DEPENDENCIES | set(scripted.dependencies()) - if (not required <= set(sources) or sources != qualification_dependencies() - or q["dependencies_sha256"] != fingerprint(sources) - or receipt.get("dependencies_sha256") != fingerprint(sources) - or any(not _hex(digest) - or sha(kv.read_regular(ROOT / relative(name), MAX_JSON)) != digest - for name, digest in sources.items())): - raise ValueError("Actor qualification dependency closure changed") - archive = artifact(q["archive"], limit=64 * MAX_JSON) - with zipfile.ZipFile(io.BytesIO(archive)) as bundle: - entries = bundle.infolist() - if (len(entries) != 2 or {e.filename for e in entries} - != {"qualification.json", "raw-evidence.zip"} - or any(e.flag_bits & 1 or e.file_size > 64 * MAX_JSON for e in entries) - or bundle.getinfo("qualification.json").file_size > MAX_JSON - or bundle.read("qualification.json") != raw): - raise ValueError("Hosted artifact does not contain the exact qualification receipt") - evidence = bundle.read("raw-evidence.zip") - if (receipt.get("schema") != "kv-scripted-actor-qualification-v1" - or receipt.get("revision") != q["revision"] or receipt.get("verdict") != "PASS" - or receipt.get("software_checks_passed") is not True - or receipt.get("source_unchanged") is not True - or receipt.get("model_calls") != 0 or receipt.get("provider_charges_usd") != 0 - or receipt.get("cleanup", {}).get("verdict") != "PASS" - or receipt.get("runtime_isolation", {}).get("verdict") != "PASS" - or not _hex(receipt.get("runtime_isolation", {}).get("inventory_sha256")) - or not _positive(receipt.get("runtime_isolation", {}).get("mount_count")) - or receipt.get("raw_evidence", {}).get("sha256") != sha(evidence) - or set(receipt.get("arms", {})) != {"A2", "A3"}): - raise ValueError("Exact model-free actor qualification required") - for arm in ("A2", "A3"): - result = receipt["arms"][arm] - config = read_json(ROOT / plan["actors"][arm]["config"]["path"]) - if (result.get("executor_status") != "completed" - or result.get("worker_status") != "completed" - or result.get("request_completion_known") is not True - or result.get("simulated_slots_consumed") != 4 - or result.get("visible_workbench_verified") is not True - or result.get("two_revision_receipts_retained") is not True - or result.get("hidden_correctness", {}).get("verdict") != "PASS" - or result.get("common_actor_contract_sha256") - != inputs.common_actor_contract_sha256(config) - or result.get("exposure") != config["edit_plan"]["files"] - or result.get("prompt_sha256") != sha(config["prompt"].encode())): - raise ValueError("Hosted actor exposure/tool evidence differs from frozen live arm") - return receipt - - -def additional_attempt(plan): - """Resolve only exact frozen v2 data; no ledger/worker/CLI cap authority.""" - if plan.get("schema", "kv-live-plan-v1") == "kv-live-plan-v1": - if "budget_exception" in plan: - raise ValueError("Legacy plan cannot carry a budget exception") - return None - value = plan.get("budget_exception") - fields = {"schema", "key", "pr_lifetime_cap_microusd", "pilot_lifetime_cap_microusd", - "additional_attempts", "prior_binding", "pr_checkpoint", "daily_checkpoint", - "campaign_id", "execution_id"} - if (plan.get("schema") != "kv-live-plan-v2" or type(value) is not dict - or set(value) != fields or value["schema"] != "kv-one-additional-attempt-v1" - or type(value["additional_attempts"]) is not int or value["additional_attempts"] != 1 - or type(value["pilot_lifetime_cap_microusd"]) is not int - or value["pilot_lifetime_cap_microusd"] != admission.PILOT_LIMIT_MICROUSD - or type(value["key"]) is not dict - or value["key"] != {"repository_id": REPOSITORY_ID, "pr_number": plan["charge_pr"]} - or type(value["prior_binding"]) is not dict - or set(value["prior_binding"]) != {"plan_digest", "run_id", "worst_case"} - or value["campaign_id"] != plan["campaign_id"] - or value["execution_id"] != plan["execution_id"] - or value["pr_checkpoint"] != plan["ledger"]["checkpoint_commit"] - or value["daily_checkpoint"] != plan["daily_ledger"]["checkpoint_commit"]): - raise ValueError("Invalid exact one-additional-attempt scope") - return admission.AdditionalAttempt( - budget.TrustedPRLimit(budget.PRKey(**value["key"]), value["pr_lifetime_cap_microusd"]), - budget.Binding(**value["prior_binding"]), value["campaign_id"], value["execution_id"], - value["pr_checkpoint"], value["daily_checkpoint"]) - - -def policy(plan, revision): - actors = [read_json(ROOT / plan["actors"][arm]["config"]["path"]) for arm in ("A2", "A3")] - return admission.PilotPolicy( - plan["campaign_id"], budget.PRKey(REPOSITORY_ID, plan["charge_pr"]), - plan["execution_id"], fingerprint(plan), plan["charge_head"], revision, - tuple(a["actor_sha256"] for a in actors), provider.canonical(plan["provider"]), - plan["expires_at"], additional_attempt(plan)) - - -def runtime(environment=None): - env = os.environ if environment is None else environment - expected = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", - "GITHUB_REPOSITORY": REPOSITORY, "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_ACTOR": OPERATOR, "GITHUB_TRIGGERING_ACTOR": OPERATOR} - if any(env.get(k) != v for k, v in expected.items()): - raise ValueError("Only fixed first-attempt owner main dispatch is supported") - revision, run_id = env.get("GITHUB_SHA", ""), env.get("GITHUB_RUN_ID", "") - if not _hex(revision, 40) or not re.fullmatch(r"[1-9][0-9]*", run_id): - raise ValueError("Invalid runtime identity") - head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() - dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT, text=True) - if head != revision or dirty: - raise ValueError("Execution checkout differs or is dirty") - return revision, run_id - - -def verify_live(api, plan, revision, run_id): - if api.request("GET", "").get("id") != REPOSITORY_ID: - raise ValueError("Live repository changed") - main = api.request("GET", "/git/ref/heads/main") - if (main.get("ref") != "refs/heads/main" or main.get("object", {}).get("type") != "commit" - or main["object"].get("sha") != revision): - raise ValueError("Only exact current main may execute") - current = api.request("GET", f"/pulls/{plan['charge_pr']}") - if (current.get("number") != plan["charge_pr"] or current.get("head", {}).get("sha") - != plan["charge_head"] or current.get("draft") is not False - or current.get("base", {}).get("ref") != "main" - or current.get("base", {}).get("repo", {}).get("id") != REPOSITORY_ID - or current.get("merged") is not True): - raise ValueError("Final reviewed implementation charge PR changed or is not merged") - run = api.request("GET", f"/actions/runs/{run_id}") - if (run.get("event") != "workflow_dispatch" or run.get("head_branch") != "main" - or run.get("head_sha") != revision or run.get("run_attempt") != 1 - or run.get("status") != "in_progress" or run.get("path") != WORKFLOW - or run.get("repository", {}).get("id") != REPOSITORY_ID - or any(run.get(k, {}).get("login") != OPERATOR for k in ("actor", "triggering_actor"))): - raise ValueError("Live workflow identity changed") - qualified = api.request("GET", f"/actions/runs/{plan['qualification']['run_id']}") - if (qualified.get("head_sha") != plan["qualification"]["head"] - or qualified.get("path") != ".github/workflows/ci-ready.yml" - or qualified.get("status") != "completed" or qualified.get("conclusion") != "success" - or qualified.get("repository", {}).get("id") != REPOSITORY_ID - or qualified.get("run_attempt") != plan["qualification"]["run_attempt"]): - raise ValueError("Actor qualification run is not successful for the pinned source") - - q = plan["qualification"] - artifact_record = api.request("GET", f"/actions/artifacts/{q['artifact_id']}") - expected_name = f"kv-actor-qualification-{q['run_id']}-{q['run_attempt']}" - if (artifact_record.get("id") != q["artifact_id"] - or artifact_record.get("name") != expected_name - or artifact_record.get("expired") is not False - or artifact_record.get("digest") != "sha256:" + q["archive"]["sha256"] - or artifact_record.get("workflow_run", {}).get("id") != q["run_id"] - or artifact_record.get("workflow_run", {}).get("head_sha") != q["head"] - or artifact_record.get("workflow_run", {}).get("head_repository_id") != REPOSITORY_ID): - raise ValueError("Qualification archive is not the exact successful hosted run artifact") diff --git a/tools/kv_live_worker.py b/tools/kv_live_worker.py deleted file mode 100644 index 3f200fbf2..000000000 --- a/tools/kv_live_worker.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Trusted fixed KV live worker. Not a candidate executor or scripted-CLI switch. - -Only a reviewed main controller may supply its frozen inputs and runtime key. -The worker has no GitHub authority; submitted source executes only via kv.invoke. -""" - -import argparse -import os -import sys -import time -from dataclasses import asdict, replace -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -if __package__ in {None, ""}: - sys.path.insert(0, str(ROOT)) - sys.path.insert(0, str(ROOT / "src")) - -from inferrsi.harness import kv_edit_workbench as workbench # noqa: E402 -from inferrsi.harness import kv_reclamation as kv # noqa: E402 -from inferrsi.infra.artifacts import atomic_write_json # noqa: E402 -from inferrsi.infra.executors.local import execute_command # noqa: E402 -from inferrsi.model import anthropic_agent as provider # noqa: E402 -from inferrsi.model import anthropic_edit_agent as edit # noqa: E402 -from inferrsi.model.request_guard import ENV_KEYS, authorize_request, gate_environment # noqa: E402 -from tools import kv_edit_pilot as scripted # noqa: E402 -from tools.kv_worker_supervisor import supervise # noqa: E402 - -SCHEMA = "kv-live-worker-v1" -REASONS = frozenset({ - "unknown", "sandbox_unavailable", "live_worker_error", "provider_transport_error", - "transport_error", "invalid_response_json", "response_bytes_limit", "credential_echo", - "invalid_credential_shape", "credential_in_payload", "request_authorization_denied", - "request_without_exact_grant", "request_bytes_limit", "timeout", "unknown_usage", - "incompatible_response", "input_token_limit", "output_token_limit", "request_limit", - "request_allocation_unavailable", "workspace_symlink", "invalid_workspace_file", - "tool_read_limit", "protected_workspace_changed", "unexposed_path", "invalid_candidate", - "candidate_bytes_limit", "invalid_visible_feedback", "feedback_bytes_limit", - "revision_retention_error", "deadline_exceeds_frozen_limit", "tool_call_limit", - "invalid_tool_call", "unsupported_tool", "task_not_read", "visible_callback_error", - "stale_or_missing_receipt", "edit_agent_error", "feedback_retention_limit", -}) -FIELDS = {"schema", "arm", "model", "client", "edit_plan", "input_token_ceiling", - "suite_sha256", "baseline_sha256", "actor_sha256", "dependencies", "prompt"} - - -def closed_reason(value): - """Fixed codes only; never exception text, provider bodies or credentials.""" - if type(value) is str and (value in REASONS or ( - len(value) == 8 and value.startswith("http_") and value[5:].isascii() - and value[5:].isdigit() and 100 <= int(value[5:]) <= 599)): - return value - return "unknown" - - -def dependencies(): - return {**scripted.dependencies(), - "tools/kv_live_worker.py": edit.digest(Path(__file__).read_bytes())} - - -def configuration(workspace, plan, suite, baseline, *, arm, model, spec, prompt): - """Offline freeze constructor; no root-plan digest cycle or credential access.""" - if (arm not in {"A2", "A3"} or type(prompt) is not str - or edit.freeze_files(workspace, [f.path for f in plan.files]) != plan.files - or kv.source_bytes(Path(workspace) / "candidate.py") != baseline): - raise ValueError("Wrong frozen live actor inputs") - deps = dependencies() - binding = {"dependencies": deps, "suite_sha256": edit.digest(suite.serialized), - "baseline_sha256": edit.digest(baseline), "prompt": prompt} - plan = replace(plan, callbacks_sha256=edit.fingerprint(binding)) - identity = edit.frozen_identity(plan, model, spec, 1000000) - return {"schema": SCHEMA, "arm": arm, "model": model, "client": spec, - "edit_plan": asdict(plan), "input_token_ceiling": 1000000, - "suite_sha256": binding["suite_sha256"], "baseline_sha256": binding["baseline_sha256"], - "actor_sha256": edit.fingerprint(identity), "dependencies": deps, "prompt": prompt} - - -def checked_inputs(inputs): - inputs = Path(inputs) - config = scripted._read_json(inputs / "worker.json", 2 * 1024 * 1024) - if (type(config) is not dict or set(config) != FIELDS or config["schema"] != SCHEMA - or config["arm"] not in {"A2", "A3"} or config["dependencies"] != dependencies() - or config["model"] != "claude-opus-5-5" or config["input_token_ceiling"] != 1000000 - or config["client"].get("max_requests") != 8 - or config["client"].get("max_output_tokens") != 4096): - raise ValueError("Live worker identity or limits changed") - plan = scripted._plan(config["edit_plan"]) - binding = {k: config[k] for k in ("dependencies", "suite_sha256", "baseline_sha256", "prompt")} - if (plan.callbacks_sha256 != edit.fingerprint(binding) - or edit.fingerprint(edit.frozen_identity(plan, config["model"], config["client"], - config["input_token_ceiling"])) != config["actor_sha256"]): - raise ValueError("Live actor callback/protocol identity changed") - suite = kv.read_regular(inputs / "suite.json", 16 * 1024 * 1024) - baseline = kv.source_bytes(inputs / "baseline.py") - if (edit.digest(suite) != config["suite_sha256"] - or edit.digest(baseline) != config["baseline_sha256"]): - raise ValueError("Live visible suite/baseline changed") - run = scripted._read_json(inputs / "binding.json", 8192) - if (type(run) is not dict or set(run) != {"plan_sha256", "execution_id"} - or not edit._sha(run["plan_sha256"]) or type(run["execution_id"]) is not str): - raise ValueError("Invalid outer binding") - return {**config, **run}, plan, workbench.VisibleSuite(suite), baseline - - -class GrantedSender: - """One IPC authorization and one exact HTTP attempt per consumed grant.""" - - def __init__(self, config, bridge, deadline, *, key, gate, - transport=provider.send_message, authorize_ipc=authorize_request): - self.config, self.bridge, self.deadline = config, Path(bridge), deadline - self.key, self.gate = key, gate_environment(gate) - self.transport, self.authorize_ipc = transport, authorize_ipc - self.grant, self.ready, self.sent, self.attempts = None, False, False, 0 - self.transport_error_code = None - - def authorize(self, grant, deadline): - if (deadline != self.deadline or time.monotonic() >= deadline - or (self.grant is not None and not self.sent)): - return False - self.grant, self.ready, self.sent = dict(grant), False, False - atomic_write_json(self.bridge / "pending.json", self.grant) - self.authorize_ipc(provider.canonical(grant).encode(), environment=self.gate, - timeout=max(0.001, deadline - time.monotonic())) - self.ready = True - return True - - def send(self, payload, spec, remaining): - left = self.deadline - time.monotonic() - if (not self.ready or self.sent or self.grant is None or left <= 0 - or spec != self.config["client"] - or edit.digest(provider.canonical(payload).encode()) != self.grant["request_sha256"] - or self.attempts >= self.config["client"]["max_requests"]): - raise provider.AgentError("request_without_exact_grant") - self.sent = True # Burn BEFORE transport; exceptions never permit resend. - self.attempts += 1 - try: - # authorize() already consumed the sole gate. Explicit empty runtime - # prevents send_message's implicit environment gate from double-counting. - value = self.transport(payload, spec, self.key, min(remaining, left), - environment={}, require_authorization=False) - except Exception as error: - # Keep a separate closed diagnostic before the edit agent generalizes - # transport failures. Counting this invocation does not prove HTTP delivery. - self.transport_error_code = closed_reason(error.args[0] if ( - isinstance(error, provider.AgentError) and len(error.args) == 1) else None) - atomic_write_json(self.bridge / "complete.json", {"grant": self.grant, "usage": None}) - raise provider.AgentError("provider_transport_error") from None - atomic_write_json(self.bridge / "complete.json", {"grant": self.grant, - "usage": scripted._known_usage(value, self.config, self.deadline)}) - return value - - -def live_worker(inputs, workspace, output, deadline, *, environment, emit, - transport=provider.send_message): - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - result = {"schema": SCHEMA, "status": "failed", "provider_attempts": 0, - "scope": "one synthetic editing feasibility arm; no efficacy claim"} - sender = None - try: - config, plan, suite, baseline = checked_inputs(inputs) - safe = provider.execution_environment(environment) - bridge = output.parent / "bridge" - result["isolation"] = kv.probe_isolation(output / "isolation", deadline, - [str(Path(inputs).resolve()), str(bridge.resolve()), str(ROOT), - f"/proc/{os.getppid()}/environ"]) - bench = workbench.VisibleWorkbench(plan, suite, baseline, output / "workbench") - sender = GrantedSender(config, bridge, deadline, key=safe[provider.KEY_ENV], - gate=safe, transport=transport) - slots = edit.RequestSlots(config["plan_sha256"], {config["arm"]: 8}, - 10170000, 81360000, execution_id=config["execution_id"]) - (output / "feedback").mkdir() - feedback_count = 0 - - def retain_feedback(event): - nonlocal feedback_count - if event.get("name") in {"run_visible_checks", "run_visible_benchmark"}: - receipt = event["receipt"] - feedback_count += 1 - if (feedback_count > plan.limits.max_checks + plan.limits.max_benchmarks - or len(provider.canonical(receipt).encode()) - > plan.limits.max_feedback_bytes): - raise provider.AgentError("feedback_retention_limit") - atomic_write_json(output / "feedback" / f"{feedback_count:04d}.json", receipt) - emit(event) - - answer = edit.run_edit_agent(workspace, plan, config["prompt"], config["model"], - config["client"], deadline, retain_feedback, arm=config["arm"], slots=slots, - authorize=sender.authorize, send=sender.send, visible_checks=bench.visible_checks, - visible_benchmark=bench.visible_benchmark, retain_revision=bench.retain_revision, - input_token_ceiling=1000000) - source = answer.pop("artifact_source").encode("utf-8") - final = output / "final" - final.mkdir() - with (final / "candidate.py").open("xb") as stream: - stream.write(source) - stream.flush() - os.fsync(stream.fileno()) - atomic_write_json(final / "receipt.json", answer) - if time.monotonic() >= deadline: - raise provider.AgentError("timeout") - result.update(status="completed", artifact=answer) - except kv.SandboxUnavailable: - result.update(status="inconclusive", reason="sandbox_unavailable") - except provider.AgentError as error: - result["reason"] = closed_reason(error.args[0] if len(error.args) == 1 else None) - except Exception: - result["reason"] = "live_worker_error" - result["provider_attempts"] = sender.attempts if sender else 0 - result["transport_error_code"] = sender.transport_error_code if sender else None - atomic_write_json(output / "result.json", result) - return 0 if result["status"] == "completed" else 1 - - -def run_supervised(inputs, workspace, output, deadline, *, admission, key, - execute=execute_command): - if admission is None: - raise ValueError("Live worker requires parent-owned durable admission") - config, plan, _, _ = checked_inputs(inputs) - safe = provider.execution_environment({provider.KEY_ENV: key, "PATH": os.defpath, "LANG": "C"}) - - def command(inputs, workspace, worker_output, inherited): - return (sys.executable, "-I", str(Path(__file__).resolve()), - "--inputs", str(inputs.resolve()), "--workspace", str(workspace.resolve()), - "--output", str(worker_output), "--deadline", repr(inherited)) - - return supervise(config, plan, inputs, workspace, output, deadline, checkout=ROOT, - command_factory=command, environment_factory=lambda gate: { - k: v for k, v in {**safe, **gate}.items() if k != "PYTHONUTF8"}, - admission=admission, execute=execute) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--inputs", type=Path, required=True) - parser.add_argument("--workspace", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--deadline", type=float, required=True) - args = parser.parse_args() - environment = {k: os.environ.get(k, "") for k in (*ENV_KEYS, provider.KEY_ENV)} - return live_worker(args.inputs, args.workspace, args.output, args.deadline, - environment=environment, emit=lambda e: print(provider.canonical(e), flush=True)) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/kv_pilot_admission.py b/tools/kv_pilot_admission.py deleted file mode 100644 index 6f01a7193..000000000 --- a/tools/kv_pilot_admission.py +++ /dev/null @@ -1,376 +0,0 @@ -"""Offline-capable admission seam for ONE paired KV editing feasibility pilot. - -No CLI, ledger bootstrap, credential discovery, transport, workflow dispatch or -new storage schema. Only trusted controller code may supply policy/stores/checks -or call these methods; candidate/author artifacts are never spending authority. -The existing ledgers retain the entire bound even after errors and completion. -""" - -import hashlib -import json -import math -import re -import threading -import time -from dataclasses import asdict, dataclass -from datetime import datetime, timezone -from decimal import Decimal - -from inferrsi.model import anthropic_agent as provider -from tools import skill_eval_budget as budget - -ARMS = ("A2", "A3") -CAMPAIGN_NAMESPACE = "kv-reclamation-one-pair-v1:" -MAX_ATTEMPTS_PER_ARM = 8 -PER_REQUEST_MICROUSD = 10_170_000 -FULL_RESERVATION_MICROUSD = 162_720_000 -MAX_ARM_SECONDS = 600 -PILOT_LIMIT_MICROUSD = 1_000_000_000 - - -def _identifier(value): - return type(value) is str and re.fullmatch(r"[A-Za-z0-9._-]{1,128}", value) is not None - - -def _sha(value, size=64): - return type(value) is str and re.fullmatch(r"[0-9a-f]{" + str(size) + "}", value) is not None - - -def _fingerprint(value): - return hashlib.sha256(provider.canonical(value).encode()).hexdigest() - - -@dataclass(frozen=True) -class AdditionalAttempt: - """One explicit issue-scoped continuation, supplied only by the trusted root. - - Old claims remain immutable. The fixed new identities are not caller-chosen - retry IDs, and checkpoints must be used by the independently anchored stores. - """ - - limit: budget.TrustedPRLimit - prior_binding: budget.Binding - campaign_id: str - execution_id: str - pr_checkpoint: str - daily_checkpoint: str - - def __post_init__(self): - if (type(self.limit) is not budget.TrustedPRLimit - or not budget.PR_LIMIT_MICROUSD < self.limit.limit_microusd <= PILOT_LIMIT_MICROUSD - or type(self.prior_binding) is not budget.Binding - or self.prior_binding.worst_case != FULL_RESERVATION_MICROUSD - or not self.prior_binding.run_id.startswith(CAMPAIGN_NAMESPACE) - or not _identifier(self.campaign_id) or not _identifier(self.execution_id) - or self.prior_binding.run_id == CAMPAIGN_NAMESPACE + self.campaign_id - or not _sha(self.pr_checkpoint, 40) or not _sha(self.daily_checkpoint, 40)): - raise budget.BudgetError("Invalid trusted one-additional-attempt exception") - - -@dataclass(frozen=True) -class PilotPolicy: - """Reviewed root configuration, NOT data accepted from a worker or PR. - - The plan digest binds visible task/K, verifier, schedule, artifact/isolation - qualification and all source revisions. Per-arm actor digests bind the exact - edit protocol/manifests. ``check_current`` independently revalidates this - policy, authorized PR/head, source revision and qualification before spending. - ``provider_json`` is canonical immutable JSON; mutable dicts are not retained. - """ - - campaign_id: str - key: budget.PRKey - execution_id: str - plan_sha256: str - pr_head: str - source_revision: str - actor_sha256: tuple[str, str] - provider_json: str - expires_at: int - additional_attempt: AdditionalAttempt | None = None - - def __post_init__(self): - if (not _identifier(self.campaign_id) or not _identifier(self.execution_id) - or type(self.key) is not budget.PRKey or not _sha(self.plan_sha256) - or not _sha(self.pr_head, 40) or not _sha(self.source_revision, 40) - or type(self.actor_sha256) is not tuple or len(self.actor_sha256) != 2 - or any(not _sha(value) for value in self.actor_sha256) - or type(self.expires_at) is not int or self.expires_at <= 0): - raise budget.BudgetError("Invalid trusted pilot binding") - extra = self.additional_attempt - if extra is not None and (type(extra) is not AdditionalAttempt - or extra.limit.key != self.key or extra.campaign_id != self.campaign_id - or extra.execution_id != self.execution_id): - raise budget.BudgetError("Additional attempt differs from exact reviewed scope") - self.reservation(now=0) - - @property - def trusted_limit(self): - return None if self.additional_attempt is None else self.additional_attempt.limit - - def reservation(self, *, now): - try: - config = json.loads(self.provider_json) - if (provider.canonical(config) != self.provider_json - or set(config) != {"model", "client", "pricing", "spend_limit"} - or config["model"] != {"name": "claude-opus-5-5", - "effort": "not_applicable"} - or config["client"]["kind"] != provider.EDIT_KIND - or config["client"]["max_requests"] != MAX_ATTEMPTS_PER_ARM - or config["client"]["max_output_tokens"] != 4096 - or config["pricing"]["currency"] != "USD" - or config["pricing"]["max_input_tokens"] != 1_000_000 - or self.expires_at > config["pricing"]["expires_at"] - or self.expires_at <= now): - raise ValueError("Unqualified pilot envelope") - result = provider.reservation(config["model"]["name"], config["client"], - config["pricing"], config["spend_limit"], 2, now=now) - if (result["requests"] != 16 - or Decimal(result["per_request"]) * 1_000_000 != PER_REQUEST_MICROUSD - or Decimal(result["amount"]) * 1_000_000 != FULL_RESERVATION_MICROUSD - or Decimal(result["limit"]) * 1_000_000 > budget.PR_LIMIT_MICROUSD): - raise ValueError("Changed pilot reservation") - return result - except (AttributeError, KeyError, TypeError, ValueError) as exc: - raise budget.BudgetError("Missing, changed or expired pilot pricing/bounds") from exc - - @property - def binding(self): - # Global run uniqueness plus the fixed namespace below prevents a new - # digest, PR, date, workflow run/attempt or campaign ID from resetting it. - value = asdict(self) - if self.additional_attempt is None: - value.pop("additional_attempt") # Preserve every historical v1 binding digest. - return budget.Binding(_fingerprint(value), - CAMPAIGN_NAMESPACE + self.campaign_id, - FULL_RESERVATION_MICROUSD) - - -def _now(clock): - now = clock() - budget.shanghai_day(now) # Reject naive or malformed clocks before timestamp(). - return now - - -def _read_pair(policy, daily_store, pr_store): - daily_key = budget.DailyKey(policy.key.repository_id) - daily = daily_store.read_verified(daily_key) - pr = pr_store.read_verified(policy.key) - if (type(daily) is not budget.DailySnapshot or daily.ledger.key != daily_key - or type(pr) is not budget.Snapshot or pr.ledger.key != policy.key - or pr.ledger.trusted_limit != policy.trusted_limit): - raise budget.BudgetError("Wrong verified ledger identity") - daily.ledger.require_active() - # An audited PR genesis is not enough if global historical obligations - # disagree. In particular, an old PR tip subtotal cannot erase daily imports. - retained_pr = sum(e.binding.worst_case for e in daily.ledger.entries if e.key == policy.key) - if retained_pr != pr.ledger.charged: - raise budget.BudgetError("Inconsistent historical daily/PR charges") - return daily, pr - - -def _campaign_history(policy, daily, pr, *, admitted=False): - """Namespace-wide one-shot guard; partial writes count, regardless of day/PR.""" - extra = policy.additional_attempt - expected = (() if extra is None else (extra.prior_binding,)) - if admitted: - expected += (policy.binding,) - entries = tuple(e for e in daily.entries if e.binding.run_id.startswith(CAMPAIGN_NAMESPACE)) - if (tuple(e.binding for e in entries) != expected - or any(e.key != policy.key or e.phase != "claimed" or e.actual_cost is not None - for e in entries)): - raise budget.BudgetError("Pilot campaign already bound or exact prior claim missing") - if extra is not None and (pr.genesis.legacy_charge != 0 - or tuple(e.binding for e in pr.entries) != expected - or any(e.phase != "claimed" or e.actual_cost is not None for e in pr.entries)): - raise budget.BudgetError("Additional attempt must preserve the exact prior PR claim") - total = sum(e.binding.worst_case for e in entries) - if total + (0 if admitted else FULL_RESERVATION_MICROUSD) > PILOT_LIMIT_MICROUSD: - raise budget.BudgetError("Pilot lifetime budget exhausted") - - -def _current(policy, check_current, wall_clock, admitted_day=None): - now = _now(wall_clock) - policy.reservation(now=now.timestamp()) - if check_current(policy) is not True: - raise budget.BudgetError("Trusted pilot authorization or qualification changed") - # The trusted live check can take time. Do not issue a stale boundary grant. - now = _now(wall_clock) - policy.reservation(now=now.timestamp()) - day = budget.shanghai_day(now) - if admitted_day is not None and day != admitted_day: - raise budget.BudgetError("Shanghai admission day ended") - return day - - -def admit(policy, daily_store, pr_store, *, check_current, - wall_clock=lambda: datetime.now(timezone.utc), monotonic_clock=time.monotonic): - """Reserve/claim once, daily first, then return a process-local controller. - - Stores implement the EXISTING verified-read/CAS contracts and must already - have audited genesis/checkpoints. No retries, missing-ledger initialization, - takeover or reconciliation-to-grant is performed. Persisted partial writes - burn capacity. A caller receiving an uncertain first write must stop, not - blindly retry: an absent observation does not prove a write cannot arrive. - """ - if (type(policy) is not PilotPolicy or not callable(check_current) - or not callable(wall_clock) or not callable(monotonic_clock)): - raise budget.BudgetError("Explicit trusted admission dependencies required") - admitted_day = _current(policy, check_current, wall_clock) - daily, pr = _read_pair(policy, daily_store, pr_store) - _campaign_history(policy, daily.ledger, pr.ledger) - # The explicit exception allows only its one additional immutable campaign. - # Unrelated obligations still count in daily_reserve; nothing is refunded. - binding = policy.binding - day_next = budget.daily_reserve(daily.ledger, policy.key, binding, admitted_day) - pr_next = budget.reserve(pr.ledger, binding) # Check BOTH caps before any write. - _current(policy, check_current, wall_clock, admitted_day) - for store, before, after in ((daily_store, daily, day_next), (pr_store, pr, pr_next)): - if (after == before.ledger - or store.compare_and_swap(before, after) is not budget.WriteResult.COMMITTED): - raise budget.BudgetError("Duplicate, conflicted or uncertain pilot reservation") - _current(policy, check_current, wall_clock, admitted_day) - budget.daily_claim_once(daily_store, policy.key, binding, - budget.shanghai_day(_now(wall_clock))) - budget.claim_once(pr_store, policy.key, binding) - session = _Admission(policy, daily_store, pr_store, admitted_day, - check_current, wall_clock, monotonic_clock) - session._check() # Both durable claims, price and day checked AFTER all writes. - return session - - -class _Admission: - """Trusted supervisor state; never serialize or pass this object to workers. - - Call start_arm BEFORE disclosure/launch. authorize matches the edit protocol's - exact grant and returns True, not a transferable token. complete requires the - SAME grant plus RAW four-class usage from trusted transport/worker observation, - never candidate-authored JSON or an unverified trace. Only one request may be - pending; missing/unknown usage in either arm stops both. No method refunds or - closes durable ledger charges. A process crash burns both durable claims. - """ - - def __init__(self, policy, daily_store, pr_store, day, check, wall, monotonic): - self.policy, self._daily_store, self._pr_store = policy, daily_store, pr_store - self._day, self._check_current = day, check - self._wall, self._monotonic = wall, monotonic - self._used = dict.fromkeys(ARMS, 0) - self._deadlines = {} - self._finished = set() - self._pending = None - self._stopped = False - self._lock = threading.Lock() - - def __reduce__(self): - raise TypeError("Pilot admission is process-local and nontransferable") - - def _check(self): - if self._stopped: - raise budget.BudgetError("Pilot admission stopped") - daily, pr = _read_pair(self.policy, self._daily_store, self._pr_store) - binding = self.policy.binding - if (not any(e.binding == binding and e.phase == "claimed" for e in pr.ledger.entries) - or not any(e.key == self.policy.key and e.binding == binding - and e.phase == "claimed" and e.admitted_day == self._day - for e in daily.ledger.entries)): - raise budget.BudgetError("Both exact durable pilot claims are required") - _campaign_history(self.policy, daily.ledger, pr.ledger, admitted=True) - _current(self.policy, self._check_current, self._wall, self._day) - - def context(self, arm): - if arm not in ARMS: - raise budget.BudgetError("Unknown pilot arm") - return f"{self.policy.binding.plan_digest}:{self.policy.binding.run_id}:{arm}" - - def start_arm(self, arm, absolute_deadline): - """Trusted supervisor only, immediately BEFORE task disclosure.""" - with self._lock: - try: - self._check() - remaining = absolute_deadline - self._monotonic() - if (arm not in ARMS or arm in self._deadlines or self._pending is not None - or set(self._deadlines) - self._finished - or type(absolute_deadline) not in (int, float) - or not math.isfinite(absolute_deadline) - or not 0 < remaining <= MAX_ARM_SECONDS): - raise budget.BudgetError("Invalid, repeated or overlapping arm launch") - self._deadlines[arm] = absolute_deadline - except BaseException: - self._stopped = True - raise - - def authorize(self, grant, absolute_deadline): - """Burn one per-arm/global attempt before live checks, then permit ONE send.""" - with self._lock: - try: - arm = grant.get("arm") if type(grant) is dict else None - if (self._stopped or self._pending is not None or arm not in self._deadlines - or arm in self._finished or self._used[arm] >= MAX_ATTEMPTS_PER_ARM): - raise budget.BudgetError("Pilot request allocation unavailable") - self._used[arm] += 1 # No failure after this point restores a slot. - expected = {"plan_sha256": self.policy.plan_sha256, - "execution_id": self.policy.execution_id, "arm": arm, - "slot": self._used[arm], - "request_sha256": grant.get("request_sha256"), - "actor_sha256": self.policy.actor_sha256[ARMS.index(arm)], - "reserved_microusd": PER_REQUEST_MICROUSD} - if (grant != expected or type(grant.get("slot")) is not int - or type(grant.get("reserved_microusd")) is not int - or not _sha(grant.get("request_sha256")) - or absolute_deadline != self._deadlines[arm]): - raise budget.BudgetError("Request differs from frozen pilot binding") - self._pending = dict(expected) # Lost/ambiguous grant stays unresolved. - self._check() - if self._monotonic() >= absolute_deadline: - raise budget.BudgetError("Frozen arm deadline ended") - return True - except BaseException: - self._stopped = True - raise - - def complete(self, grant, raw_usage): - """Record trusted observed usage only; never a refund or restart permit.""" - with self._lock: - try: - if (self._stopped or self._pending is None or type(grant) is not dict - or provider.canonical(grant) != provider.canonical(self._pending)): - raise budget.BudgetError("Missing or inconsistent request completion") - usage = provider.token_usage(raw_usage) - if usage["input_tokens"] > 1_000_000 or usage["output_tokens"] > 4096: - raise budget.BudgetError("Observed usage exceeds reserved request bound") - self._pending = None - except BaseException: - self._stopped = True - raise - - def finish_arm(self, arm): - """Keep failure/missing artifacts externally; unknown usage cannot finish.""" - with self._lock: - if (self._stopped or self._pending is not None or arm not in self._deadlines - or arm in self._finished): - self._stopped = True - raise budget.BudgetError("Unresolved, unknown or repeated arm completion") - self._finished.add(arm) - - def attempts(self): - with self._lock: - return dict(self._used) - - def close(self): - with self._lock: - self._stopped = True - - -def run_admitted(policy, daily_store, pr_store, *, launch, **dependencies): - """Invoke the injected TRUSTED launch callback once after both durable claims. - - Intended for fake/offline integration until independently reviewed real launch - wiring is supplied. This module itself performs no provider or process call. - """ - if not callable(launch): - raise budget.BudgetError("A trusted launch callback is required") - session = admit(policy, daily_store, pr_store, **dependencies) - try: - return launch(session) - finally: - session.close() diff --git a/tools/kv_runtime_qualification.py b/tools/kv_runtime_qualification.py deleted file mode 100644 index 397dcd3cb..000000000 --- a/tools/kv_runtime_qualification.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Inventory only the production sandbox runtime; probe sensitive files fail-closed. - -This is a trusted software-control check, not a candidate execution alternative. -No environment values or raw absolute inventory paths are retained. -""" - -import csv -import hashlib -import importlib.metadata -import importlib.util -import io -import json -import os -import stat -import time -from pathlib import Path, PurePosixPath -from urllib.parse import urlsplit -from urllib.request import url2pathname - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.harness.skill_eval_engineering import sandbox_command -from inferrsi.model import anthropic_edit_agent as edit - -FORBIDDEN_NAMES = {"inferrsi", "skill_library", "kv_reclamation", "oracle.py", "fixtures.json"} -MAX_NODES = 200000 -MAX_ESCAPES = 8192 -MAX_SENSITIVE = 20000 -MAX_DISTRIBUTIONS = 2000 -MAX_HASH_BYTES = 128 * 1024 * 1024 -MAX_SENSITIVE_BYTES = 32 * 1024 * 1024 - - -def _distribution_package(distribution): - """Resolve one declared package, never interpret SOURCES.txt as installed paths.""" - direct = distribution.read_text("direct_url.json") - direct = {} if direct is None else json.loads(direct) - if direct.get("dir_info", {}).get("editable") is True: - parsed = urlsplit(direct["url"]) - if (parsed.scheme != "file" or parsed.netloc not in {"", "localhost"} - or parsed.query or parsed.fragment): - raise ValueError("Unsupported editable source mapping") - source = Path(url2pathname(parsed.path)) - if not source.is_absolute(): - raise ValueError("Editable source must be absolute") - candidates = [source / "src/inferrsi", source / "inferrsi"] - candidates = [p for p in candidates if p.is_dir()] - if len(candidates) != 1: - raise ValueError("Editable package layout is missing or ambiguous") - return candidates[0] - return Path(distribution.locate_file("inferrsi")) - - -def runtime_mounts(work): - command = sandbox_command(work) - mounts = [] - for index, value in enumerate(command): - if value == "--ro-bind" and command[index + 2] != "/work": - mounts.append((Path(command[index + 1]), Path(command[index + 2]))) - if not mounts: - raise ValueError("No frozen runtime mounts") - return mounts - - -def sensitive_files(root, independent, fixtures, knowledge_root, deadline): - """Actual regular package, K, oracle and optimized-reference paths, not directories.""" - paths = [ - root / "src/inferrsi/__init__.py", - fixtures / "reference.py", - independent / "oracle.py", - independent / "fixtures.json", - knowledge_root / "SKILL.md", - knowledge_root / "metadata.json", - knowledge_root / "references/source-notes.md", - ] - if any(not p.is_file() or p.is_symlink() for p in paths): - raise ValueError("Required sensitive-file anchor missing") - - def add_tree(directory): - stack = [directory] - while stack: - if time.monotonic() >= deadline or len(paths) + len(stack) >= MAX_SENSITIVE: - raise ValueError("Sensitive-file inventory limit") - parent = stack.pop() - with os.scandir(parent) as children: - for item in children: - if item.is_symlink(): - raise ValueError("Sensitive inventory symlink") - if item.is_dir(follow_symlinks=False): - if item.name != "__pycache__": - stack.append(Path(item.path)) - elif item.is_file(follow_symlinks=False): - paths.append(Path(item.path)) - else: - raise ValueError("Sensitive inventory special node") - if len(paths) + len(stack) >= MAX_SENSITIVE: - raise ValueError("Sensitive-file inventory limit") - - def add_package(directory): - if (any(p.is_symlink() for p in (directory, *directory.parents)) - or (directory / "__init__.py").is_symlink() - or (directory / "data").is_symlink() - or not (directory / "__init__.py").is_file() - or not (directory / "data").is_dir()): - raise ValueError("Actual package source anchors incomplete") - add_tree(directory) - - add_tree(independent) - add_tree(knowledge_root) - spec = importlib.util.find_spec("inferrsi") - if spec is None or spec.origin is None: - raise ValueError("Cannot locate actual imported package") - paths.append(Path(spec.origin)) - for directory in spec.submodule_search_locations or (): - add_package(Path(directory)) - # Discover installed copies independently of the source checkout import path. - for index, distribution in enumerate(importlib.metadata.distributions()): - if index >= MAX_DISTRIBUTIONS or time.monotonic() >= deadline: - raise ValueError("Installed distribution inventory limit") - if distribution.metadata.get("Name", "").lower().replace("_", "-") == "inferrsi": - record = distribution.read_text("RECORD") - if record is not None: - # .files may silently filter missing nodes on newer Python. - # RECORD is the installed-file authority; parse its raw rows. - for row_number, row in enumerate(csv.reader(io.StringIO(record), strict=True)): - if (row_number >= MAX_SENSITIVE or len(paths) >= MAX_SENSITIVE - or time.monotonic() >= deadline): - raise ValueError("Installed package inventory limit") - if len(row) != 3 or not row[0]: - raise ValueError("Malformed installed package RECORD") - entry = PurePosixPath(row[0]) - if entry.parts[0] == "inferrsi": - if ".." in entry.parts or str(entry) != row[0]: - raise ValueError("Unsafe installed package RECORD path") - path = Path(distribution.locate_file(entry)) - if (any(p.is_symlink() for p in (path, *path.parents)) - or not path.is_file()): - raise ValueError("Installed package inventory incomplete") - paths.append(path) - elif "inferrsi" in entry.parts and entry.name != "inferrsi": - raise ValueError("Unsupported installed package RECORD mapping") - elif (distribution.read_text("PKG-INFO") is None - or distribution.read_text("installed-files.txt") is not None): - raise ValueError("Unsupported installed package inventory metadata") - # Editable/source egg-info SOURCES.txt lists source-archive paths, - # not installed nodes. Inventory the actual anchored package tree; - # retain independently installed wheel/editable copies as well. - add_package(_distribution_package(distribution)) - result = sorted({p.resolve(strict=True) for p in paths}) - if not result: - raise ValueError("No sensitive package/data inventory") - return result - - -def inventory(mounts, sensitive, deadline): - """Enumerate without following symlink directories; detect copies and escapes. - - Matching-size regular runtime files are compared to sensitive file hashes, - including the complete large hidden corpus, so renaming a complete file does - not evade the path-name check. Hash work is bounded and fails closed. - This attests this trusted host layout, not arbitrary hostile host integrity. - """ - roots = [source.resolve(strict=True) for source, _ in mounts] - sensitive = [p.resolve(strict=True) for p in sensitive] - sensitive_ancestors = {parent for p in sensitive for parent in (p, *p.parents)} - sensitive_hashes, sizes, hashed_bytes = set(), set(), 0 - - def bounded_hash(path, size): - nonlocal hashed_bytes - if size > MAX_SENSITIVE_BYTES or hashed_bytes + size > MAX_HASH_BYTES: - raise ValueError("Runtime hash byte limit") - hashed_bytes += size - data = kv.read_regular(path, MAX_SENSITIVE_BYTES) - if len(data) != size or time.monotonic() >= deadline: - raise ValueError("Runtime hash inventory incomplete") - return edit.digest(data) - - for path in sensitive: - if time.monotonic() >= deadline or len(sensitive) > MAX_SENSITIVE + 2: - raise ValueError("Sensitive hash inventory limit") - size = path.stat().st_size - if size: - sizes.add(size) - sensitive_hashes.add(bounded_hash(path, size)) - rows, escaped, rejected = [], [], False - for index, (source, destination) in enumerate(mounts): - # A bind source is resolved by the mount operation. Its target is the - # mounted tree, not an escaped descendant alias: enumerate its contents - # even when the source pathname itself is a symlink. - stack = [(source.resolve(strict=True), Path("."))] - while stack: - if time.monotonic() >= deadline or len(rows) >= MAX_NODES: - raise ValueError("Runtime inventory limit") - path, relative = stack.pop() - info = path.lstat() - row = { - "mount": index, - "relative": relative.as_posix(), - "mode": stat.S_IFMT(info.st_mode), - } - resolved = path.resolve(strict=False) - if ( - any(name.lower() in FORBIDDEN_NAMES for name in path.parts) - or resolved in sensitive_ancestors - ): - rejected = True - if stat.S_ISLNK(info.st_mode): - # Never traverse an escaped target. Test its in-sandbox alias for denial. - row["target_sha256"] = hashlib.sha256(os.readlink(path).encode()).hexdigest() - if not any(resolved == root or root in resolved.parents for root in roots): - escaped.append(str(destination / relative)) - if len(escaped) > MAX_ESCAPES: - raise ValueError("Runtime symlink limit") - row["escaped"] = True - elif stat.S_ISDIR(info.st_mode): - with os.scandir(path) as children: - stack.extend((Path(p.path), relative / p.name) for p in children) - elif stat.S_ISREG(info.st_mode): - row["bytes"] = info.st_size - if info.st_size in sizes: - row["sha256"] = bounded_hash(path, info.st_size) - if row["sha256"] in sensitive_hashes: - rejected = True - else: - raise ValueError("Special node in runtime mount") - rows.append(row) - rows.sort(key=lambda r: (r["mount"], r["relative"])) - return { - "verdict": "FAIL" if rejected else "PASS", - "nodes": len(rows), - "mount_count": len(mounts), - "inventory_sha256": edit.fingerprint({"nodes": rows}), - "escaped_symlinks": len(escaped), - "sensitive_files": len(sensitive), - }, escaped - - -def qualify(output, deadline, *, root, independent, fixtures, knowledge_root): - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - result = {"verdict": "INCONCLUSIVE", "scope": "actual production candidate runtime mounts"} - try: - sensitive = sensitive_files(root, independent, fixtures, knowledge_root, deadline) - for name in ("knowledge-canary.txt", "oracle-canary.txt"): - path = output / name - path.write_text("synthetic sensitive-path isolation canary\n", encoding="utf-8") - sensitive.append(path) - result, escaped = inventory(runtime_mounts(output), sensitive, deadline) - if result["verdict"] == "PASS": - result["probe"] = kv.probe_isolation( - output / "probe", deadline, [str(p) for p in sensitive] + escaped - ) - except kv.SandboxUnavailable: - result.update(verdict="INCONCLUSIVE", reason="sandbox_unavailable") - except Exception: - result.update(verdict="INCONCLUSIVE", reason="runtime_inventory_incomplete") - return result diff --git a/tools/kv_worker_supervisor.py b/tools/kv_worker_supervisor.py deleted file mode 100644 index 4dd58e741..000000000 --- a/tools/kv_worker_supervisor.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Private trusted worker supervision, shared by scripted and fixed live launchers. - -Factories are reviewed Python callbacks, never CLI/config/model command inputs. -No credential discovery, provider transport, ledger bootstrap or execution CLI. -""" - -import json -import math -import time -from pathlib import Path - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.infra.executors.local import execute_command -from inferrsi.model import anthropic_agent as provider -from inferrsi.model import anthropic_edit_agent as edit -from inferrsi.model.request_guard import ENV_KEYS, authorization_broker - - -def _read_json(path, limit): - return json.loads(kv.read_regular(Path(path), limit), object_pairs_hook=provider.unique_object, - parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON"))) - - -def supervise(config, plan, inputs, workspace, output, deadline, *, checkout, - command_factory, environment_factory, admission=None, execute=execute_command): - """One exact inherited deadline and parent-owned grant/completion latch.""" - if (not math.isfinite(deadline) - or not 0 < deadline - time.monotonic() <= plan.limits.max_seconds): - raise ValueError("Invalid inherited absolute deadline") - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - bridge = output / "bridge" - bridge.mkdir(mode=0o700) - pending, consumed, failed = None, 0, False - completions = [] - - def complete(): - nonlocal pending, failed - if pending is None: - return - try: - value = _read_json(bridge / "complete.json", 8192) - if (type(value) is not dict or set(value) != {"grant", "usage"} - or value["grant"] != pending): - raise ValueError("Wrong usage receipt") - if admission is not None: - admission.complete(pending, value["usage"]) - elif value["usage"] is None: - raise ValueError("Unknown simulated usage") - else: - usage = provider.token_usage(value["usage"]) - if (usage["input_tokens"] > config["input_token_ceiling"] - or usage["output_tokens"] > config["client"]["max_output_tokens"]): - raise ValueError("Out-of-bound simulated usage") - completions[-1].update(status="known", - usage={k: value["usage"][k] for k in ("input_tokens", "output_tokens", - "cache_read_input_tokens", "cache_creation_input_tokens")}) - pending = None - (bridge / "complete.json").unlink() - except Exception: - failed = True - if admission is not None: - # Unknown, lost and malformed completion is never a refund. - try: - admission.complete(pending, None) - except Exception: - pass - raise ValueError("Unresolved request completion") from None - - def check(): - nonlocal pending, consumed, failed - try: - complete() - value = _read_json(bridge / "pending.json", 8192) - expected = {"plan_sha256", "execution_id", "arm", "slot", "request_sha256", - "actor_sha256", "reserved_microusd"} - if (failed or time.monotonic() >= deadline or type(value) is not dict - or set(value) != expected or value["plan_sha256"] != config["plan_sha256"] - or value["execution_id"] != config["execution_id"] - or value["arm"] != config["arm"] - or value["actor_sha256"] != config["actor_sha256"] - or value["slot"] != consumed + 1 or not edit._sha(value["request_sha256"]) - or value["reserved_microusd"] != 10170000 - or consumed >= config["client"]["max_requests"]): - raise ValueError("Wrong bounded grant") - consumed += 1 # Burn before any uncertain live admission check. - pending = value - completions.append({"grant": dict(value), "status": "unresolved", "usage": None}) - if admission is not None and admission.authorize(value, deadline) is not True: - raise ValueError("Admission denied") - return True - except Exception: - failed = True - raise ValueError("Scripted request authorization denied") from None - - try: - if admission is not None: - admission.start_arm(config["arm"], deadline) - context = f"{config['plan_sha256']}:{config['execution_id']}:{config['arm']}" - with authorization_broker(check, config["client"]["max_requests"], context=context) as gate: - environment = environment_factory(gate) - allowed = {"PATH", "LANG", provider.KEY_ENV, *ENV_KEYS} - if (type(environment) is not dict or set(environment) - allowed - or any(environment.get(k) != v for k, v in gate.items())): - raise ValueError("Worker environment exceeds fixed allowlist") - command = command_factory(Path(inputs), Path(workspace), output / "worker", deadline) - record = execute(command, checkout, output / "executor", deadline, - environment=environment) - try: - complete() - except ValueError: - pass - # Worker JSON cannot upgrade a failed or unknown outer execution outcome. - # Only explicit successful execution can authorize the following arm. - try: - worker = _read_json(output / "worker/result.json", 1024 * 1024) - except (ValueError, OSError): - worker = {"status": "missing", "reason": "worker_result_unavailable"} - returncode = record.get("returncode") - if (record.get("status") != "completed" - or type(returncode) is not int or returncode != 0 - or type(worker) is not dict or worker.get("status") not in { - "completed", "failed", "inconclusive"}): - failed = True - if admission is not None: - admission.close() - if admission is not None and not failed: - admission.finish_arm(config["arm"]) - except BaseException: - if admission is not None: - admission.close() - raise - result = {"schema": "kv-worker-supervision-v1", "execution": record, "worker": worker, - "request_completion_known": not failed and pending is None, - "slots_consumed": consumed, "completions": completions} - atomic_write_json(output / "supervision.json", result) - return result - diff --git a/tools/qualify_kv_actor_cleanup.py b/tools/qualify_kv_actor_cleanup.py deleted file mode 100644 index 1f9ac92bd..000000000 --- a/tools/qualify_kv_actor_cleanup.py +++ /dev/null @@ -1,329 +0,0 @@ -"""Model-free qualification of namespace cleanup with a trusted fork/setsid probe. - -No candidate input is accepted. Only this authored probe may fork; the real -candidate profile remains RLIMIT_NPROC=0. Every execution still uses the existing -bubblewrap boundary and execute_command. Missing isolation/visibility is not PASS. -""" - -import argparse -import hashlib -import json -import math -import os -import signal -import sys -import tempfile -import threading -import time -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -if __package__ in {None, ""}: - sys.path.insert(0, str(ROOT / "src")) - -from inferrsi.harness.skill_eval_engineering import ( # noqa: E402 - SandboxUnavailable, - read_regular, - sandbox_command, -) -from inferrsi.infra.artifacts import atomic_write_json # noqa: E402 -from inferrsi.infra.executors.local import execute_command # noqa: E402 -from inferrsi.infra.targets import repository_identity # noqa: E402 - -PROBE = '''import json, os, resource, sys, time -resource.setrlimit(resource.RLIMIT_CPU, (4, 4)) -resource.setrlimit(resource.RLIMIT_AS, (268435456, 268435456)) -resource.setrlimit(resource.RLIMIT_FSIZE, (65536, 65536)) -resource.setrlimit(resource.RLIMIT_NOFILE, (32, 32)) -resource.setrlimit(resource.RLIMIT_NPROC, (64, 64)) -resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) -try: - child = os.fork() -except OSError: - print(json.dumps({"role": "unavailable"}), flush=True) - raise SystemExit(1) -if child == 0: - try: - os.setsid() - except OSError: - print(json.dumps({"role": "unavailable"}), flush=True) - os._exit(1) - with open("/proc/self/stat") as stream: - start = int(stream.read().rsplit(")", 1)[1].split()[19]) - print(json.dumps({"role": "detached", "own_session": os.getsid(0) == os.getpid(), - "namespace_pid": os.getpid(), "start": start, - "pid_namespace": os.readlink("/proc/self/ns/pid")}), flush=True) - time.sleep(30) - os._exit(0) -print(json.dumps({"role": "parent"}), flush=True) -time.sleep(0.5 if sys.argv[1] == "normal" else 30) -''' - - -def _stat(path): - value = path.read_text().rsplit(")", 1)[1].split() - return {"state": value[0], "ppid": int(value[1]), "start": int(value[19])} - - -def _owned_snapshot(parent, known, *, proc=Path("/proc")): - """Read stat only; retain information only for this probe's descendants.""" - current, visible = {}, True - try: - entries = list(proc.iterdir()) - except OSError: - return {}, False - for entry in entries: - if not entry.name.isdigit(): - continue - try: - current[int(entry.name)] = _stat(entry / "stat") - except (FileNotFoundError, ProcessLookupError): - continue - except (OSError, ValueError, IndexError): - visible = False - selected = {pid: item for pid, item in current.items() - if known.get(pid) == item["start"]} - parents = {parent, *selected} - while True: - added = {pid: item for pid, item in current.items() - if item["ppid"] in parents and pid != parent and pid not in selected} - if not added: - break - selected.update(added) - parents.update(added) - return selected, visible - - -def _events(path): - try: - return [json.loads(line) for line in read_regular(path, 65536).decode().splitlines()] - except (OSError, ValueError, UnicodeError): - return [] - - -def _bind_detached(events, owned, *, proc=Path("/proc")): - """Match a trusted marker to an already-owned host PID, never a count heuristic.""" - markers = [e for e in events if e.get("role") == "detached"] - if len(markers) != 1: - return None - marker = markers[0] - if (set(marker) != {"role", "own_session", "namespace_pid", "start", "pid_namespace"} - or marker["own_session"] is not True - or type(marker["namespace_pid"]) is not int or marker["namespace_pid"] <= 0 - or type(marker["start"]) is not int or marker["start"] < 0 - or type(marker["pid_namespace"]) is not str): - return None - matches = [] - for pid, item in owned.items(): - if item["start"] != marker["start"]: - continue - try: - fields = (proc / str(pid) / "status").read_text().splitlines() - nspid = next(line.split()[1:] for line in fields if line.startswith("NSpid:")) - namespace = os.readlink(proc / str(pid) / "ns/pid") - if (int(nspid[-1]) == marker["namespace_pid"] - and namespace == marker["pid_namespace"]): - matches.append({"pid": pid, "start": item["start"]}) - except (OSError, ValueError, IndexError, StopIteration): - continue - return matches[0] if len(matches) == 1 else None - - -def _bind_chain(worker_events, detached, owned): - markers = [e for e in worker_events if e.get("role") == "worker"] - if detached is None or len(markers) != 1: - return None - worker = markers[0] - if (set(worker) != {"role", "pid", "start"} - or type(worker["pid"]) is not int or type(worker["start"]) is not int - or owned.get(worker["pid"], {}).get("start") != worker["start"]): - return None - chain, pid = [], detached["pid"] - while pid in owned and pid not in chain: - chain.append(pid) - if pid == worker["pid"]: - # At least one nested sandbox process must separate worker and probe. - return {"worker": {k: worker[k] for k in ("pid", "start")}, - "detached": detached, "chain_length": len(chain)} if len(chain) >= 3 else None - pid = owned[pid]["ppid"] - return None - - -def nested_worker(mode, work, output, deadline): - """Trusted stand-in for the actor's nested executor; no candidate input accepted.""" - work = Path(work) - if (mode not in {"normal", "timeout", "cancel"} or work.is_symlink() - or {p.name for p in work.iterdir()} != {"runner.py"} - or read_regular(work / "runner.py", 65536) != PROBE.encode()): - raise ValueError("Only the exact authored descendant probe is accepted") - own = _stat(Path("/proc/self/stat")) - print(json.dumps({"role": "worker", "pid": os.getpid(), "start": own["start"]}), flush=True) - record = execute_command((*sandbox_command(work), mode), work, output, deadline, - environment={"PATH": "/usr/bin:/bin", "LANG": "C"}) - return 0 if record["status"] == "completed" else 1 - - -def probe_worker(mode, output, deadline): - """Run only in the dedicated qualifier child; cancellation signals THIS child.""" - if mode not in {"normal", "timeout", "cancel"}: - raise ValueError("Unknown trusted cleanup probe") - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - known, visibility, stop = {}, [True], threading.Event() - cancelled, bound_chain = [False], [None] - cancel_at = [None] - stdout = output / "sandbox/stdout.log" - worker_stdout = output / "worker/stdout.log" - - def observe(): - while not stop.is_set(): - found, visible = _owned_snapshot(os.getpid(), known) - visibility[0] = visibility[0] and visible - known.update({pid: item["start"] for pid, item in found.items()}) - observed = _bind_chain(_events(worker_stdout), - _bind_detached(_events(stdout), found), found) - if observed is not None: - bound_chain[0] = observed - if (mode == "cancel" and cancel_at[0] is None - and observed is not None): - cancel_at[0] = time.monotonic() + 0.05 - if (cancel_at[0] is not None and not cancelled[0] - and time.monotonic() >= cancel_at[0] and not stop.is_set()): - cancelled[0] = True - os.kill(os.getpid(), signal.SIGINT) - stop.wait(0.02) - - result = {"mode": mode, "verdict": "INCONCLUSIVE", "model_calls": 0, - "scope": "outer-worker nested sandbox cleanup; trusted probe, NOT candidate profile"} - observer = threading.Thread(target=observe, daemon=True) - try: - with tempfile.TemporaryDirectory(prefix="kv-tree-probe-") as temporary: - work = Path(temporary) - (work / "runner.py").write_text(PROBE) - sandbox_command(work) # Fail closed before spawning if unavailable. - command = (sys.executable, "-I", str(Path(__file__).resolve()), - "--nested-worker", mode, "--work", str(work), - "--output", str(output / "sandbox"), "--deadline", repr(deadline)) - observer.start() - try: - record = execute_command(command, work, output / "worker", deadline, - environment={"PATH": "/usr/bin:/bin", "LANG": "C"}) - finally: - stop.set() - observer.join(timeout=1) - found, visible = _owned_snapshot(os.getpid(), known) - visibility[0] = visibility[0] and visible and not observer.is_alive() - alive = [pid for pid, item in found.items() if item["state"] != "Z"] - expected_status = {"normal": "completed", "timeout": "timeout", "cancel": "cancelled"} - events = _events(stdout) - detached = bound_chain[0] is not None - ready = {"role": "parent"} in events and detached - result.update(executor_status=record["status"], observed_descendants=len(known), - detached_session_observed=detached, visibility_complete=visibility[0], - surviving_descendants=len(alive), explicit_cancel_sent=cancelled[0], - candidate_profile_unchanged=True, - outer_worker_supervised=True, bound_chain=bound_chain[0]) - if alive: - result["verdict"] = "FAIL" - # Recovery does not turn a failed cleanup test into PASS. Kill - # only PID+starttime identities observed as this probe's children. - killed = 0 - for pid in alive: - try: - if _stat(Path("/proc") / str(pid) / "stat")["start"] == known[pid]: - os.kill(pid, signal.SIGKILL) - killed += 1 - except (OSError, ValueError, IndexError): - pass - result["recovery_kills"] = killed - # SIGKILL delivery is asynchronous. Establish the recovery outcome - # without changing the failed verdict or granting more worker time. - recovery_deadline = time.monotonic() + 1 - while True: - remaining, visible = _owned_snapshot(os.getpid(), known) - survivors = [p for p, item in remaining.items() if item["state"] != "Z"] - if not survivors or time.monotonic() >= recovery_deadline: - result["recovery_survivors"] = len(survivors) - result["recovery_visibility_complete"] = visible - break - time.sleep(0.01) - elif (ready and visibility[0] and record["status"] == expected_status[mode] - and (mode == "normal" or record["returncode"] == -signal.SIGKILL)): - result["verdict"] = "PASS" - else: - result["reason"] = "isolation_or_observation_incomplete" - except (SandboxUnavailable, OSError): - result["reason"] = "sandbox_unavailable" - finally: - stop.set() - if observer.is_alive(): - observer.join(timeout=1) - atomic_write_json(output / "result.json", result) - return result - - -def qualify(output, deadline, probe_seconds): - """Own a fresh child for each normal/timeout/cancel case, no parent SIGINT.""" - if (type(probe_seconds) not in (float, int) or not 0 < probe_seconds <= 30 - or type(deadline) not in (float, int) or not math.isfinite(deadline) - or deadline <= time.monotonic()): - raise ValueError("Explicit bounded probe duration required") - identity = repository_identity(ROOT) - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - result = {"schema": "kv-actor-descendant-qualification-v1", "model_calls": 0, - "probe_sha256": hashlib.sha256(PROBE.encode()).hexdigest(), "cases": {}, - "qualifier_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), - "revision": identity.get("head"), "snapshot_hash": identity.get("snapshot_hash"), - "dirty": identity.get("dirty"), - "scope": "outer-worker nested namespace cleanup; no actor efficacy or provider proof"} - for mode in ("normal", "timeout", "cancel"): - worker_deadline = min(deadline - 1, time.monotonic() + probe_seconds) - command = (sys.executable, "-I", str(Path(__file__).resolve()), "--probe-worker", mode, - "--output", str(output / mode), "--deadline", repr(worker_deadline)) - record = execute_command(command, ROOT, output / f"{mode}-supervisor", deadline, - environment={"PATH": "/usr/bin:/bin", "LANG": "C"}) - try: - case = json.loads(read_regular(output / mode / "result.json", 65536)) - except (OSError, ValueError): - case = {"verdict": "INCONCLUSIVE", "reason": "probe_worker_incomplete"} - if record["status"] != "completed": - case = {**case, "verdict": "INCONCLUSIVE", "worker_status": record["status"]} - result["cases"][mode] = case - verdicts = [v["verdict"] for v in result["cases"].values()] - result["verdict"] = ("FAIL" if "FAIL" in verdicts else "PASS" if set(verdicts) == {"PASS"} - else "INCONCLUSIVE") - after = repository_identity(ROOT) - result["source_unchanged"] = (identity.get("available") and after.get("available") - and identity.get("snapshot_hash") == after.get("snapshot_hash")) - if not result["source_unchanged"]: - result["verdict"] = "INCONCLUSIVE" - atomic_write_json(output / "qualification.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--probe-worker", choices=("normal", "timeout", "cancel")) - parser.add_argument("--nested-worker", choices=("normal", "timeout", "cancel")) - parser.add_argument("--work", type=Path) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--deadline", type=float, required=True) - parser.add_argument("--probe-seconds", type=float) - args = parser.parse_args() - if args.nested_worker: - if args.probe_worker or args.work is None: - parser.error("Nested trusted worker requires its private probe workspace") - return nested_worker(args.nested_worker, args.work, args.output, args.deadline) - if args.probe_worker: - probe_worker(args.probe_worker, args.output, args.deadline) - else: - if args.probe_seconds is None: - parser.error("--probe-seconds must be explicit") - result = qualify(args.output, args.deadline, args.probe_seconds) - print(json.dumps({"verdict": result["verdict"], "model_calls": 0})) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/qualify_kv_edit_actor.py b/tools/qualify_kv_edit_actor.py deleted file mode 100644 index d388cd8c7..000000000 --- a/tools/qualify_kv_edit_actor.py +++ /dev/null @@ -1,446 +0,0 @@ -"""Fixed, secret-free actor-host qualification. Scripted data, never model calls.""" - -import argparse -import hashlib -import json -import re -import tempfile -import time -import zipfile -from pathlib import Path - -from inferrsi.harness import kv_edit_workbench as workbench -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.infra.targets import repository_identity -from inferrsi.model import anthropic_edit_agent as edit -from tools import kv_edit_inputs as shared_inputs -from tools import kv_edit_pilot as pilot -from tools import kv_runtime_qualification as runtime -from tools import qualify_kv_actor_cleanup as cleanup -from tools import qualify_kv_reclamation as task - -ROOT = Path(__file__).resolve().parents[1] -TASK_PLAN = task.EVIDENCE / "plan/qualification-plan.json" - - -def checked_plan(path): - value = pilot._read_json(path) - if ( - value.get("schema") != "kv-actor-model-free-qualification-v1" - or task.file_hash(TASK_PLAN) != value["task_plan_sha256"] - or type(value["qualification_max_seconds"]) is not int - or not 0 < value["qualification_max_seconds"] <= 600 - or type(value["visible_workload_count"]) is not int - or not 0 < value["visible_workload_count"] <= 10000 - ): - raise ValueError("Unsupported actor qualification plan") - task.checked_plan(TASK_PLAN) # Includes independent original + supplemental verifier hashes. - expected = {"SKILL.md", "metadata.json", "references/source-notes.md"} - if ( - value["knowledge_root"] - != "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes" - or set(value["knowledge_files_sha256"]) != expected - ): - raise ValueError("Only the explicit complete pinned K owner is supported") - owner = ROOT / value["knowledge_root"] - actual_files = {p.relative_to(owner).as_posix() for p in owner.rglob("*") if p.is_file()} - if actual_files != expected or any(p.is_symlink() for p in owner.rglob("*")): - raise ValueError("Pinned K owner is not the complete declared regular-file set") - for name, digest in value["knowledge_files_sha256"].items(): - if task.file_hash(ROOT / value["knowledge_root"] / name) != digest: - raise ValueError("Pinned K bytes changed") - return value - - -def _message(calls): - return { - "type": "message", - "role": "assistant", - "model": "claude-opus-5-5", - "stop_reason": "tool_use", - "content": [ - {"type": "tool_use", "id": ident, "name": name, "input": inputs} - for ident, name, inputs in calls - ], - "usage": { - "input_tokens": 1, - "output_tokens": 1, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, - }, - "fixture_origin": "scripted software control, not a provider response", - } - - -def _scripts(paths, baseline): - messages = [ - _message([(f"read-{i}", "read_file", {"path": path}) for i, path in enumerate(paths)]) - ] - for revision in (1, 2): - messages.append( - _message( - [ - ( - f"write-{revision}", - "write_candidate", - { - "source": baseline.decode() - + f"\n# Scripted revision {revision}; unchanged algorithm.\n" - }, - ), - (f"check-{revision}", "run_visible_checks", {}), - (f"benchmark-{revision}", "run_visible_benchmark", {}), - ] - ) - ) - messages.append(_message([("submit", "submit_candidate", {})])) - return messages - - -def _archive(private, output): - """Public generated controls/K only; never captures environment or gate tokens.""" - archive = output / "raw-evidence.zip" - names = [] - with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as bundle: - for path in sorted(private.rglob("*")): - if not path.is_file() or path.suffix not in { - ".json", - ".log", - ".py", - ".md", - ".html", - ".svg", - }: - continue - content = kv.read_regular(path, 32 * 1024 * 1024).decode("utf-8", errors="replace") - content = content.replace(str(ROOT), "") - content = content.replace(str(private), "") - content = content.replace(str(output), "") - content = re.sub(r"/tmp/kv-replay-[A-Za-z0-9_-]+", "", content) - content = re.sub(r"/tmp/kv-tree-probe-[A-Za-z0-9_-]+", "", content) - name = path.relative_to(private).as_posix() - names.append(name) - bundle.writestr(name, content) - bundle.writestr( - "REDACTIONS.txt", - "Paths replaced by , , " - ", , . No provider transport, " - "provider key, GitHub token or gate-token capture. All message/usage fixtures " - "are scripted and establish no real API usage or K efficacy.\n", - ) - return {"path": archive.name, "sha256": task.file_hash(archive), "files": names} - - -def visible_receipts(path, artifact, actor_sha256, repetitions, revisions): - """Independently read back exact current receipts and all expected control checks.""" - events = [ - json.loads(line) for line in kv.read_regular(path, 16 * 1024 * 1024).decode().splitlines() - ] - receipts = [ - event["receipt"] - for event in events - if event.get("type") == "agent.tool" and "receipt" in event - ] - if [(r["revision"], r["kind"]) for r in receipts] != [ - (1, "checks"), (1, "benchmark"), (2, "checks"), (2, "benchmark") - ] or artifact["revision"] != 2: - return False - retained = { - revision: edit.digest(kv.source_bytes(revisions / f"revision-{revision:04d}/candidate.py")) - for revision in (0, 1, 2) - } - artifact_events = [e for e in events if e.get("type") == "edit.artifact"] - if [e["revision"] for e in artifact_events] != [0, 1, 2]: - return False - for event in artifact_events: - directory = revisions / f"revision-{event['revision']:04d}" - metadata = {k: v for k, v in event.items() if k not in {"type", "bytes"}} - if ( - metadata != pilot._read_json(directory / "receipt.json") - or event["sha256"] != retained[event["revision"]] - or event["bytes"] != len(kv.source_bytes(directory / "candidate.py")) - or event["actor_sha256"] != actor_sha256 - or event["artifact"] != "candidate.py" - or any(event[k] != artifact[k] for k in ("plan_sha256", "execution_id", "arm")) - ): - return False - for receipt in receipts: - if ( - receipt["artifact_sha256"] != retained[receipt["revision"]] - or any(receipt[k] != artifact[k] for k in ("plan_sha256", "execution_id", "arm")) - ): - return False - body = {k: v for k, v in receipt.items() if k != "receipt_sha256"} - if receipt["receipt_sha256"] != edit.fingerprint({**body, "actor_sha256": actor_sha256}): - return False - feedback = receipt["feedback"] - if receipt["kind"] == "checks": - if feedback["verdict"] != "PASS": - return False - elif receipt["kind"] == "benchmark": - if ( - feedback["verdict"] == "INCONCLUSIVE" - or feedback["quality_verdict"] != "PASS" - or len(feedback["baseline_values"]) != repetitions - or len(feedback["candidate_values"]) != repetitions - ): - return False - else: - return False - final = {r["kind"]: r for r in receipts if r["revision"] == artifact["revision"]} - if set(final) != {"checks", "benchmark"}: - return False - return all( - r["artifact_sha256"] == artifact["sha256"] - and r["plan_sha256"] == artifact["plan_sha256"] - and r["execution_id"] == artifact["execution_id"] - and r["arm"] == artifact["arm"] - and r["receipt_sha256"] == artifact[field] - for kind, field in (("checks", "check_receipt"), ("benchmark", "benchmark_receipt")) - for r in [final[kind]] - ) - - -def dependencies(): - names = set(pilot.FILES) | { - "src/inferrsi/model/anthropic_edit_agent.py", - "src/inferrsi/model/anthropic_agent.py", - "src/inferrsi/model/request_guard.py", - "src/inferrsi/model/protocols/metrics.py", - "src/inferrsi/harness/scenarios/external.py", - "src/inferrsi/infra/targets/git.py", - "tools/qualify_kv_edit_actor.py", - "tools/qualify_kv_actor_cleanup.py", - "tools/kv_runtime_qualification.py", - "tools/kv_pilot_admission.py", - "tools/skill_eval_budget.py", - "tools/kv_edit_inputs.py", - "tools/qualify_kv_reclamation.py", - } - # Imported package initializers are executable dependencies too, including - # target identity and transitive runner metric/scenario imports. - for name in tuple(names): - for parent in Path(name).parents: - initializer = parent / "__init__.py" - if (ROOT / initializer).is_file(): - names.add(initializer.as_posix()) - return {name: task.file_hash(ROOT / name) for name in sorted(names)} - - -def qualify(plan_path, revision, output): - plan = checked_plan(plan_path) - identity = repository_identity(ROOT) - if ( - not re.fullmatch(r"[0-9a-f]{40}", revision) - or not identity.get("available") - or identity["head"] != revision - or identity["dirty"] - ): - raise ValueError("Actor qualification requires the exact clean Git revision") - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - deadline = time.monotonic() + plan["qualification_max_seconds"] - result = { - "schema": "kv-scripted-actor-qualification-v1", - "revision": revision, - "snapshot_hash": identity["snapshot_hash"], - "plan_sha256": task.file_hash(plan_path), - "qualifier_sha256": task.file_hash(Path(__file__)), - "dependencies": dependencies(), - "dependencies_sha256": edit.fingerprint(dependencies()), - "model_calls": 0, - "provider_charges_usd": 0, - "verdict": "INCONCLUSIVE", - "software_checks_passed": False, - "scope": "scripted actual-host actor mechanics; NOT a paid A2/A3 or efficacy result", - "arms": {}, - } - with tempfile.TemporaryDirectory(prefix="kv-actor-qualification-") as temporary: - private = Path(temporary) - try: - result["cleanup"] = cleanup.qualify( - private / "cleanup", deadline, plan["descendant_probe_seconds"] - ) - result["runtime_isolation"] = runtime.qualify( - private / "runtime", - deadline, - root=ROOT, - independent=task.INDEPENDENT, - fixtures=task.FIXTURES, - knowledge_root=ROOT / plan["knowledge_root"], - ) - traces = task.load_module( - "fixed_verifier", task.INDEPENDENT / "verifier.py" - ).load_traces() - - def selected(split): - return [ - {k: t[k] for k in ("id", "split", "events", "expected")} - for t in traces - if t["split"] == split - ] - - development, hidden = selected("development"), selected("grader") - workload = task.load_module("fixed_workload", task.FIXTURES / "workload.py") - oracle = task.load_module("fixed_benchmark", task.EVIDENCE / "validator/benchmark.py") - baseline = kv.source_bytes(task.FIXTURES / "candidate.py") - common, prompt, suite = shared_inputs.build_inputs( - baseline, (task.INDEPENDENT / "CONTRACT.md").read_bytes(), development, - workload.workload(plan["visible_workload_count"]), - oracle.benchmark_expected(plan["visible_workload_count"]), - plan["visible_min_improvement_pct"], - ) - for arm in ("A2", "A3"): - root = private / arm - workspace = root / "workspace" - workspace.mkdir(parents=True) - for name, data in common.items(): - (workspace / name).write_bytes(data) - if arm == "A3": - for name in plan["knowledge_files_sha256"]: - target = workspace / "knowledge" / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes((ROOT / plan["knowledge_root"] / name).read_bytes()) - paths = [ - "task.json", # First read is identical in both scripted arms. - *sorted(shared_inputs.COMMON - {"task.json"}), - *( - sorted("knowledge/" + n for n in plan["knowledge_files_sha256"]) - if arm == "A3" - else [] - ), - ] - actor = edit.EditPlan( - edit.freeze_files(workspace, paths), - edit.EditLimits(**plan["actor_limits"]), - tuple(t["id"] for t in development), - "elapsed_seconds", - "seconds", - "minimize", - "0" * 64, - ) - config = pilot.prepare_scripted( - root / "inputs", - workspace, - actor, - suite, - baseline, - _scripts(paths, baseline), - plan_sha256=result["plan_sha256"], - execution_id="scripted-host-qualification", - arm=arm, - model=plan["model"], - spec=plan["client"], - prompt=prompt, - input_token_ceiling=plan["input_token_ceiling"], - ) - run = pilot.run_supervised( - root / "inputs", - workspace, - root / "run", - min(deadline, time.monotonic() + actor.limits.max_seconds), - ) - item = { - "executor_status": run["execution"]["status"], - "worker_status": run["worker"]["status"], - "request_completion_known": run["request_completion_known"], - "simulated_slots_consumed": run["simulated_slots_consumed"], - "actor_sha256": config["actor_sha256"], - "common_actor_contract_sha256": shared_inputs.common_actor_contract_sha256( - config - ), - "exposure": config["edit_plan"]["files"], - "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), - } - result["arms"][arm] = item - if run["worker"]["status"] != "completed": - item["reason"] = run["worker"].get("reason", "worker_incomplete") - continue - receipt = run["worker"]["artifact"] - item["visible_workbench_verified"] = visible_receipts( - root / "run/executor/stdout.log", - receipt, - config["actor_sha256"], - actor.limits.benchmark_repetitions, - root / "run/worker/workbench/revisions", - ) - source = kv.source_bytes(root / "run/worker/final/candidate.py") - item["final_artifact_sha256"] = receipt["sha256"] - item["hidden_correctness"] = workbench.grade_final( - source, receipt["sha256"], hidden, root / "post-submission-grading", deadline - ) - item["two_revision_receipts_retained"] = all( - ( - root / f"run/worker/workbench/revisions/revision-{i:04d}/candidate.py" - ).is_file() - for i in (0, 1, 2) - ) - complete = ( - all( - a["executor_status"] == "completed" - and a["worker_status"] == "completed" - and a["request_completion_known"] - and a["simulated_slots_consumed"] == 4 - and a.get("two_revision_receipts_retained") - and a.get("visible_workbench_verified") - and a.get("hidden_correctness", {}).get("verdict") == "PASS" - for a in result["arms"].values() - ) - and len(result["arms"]) == 2 - ) - result["software_checks_passed"] = ( - complete - and result["cleanup"]["verdict"] == "PASS" - and result["runtime_isolation"]["verdict"] == "PASS" - ) - if result["software_checks_passed"]: - result["verdict"] = "PASS" - elif ( - result["cleanup"]["verdict"] == "FAIL" - or result["runtime_isolation"]["verdict"] == "FAIL" - ) or any( - a.get("hidden_correctness", {}).get("verdict") == "FAIL" - for a in result["arms"].values() - ): - result["verdict"] = "FAIL" - finally: - try: - result["raw_evidence"] = _archive(private, output) - except Exception: - result.update( - verdict="INCONCLUSIVE", - software_checks_passed=False, - retention_error="raw_evidence_retention_failed", - ) - after = repository_identity(ROOT) - result["source_unchanged"] = after.get("available") and identity[ - "snapshot_hash" - ] == after.get("snapshot_hash") - if not result["source_unchanged"]: - result.update(verdict="INCONCLUSIVE", software_checks_passed=False) - atomic_write_json(output / "qualification.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--plan", type=Path, required=True) - parser.add_argument("--revision", required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - result = qualify(args.plan, args.revision, args.output) - print( - json.dumps( - { - "verdict": result["verdict"], - "model_calls": 0, - "software_checks_passed": result["software_checks_passed"], - } - ) - ) - return 0 if result["software_checks_passed"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/qualify_kv_reclamation.py b/tools/qualify_kv_reclamation.py deleted file mode 100644 index d8f57b9a6..000000000 --- a/tools/qualify_kv_reclamation.py +++ /dev/null @@ -1,295 +0,0 @@ -"""Model-free synthetic task qualification through existing execution primitives. - -No provider, submitted artifact, arbitrary command or custom grader is accepted. -The numeric plan is explicit evidence, never a workflow/runtime default. -""" - -import argparse -import hashlib -import importlib.util -import json -import platform -import re -import tempfile -import time -import zipfile -from contextlib import contextmanager -from pathlib import Path - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.harness.runner import run_experiment -from inferrsi.infra.artifacts import digest, write_json -from inferrsi.infra.targets import repository_identity -from inferrsi.runspec import RunSpec - -ROOT = Path(__file__).resolve().parents[1] -FIXTURES = ROOT / "tests/fixtures/kv_reclamation" -EVIDENCE = ROOT / "evidence/kv-reclamation-20261030" -INDEPENDENT = EVIDENCE / "validator/independent" - - -def load_module(name, path): - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def file_hash(path): - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def retain_raw_evidence(private, output): - """Retain actual outputs with labelled path redactions, never host environment.""" - archive = output / "raw-evidence.zip" - files = [] - with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as bundle: - for path in sorted(private.rglob("*")): - if path.suffix not in {".json", ".log", ".html", ".svg"}: - continue - data = kv.read_regular(path, 16 * 1024 * 1024).decode("utf-8", errors="replace") - data = data.replace(str(ROOT), "").replace(str(private), "") - data = data.replace(str(output), "") - data = re.sub(r"/tmp/kv-replay-[A-Za-z0-9_-]+", "", data) - name = path.relative_to(private).as_posix() - files.append(name) - bundle.writestr(name, data) - bundle.writestr("REDACTIONS.txt", "Labelled path redactions: , , " - ", . Original paths are omitted. " - "No environment dump or provider credentials were captured.\n") - return {"path": archive.name, "sha256": file_hash(archive), "files": files, - "scope": "Actual control outputs and runner evidence; labelled path redactions"} - - -@contextmanager -def retained_workspace(output, result): - with tempfile.TemporaryDirectory(prefix="kv-qualification-") as temporary: - private = Path(temporary) - try: - yield private - finally: - try: - result["raw_evidence"] = retain_raw_evidence(private, output) - except (OSError, ValueError): - result["verdict"] = "INCONCLUSIVE" - result["software_checks_passed"] = False - result["retention_error"] = "Raw evidence retention failed" - raise - - -def checked_plan(path): - plan = json.loads(path.read_text(encoding="utf-8")) - if plan.get("schema") != "kv-reclamation-model-free-plan-v1": - raise ValueError("Unsupported qualification plan") - for name, actual in (("verifier_freeze_sha256", file_hash(INDEPENDENT / "FREEZE.json")), - ("starter_sha256", file_hash(FIXTURES / "candidate.py")), - ("workload_sha256", file_hash(FIXTURES / "workload.py"))): - if actual != plan[name]: - raise ValueError("Frozen qualification input changed: " + name) - frozen = json.loads((INDEPENDENT / "FREEZE.json").read_text()) - for name, expected in frozen["files_sha256"].items(): - if file_hash(INDEPENDENT / name) != expected: - raise ValueError("Independent frozen verifier bytes changed") - supplement = json.loads((EVIDENCE / "validator/supplemental-freeze.json").read_text()) - if supplement["parent_freeze_sha256"] != plan["verifier_freeze_sha256"]: - raise ValueError("Benchmark supplement belongs to a different verifier freeze") - for name, expected in supplement["files_sha256"].items(): - if file_hash(EVIDENCE / "validator" / name) != expected: - raise ValueError("Independent benchmark supplement bytes changed") - return plan - - -def control_sources(): - starter = kv.source_bytes(FIXTURES / "candidate.py").decode("utf-8") - - def replace(old, new): - if starter.count(old) != 1: - raise ValueError("Fault injection no longer matches frozen starter") - return starter.replace(old, new).encode("utf-8") - - return { - "starter": starter.encode("utf-8"), - "reference": kv.source_bytes(FIXTURES / "reference.py"), - "ignore_save": replace('(not lease["save_required"] or lease["saved"])', "True"), - "ignore_fence": replace('and lease["write_step"] <= self._frontier', "and True"), - "ignore_generation": replace( - '(lease["request_id"], lease["generation"]) == (\n' - ' event["request_id"], event["generation"])', - 'lease["request_id"] == event["request_id"]'), - "empty_answers": replace("return reclaimed", "return []"), - "skip_events": replace(" def apply(self, event):\n", - " def apply(self, event):\n" - " if event.get('op') == 'save_complete':\n" - " return []\n"), - "missing_output": b"raise SystemExit(0)\n", - "spoof_output": (b"import json\nprint(json.dumps({'observations': [], 'pending': []}))\n" - b"raise SystemExit(0)\n"), - } - - -def protected_path_control(output): - output.mkdir() - (output / "candidate.py").write_bytes(b"# starter\n") - (output / "contract.md").write_bytes(b"frozen contract\n") - frozen = kv.workspace_hashes(output) - (output / "contract.md").write_bytes(b"tampered\n") - try: - kv.capture_artifact(output, frozen) - except ValueError: - return True - return False - - -def measure(name, before, after, events, expected, plan, output, deadline): - """Adapter only: the shared runner owns AB/BA, snapshots and adjudication.""" - sources = {"baseline": before, "candidate": after} - public_records = [] - - def execute(command, cwd, trial_dir, runner_deadline, metric_key): - label = command[1] - record, actual = kv.invoke(sources[label], {"mode": "benchmark", "events": events}, - trial_dir, min(deadline, runner_deadline)) - complete = digest(actual) == digest(expected) - record["metric"] = record["elapsed_seconds"] if complete else None - record["metric_error"] = None if complete else "complete trace output mismatch" - if record["status"] == "completed" and not complete: - record["status"] = "failed" - public_records.append({"side": label, "quality_check": metric_key is None, - "status": record["status"], "complete_trace": complete, - "elapsed_seconds": record["elapsed_seconds"], - "source_sha256": hashlib.sha256(sources[label]).hexdigest()}) - write_json(trial_dir / "execution.json", record) - return record - - timing = plan["timing"] - spec = RunSpec( - repo=str(ROOT), repo_type="synthetic-cpu-kv", backend="cpu", mode="code", - scenario="kv-reclamation-qualification", metric_key="elapsed_seconds", metric_unit="s", - comparison_scope="whole CPU task command including sandbox and interpreter startup", - baseline_command=("trusted-kv-control", "baseline"), - candidate_command=("trusted-kv-control", "candidate"), - validation_command=("trusted-kv-control", "candidate"), - repetitions=timing["paired_repetitions"], - min_improvement_pct=timing["min_improvement_pct"], - max_minutes=min(timing["comparison_max_seconds"], max(0.001, deadline - time.monotonic())) - / 60, - output_dir=str(output / name), agent="scripted-software-control", - ) - run = run_experiment(spec, {"platform": platform.system(), - "scope": "model-free trusted controls"}, execute=execute) - return {"evaluation": json.loads((run / "evaluation.json").read_text()), - "records": public_records} - - -def qualify(plan_path, revision, output): - plan = checked_plan(plan_path) - identity = repository_identity(ROOT) - if (not re.fullmatch(r"[0-9a-f]{40}", revision) or not identity.get("available") - or identity["head"] != revision): - raise ValueError("Qualification revision must equal the exact checked-out Git head") - started = time.monotonic() - deadline = started + plan["qualification_max_seconds"] - result = { - "schema": "kv-reclamation-qualification-v1", "revision": revision, - "snapshot_hash": identity["snapshot_hash"], "dirty": identity["dirty"], - "plan_sha256": file_hash(plan_path), "verifier_freeze_sha256": plan[ - "verifier_freeze_sha256"], "controller_sha256": file_hash(Path(kv.__file__)), - "qualifier_sha256": file_hash(Path(__file__)), - "benchmark_supplement_sha256": file_hash(EVIDENCE / "validator/supplemental-freeze.json"), - "python": platform.python_version(), "platform": platform.system(), - "verdict": "INCONCLUSIVE", "software_checks_passed": False, - "controls": {}, "measurements": {}, "model_calls": 0, "provider_charges_usd": 0, - "scope": "Synthetic model-free artifact qualification; actor isolation NOT qualified", - "chronology": "Independent and plan local freeze preceded reference; public plan posting " - "followed reference authorship and preceded all timing; independent " - "benchmark supplement frozen after reference authorship and before timing", - } - output.mkdir(parents=True, exist_ok=False) - try: - with retained_workspace(output, result) as private: - sentinel = private / "not-mounted-oracle.txt" - sentinel.write_text("synthetic isolation sentinel\n") - result["isolation"] = kv.probe_isolation(private / "probe", deadline, [ - str(sentinel), str(INDEPENDENT / "fixtures.json"), - str(ROOT / "src/inferrsi/data/skill_library/skill-index.json"), - ]) - sources = control_sources() - result["control_source_sha256"] = { - name: hashlib.sha256(source).hexdigest() for name, source in sources.items()} - verifier = load_module("kv_independent_verifier", INDEPENDENT / "verifier.py") - traces = verifier.load_traces() - result["trace_counts"] = {split: sum(t["split"] == split for t in traces) - for split in ("development", "grader")} - for name in plan["required_good"] + plan["required_bad"]: - result["controls"][name] = kv.verify_source( - sources[name], traces, private / name, deadline, verifier.compare) - result["protected_edit_rejected"] = protected_path_control(private / "protected") - record, _ = kv.invoke(b"while True: pass\n", {"mode": "verify", "events": []}, - private / "deadline", min(deadline, time.monotonic() - + plan["deadline_probe_seconds"])) - result["deadline"] = {"status": record["status"], - "elapsed_seconds": record["elapsed_seconds"], - "process_group_gone": kv.process_group_gone(record)} - controls = result["controls"] - good = all(controls[name]["verdict"] == "PASS" for name in plan["required_good"]) - bad = all(controls[name]["verdict"] == "FAIL" for name in plan["required_bad"]) - result["software_checks_passed"] = (good and bad and result["protected_edit_rejected"] - and record["status"] == "timeout" - and result["deadline"]["process_group_gone"]) - if not result["software_checks_passed"]: - result["verdict"] = ("INCONCLUSIVE" if any( - item["verdict"] == "INCONCLUSIVE" for item in controls.values()) else "FAIL") - return result - workload = load_module("kv_workload", FIXTURES / "workload.py") - benchmark = load_module("kv_benchmark_oracle", EVIDENCE / "validator/benchmark.py") - result["benchmark_oracle_sha256"] = file_hash(EVIDENCE / "validator/benchmark.py") - events = workload.workload(plan["timing"]["lease_count"]) - expected = benchmark.benchmark_expected(plan["timing"]["lease_count"]) - result["benchmark_input_sha256"] = digest(events) - result["benchmark_expected_sha256"] = digest(expected) - for name, after in (("reference", sources["reference"]), - ("unchanged", sources["starter"]), - ("no_op", sources["starter"])): - result["measurements"][name] = measure( - name, sources["starter"], after, events, expected, plan, private, deadline) - measured = result["measurements"] - reference = measured["reference"]["evaluation"] - negatives = [measured[name]["evaluation"] - for name in plan["required_negative_comparisons"]] - complete = all(e["improvement_pct"] is not None - and e["verdict"] != "INCONCLUSIVE" for e in [reference, *negatives]) - passed = (reference["verdict"] == "PASS" and all( - abs(e["improvement_pct"]) < plan["timing"]["negative_absolute_gain_limit_pct"] - for e in negatives)) if complete else False - result["verdict"] = "PASS" if passed else "FAIL" if complete else "INCONCLUSIVE" - except kv.SandboxUnavailable as error: - result["error"] = str(error) - result["software_checks_passed"] = False - finally: - after = repository_identity(ROOT) - result["source_unchanged"] = (after.get("available") and after.get("snapshot_hash") - == identity["snapshot_hash"]) - if not result["source_unchanged"]: - result["verdict"] = "INCONCLUSIVE" - result["software_checks_passed"] = False - result["elapsed_seconds"] = time.monotonic() - started - write_json(output / "qualification.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--plan", required=True, type=Path) - parser.add_argument("--revision", required=True) - parser.add_argument("--output", required=True, type=Path) - args = parser.parse_args() - result = qualify(args.plan, args.revision, args.output) - print(json.dumps({"qualification_verdict": result["verdict"], - "software_checks_passed": result["software_checks_passed"]})) - # CI checks software controls, not a campaign performance/efficacy threshold. - # The full qualification verdict must be read separately from a green check. - return 0 if result["software_checks_passed"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) From 10f70e09c3874629c38a56b308fbba32ab7e93b4 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:41:04 +0800 Subject: [PATCH 2/2] Resolve the review points on the retirement page Record where the removed paths remain readable, and bind each claim on the page to the receipt that supports it. The retained evidence is hash-pinned, so it still names eleven files and modules that the removal deletes. Name them explicitly and point at the last revision shared with main, so every reference stays resolvable through git history rather than appearing as a broken bundle. Correct the sourcing of the outcome claims. The live-pilot bundle records "not executed" and holds no arm outcome; the INCONCLUSIVE results and the HTTP 403 come from the workflow run records, not from the bundle. Separate the two and link each. Also record that A3 never started on the second attempt. Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> Agent-assisted-by: Claude Code / claude-opus-5 (author) --- docs/kv-reclamation-qualification.md | 43 ++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/kv-reclamation-qualification.md b/docs/kv-reclamation-qualification.md index 50bab84b9..ff75703a2 100644 --- a/docs/kv-reclamation-qualification.md +++ b/docs/kv-reclamation-qualification.md @@ -20,6 +20,16 @@ the `kv-edit-pilot` workflow and the `kv-task-qualification` job in the ready lane. None of the commands named on this page can be run from the repository any more. +Every path named above still exists at +[`e843b9d3`](https://github.com/InferMatrixLab/InferRSI/commit/e843b9d31cb86e1fd12d00890c89798b0bedd1fe), +the last revision this branch shared with `main` before the removal, which +remains an ancestor of `main`. Read any of them from history: + +``` +git show e843b9d3:tools/qualify_kv_reclamation.py +git show e843b9d3:tests/fixtures/kv_reclamation/candidate.py +``` + ## What was kept The retained evidence stays in place and unmodified: @@ -32,14 +42,37 @@ The retained evidence stays in place and unmodified: Their entries in `tools/evidence_inventory.json` are unchanged, so `python -m tools.validate_evidence` still verifies them. +Because that evidence is hash-pinned, it still names paths and modules that the +removal deleted. Eleven files do so: + +| Bundle | Files | Reference | +| --- | --- | --- | +| `kv-live-pilot-20261031` | `patch/README.md` | `tests/fixtures/kv_reclamation/candidate.py` | +| `kv-live-pilot-20261031` | `plan/plan.json`, `plan/A2.json`, `plan/A3.json`, `plan/second-attempt/{plan,A2,A3}.json`, `plan/second-attempt-1200/plan.json` | `tools.qualify_kv_reclamation`, `tools.kv_live_job` | +| `kv-actor-qualification-20261030` | `results/hosted-41154288295/qualification.json`, `results/hosted-41182807900/qualification.json` | `kv_edit_workbench` | +| `kv-reclamation-20261030` | `patch/README.md` | `tests/fixtures/kv_reclamation/candidate.py` | + +Each of those references resolves at `e843b9d3` through the command above. None +resolves in the working tree, which is the expected result of the removal rather +than a broken bundle. + ## What the evidence does and does not show The qualification receipts record **software and scripted-control execution** -only. Both live A2/A3 attempts on 2026-10-31 ended `INCONCLUSIVE`, the second -stopping at its first transport call on HTTP 403, so no completed arm, per-arm -outcome or A/A variance exists. The recorded dollar figures are reserved upper -bounds, not observed spend. Nothing here is a model-effect, knowledge-effect or -hardware result. +only. + +| Claim | Where to check it | +| --- | --- | +| No completed arm, per-arm outcome or A/A variance exists | [`evidence/kv-live-pilot-20261031/results/README.md`](../evidence/kv-live-pilot-20261031/results/README.md): "Prospective pair: not executed … No paid model output, timing result or efficacy verdict exists in this bundle" | +| The dollar figures are reserved upper bounds, not observed spend | `evidence/kv-live-pilot-20261031/plan/second-attempt-budget.json`: `worst_case` 162,720,000 microUSD against the PR and pilot lifetime caps | +| Both authorized attempts on 2026-10-31 ended `INCONCLUSIVE`, the second stopping at its first transport call on HTTP 403 with A3 never started | Runs [41159550621](https://github.com/InferMatrixLab/InferRSI/actions/runs/41159550621) and [41194057284](https://github.com/InferMatrixLab/InferRSI/actions/runs/41194057284), assessed in [issue 248](https://github.com/InferMatrixLab/InferRSI/issues/248#issuecomment-6648189616) | + +The third row is not recorded inside the retained bundles: they hold the frozen +inputs and the not-executed marker, while the attempt outcomes live in the run +records. A workflow conclusion of `success` on those runs means orchestration +and evidence retention finished, not that a request was delivered or charged. + +Nothing here is a model-effect, knowledge-effect or hardware result. The general SkillEval capability is unaffected: nothing in `src/inferrsi/harness/skill_eval*.py`, the runner, the evaluator or