From 9299249fc5d22ecb52d6ef4c146b9ac861b7451e Mon Sep 17 00:00:00 2001 From: ICanHazCrypto Date: Wed, 9 Sep 2026 09:26:16 -0700 Subject: [PATCH] feat: verify a run package from a shell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pnpm verify ` verifies a run and writes the same JSON and Markdown evidence pack the product exports, so an evaluation pipeline or a CI job can do what previously needed the import screen. It is the import screen's own code rather than a second implementation: the same `importRun`, the same `readContractArtifact`, and the same deterministic verifier behind `buildRunView`. A package that verifies here verifies identically in the product, because there is only one of each check. The contract comes from whichever of three sources is available, in the order the product prefers: named with `--contract`, shipped inside the package, or the frozen sample contract whose task fingerprint the run reproduces. When none is, the command says so and stops. It never compiles one, so no model is called on any path — verification never calls one either, which is the product's central claim. Exit codes separate a failed run from a broken invocation: 0 for PASS, 1 for FAIL, 2 when the package, the contract or the invocation did not validate. NEEDS_REVIEW is a real outcome rather than an error, so whether it fails the command is the caller's choice via `--fail-on-needs-review`. `parseContractDocument` is extracted from the importer and shared, so a contract passed on the command line reports the same problems, field by field, as one uploaded through the screen. Two readers would drift. Five integration tests run the real command rather than calling the module, since an exit code is only worth asserting if a process produced it: the sample package end to end including both written files and `verificationModelCalls: 0`, an unreadable package, a contract that does not resolve, `--help`, and an unknown option. Full suite: 542 passing. Refs #2. --- README.md | 10 ++ apps/product/src/server/importer.ts | 46 ++++-- apps/product/test/verify-cli.test.ts | 98 +++++++++++ package.json | 1 + scripts/verify.ts | 235 +++++++++++++++++++++++++++ 5 files changed, 375 insertions(+), 15 deletions(-) create mode 100644 apps/product/test/verify-cli.test.ts create mode 100644 scripts/verify.ts diff --git a/README.md b/README.md index 9306dcd..e095398 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,16 @@ pnpm product:dev Open , click **Run the verification demo**, then **Verify this run**. No API key, no model call, no network. +Verify a run package from a shell, for an evaluation pipeline or a CI job: + +```bash +pnpm verify samples/stateproof-sample-run.zip --out evidence/ +``` + +It writes the same JSON and Markdown evidence pack the product exports, and exits +non-zero on `FAIL` (add `--fail-on-needs-review` to treat `NEEDS_REVIEW` the same +way). It calls no model: a run with no contract is reported, never compiled. + Reproduce the published evaluation offline: ```bash diff --git a/apps/product/src/server/importer.ts b/apps/product/src/server/importer.ts index 7cfaa7e..e9559dd 100644 --- a/apps/product/src/server/importer.ts +++ b/apps/product/src/server/importer.ts @@ -249,6 +249,32 @@ function filesFromInput(input: ImportInput): Record { return files; } +/** + * Read a compiled contract, accepting either a bare contract or the full artifact + * that wraps one. + * + * Exported because the verify CLI takes a contract as a file argument and has to + * report the same problems, field by field, as the import screen. Two readers would + * drift; one cannot. + */ +export function parseContractDocument( + text: string | undefined, + field: string, + problems: ImportProblem[], +): CompiledContractV2 | null { + if (text === undefined) return null; + const parsed = parseJson(field, text, problems); + if (parsed === null) return null; + + const bare = CompiledContractV2Schema.safeParse(parsed); + if (bare.success) return bare.data; + const wrapped = CompiledContractV2Schema.safeParse((parsed as { contract?: unknown }).contract); + if (wrapped.success) return wrapped.data; + + problems.push(...describeZodError(bare.error, field)); + return null; +} + export function importRun( input: ImportInput, repoRoot: string, @@ -305,21 +331,11 @@ export function importRun( const warnings = checkDomain(agentVisible, problems); if (problems.length > 0) throw new ImportError(problems); - let uploadedContract: CompiledContractV2 | null = null; - const contractText = files['compiled-contract.json']; - if (contractText !== undefined) { - const parsed = parseJson('compiled-contract.json', contractText, problems); - if (parsed !== null) { - // Accept either a bare contract or a full compiled-contract artifact. - const bare = CompiledContractV2Schema.safeParse(parsed); - const wrapped = CompiledContractV2Schema.safeParse( - (parsed as { contract?: unknown }).contract, - ); - if (bare.success) uploadedContract = bare.data; - else if (wrapped.success) uploadedContract = wrapped.data; - else problems.push(...describeZodError(bare.error, 'compiled-contract.json')); - } - } + const uploadedContract = parseContractDocument( + files['compiled-contract.json'], + 'compiled-contract.json', + problems, + ); if (problems.length > 0) throw new ImportError(problems); // Does this task match one of the three frozen sample tasks? The fingerprint diff --git a/apps/product/test/verify-cli.test.ts b/apps/product/test/verify-cli.test.ts new file mode 100644 index 0000000..1933a9c --- /dev/null +++ b/apps/product/test/verify-cli.test.ts @@ -0,0 +1,98 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +/** + * `pnpm verify` end to end, as a shell caller experiences it. + * + * The point of the command is that an evaluation pipeline can trust its exit code and + * its evidence pack without driving the import screen, so these run the real binary + * rather than calling into the module: an exit code is only worth asserting if the + * process actually produced it. + */ + +const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url)); +const SAMPLE = path.join(REPO_ROOT, 'samples', 'stateproof-sample-run.zip'); + +function runVerify(args: readonly string[]): { status: number; stdout: string; stderr: string } { + const result = spawnSync( + process.execPath, + [path.join(REPO_ROOT, 'node_modules', 'tsx', 'dist', 'cli.mjs'), 'scripts/verify.ts', ...args], + { cwd: REPO_ROOT, encoding: 'utf8', env: { ...process.env } }, + ); + return { status: result.status ?? -1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; +} + +describe('pnpm verify', () => { + + it('verifies the sample package and writes both evidence files', () => { + const out = path.join(mkdtempSync(path.join(tmpdir(), 'stateproof-verify-')), 'evidence'); + const result = runVerify([SAMPLE, '--out', out]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('verdict: PASS'); + // The contract is not in the package; it is matched by task fingerprint, which is + // the path a pipeline hits when it verifies a run of one of the frozen tasks. + expect(result.stdout).toContain('contract: frozen'); + + const written = result.stdout.match(/evidence-(\S+)\.json/); + expect(written).not.toBeNull(); + const runId = written?.[1] ?? ''; + const jsonPath = path.join(out, `evidence-${runId}.json`); + expect(existsSync(jsonPath)).toBe(true); + expect(existsSync(path.join(out, `evidence-${runId}.md`))).toBe(true); + + // The same pack the product exports, not a second rendering of the same idea. + const pack = JSON.parse(readFileSync(jsonPath, 'utf8')) as { + schemaVersion: string; + verdict: string; + usage: { verificationModelCalls: number }; + requirements: unknown[]; + }; + expect(pack.schemaVersion).toBe('1.0.0'); + expect(pack.verdict).toBe('PASS'); + expect(pack.requirements).toHaveLength(4); + // The product's central claim: verification itself calls no model. + expect(pack.usage.verificationModelCalls).toBe(0); + }); + + it('reports an unreadable package field by field and exits 2', () => { + const broken = path.join(mkdtempSync(path.join(tmpdir(), 'stateproof-verify-')), 'broken.zip'); + writeFileSync(broken, 'not an archive', 'utf8'); + + const result = runVerify([broken]); + + // 2, not 1: a caller has to be able to tell a bad invocation from a failed run. + expect(result.status).toBe(2); + expect(result.stderr).toContain('This run package did not validate.'); + expect(result.stderr).toContain('archive:'); + }); + + it('refuses to compile a missing contract rather than reaching for a model', () => { + // A contract path that does not resolve is the closest reachable stand-in for a + // package with no contract and no fingerprint match; either way the command must + // stop rather than fall back to compilation. + const result = runVerify([SAMPLE, '--contract', path.join(REPO_ROOT, 'no-such-contract.json')]); + + expect(result.status).toBe(2); + expect(result.stderr).toContain('no-such-contract.json'); + }); + + it('prints usage and exits cleanly for --help', () => { + const result = runVerify(['--help']); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('pnpm verify '); + expect(result.stdout).toContain('--fail-on-needs-review'); + }); + + it('rejects an unknown option instead of ignoring it', () => { + const result = runVerify([SAMPLE, '--not-an-option']); + + expect(result.status).toBe(2); + expect(result.stderr).toContain('unknown option --not-an-option'); + }); +}); diff --git a/package.json b/package.json index 536941b..9c649ed 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dashboard:build": "pnpm --filter dashboard build", "dev": "pnpm --filter dashboard dev", "reproduce": "tsx scripts/reproduce.ts", + "verify": "tsx scripts/verify.ts", "reproduce:check": "tsx scripts/reproduce.ts --check-only", "test:clean-reproduction": "tsx scripts/clean-reproduction.ts", "submission:finalize": "tsx scripts/finalize-submission.ts", diff --git a/scripts/verify.ts b/scripts/verify.ts new file mode 100644 index 0000000..2ae97db --- /dev/null +++ b/scripts/verify.ts @@ -0,0 +1,235 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { CompiledContractV2 } from '@stateproof/core'; +import { buildEvidencePack, renderEvidenceMarkdown } from '../apps/product/src/server/evidence'; +import { + ImportError, + type ImportProblem, + importRun, + parseContractDocument, +} from '../apps/product/src/server/importer'; +import { buildRunView, readContractArtifact } from '../apps/product/src/server/runs'; + +/** + * `pnpm verify [options]` + * + * Verifies a run package from a shell and writes the same evidence pack the product + * exports, for evaluation pipelines and CI jobs that cannot drive the import screen. + * + * It is the import screen's own code: the same importer, the same contract reader and + * the same deterministic verifier behind `buildRunView`. Nothing here re-implements a + * check, so a package that verifies here verifies identically in the product. + * + * Read-only, and no model is called. Verification never calls one, and this command + * will not compile a missing contract — it reports that one is needed and stops. + */ + +const REPO_ROOT = fileURLToPath(new URL('../', import.meta.url)); + +/** Exit codes, so a CI job can tell a failed run from a broken invocation. */ +const EXIT_OK = 0; +const EXIT_VERDICT = 1; +const EXIT_USAGE = 2; + +interface Options { + readonly packagePath: string; + readonly contractPath: string | null; + readonly outDir: string; + /** NEEDS_REVIEW is a real outcome, so whether it fails the command is the caller's call. */ + readonly failOnNeedsReview: boolean; +} + +const USAGE = `stateproof verify — verify a run package and write its evidence pack + + pnpm verify [options] + +Options + --contract compiled contract to verify against, when the package + carries none + --out where to write the evidence pack (default: evidence) + --fail-on-needs-review exit non-zero on NEEDS_REVIEW as well as FAIL + -h, --help show this + +Exit codes + 0 PASS (and NEEDS_REVIEW unless --fail-on-needs-review) + 1 FAIL + 2 the package, the contract or the invocation did not validate +`; + +function parseArgs(argv: readonly string[]): Options | 'help' { + let packagePath: string | null = null; + let contractPath: string | null = null; + let outDir = 'evidence'; + let failOnNeedsReview = false; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ''; + if (arg === '-h' || arg === '--help') return 'help'; + if (arg === '--fail-on-needs-review') { + failOnNeedsReview = true; + continue; + } + if (arg === '--contract' || arg === '--out') { + const value = argv[index + 1]; + if (value === undefined || value.startsWith('-')) { + throw new UsageError(`${arg} needs a value`); + } + if (arg === '--contract') contractPath = value; + else outDir = value; + index += 1; + continue; + } + if (arg.startsWith('-')) throw new UsageError(`unknown option ${arg}`); + if (packagePath !== null) throw new UsageError('give exactly one run package'); + packagePath = arg; + } + + if (packagePath === null) throw new UsageError('a run package is required'); + return { packagePath, contractPath, outDir, failOnNeedsReview }; +} + +class UsageError extends Error {} + +function readPackage(packagePath: string): string { + try { + return readFileSync(packagePath).toString('base64'); + } catch (error) { + throw new UsageError( + `could not read ${packagePath}: ${error instanceof Error ? error.message : 'unreadable'}`, + ); + } +} + +/** Field-by-field, in the shape the import screen shows them. */ +function reportProblems(heading: string, problems: readonly ImportProblem[]): void { + process.stderr.write(`${heading}\n`); + for (const problem of problems) { + process.stderr.write(` ${problem.field}: ${problem.message}\n`); + } +} + +function main(): number { + let options: Options | 'help'; + try { + options = parseArgs(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : 'bad arguments'}\n\n${USAGE}`); + return EXIT_USAGE; + } + if (options === 'help') { + process.stdout.write(USAGE); + return EXIT_OK; + } + + const zipBase64 = readPackage(options.packagePath); + + let imported; + try { + ({ imported } = importRun({ zipBase64 }, REPO_ROOT)); + } catch (error) { + if (error instanceof ImportError) { + reportProblems('This run package did not validate.', error.problems); + return EXIT_USAGE; + } + throw error; + } + for (const warning of imported.warnings) process.stderr.write(`warning: ${warning}\n`); + + // Three legitimate sources, in the order the product prefers them: a contract named + // on the command line, one shipped inside the package, or the frozen sample contract + // whose task fingerprint this run reproduces. + let contract: CompiledContractV2 | null = null; + let source: 'supplied' | 'package' | 'frozen' = 'supplied'; + let artifact: ReturnType | null = null; + + if (options.contractPath !== null) { + const problems: ImportProblem[] = []; + let text: string; + try { + text = readFileSync(options.contractPath, 'utf8'); + } catch (error) { + process.stderr.write( + `could not read ${options.contractPath}: ${error instanceof Error ? error.message : 'unreadable'}\n`, + ); + return EXIT_USAGE; + } + contract = parseContractDocument(text, options.contractPath, problems); + if (problems.length > 0 || contract === null) { + reportProblems('That contract did not validate.', problems); + return EXIT_USAGE; + } + } else if (imported.uploadedContract !== null) { + contract = imported.uploadedContract; + source = 'package'; + } else if (imported.matchedContractPath !== null) { + artifact = readContractArtifact(REPO_ROOT, imported.matchedContractPath); + contract = artifact.contract; + source = 'frozen'; + } else { + reportProblems('No compiled contract is available for this run.', [ + { + field: 'contract', + message: + 'pass --contract , or include compiled-contract.json in the package. ' + + 'This command never compiles one, so no model is called.', + }, + ]); + return EXIT_USAGE; + } + + const run = + source === 'frozen' && artifact !== null + ? buildRunView({ + label: imported.agentVisible.task.title, + caseId: null, + agentVisible: imported.agentVisible, + contract: artifact.contract, + contractHash: artifact.contractHash, + taskFingerprint: artifact.taskFingerprint, + promptPath: artifact.promptPath, + promptHash: artifact.promptHash, + assertionSchemaVersion: artifact.assertionSchemaVersion, + contractSource: 'frozen-bundle', + imported: true, + }) + : buildRunView({ + label: imported.agentVisible.task.title, + caseId: null, + agentVisible: imported.agentVisible, + contract, + contractHash: 'uploaded-contract', + taskFingerprint: 'not-computed-for-uploaded-contracts', + promptPath: options.contractPath ?? 'supplied with the run package', + promptHash: 'not applicable', + assertionSchemaVersion: '2.1.0', + contractSource: 'uploaded', + imported: true, + }); + + const pack = buildEvidencePack(run); + const outDir = path.resolve(options.outDir); + mkdirSync(outDir, { recursive: true }); + const jsonPath = path.join(outDir, `evidence-${run.runId}.json`); + const markdownPath = path.join(outDir, `evidence-${run.runId}.md`); + writeFileSync(jsonPath, `${JSON.stringify(pack, null, 2)}\n`, 'utf8'); + writeFileSync(markdownPath, renderEvidenceMarkdown(pack), 'utf8'); + + const verdict = run.verdict; + process.stdout.write(`run: ${run.runId}\n`); + process.stdout.write(`task: ${run.label}\n`); + process.stdout.write(`contract: ${source}\n`); + process.stdout.write(`verdict: ${verdict}\n\n`); + for (const requirement of run.requirements) { + const mark = + requirement.status === 'PASS' ? 'ok ' : requirement.status === 'FAIL' ? 'FAIL' : 'rvw '; + process.stdout.write(` ${mark} ${requirement.requirementKey} ${requirement.description}\n`); + } + process.stdout.write(`\nevidence: ${jsonPath}\n ${markdownPath}\n`); + + if (verdict === 'FAIL') return EXIT_VERDICT; + if (verdict === 'NEEDS_REVIEW' && options.failOnNeedsReview) return EXIT_VERDICT; + return EXIT_OK; +} + +process.exitCode = main();