diff --git a/README.md b/README.md index 9306dcd..e095398 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,16 @@ pnpm product:dev Open , click **Run the verification demo**, then **Verify this run**. No API key, no model call, no network. +Verify a run package from a shell, for an evaluation pipeline or a CI job: + +```bash +pnpm verify samples/stateproof-sample-run.zip --out evidence/ +``` + +It writes the same JSON and Markdown evidence pack the product exports, and exits +non-zero on `FAIL` (add `--fail-on-needs-review` to treat `NEEDS_REVIEW` the same +way). It calls no model: a run with no contract is reported, never compiled. + Reproduce the published evaluation offline: ```bash diff --git a/apps/product/src/server/importer.ts b/apps/product/src/server/importer.ts index 7cfaa7e..e9559dd 100644 --- a/apps/product/src/server/importer.ts +++ b/apps/product/src/server/importer.ts @@ -249,6 +249,32 @@ function filesFromInput(input: ImportInput): Record { return files; } +/** + * Read a compiled contract, accepting either a bare contract or the full artifact + * that wraps one. + * + * Exported because the verify CLI takes a contract as a file argument and has to + * report the same problems, field by field, as the import screen. Two readers would + * drift; one cannot. + */ +export function parseContractDocument( + text: string | undefined, + field: string, + problems: ImportProblem[], +): CompiledContractV2 | null { + if (text === undefined) return null; + const parsed = parseJson(field, text, problems); + if (parsed === null) return null; + + const bare = CompiledContractV2Schema.safeParse(parsed); + if (bare.success) return bare.data; + const wrapped = CompiledContractV2Schema.safeParse((parsed as { contract?: unknown }).contract); + if (wrapped.success) return wrapped.data; + + problems.push(...describeZodError(bare.error, field)); + return null; +} + export function importRun( input: ImportInput, repoRoot: string, @@ -305,21 +331,11 @@ export function importRun( const warnings = checkDomain(agentVisible, problems); if (problems.length > 0) throw new ImportError(problems); - let uploadedContract: CompiledContractV2 | null = null; - const contractText = files['compiled-contract.json']; - if (contractText !== undefined) { - const parsed = parseJson('compiled-contract.json', contractText, problems); - if (parsed !== null) { - // Accept either a bare contract or a full compiled-contract artifact. - const bare = CompiledContractV2Schema.safeParse(parsed); - const wrapped = CompiledContractV2Schema.safeParse( - (parsed as { contract?: unknown }).contract, - ); - if (bare.success) uploadedContract = bare.data; - else if (wrapped.success) uploadedContract = wrapped.data; - else problems.push(...describeZodError(bare.error, 'compiled-contract.json')); - } - } + const uploadedContract = parseContractDocument( + files['compiled-contract.json'], + 'compiled-contract.json', + problems, + ); if (problems.length > 0) throw new ImportError(problems); // Does this task match one of the three frozen sample tasks? The fingerprint diff --git a/apps/product/test/verify-cli.test.ts b/apps/product/test/verify-cli.test.ts new file mode 100644 index 0000000..1933a9c --- /dev/null +++ b/apps/product/test/verify-cli.test.ts @@ -0,0 +1,98 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +/** + * `pnpm verify` end to end, as a shell caller experiences it. + * + * The point of the command is that an evaluation pipeline can trust its exit code and + * its evidence pack without driving the import screen, so these run the real binary + * rather than calling into the module: an exit code is only worth asserting if the + * process actually produced it. + */ + +const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url)); +const SAMPLE = path.join(REPO_ROOT, 'samples', 'stateproof-sample-run.zip'); + +function runVerify(args: readonly string[]): { status: number; stdout: string; stderr: string } { + const result = spawnSync( + process.execPath, + [path.join(REPO_ROOT, 'node_modules', 'tsx', 'dist', 'cli.mjs'), 'scripts/verify.ts', ...args], + { cwd: REPO_ROOT, encoding: 'utf8', env: { ...process.env } }, + ); + return { status: result.status ?? -1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; +} + +describe('pnpm verify', () => { + + it('verifies the sample package and writes both evidence files', () => { + const out = path.join(mkdtempSync(path.join(tmpdir(), 'stateproof-verify-')), 'evidence'); + const result = runVerify([SAMPLE, '--out', out]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('verdict: PASS'); + // The contract is not in the package; it is matched by task fingerprint, which is + // the path a pipeline hits when it verifies a run of one of the frozen tasks. + expect(result.stdout).toContain('contract: frozen'); + + const written = result.stdout.match(/evidence-(\S+)\.json/); + expect(written).not.toBeNull(); + const runId = written?.[1] ?? ''; + const jsonPath = path.join(out, `evidence-${runId}.json`); + expect(existsSync(jsonPath)).toBe(true); + expect(existsSync(path.join(out, `evidence-${runId}.md`))).toBe(true); + + // The same pack the product exports, not a second rendering of the same idea. + const pack = JSON.parse(readFileSync(jsonPath, 'utf8')) as { + schemaVersion: string; + verdict: string; + usage: { verificationModelCalls: number }; + requirements: unknown[]; + }; + expect(pack.schemaVersion).toBe('1.0.0'); + expect(pack.verdict).toBe('PASS'); + expect(pack.requirements).toHaveLength(4); + // The product's central claim: verification itself calls no model. + expect(pack.usage.verificationModelCalls).toBe(0); + }); + + it('reports an unreadable package field by field and exits 2', () => { + const broken = path.join(mkdtempSync(path.join(tmpdir(), 'stateproof-verify-')), 'broken.zip'); + writeFileSync(broken, 'not an archive', 'utf8'); + + const result = runVerify([broken]); + + // 2, not 1: a caller has to be able to tell a bad invocation from a failed run. + expect(result.status).toBe(2); + expect(result.stderr).toContain('This run package did not validate.'); + expect(result.stderr).toContain('archive:'); + }); + + it('refuses to compile a missing contract rather than reaching for a model', () => { + // A contract path that does not resolve is the closest reachable stand-in for a + // package with no contract and no fingerprint match; either way the command must + // stop rather than fall back to compilation. + const result = runVerify([SAMPLE, '--contract', path.join(REPO_ROOT, 'no-such-contract.json')]); + + expect(result.status).toBe(2); + expect(result.stderr).toContain('no-such-contract.json'); + }); + + it('prints usage and exits cleanly for --help', () => { + const result = runVerify(['--help']); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('pnpm verify '); + expect(result.stdout).toContain('--fail-on-needs-review'); + }); + + it('rejects an unknown option instead of ignoring it', () => { + const result = runVerify([SAMPLE, '--not-an-option']); + + expect(result.status).toBe(2); + expect(result.stderr).toContain('unknown option --not-an-option'); + }); +}); diff --git a/package.json b/package.json index 536941b..9c649ed 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dashboard:build": "pnpm --filter dashboard build", "dev": "pnpm --filter dashboard dev", "reproduce": "tsx scripts/reproduce.ts", + "verify": "tsx scripts/verify.ts", "reproduce:check": "tsx scripts/reproduce.ts --check-only", "test:clean-reproduction": "tsx scripts/clean-reproduction.ts", "submission:finalize": "tsx scripts/finalize-submission.ts", diff --git a/scripts/verify.ts b/scripts/verify.ts new file mode 100644 index 0000000..2ae97db --- /dev/null +++ b/scripts/verify.ts @@ -0,0 +1,235 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { CompiledContractV2 } from '@stateproof/core'; +import { buildEvidencePack, renderEvidenceMarkdown } from '../apps/product/src/server/evidence'; +import { + ImportError, + type ImportProblem, + importRun, + parseContractDocument, +} from '../apps/product/src/server/importer'; +import { buildRunView, readContractArtifact } from '../apps/product/src/server/runs'; + +/** + * `pnpm verify [options]` + * + * Verifies a run package from a shell and writes the same evidence pack the product + * exports, for evaluation pipelines and CI jobs that cannot drive the import screen. + * + * It is the import screen's own code: the same importer, the same contract reader and + * the same deterministic verifier behind `buildRunView`. Nothing here re-implements a + * check, so a package that verifies here verifies identically in the product. + * + * Read-only, and no model is called. Verification never calls one, and this command + * will not compile a missing contract — it reports that one is needed and stops. + */ + +const REPO_ROOT = fileURLToPath(new URL('../', import.meta.url)); + +/** Exit codes, so a CI job can tell a failed run from a broken invocation. */ +const EXIT_OK = 0; +const EXIT_VERDICT = 1; +const EXIT_USAGE = 2; + +interface Options { + readonly packagePath: string; + readonly contractPath: string | null; + readonly outDir: string; + /** NEEDS_REVIEW is a real outcome, so whether it fails the command is the caller's call. */ + readonly failOnNeedsReview: boolean; +} + +const USAGE = `stateproof verify — verify a run package and write its evidence pack + + pnpm verify [options] + +Options + --contract compiled contract to verify against, when the package + carries none + --out where to write the evidence pack (default: evidence) + --fail-on-needs-review exit non-zero on NEEDS_REVIEW as well as FAIL + -h, --help show this + +Exit codes + 0 PASS (and NEEDS_REVIEW unless --fail-on-needs-review) + 1 FAIL + 2 the package, the contract or the invocation did not validate +`; + +function parseArgs(argv: readonly string[]): Options | 'help' { + let packagePath: string | null = null; + let contractPath: string | null = null; + let outDir = 'evidence'; + let failOnNeedsReview = false; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ''; + if (arg === '-h' || arg === '--help') return 'help'; + if (arg === '--fail-on-needs-review') { + failOnNeedsReview = true; + continue; + } + if (arg === '--contract' || arg === '--out') { + const value = argv[index + 1]; + if (value === undefined || value.startsWith('-')) { + throw new UsageError(`${arg} needs a value`); + } + if (arg === '--contract') contractPath = value; + else outDir = value; + index += 1; + continue; + } + if (arg.startsWith('-')) throw new UsageError(`unknown option ${arg}`); + if (packagePath !== null) throw new UsageError('give exactly one run package'); + packagePath = arg; + } + + if (packagePath === null) throw new UsageError('a run package is required'); + return { packagePath, contractPath, outDir, failOnNeedsReview }; +} + +class UsageError extends Error {} + +function readPackage(packagePath: string): string { + try { + return readFileSync(packagePath).toString('base64'); + } catch (error) { + throw new UsageError( + `could not read ${packagePath}: ${error instanceof Error ? error.message : 'unreadable'}`, + ); + } +} + +/** Field-by-field, in the shape the import screen shows them. */ +function reportProblems(heading: string, problems: readonly ImportProblem[]): void { + process.stderr.write(`${heading}\n`); + for (const problem of problems) { + process.stderr.write(` ${problem.field}: ${problem.message}\n`); + } +} + +function main(): number { + let options: Options | 'help'; + try { + options = parseArgs(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : 'bad arguments'}\n\n${USAGE}`); + return EXIT_USAGE; + } + if (options === 'help') { + process.stdout.write(USAGE); + return EXIT_OK; + } + + const zipBase64 = readPackage(options.packagePath); + + let imported; + try { + ({ imported } = importRun({ zipBase64 }, REPO_ROOT)); + } catch (error) { + if (error instanceof ImportError) { + reportProblems('This run package did not validate.', error.problems); + return EXIT_USAGE; + } + throw error; + } + for (const warning of imported.warnings) process.stderr.write(`warning: ${warning}\n`); + + // Three legitimate sources, in the order the product prefers them: a contract named + // on the command line, one shipped inside the package, or the frozen sample contract + // whose task fingerprint this run reproduces. + let contract: CompiledContractV2 | null = null; + let source: 'supplied' | 'package' | 'frozen' = 'supplied'; + let artifact: ReturnType | null = null; + + if (options.contractPath !== null) { + const problems: ImportProblem[] = []; + let text: string; + try { + text = readFileSync(options.contractPath, 'utf8'); + } catch (error) { + process.stderr.write( + `could not read ${options.contractPath}: ${error instanceof Error ? error.message : 'unreadable'}\n`, + ); + return EXIT_USAGE; + } + contract = parseContractDocument(text, options.contractPath, problems); + if (problems.length > 0 || contract === null) { + reportProblems('That contract did not validate.', problems); + return EXIT_USAGE; + } + } else if (imported.uploadedContract !== null) { + contract = imported.uploadedContract; + source = 'package'; + } else if (imported.matchedContractPath !== null) { + artifact = readContractArtifact(REPO_ROOT, imported.matchedContractPath); + contract = artifact.contract; + source = 'frozen'; + } else { + reportProblems('No compiled contract is available for this run.', [ + { + field: 'contract', + message: + 'pass --contract , or include compiled-contract.json in the package. ' + + 'This command never compiles one, so no model is called.', + }, + ]); + return EXIT_USAGE; + } + + const run = + source === 'frozen' && artifact !== null + ? buildRunView({ + label: imported.agentVisible.task.title, + caseId: null, + agentVisible: imported.agentVisible, + contract: artifact.contract, + contractHash: artifact.contractHash, + taskFingerprint: artifact.taskFingerprint, + promptPath: artifact.promptPath, + promptHash: artifact.promptHash, + assertionSchemaVersion: artifact.assertionSchemaVersion, + contractSource: 'frozen-bundle', + imported: true, + }) + : buildRunView({ + label: imported.agentVisible.task.title, + caseId: null, + agentVisible: imported.agentVisible, + contract, + contractHash: 'uploaded-contract', + taskFingerprint: 'not-computed-for-uploaded-contracts', + promptPath: options.contractPath ?? 'supplied with the run package', + promptHash: 'not applicable', + assertionSchemaVersion: '2.1.0', + contractSource: 'uploaded', + imported: true, + }); + + const pack = buildEvidencePack(run); + const outDir = path.resolve(options.outDir); + mkdirSync(outDir, { recursive: true }); + const jsonPath = path.join(outDir, `evidence-${run.runId}.json`); + const markdownPath = path.join(outDir, `evidence-${run.runId}.md`); + writeFileSync(jsonPath, `${JSON.stringify(pack, null, 2)}\n`, 'utf8'); + writeFileSync(markdownPath, renderEvidenceMarkdown(pack), 'utf8'); + + const verdict = run.verdict; + process.stdout.write(`run: ${run.runId}\n`); + process.stdout.write(`task: ${run.label}\n`); + process.stdout.write(`contract: ${source}\n`); + process.stdout.write(`verdict: ${verdict}\n\n`); + for (const requirement of run.requirements) { + const mark = + requirement.status === 'PASS' ? 'ok ' : requirement.status === 'FAIL' ? 'FAIL' : 'rvw '; + process.stdout.write(` ${mark} ${requirement.requirementKey} ${requirement.description}\n`); + } + process.stdout.write(`\nevidence: ${jsonPath}\n ${markdownPath}\n`); + + if (verdict === 'FAIL') return EXIT_VERDICT; + if (verdict === 'NEEDS_REVIEW' && options.failOnNeedsReview) return EXIT_VERDICT; + return EXIT_OK; +} + +process.exitCode = main();