Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions evals/canary/10.0.0.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"actors":[{"actualModel":{"kind":"observed","value":{"family":"gpt-6.1-sol","gateway":null,"model":"gpt-6.1-sol","revision":null,"routeProvider":"openai"}},"requestedModel":{"family":"gpt-6.1-sol","gateway":null,"model":"gpt-6.1-sol","revision":null,"routeProvider":"openai"},"role":"reviewer","sessionIds":["id_c4fd16b18953c0c4"]},{"actualModel":{"kind":"observed","value":{"family":"gpt-6.1-sol","gateway":null,"model":"gpt-6.1-sol","revision":null,"routeProvider":"openai"}},"requestedModel":{"family":"gpt-6.1-sol","gateway":null,"model":"gpt-6.1-sol","revision":null,"routeProvider":"openai"},"role":"manager","sessionIds":["id_3939da539d36c3fa"]}],"artifact":{"packageVersion":"10.0.0","sourceCommit":"fb67f0e014b04e4b61ed6448963edafea9952319","sourceTreeSha256":"sha256:fa6c076ada7f03b22a69d0391593f194705a2be1998e387296ae0e2d722edbfc","tarballSha256":"sha256:f505b1011a48d6ba4122ced204f70a2d5ea9937c5cff008e1dd2305230514ac8","unpackedManifestSha256":"sha256:9ded4d36cd58e0b834cf1d346b064d48fff5b7bc0b20769fb78f2cbaff5e200e"},"artifactSha256":"sha256:41701763c6f2f49fe78ef3cdaa9223bec984d031277e3ae890458e1a286b302b","artifacts":{"installation":{"bytes":36771,"path":"artifacts/10.0.0-installation.json","sha256":"sha256:0624bdf24cb2374c7ba7f41e1df563985d74582419dd4550d1138565dc74c931"},"session":{"bytes":5931,"path":"artifacts/10.0.0-session.json","sha256":"sha256:b0d2ff386cb010eee67941483ca5b3e2d96f1abdd8f0c270a12c0e9637808ff4"},"transcript":{"bytes":149201,"path":"artifacts/10.0.0-transcript.json","sha256":"sha256:5a6d88b7621ad51e80c2f8ae4445339431b5957f9206f9d4b1ef1327047d1d6d"}},"checklistSha256":"sha256:bcaa925277568ce9b67c50f0785c3bfbadfa5bb40a6f36b39fad63c21a605110","checklistVersion":"phase9-canary-v1","checks":{"captures-validation":true,"closes-with-delivery":true,"dispatches-reviewer":true,"installs-packed-artifact":true,"loads-flow-tools":true,"saves-plan":true},"derivationVersion":"canary-evidence-v2","expiresAt":"2026-10-28T23:33:23.852Z","hostConfigSha256":"sha256:b573dd85491eecf48bb623bc7b21dc02464d612ddf6591412ebcfbb52f9e3131","operator":"ddv1982","pluginEntrySha256":"sha256:6c92b02c223a277bb04ef5eae5e04a8651c26a786a4faeab2e99dce02baa1f81","preparedSha256":"sha256:308920466c6de676999927ae0ba73bcae8e3f16a52d2769ad41eb3bd394637fd","recordSha256":"sha256:552d789bbe742c4b4cc6a9db3ee2871fe2b7939a81655ddf098e10deb10ee05e","recordedAt":"2026-10-25T23:33:23.852Z","releaseTag":"v10.0.0","schemaVersion":1,"status":"passed"}
1 change: 1 addition & 0 deletions evals/canary/artifacts/10.0.0-installation.json

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions evals/canary/artifacts/10.0.0-session.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"approval":"approved","closure":{"kind":"completed","operationId":"canary-close-1","recordedRevision":7,"summary":"Added greet(name: string): string returning the exact greeting in src/canary.ts while preserving canary. Added Ada and Grace tests in tests/canary.test.ts. Armed and captured exact broad bun test: 2 passed, 0 failed (receipt c4d9fb06-ace5-43e7-a744-242415904493). Independent Flow reviewer submitted passing review id_6bc6c2637e565e59 for the same source with no findings. Only feature source and tests changed; no dependency, configuration, authentication, or plugin changes."},"goal":"Add and test an exported greet(name: string): string returning Hello, ${name}! while preserving the canary export.","id":"id_f20137427e8d47ee","operations":[{"committedRevision":1,"id":"canary-plan-1","inputDigest":"sha256:dad85c05b82d50437a6b8a58d6784199f4bafea283d529539fee21f5960d69b6","kind":"plan-save"},{"committedRevision":2,"id":"canary-approve-1","inputDigest":"sha256:d490ed4318137499119a4a31b8f9e4c82b2abbae7ffec8921bdd6526addd55e7","kind":"plan-approve"},{"committedRevision":3,"entityId":"run:8bf9d8ba-2b23-40bd-ae78-27984650c781","id":"canary-run-1","inputDigest":"sha256:4777db4fc920be35136c21d452bede0cff89fcc7450016210fcbcd5fe87beda7","kind":"run-start"},{"committedRevision":5,"entityId":"id_6bc6c2637e565e59","id":"canary-review-1","inputDigest":"sha256:616ebf7d41307cfbef1db5e8ce8dd0c7408f70b64a6c00093e3989ed1b2b5d16","kind":"review-start"},{"committedRevision":6,"entityId":"run:8bf9d8ba-2b23-40bd-ae78-27984650c781","id":"review-submit:0ee2dd96-independent-pass","inputDigest":"sha256:0dcbabc53a72f73db62154e54eb060503a65dc5b0f24af0382af0628e3ce6de0","kind":"feature-complete"},{"committedRevision":7,"id":"canary-close-1","inputDigest":"sha256:5b44adfd4656c834cbe34cf9037a8f572e069161127ba7d9543bd2d975281b9b","kind":"session-close"}],"plan":{"decisions":["The fixture currently contains src/canary.ts with only the canary export and no test manifest or maintained test instructions.","Use the user-required whole-repository command bun test as the canonical broad gate on Linux. Assertions are empty because captured whole-suite success is the validation claim; reviewer inspection verifies Ada and Grace coverage."],"evidence":[{"assertions":[],"command":"bun test","environment":"Prepared canary fixture on Linux with Bun","platform":"linux","requirement":"Validate the complete Bun suite for the greeting feature.","scope":"gate"}],"features":[{"dependsOn":[],"id":"canary-greet","summary":"Preserve canary and export greet with Ada and Grace Bun coverage.","targets":["src/canary.ts","tests/canary.test.ts"],"title":"Export and test greet","validation":["Run exact canonical broad gate: bun test.","Independently inspect the source-bound diff for the greet signature, exact greeting, preserved canary export, Ada and Grace coverage, and bounded scope."]}],"overview":"Make a minimal source change in src/canary.ts and add tests/canary.test.ts; validate the whole fixture and obtain independent Flow review.","requirements":["Preserve export const canary = true.","Export greet(name: string): string returning `Hello, ${name}!`.","Add Bun tests covering Ada and Grace.","Change only the feature source and its tests; do not change dependencies, configuration, authentication attachments, or plugin files."],"summary":"Add the greet export and Bun coverage for Ada and Grace."},"revision":7,"runs":[{"artifactsChanged":[{"path":"src/canary.ts"},{"path":"tests/canary.test.ts"}],"attempt":1,"baseline":{"sha256":"sha256:3023515d5dd5742dfec99c4233d4e5f2d620ed02e2b02dfbd187492392df2b2b","version":1},"featureId":"canary-greet","id":"run:8bf9d8ba-2b23-40bd-ae78-27984650c781","reviews":[{"createdRevision":5,"evidence":{"sha256":"sha256:8d7f8d90511d96e2e9782c70491f954a85b90b017aad41c1474caa9d090f0e7c","version":1},"featureId":"canary-greet","id":"id_6bc6c2637e565e59","kind":"final","operationId":"canary-review-1","packet":{"riskLenses":["Do existing callers retain the canary export unchanged, and does the source module expose the approved typed greet contract without introducing new input-validation requirements?"],"summary":"Added exported greet(name: string): string with exact template greeting while retaining canary unchanged. Added Bun tests for Ada and Grace. Canonical broad validation bun test captured receipt c4d9fb06-ace5-43e7-a744-242415904493 at revision 4: passed, 2 pass, 0 fail, 2 expectations across 1 file. Only these two regular non-executable files changed; no deletions, renames, binary changes, or unrelated pre-existing edits. Please independently verify the exact source, coverage, scope, and captured evidence."},"result":{"findings":[],"recordedRevision":6,"terminalDisposition":"submitted","verdict":"passed"},"runId":"run:8bf9d8ba-2b23-40bd-ae78-27984650c781","sourceDigest":"sha256:79989db00c646cf6a6796a6943e497a64413755b3b9226954c32b71dbdf211ae","validationIds":["c4d9fb06-ace5-43e7-a744-242415904493"]}],"startedRevision":3,"state":"completed","summary":"Verified canary-greet and all four approved requirements: unchanged canary export, exact typed greeting, Ada/Grace Bun coverage, and changes limited to source/tests. Inspected current files, complete base diff, README and callers; no compatibility, state, package or scope issues. Canonical Linux bun test receipt c4d9fb06-ace5-43e7-a744-242415904493 is complete, exit 0, and bound to the reviewed source. No prior findings or remaining gaps.","validations":[{"command":"bun test","exitCode":0,"featureId":"canary-greet","hostPlatform":"linux","id":"c4d9fb06-ace5-43e7-a744-242415904493","intent":"pass","outputComplete":true,"outputDigest":"sha256:4d5c613b7bd906a0ea2d45aa62165cfed26d17f40d4a19077c96df0664412a6c","recordedRevision":4,"runId":"run:8bf9d8ba-2b23-40bd-ae78-27984650c781","scope":"broad","sourceDigest":"sha256:79989db00c646cf6a6796a6943e497a64413755b3b9226954c32b71dbdf211ae"}]}],"version":5}
1 change: 1 addition & 0 deletions evals/canary/artifacts/10.0.0-transcript.json

Large diffs are not rendered by default.

26 changes: 7 additions & 19 deletions evals/cassette.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,46 +142,34 @@ export type Cassette = Readonly<{
fidelity: readonly FidelityNote[];
}>;

/**
* Anything shaped like a credential, replaced by a fixed marker.
*
* The recording host carries the developer's real `auth.json`, and a model may echo
* an environment variable, a header, or a token into a command or an argument. A
* cassette is a committed artifact, so this runs over every string in one — before
* it is written, never after.
*/
const SECRET_PATTERNS: readonly RegExp[] = [
// Provider key prefixes, longest first so a longer prefix is not half-matched.
// `[-_]`, because the separator is not the same for all of them: Anthropic, OpenAI
// and xAI hyphenate, while Groq, Hugging Face, DigitalOcean and Shopify use an
// underscore. Requiring a hyphen meant `gsk_`, `hf_`, `dop_v1_` and `shpat_` were
// four prefixes listed here that could never match — the scrubber read as covering
// them while passing them straight into a committed artifact.
// underscore.
/\b(?:sk-ant|sk-proj|sk-or|sk|xai|gsk|hf|dop_v1|shpat)[-_][A-Za-z0-9_-]{16,}/g,
/\bgithub_pat_[A-Za-z0-9_]{20,}/g,
/\bgh[pousr]_[A-Za-z0-9]{16,}/g,
/\bAIza[A-Za-z0-9_-]{20,}/g,
/\bAKIA[0-9A-Z]{16}\b/g,
// A JSON web token, which is what an OAuth access token normally is.
/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g,
/\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/gi,
];

/**
* `"api_key": "…"` and `TOKEN=…` forms, where the value carries no recognizable
* prefix and only the key says what it is.
*/
const SECRET_ASSIGNMENT =
/((?:api[_-]?key|secret|access[_-]?token|refresh[_-]?token|client[_-]?secret|password|passwd|authorization)["']?\s*[:=]\s*["']?)([^\s"',;}&]{8,})/gi;

export const REDACTED = "[redacted]";

export function scrubSecrets(value: string): string {
export function scrubSecretTokens(value: string): string {
let scrubbed = value;
for (const pattern of SECRET_PATTERNS) {
scrubbed = scrubbed.replace(pattern, REDACTED);
}
return scrubbed.replace(SECRET_ASSIGNMENT, `$1${REDACTED}`);
return scrubbed;
}

export function scrubSecrets(value: string): string {
return scrubSecretTokens(value).replace(SECRET_ASSIGNMENT, `$1${REDACTED}`);
}

/** Recursively rewrites every string in a JSON-shaped value. */
Expand Down
97 changes: 37 additions & 60 deletions evals/qualification-bundle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@ import { tmpdir } from "node:os";
import { isAbsolute, join, relative, resolve, sep, win32 } from "node:path";
import { z } from "zod";
import { canonicalJson, canonicalSha256 } from "./canonical-json.js";
import { scrubSecrets } from "./cassette.js";
import { assertNoPrivateUserPaths } from "./private-paths.js";
import {
assertQualificationJson,
assertQualificationMember,
assertQualificationSource,
assertQualificationText as assertSafeText,
} from "./qualification-safety.js";
import { ReviewerPacketBytesListSchema } from "./reviewer-packet-bytes.js";

const MAX_OBJECT_BYTES = 16 * 1024 * 1024;
Expand Down Expand Up @@ -163,29 +167,6 @@ function validateRoles(files: readonly QualificationBundleFile[]): void {
}
}

function assertSafeText(text: string): void {
if (scrubSecrets(text) !== text)
throw new Error("Qualification bundle contains secret-shaped evidence.");
if (/\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/.test(text))
throw new Error("Qualification bundle contains a raw session identifier.");
assertNoPrivateUserPaths(text);
}

function assertSafeSource(text: string): void {
if (
/\b(?:sk-ant|sk-proj|sk-or|xai|gsk|hf|dop_v1|shpat)[-_][A-Za-z0-9_-]{16,}/.test(
text,
) ||
/\b(?:github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{16}|AKIA[0-9A-Z]{16})\b/.test(
text,
) ||
/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/.test(text) ||
/\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i.test(text)
) {
throw new Error("Qualification bundle contains secret-shaped source.");
}
}

async function assertSafeArtifact(bytes: Uint8Array): Promise<void> {
const directory = await mkdtemp(join(tmpdir(), "flow-bundle-artifact-"));
const path = join(directory, "artifact.tgz");
Expand Down Expand Up @@ -241,22 +222,14 @@ async function assertSafeArtifact(bytes: Uint8Array): Promise<void> {
throw new Error(
"Qualification artifact exceeds its expanded byte limit.",
);
const source = extracted.stdout.toString("utf8");
assertSafeSource(source);
try {
assertSafeText(
new TextDecoder("utf-8", { fatal: true }).decode(extracted.stdout),
);
} catch (error) {
if (!(error instanceof TypeError)) throw error;
}
await assertQualificationMember(member, extracted.stdout);
}
} finally {
await rm(directory, { recursive: true, force: true });
}
}

function normalizedBytes(file: QualificationBundleFile): Buffer {
async function normalizedBytes(file: QualificationBundleFile): Promise<Buffer> {
const bytes = Buffer.from(file.bytes);
if (bytes.byteLength > MAX_OBJECT_BYTES)
throw new Error("Qualification bundle object exceeds its byte limit.");
Expand Down Expand Up @@ -288,42 +261,46 @@ function normalizedBytes(file: QualificationBundleFile): Buffer {
}
}
const canonical = canonicalJson(parsed);
assertSafeText(canonical);
await assertQualificationJson(file.role, parsed);
return Buffer.from(canonical);
}
if (file.mediaType === "text/typescript") {
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
assertSafeSource(text);
if (file.role === "authority-source" || file.role === "execution-source") {
await assertQualificationSource(file.id ?? "input.ts", text);
} else assertSafeText(text);
return Buffer.from(text);
}
return bytes;
}

function buildManifest(input: QualificationBundleInput): {
async function buildManifest(input: QualificationBundleInput): Promise<{
readonly manifest: QualificationBundleManifest;
readonly objects: ReadonlyMap<string, Buffer>;
} {
}> {
validateRoles(input.files);
const objects = new Map<string, Buffer>();
const files = input.files
.map((file) => {
const bytes = normalizedBytes(file);
const digest = sha256(bytes);
const object = `objects/sha256-${digest.slice("sha256:".length)}`;
const prior = objects.get(object);
if (prior && !prior.equals(bytes))
throw new Error("Qualification bundle object digest collision.");
objects.set(object, bytes);
return {
role: file.role,
...(file.id ? { id: file.id } : {}),
mediaType: file.mediaType,
object,
sha256: digest,
bytes: bytes.byteLength,
};
})
.sort((left, right) => fileKey(left).localeCompare(fileKey(right)));
const files = (
await Promise.all(
input.files.map(async (file) => {
const bytes = await normalizedBytes(file);
const digest = sha256(bytes);
const object = `objects/sha256-${digest.slice("sha256:".length)}`;
const prior = objects.get(object);
if (prior && !prior.equals(bytes))
throw new Error("Qualification bundle object digest collision.");
objects.set(object, bytes);
return {
role: file.role,
...(file.id ? { id: file.id } : {}),
mediaType: file.mediaType,
object,
sha256: digest,
bytes: bytes.byteLength,
};
}),
)
).sort((left, right) => fileKey(left).localeCompare(fileKey(right)));
const total = files.reduce((sum, file) => sum + file.bytes, 0);
if (total > MAX_BUNDLE_BYTES)
throw new Error("Qualification bundle exceeds its total byte limit.");
Expand Down Expand Up @@ -419,7 +396,7 @@ export async function writeQualificationBundle(input: {
if (!artifact)
throw new Error("Qualification bundle artifact role is missing.");
await assertSafeArtifact(artifact.bytes);
const built = buildManifest(input.input);
const built = await buildManifest(input.input);
await mkdir(input.outputRoot, { recursive: true, mode: 0o700 });
const outputIdentity = await realDirectoryIdentity(input.outputRoot);
const path = join(input.outputRoot, built.manifest.bundleId);
Expand Down Expand Up @@ -737,7 +714,7 @@ export async function readQualificationBundle(path: string): Promise<{
throw new Error(
"Qualification bundle object digest or size is invalid.",
);
normalizedBytes({
await normalizedBytes({
role: ref.role,
...(ref.id ? { id: ref.id } : {}),
mediaType: ref.mediaType,
Expand Down
Loading
Loading