Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions scripts/canary-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,10 @@ export function canaryLaunchEnvironment(
delete environment.TYPESAFE_API_KEY;
delete environment.FLOW_EVAL_NATIVE_AUTH;
for (const name of Object.keys(environment))
if (name.startsWith("OPENCODE_") && name !== "OPENCODE_API_KEY")
if (
(name.startsWith("OPENCODE_") && name !== "OPENCODE_API_KEY") ||
name.startsWith("GIT_")
)
delete environment[name];
for (const name of [
"NODE_OPTIONS",
Expand Down Expand Up @@ -354,6 +357,24 @@ async function launchCanary(
await realpath(join(dirname(directory), CANARY_PLUGIN_DIRECTORY)),
attachment.plugin.package,
]);
const entryRoots = new Map([
[
await realpath(
join(dirname(directory), CANARY_PLUGIN_DIRECTORY, "index.js"),
),
await realpath(join(dirname(directory), CANARY_PLUGIN_DIRECTORY)),
],
[
await realpath(
join(dirname(directory), CANARY_PLUGIN_DIRECTORY, "dist/index.js"),
),
await realpath(join(dirname(directory), CANARY_PLUGIN_DIRECTORY)),
],
[
await realpath(join(attachment.plugin.package, "index.ts")),
attachment.plugin.package,
],
]);
for (;;) {
signal.throwIfAborted();
if (Date.now() >= deadline)
Expand All @@ -372,12 +393,8 @@ async function launchCanary(
if (plugin.source.type !== "local")
throw new Error("Unprepared canary plugin loaded.");
const path = await realpath(plugin.source.path);
const base = expected.has(path)
? path
: /[/\\]index\.(?:ts|js)$/.test(path)
? dirname(path)
: path;
if (!expected.has(base) || plugin.state.status !== "active")
const base = expected.has(path) ? path : entryRoots.get(path);
if (!base || !expected.has(base) || plugin.state.status !== "active")
throw new Error("Unprepared or failed canary plugin loaded.");
active.add(base);
}
Expand Down
67 changes: 67 additions & 0 deletions scripts/canary-workspace.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { spawnSync } from "node:child_process";
import { mkdir, realpath, writeFile } from "node:fs/promises";
import { join } from "node:path";

function git(directory: string, args: readonly string[]): string {
const env = { ...process.env };
for (const name of Object.keys(env)) {
if (name.startsWith("GIT_")) delete env[name];
}
env.GIT_CONFIG_NOSYSTEM = "1";
env.GIT_CONFIG_GLOBAL = process.platform === "win32" ? "NUL" : "/dev/null";
const result = spawnSync(
"git",
[
"-c",
`core.hooksPath=${join(directory, ".git", "disabled-hooks")}`,
"-C",
directory,
...args,
],
{ env, encoding: "utf8", timeout: 10000, maxBuffer: 1024 * 1024 },
);
if (result.status !== 0 || typeof result.stdout !== "string") {
throw new Error("Canary fixture requires a committed Git workspace.");
}
return result.stdout.trim();
}

export async function assertCanaryWorkspace(directory: string): Promise<void> {
try {
const expected = await realpath(directory);
if (
(await realpath(git(expected, ["rev-parse", "--show-toplevel"]))) !==
expected
) {
throw new Error("Workspace root differs.");
}
git(expected, ["rev-parse", "--verify", "HEAD^{commit}"]);
} catch {
throw new Error("Canary fixture requires a committed Git workspace.");
}
}

export async function prepareCanaryWorkspace(directory: string): Promise<void> {
const workspace = await realpath(directory);
await mkdir(join(workspace, ".git"));
await writeFile(
join(workspace, ".gitignore"),
".flow/\n.opencode/\nnode_modules/\n",
{ flag: "wx" },
);
git(workspace, ["init", "--initial-branch=main"]);
git(workspace, ["add", "--all"]);
git(workspace, [
"-c",
"user.name=Flow Canary",
"-c",
"user.email=flow-canary@example.invalid",
"-c",
"commit.gpgsign=false",
"commit",
"--no-verify",
"-m",
"Prepare canary fixture",
]);
await assertCanaryWorkspace(workspace);
}
50 changes: 35 additions & 15 deletions scripts/eval-canary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ import {
} from "../src/domain/operation.js";
import { currentRun } from "../src/domain/session.js";
import { readWorkspaceTestReport } from "../src/infrastructure/fs/workspace-validation.js";
import {
assertCanaryWorkspace,
prepareCanaryWorkspace,
} from "./canary-workspace.js";

export const CANARY_CHECKLIST_VERSION = "phase9-canary-v1";
export const CANARY_DERIVATION_VERSION = "canary-evidence-v1";
Expand All @@ -70,6 +74,11 @@ export const CANARY_CHECK_IDS = [
"closes-with-delivery",
] as const;
export const CANARY_MAX_AGE_MS = 72 * 60 * 60 * 1_000;
export const CANARY_PLUGIN_LOADER =
'export { default } from "./dist/index.js";\n';
export const CANARY_PLUGIN_LOADER_SHA256 = sha256(
Buffer.from(CANARY_PLUGIN_LOADER),
);

const checklist = {
version: CANARY_CHECKLIST_VERSION,
Expand Down Expand Up @@ -223,6 +232,7 @@ const NativeInstallationEvidenceSchema =
sourcePackageSha256: DigestSchema,
tuiEntrySha256: DigestSchema,
pluginPackageSha256: DigestSchema,
loaderEntrySha256: z.literal(CANARY_PLUGIN_LOADER_SHA256),
configSha256: DigestSchema,
nativeAuth: NativeAuthInstallationSchema.optional(),
}).strict();
Expand Down Expand Up @@ -1170,7 +1180,8 @@ function canaryPluginPackage(
name: metadata.name,
version: metadata.version,
type: "module",
exports: { ".": "./index.js", "./tui": "./tui.js" },
main: "./dist/index.js",
exports: { ".": "./dist/index.js", "./tui": "./dist/tui.js" },
dependencies: {
...metadata.dependencies,
"@opencode/plugin": OPENCODE_HOST_VERSION,
Expand Down Expand Up @@ -1267,6 +1278,7 @@ const NativePreparationSchema = z
sourcePackageSha256: DigestSchema,
tuiEntrySha256: DigestSchema,
pluginPackageSha256: DigestSchema,
loaderEntrySha256: z.literal(CANARY_PLUGIN_LOADER_SHA256),
configSha256: DigestSchema,
nativeAuth: CanaryNativeAuthSchema.optional(),
})
Expand Down Expand Up @@ -1528,7 +1540,7 @@ export async function prepareCanary(input: {
}
const artifact = input.expectedArtifact ?? inspectedArtifact;
const fixture = join(input.outputDirectory, "fixture");
await mkdir(join(input.outputDirectory, CANARY_PLUGIN_DIRECTORY), {
await mkdir(join(input.outputDirectory, CANARY_PLUGIN_DIRECTORY, "dist"), {
recursive: true,
});
const copiedArtifactPath = join(input.outputDirectory, "artifact.tgz");
Expand Down Expand Up @@ -1564,8 +1576,9 @@ export async function prepareCanary(input: {
const pluginPackage = canaryPluginPackage(packageMetadata);
const config = canaryConfig();
for (const [name, bytes] of [
["index.js", pluginEntry],
["tui.js", tuiEntry],
["index.js", Buffer.from(CANARY_PLUGIN_LOADER)],
["dist/index.js", pluginEntry],
["dist/tui.js", tuiEntry],
["package.json", pluginPackage],
] as const)
await writeFile(
Expand All @@ -1578,6 +1591,7 @@ export async function prepareCanary(input: {
sourcePackageSha256: sha256(sourcePackage),
tuiEntrySha256: sha256(tuiEntry),
pluginPackageSha256: sha256(pluginPackage),
loaderEntrySha256: CANARY_PLUGIN_LOADER_SHA256,
configSha256: sha256(config),
};
if (input.nativeAuth) {
Expand All @@ -1600,6 +1614,7 @@ export async function prepareCanary(input: {
join(fixture, "README.md"),
`# Flow exact-artifact canary\n\nArtifact: ${artifact.tarballSha256}\n\nRun every checklist item and export sanitized JSON session and transcript evidence.\n`,
);
await prepareCanaryWorkspace(fixture);
const base = {
schemaVersion: 2 as const,
runtime,
Expand Down Expand Up @@ -1675,7 +1690,7 @@ export async function measureCanaryInstallation(
join(
directory,
prepared.schemaVersion === 2
? `${CANARY_PLUGIN_DIRECTORY}/index.js`
? `${CANARY_PLUGIN_DIRECTORY}/dist/index.js`
: "fixture/.opencode/plugins/flow.js",
),
),
Expand All @@ -1690,21 +1705,25 @@ export async function measureCanaryInstallation(
);
}
if (prepared.schemaVersion === 2) {
const [sourcePackage, tuiEntry, pluginPackage, config] = await Promise.all([
Promise.resolve(
extractArtifactFile(
join(directory, prepared.artifactFile),
"package/package.json",
await assertCanaryWorkspace(join(directory, "fixture"));
const [sourcePackage, tuiEntry, pluginPackage, config, loaderEntry] =
await Promise.all([
Promise.resolve(
extractArtifactFile(
join(directory, prepared.artifactFile),
"package/package.json",
),
),
),
readFile(join(directory, CANARY_PLUGIN_DIRECTORY, "tui.js")),
readFile(join(directory, CANARY_PLUGIN_DIRECTORY, "package.json")),
readFile(join(directory, "fixture/opencode.json")),
]);
readFile(join(directory, CANARY_PLUGIN_DIRECTORY, "dist/tui.js")),
readFile(join(directory, CANARY_PLUGIN_DIRECTORY, "package.json")),
readFile(join(directory, "fixture/opencode.json")),
readFile(join(directory, CANARY_PLUGIN_DIRECTORY, "index.js")),
]);
const measured = {
sourcePackageSha256: sha256(sourcePackage),
tuiEntrySha256: sha256(tuiEntry),
pluginPackageSha256: sha256(pluginPackage),
loaderEntrySha256: sha256(loaderEntry),
configSha256: sha256(config),
};
const {
Expand All @@ -1718,6 +1737,7 @@ export async function measureCanaryInstallation(
if (
canonicalJson(measured) !== canonicalJson(expected) ||
!pluginPackage.equals(canaryPluginPackage(metadata)) ||
!loaderEntry.equals(Buffer.from(CANARY_PLUGIN_LOADER)) ||
!config.equals(canaryConfig()) ||
sha256(
extractArtifactFile(
Expand Down
72 changes: 67 additions & 5 deletions tests/canary-run.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@ import {
canaryProviderConfiguration,
runPaidCanary,
} from "../scripts/canary-run.js";
import { prepareCanaryWorkspace } from "../scripts/canary-workspace.js";
import {
artifactIdentitySha256,
CANARY_CHECKLIST_SHA256,
CANARY_CHECKLIST_VERSION,
CANARY_PLUGIN_LOADER,
CANARY_PLUGIN_LOADER_SHA256,
preparedCanarySha256,
} from "../scripts/eval-canary.js";
import { authorizePaidRun, paidRunStatus } from "../scripts/paid-budget.js";
Expand All @@ -42,7 +45,7 @@ async function fixture() {
);
directories.push(directory);
const pluginDirectory = join(directory, "fixture/.opencode/plugins/flow");
await mkdir(pluginDirectory, { recursive: true });
await mkdir(join(pluginDirectory, "dist"), { recursive: true });
const hash = (text: string | Uint8Array) =>
`sha256:${createHash("sha256").update(text).digest("hex")}`;
const sourcePackage = JSON.stringify({
Expand All @@ -55,7 +58,8 @@ async function fixture() {
name: "opencode-plugin-flow",
version: "10.0.0",
type: "module",
exports: { ".": "./index.js", "./tui": "./tui.js" },
main: "./dist/index.js",
exports: { ".": "./dist/index.js", "./tui": "./dist/tui.js" },
dependencies: { "@opencode/plugin": "2.1.4" },
});
const config = canonicalJson({
Expand All @@ -73,8 +77,9 @@ async function fixture() {
["package/package.json", sourcePackage],
["package/dist/index.js", "plugin"],
["package/dist/tui.js", "tui"],
["fixture/.opencode/plugins/flow/index.js", "plugin"],
["fixture/.opencode/plugins/flow/tui.js", "tui"],
["fixture/.opencode/plugins/flow/index.js", CANARY_PLUGIN_LOADER],
["fixture/.opencode/plugins/flow/dist/index.js", "plugin"],
["fixture/.opencode/plugins/flow/dist/tui.js", "tui"],
["fixture/.opencode/plugins/flow/package.json", pluginPackage],
["fixture/opencode.json", config],
] as const)
Expand Down Expand Up @@ -109,13 +114,15 @@ async function fixture() {
sourcePackageSha256: hash(sourcePackage),
tuiEntrySha256: hash("tui"),
pluginPackageSha256: hash(pluginPackage),
loaderEntrySha256: CANARY_PLUGIN_LOADER_SHA256,
configSha256: hash(config),
},
};
await writeFile(
join(directory, "prepared.json"),
JSON.stringify({ ...base, sha256: preparedCanarySha256(base) }),
);
await prepareCanaryWorkspace(join(directory, "fixture"));
await writeFile(join(directory, "prompt.txt"), "Run fixture");
await authorizePaidRun(directory, {
schemaVersion: 1,
Expand Down Expand Up @@ -219,7 +226,12 @@ test.skipIf(process.platform === "win32")(
},
);

for (const path of ["index.js", "tui.js", "package.json"] as const)
for (const path of [
"index.js",
"dist/index.js",
"dist/tui.js",
"package.json",
] as const)
test(`changed prepared ${path} fails before dispatch consumption`, async () => {
const { directory, input } = await fixture();
await writeFile(
Expand Down Expand Up @@ -350,3 +362,53 @@ test("canary provider configuration uses the native HTTP transport setting for t
},
});
});

for (const state of ["absent", "without commit"] as const)
test(`canary refuses ${state} Git workspace before charging`, async () => {
const { directory, input } = await fixture();
const workspace = join(directory, "fixture");
await rm(join(workspace, ".git"), { recursive: true, force: true });
if (state === "without commit") {
const child = Bun.spawn(["git", "init", "--initial-branch=main"], {
cwd: workspace,
stdout: "ignore",
stderr: "pipe",
});
expect(await child.exited).toBe(0);
}
let launches = 0;
await expect(
runPaidCanary(input, async () => {
launches++;
return 0;
}),
).rejects.toThrow("committed Git workspace");
expect(launches).toBe(0);
expect((await paidRunStatus(directory)).consumed).toBe(0);
});

test("canary does not accept an ancestor Git repository as its fixture", async () => {
const { directory, input } = await fixture();
await rm(join(directory, "fixture/.git"), { recursive: true });
await prepareCanaryWorkspace(directory);
let launches = 0;
await expect(
runPaidCanary(input, async () => {
launches++;
return 0;
}),
).rejects.toThrow("committed Git workspace");
expect(launches).toBe(0);
expect((await paidRunStatus(directory)).consumed).toBe(0);
});

test("canary Git environment cannot redirect its prepared workspace", () => {
expect(
canaryLaunchEnvironment("/prepared/fixture", {
GIT_DIR: "/other/.git",
GIT_WORK_TREE: "/other",
GIT_CONFIG_GLOBAL: "/other/config",
FLOW_CANARY_SENTINEL: "kept",
}),
).toEqual({ PWD: "/prepared/fixture", FLOW_CANARY_SENTINEL: "kept" });
});
Loading
Loading