From b43c455c973329485cec59cdcdaa501c11cce57e Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:35:03 +0000 Subject: [PATCH 1/5] feat(action): Add a GitHub Action that runs a command through Roach A repository adds one step and a roach.json file. The action starts a run, runs the command with the proxy variables of the run, and ends the run. Only the command uses the proxy, so later steps such as cache and artifact uploads still work. The command does not get the write token. The client now keeps the certificates of NODE_EXTRA_CA_CERTS in the CA file that it writes. The deployed test runs the action in place of tests/ci-job.ts. Co-Authored-By: Junior --- README.md | 46 +++++++++++++- action.yml | 20 ++++++ knip.json | 2 +- src/action.ts | 80 ++++++++++++++++++++++++ src/client.ts | 16 ++++- tests/ci-job.ts | 49 --------------- tests/deployed.test.ts | 135 +++++++++++++++++++++++++++-------------- 7 files changed, 249 insertions(+), 99 deletions(-) create mode 100644 action.yml create mode 100644 src/action.ts delete mode 100644 tests/ci-job.ts diff --git a/README.md b/README.md index aefe809..f9b2e4f 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,45 @@ console.log(describeRecordingStats(await run.close())); A client that does not use `client.ts` can do the same with plain HTTP. See [Control API](#control-api). +## Use the GitHub Action + +A repository adds one step and a `roach.json` file: + +```yaml +- uses: getsentry/roach@main + with: + token: ${{ secrets.ROACH_TOKEN }} # without it, the run can only replay + run: pnpm test +``` + +`roach.json` is at the root of the repository. It has the `rules` of the +run, and can have `allow` (`RunConfig` in `src/service.ts`): + +```json +{ + "rules": [ + { + "name": "model", + "match": { "method": "POST", "url": "https://ai-gateway.vercel.sh/" }, + "keyHeaders": ["ai-model-id"] + } + ] +} +``` + +The action (`src/action.ts`) starts a run of the tenant +`GITHUB_REPOSITORY`, in `auto` mode with the token or `replay` mode without +it. It runs `run` in bash with the proxy variables of the run, then ends +the run and writes its stats to the job summary. + +- Only `run` uses the proxy. Later steps, such as cache and artifact + uploads, do not. `run` does not get the write token. +- `run` is one session. When it fails, the run drops its new recordings. A + miss in `replay` mode fails the step. The summary names the job, not the + test, of each miss. +- `NODE_EXTRA_CA_CERTS` works only for Node. Python and `curl` need a CA + bundle that also has the system certificates. + ## Use a local proxy ```ts @@ -80,7 +119,8 @@ await proxy.close(); - `env` has `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, and `NODE_EXTRA_CA_CERTS`. Node 24 reads the last two only at startup. A - process that is already running must set its own agents. + process that is already running must set its own agents. The CA file + also has the certificates of `NODE_EXTRA_CA_CERTS` of the caller. - `spawnRoach(config, { launcher })` runs the proxy with a command prefix, such as `sudo`. Use it when the caller cannot reach the network, but the proxy must. @@ -357,7 +397,8 @@ pnpm check # format, lint, types, unused code, and tests, as CI runs them - `tests/deployed.test.ts` runs the service as it runs in production: from the command line, behind a TLS server that does what the load balancer does, with a local GCS server, metadata server, and Sentry server. Each - CI job (`tests/ci-job.ts`) sets only the proxy variables of its run. + CI job runs the GitHub Action, and its command has only the proxy + variables of its run. ## Files @@ -380,4 +421,5 @@ The proxy is in `src/`. Tests are in `tests/`. The production setup is in - `client.ts`: starts a remote run or a local proxy, and calls the control API. - `report.ts`: text reports of a run. +- `action.ts`: the GitHub Action (`action.yml` at the root). - `cli.ts`: the command line. diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..bb2c735 --- /dev/null +++ b/action.yml @@ -0,0 +1,20 @@ +name: Roach +description: Run a command with its HTTPS traffic recorded and replayed by the Roach service. + +inputs: + run: + description: The command to run, in bash. Only this command uses the proxy. + required: true + token: + description: >- + The write token of the service. Without it, the run can only replay, + as in jobs of forks. Give it only to workflows that you trust. + required: false + url: + description: The URL of the Roach service. + required: false + default: https://roach-proxy.getsentry.net + +runs: + using: node24 + main: src/action.ts diff --git a/knip.json b/knip.json index 5742288..3a2363c 100644 --- a/knip.json +++ b/knip.json @@ -2,5 +2,5 @@ "$schema": "https://unpkg.com/knip@6/schema.json", "project": ["src/**/*.ts", "tests/**/*.ts"], "ignoreBinaries": ["openssl"], - "entry": ["tests/ci-job.ts"] + "entry": ["src/action.ts"] } diff --git a/src/action.ts b/src/action.ts new file mode 100644 index 0000000..9968e8c --- /dev/null +++ b/src/action.ts @@ -0,0 +1,80 @@ +/** + * The GitHub Action of Roach (`action.yml`). See "Use the GitHub Action" in + * `README.md`. + * + * It starts a run, runs the command of the step with the proxy variables of + * the run, and ends the run. The rules of the run come from `roach.json`. + * The runner gives each input as `INPUT_`. + */ +import { spawn } from "node:child_process"; +import { appendFile, readFile } from "node:fs/promises"; +import { startRemoteRun } from "./client.ts"; +import { describeRecordingMisses, describeRecordingStats } from "./report.ts"; +import type { RunConfig } from "./service.ts"; + +/** Variables that the command must not get: the inputs, with the token. */ +const HIDDEN_VARIABLE = /^(INPUT_.*|https?_proxy|no_proxy|all_proxy)$/i; + +/** Run the command in bash, and return its exit code. */ +function runCommand(command: string, env: NodeJS.ProcessEnv): Promise { + const child = spawn("bash", ["-e", "-o", "pipefail", "-c", command], { + env, + stdio: "inherit", + }); + // A canceled job sends a signal. Stop the command, then end the run. + process.on("SIGINT", () => child.kill("SIGINT")); + process.on("SIGTERM", () => child.kill("SIGTERM")); + return new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve(code ?? 1)); + }); +} + +const command = process.env.INPUT_RUN; +if (!command) throw new Error("The run input is required"); +// The runner gives an empty string for an input that the step does not set. +const token = + process.env.INPUT_TOKEN === "" ? undefined : process.env.INPUT_TOKEN; +// Without the token, the service only allows `replay`, as for forks. +const mode = token ? "auto" : "replay"; +const config = JSON.parse(await readFile("roach.json", "utf8")) as Pick< + RunConfig, + "allow" | "rules" +>; +const run = await startRemoteRun( + { url: process.env.INPUT_URL!, token }, + { + ...config, + tenant: process.env.GITHUB_REPOSITORY!, + mode, + name: `${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT}`, + }, +); +// The proxy URL has the run token. Keep it out of the log. +process.stdout.write(`::add-mask::${run.token}\n`); + +let exitCode = 1; +try { + const session = await run.startSession(process.env.GITHUB_JOB ?? "command"); + exitCode = await runCommand(command, { + ...Object.fromEntries( + Object.entries(process.env).filter( + ([name]) => !HIDDEN_VARIABLE.test(name), + ), + ), + ...run.env, + }); + const { missed } = await session.end(exitCode === 0); + if (missed > 0 && exitCode === 0) exitCode = 1; +} finally { + const stats = await run.close(); + const report = [ + `Roach (${mode}): ${describeRecordingStats(stats)}`, + ...describeRecordingMisses(stats.misses).map((miss) => `- ${miss}`), + ].join("\n"); + process.stdout.write(`${report}\n`); + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile(process.env.GITHUB_STEP_SUMMARY, `${report}\n`); + } +} +process.exitCode = exitCode; diff --git a/src/client.ts b/src/client.ts index 07c7ef7..8f6e361 100644 --- a/src/client.ts +++ b/src/client.ts @@ -8,7 +8,7 @@ * process, such as a test worker. */ import { spawn } from "node:child_process"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import http from "node:http"; import https from "node:https"; import { tmpdir } from "node:os"; @@ -238,7 +238,11 @@ export async function startRemoteRun( }; } -/** The proxy variables of a process, with the CA certificate in a file. */ +/** + * The proxy variables of a process, with the CA certificate in a file. The + * file also has the certificates of `NODE_EXTRA_CA_CERTS` of this process, + * so the process still trusts them. + */ async function createProxyEnv( address: Pick, noProxy: string, @@ -246,7 +250,13 @@ async function createProxyEnv( // `NODE_EXTRA_CA_CERTS` takes a file. const caDirectory = await mkdtemp(path.join(tmpdir(), "roach-")); const caFile = path.join(caDirectory, "ca.pem"); - await writeFile(caFile, address.caCert); + const extra = process.env.NODE_EXTRA_CA_CERTS; + await writeFile( + caFile, + extra + ? `${address.caCert.trimEnd()}\n${await readFile(extra, "utf8")}` + : address.caCert, + ); return { env: { HTTP_PROXY: address.url, diff --git a/tests/ci-job.ts b/tests/ci-job.ts deleted file mode 100644 index f40d11c..0000000 --- a/tests/ci-job.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * One CI job that uses a deployed Roach service. `deployed.test.ts` runs it - * in its own process, because it must trust the TLS certificate of the - * service at startup, as a CI job trusts a public one. - * - * It starts a run, then runs a test process that has only the proxy - * variables of the run. That process sends one HTTPS request with plain - * `fetch`. The job prints the response, the stats, and the CA certificate - * as one JSON line. - * - * Argument: JSON `{ service, token?, config, url, body }`. - */ -import { execFile } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; -import { promisify } from "node:util"; -import { startRemoteRun } from "../src/client.ts"; -import type { RunConfig } from "../src/service.ts"; - -const { service, token, config, url, body } = JSON.parse(process.argv[2]!) as { - service: string; - token?: string; - config: RunConfig; - url: string; - body: string; -}; - -const run = await startRemoteRun({ url: service, token }, config); -// The test process trusts the CA of Roach and the TLS certificate of the -// service, as this job does. -const caFile = run.env.NODE_EXTRA_CA_CERTS!; -await writeFile( - caFile, - `${await readFile(caFile, "utf8")}${await readFile(process.env.NODE_EXTRA_CA_CERTS!, "utf8")}`, -); -const session = await run.startSession("test"); -const script = ` -const response = await fetch(${JSON.stringify(url)}, { method: "POST", body: ${JSON.stringify(body)} }); -console.log(JSON.stringify({ status: response.status, source: response.headers.get("x-roach"), body: await response.text() })); -`; -const { stdout } = await promisify(execFile)( - process.execPath, - ["--input-type=module", "--eval", script], - { env: { ...process.env, ...run.env } }, -); -await session.end(true); -const stats = await run.close(); -process.stdout.write( - `${JSON.stringify({ response: JSON.parse(stdout), stats, caCert: run.caCert })}\n`, -); diff --git a/tests/deployed.test.ts b/tests/deployed.test.ts index d20b98a..16a836a 100644 --- a/tests/deployed.test.ts +++ b/tests/deployed.test.ts @@ -7,8 +7,8 @@ * TLS and passes the bytes to the service. * - A local server plays the GCS JSON API and the metadata server of the * VM. Another one plays Sentry. - * - Each CI job (`ci-job.ts`) sends HTTPS through the service with only the - * proxy variables of its run. + * - Each CI job runs the GitHub Action (`src/action.ts`). Its command sends + * HTTPS through the service with only the proxy variables of its run. */ import { execFile, spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; @@ -22,7 +22,7 @@ import path from "node:path"; import tls from "node:tls"; import { promisify } from "node:util"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import type { RoachServiceConfig, RunConfig } from "../src/service.ts"; +import type { RoachServiceConfig } from "../src/service.ts"; const execFileAsync = promisify(execFile); const ROOT = path.join(import.meta.dirname, ".."); @@ -76,34 +76,73 @@ function readEnvelope(envelope: string) { } } -/** Run one CI job, and return what it printed. */ -async function ciJob(token: string | undefined, mode: RunConfig["mode"]) { - const config: RunConfig = { - tenant: TENANT, - mode, - name: `${mode}-run`, - rules: [{ name: "model", match: { method: "POST" } }], - }; - const { stdout } = await execFileAsync( +/** + * Run one CI job: the action (`src/action.ts`) with a command that sends + * one HTTPS request with plain `fetch`. Returns the exit code of the + * action, what the command saw, and the job summary. + */ +async function ciJob(job: { + token?: string; + runId: string; + prompt: string; +}): Promise<{ + exitCode: number; + response: { status: number; source: string | null; body: string }; + token: string | null; + trusted: string; + summary: string; +}> { + const workspace = await mkdtemp(path.join(directory, "job-")); + const config = { rules: [{ name: "model", match: { method: "POST" } }] }; + await writeFile(path.join(workspace, "roach.json"), JSON.stringify(config)); + await writeFile( + path.join(workspace, "job.mjs"), + `import { readFileSync } from "node:fs"; +const response = await fetch(${JSON.stringify(`${upstreamOrigin}/v1/messages`)}, { + method: "POST", + body: JSON.stringify({ prompt: ${JSON.stringify(job.prompt)} }), +}); +console.log(JSON.stringify({ + response: { status: response.status, source: response.headers.get("x-roach"), body: await response.text() }, + token: process.env.INPUT_TOKEN ?? null, + trusted: readFileSync(process.env.NODE_EXTRA_CA_CERTS, "utf8"), +})); +`, + ); + const summary = path.join(workspace, "summary.md"); + const action = spawn( process.execPath, [ "--experimental-strip-types", "--disable-warning=ExperimentalWarning", - path.join(ROOT, "tests/ci-job.ts"), - JSON.stringify({ - service: frontUrl, - token, - config, - url: `${upstreamOrigin}/v1/messages`, - body: JSON.stringify({ prompt: "hi" }), - }), + path.join(ROOT, "src/action.ts"), ], - { env: childEnv({ NODE_EXTRA_CA_CERTS: hostCertFile }) }, + { + cwd: workspace, + env: childEnv({ + // The action trusts the TLS certificate of the service, as a CI job + // trusts a public one. + NODE_EXTRA_CA_CERTS: hostCertFile, + GITHUB_REPOSITORY: TENANT, + GITHUB_RUN_ID: job.runId, + GITHUB_RUN_ATTEMPT: "1", + GITHUB_JOB: "test", + GITHUB_STEP_SUMMARY: summary, + INPUT_RUN: "node job.mjs > result.json", + INPUT_URL: frontUrl, + INPUT_TOKEN: job.token ?? "", + }), + stdio: ["ignore", "ignore", "inherit"], + }, + ); + const [exitCode] = (await once(action, "exit")) as [number]; + const result = JSON.parse( + await readFile(path.join(workspace, "result.json"), "utf8"), ); - return JSON.parse(stdout) as { - response: { status: number; source: string; body: string }; - stats: { written: number }; - caCert: string; + return { + exitCode, + ...result, + summary: await readFile(summary, "utf8"), }; } @@ -280,20 +319,29 @@ afterAll(async () => { describe("deployed roach", () => { it("records with the token, replays for a fork, and counts in Sentry", async () => { - const writer = await ciJob(TOKEN, "auto"); + const writer = await ciJob({ token: TOKEN, runId: "writer", prompt: "hi" }); + expect(writer.exitCode).toBe(0); expect(writer.response).toEqual({ status: 200, source: "live", body: JSON.stringify({ n: 1 }), }); - expect(writer.stats.written).toBe(1); - // A restart keeps the authority that clients trust. - expect(writer.caCert).toBe(caCert); + expect(writer.summary).toContain("1 recordings new or changed"); + // The command never sees the write token. + expect(writer.token).toBeNull(); + // It trusts the authority of the service, which a restart keeps, and + // still trusts the certificates that the job trusted. + expect(writer.trusted).toContain(caCert.trim()); + expect(writer.trusted).toContain( + (await readFile(hostCertFile, "utf8")).trim(), + ); expect([...objects.keys()]).toEqual([ expect.stringMatching(/^getsentry\/junior\/model\/[0-9a-f]{64}\.json$/), ]); - const fork = await ciJob(undefined, "replay"); + // Without the token, the action only replays. + const fork = await ciJob({ runId: "fork", prompt: "hi" }); + expect(fork.exitCode).toBe(0); expect(fork.response).toEqual({ status: 200, source: "replayed", @@ -301,10 +349,18 @@ describe("deployed roach", () => { }); expect(liveRequests).toBe(1); + // A request without a recording fails the step, and the summary names it. + const miss = await ciJob({ runId: "miss", prompt: "bye" }); + expect(miss.response.status).toBe(412); + expect(miss.exitCode).toBe(1); + expect(miss.summary).toMatch(/^- test: model model\/[0-9a-f]{64}\.json /m); + expect(liveRequests).toBe(1); + // The service sends its metrics when it stops. service.kill("SIGTERM"); expect(await once(service, "exit")).toEqual([0, null]); const [key] = objects.keys(); + const relativeKey = key!.slice(`${TENANT}/`.length); expect( metrics.map(({ tenant, run, result, key: metricKey }) => ({ tenant, @@ -313,23 +369,14 @@ describe("deployed roach", () => { key: metricKey, })), ).toEqual([ + { tenant: TENANT, run: "writer-1", result: "missed", key: relativeKey }, + { tenant: TENANT, run: "writer-1", result: "written", key: relativeKey }, + { tenant: TENANT, run: "fork-1", result: "replayed", key: relativeKey }, { tenant: TENANT, - run: "auto-run", + run: "miss-1", result: "missed", - key: key!.slice(`${TENANT}/`.length), - }, - { - tenant: TENANT, - run: "auto-run", - result: "written", - key: key!.slice(`${TENANT}/`.length), - }, - { - tenant: TENANT, - run: "replay-run", - result: "replayed", - key: key!.slice(`${TENANT}/`.length), + key: expect.stringMatching(/^model\//), }, ]); }); From 799e8de0be51b784db509d72063d77dd98d8be95 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:46:20 +0000 Subject: [PATCH 2/5] ref: Remove the local proxy Roach is now only the shared service. Nothing outside this repository used the local proxy, and the GitHub Action replaces it for adopters. Removed spawnRoach, startRoach, the serve and prune commands, usedFile, missDirectory, and RoachConfig. Also removed the closest-recording hint for misses: the service never passed it through, and the GCS store cannot give it. Recording tests now run against the service. Co-Authored-By: Junior --- AGENTS.md | 24 +-- README.md | 118 ++++--------- package.json | 3 +- src/certificates.ts | 5 +- src/cli.ts | 85 +++------ src/client.ts | 161 ++++------------- src/recorder.ts | 98 ++--------- src/recordings.ts | 127 +------------ src/report.ts | 9 +- src/request-key.ts | 106 +---------- src/secrets.ts | 2 +- src/server.ts | 92 +--------- src/service.ts | 2 +- src/store.ts | 42 +---- src/types.ts | 58 +----- tests/deployed.test.ts | 2 +- tests/{roach.test.ts => recording.test.ts} | 196 ++++++--------------- tests/service.test.ts | 11 +- 18 files changed, 225 insertions(+), 916 deletions(-) rename tests/{roach.test.ts => recording.test.ts} (63%) diff --git a/AGENTS.md b/AGENTS.md index d7d7f46..31e2123 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,16 +7,16 @@ ## Commands -| Task | Command | -| ---------------- | ------------------------------------------------- | -| Test file | `pnpm exec vitest run tests/roach.test.ts` | -| Test case | `pnpm exec vitest run tests/roach.test.ts -t "…"` | -| Lint file | `pnpm exec oxlint --deny-warnings src/server.ts` | -| Format | `pnpm format` | -| Typecheck | `pnpm typecheck` | -| Deployed shape | `pnpm exec vitest run tests/deployed.test.ts` | -| Unused code/deps | `pnpm knip` | -| Everything (CI) | `pnpm check` | +| Task | Command | +| ---------------- | ----------------------------------------------------- | +| Test file | `pnpm exec vitest run tests/recording.test.ts` | +| Test case | `pnpm exec vitest run tests/recording.test.ts -t "…"` | +| Lint file | `pnpm exec oxlint --deny-warnings src/server.ts` | +| Format | `pnpm format` | +| Typecheck | `pnpm typecheck` | +| Deployed shape | `pnpm exec vitest run tests/deployed.test.ts` | +| Unused code/deps | `pnpm knip` | +| Everything (CI) | `pnpm check` | ## External References @@ -29,14 +29,14 @@ ## Key Conventions -- Use Node built-ins and the `openssl` command. The only runtime dependency is `@sentry/node`, and only `src/service.ts` imports it. The local proxy and the client must not load it. +- Use Node built-ins and the `openssl` command. The only runtime dependency is `@sentry/node`, and only `src/service.ts` imports it. The client and the action must not load it. - `deploy/gcp/` is the production setup (Terraform). Change it in the same change when the service config (`RoachServiceConfig`) changes. CI runs `terraform validate` on it. - Node runs `src/` as TypeScript with type stripping. Use only erasable syntax, and import local files with the `.ts` extension. - Use functions and plain objects, not classes. - Start each source file with a comment that says what the file owns. Give each export a short JSDoc. - Update `README.md` in the same change when behavior, config, or the control API changes. - Test through a real proxy and a local upstream server in `tests/`. Do not mock modules. Do not reach the internet. -- The proxy decrypts HTTPS. Never write a credential: every recording and miss file goes through `src/secrets.ts`. +- The proxy decrypts HTTPS. Never write a credential: every recording goes through `src/secrets.ts`. - The upstream host always comes from `allow`, never from the client. - Write docs in ASD-STE100 English: common words, active voice, short sentences. diff --git a/README.md b/README.md index f9b2e4f..669aaf1 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,11 @@ replays them, so a second run of a test suite makes no live calls. The proxy owns every read and write of the recordings. A test runner only gives it rules and tells it when each test starts and ends. -Roach runs in one of two ways: - -- **A shared service** (`service.ts`): one deployed proxy for many - projects and CI runs. A client points `HTTPS_PROXY` at it and trusts its - certificate authority. It installs nothing else. The recordings are in a - GCS bucket, which deletes each one 30 days after it was written. See - [Service](#service) and [Deploy](#deploy). -- **A local proxy** (`server.ts`): a process in the process tree of the - test run, with the recordings in JSON files that you commit to git. +Roach is a shared service (`service.ts`): one deployed proxy for many +projects and CI runs. A client points `HTTPS_PROXY` at it and trusts its +certificate authority. It installs nothing else. The recordings are in a +GCS bucket, which deletes each one 30 days after it was written. See +[Service](#service) and [Deploy](#deploy). Roach started in [`getsentry/junior`](https://github.com/getsentry/junior/tree/main/packages/junior-evals/src/roach). @@ -46,9 +42,9 @@ const run = await startRemoteRun( ); spawn("vitest", { env: { ...process.env, ...run.env } }); -// Once per test, in a worker that has the `url`, `controlUrl`, and `token` -// of the run. -const session = await connectRoach({ url, controlUrl, token }).startSession( +// Once per test, in a worker that has the `controlUrl` and `token` of the +// run. +const session = await connectRoach({ controlUrl, token }).startSession( testName, ); const { missed } = await session.end(passed); @@ -58,6 +54,9 @@ const { missed } = await session.end(passed); console.log(describeRecordingStats(await run.close())); ``` +`run.env` has `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, +and `NODE_EXTRA_CA_CERTS`. Node 24 reads the last two only at startup. The +CA file also has the certificates of `NODE_EXTRA_CA_CERTS` of the caller. A client that does not use `client.ts` can do the same with plain HTTP. See [Control API](#control-api). @@ -100,38 +99,11 @@ the run and writes its stats to the job summary. - `NODE_EXTRA_CA_CERTS` works only for Node. Python and `curl` need a CA bundle that also has the system certificates. -## Use a local proxy - -```ts -import { spawnRoach } from "@sentry/roach/client"; - -const proxy = await spawnRoach({ - directory: "recordings", - mode: "auto", - allow: ["https://ai-gateway.vercel.sh"], - rules: [/* as above */], -}); -spawn("vitest", { env: { ...process.env, ...proxy.env } }); -// Sessions as above, with connectRoach({ url: proxy.url, token: proxy.token }). -console.log(describeRecordingStats(await proxy.stats())); -await proxy.close(); -``` - -- `env` has `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, - and `NODE_EXTRA_CA_CERTS`. Node 24 reads the last two only at startup. A - process that is already running must set its own agents. The CA file - also has the certificates of `NODE_EXTRA_CA_CERTS` of the caller. -- `spawnRoach(config, { launcher })` runs the proxy with a command - prefix, such as `sudo`. Use it when the caller cannot reach the network, - but the proxy must. - ## Config -A local proxy takes `RoachConfig` (`src/types.ts`). A run on the service -takes `RunConfig` (`src/service.ts`): `tenant`, `mode`, `rules`, `allow`, -and `name`. The fields below mean the same in both. +A run takes `RunConfig` (`src/service.ts`): `tenant`, `mode`, `rules`, +`allow`, and `name`. -- `directory`: where the recordings are. Each rule has a subdirectory. - `mode`: `auto` replays recordings and records misses. `replay` replays recordings and fails a miss with HTTP 412, so nothing goes live. `record` sends every request live and records it again. `off` records and replays @@ -143,20 +115,11 @@ and `name`. The fields below mean the same in both. and `-`. `match` has `method`, `url` (a prefix), and `headers`. `keyHeaders` adds request headers to the key. `values` names the values that change from run to run. -- `usedFile`: when the proxy stops, it lists here the recordings that - sessions used. A failed session uses all recordings that it recorded - before, because it stops early. `cli.ts prune` takes these files. Only - for `directory`. -- `missDirectory`: for each miss, the proxy writes the request as the key - sees it, and its diagnosis, to `//.json`. These - files contain request bodies, but no request headers except `keyHeaders`. - Do not commit them. The proxy refuses a `missDirectory` inside - `directory`. -- `secrets`: credentials that the proxy must never write. The proxy also - learns the value of each request header whose name can mean a credential, +- Credentials: the proxy never writes the `secrets` of the service config. + It also learns the value of each request header whose name can mean a credential, such as one with `auth`, `token`, `key`, `secret`, `cookie`, or `session`. - The match is broad on purpose. Recordings and miss files show - `<>` in place of each known value. + The match is broad on purpose. Recordings show `<>` in place of + each known value. Requests that match no rule go live without a change, and their responses stream. Each response has an `x-roach` header: `replayed`, `live`, @@ -164,11 +127,10 @@ stream. Each response has an `x-roach` header: `replayed`, `live`, ## Recordings -The key of a recording is `/.json`. The file store keeps it at -`/`. The hash is the hash of the -rule, the method, the URL, the key headers, and the body. A JSON body has -sorted keys. A recording keeps the response, the session that recorded it, -and a short hash of each part of the request. It does not keep the request +The key of a recording is `/.json`, under `/` in the +store. The hash is the hash of the rule, the method, the URL, the key +headers, and the body. A JSON body has sorted keys. A recording keeps the +response and the session that recorded it. It does not keep the request body. - One session is open at a time. The proxy keeps the new recordings of a @@ -220,19 +182,10 @@ real way use the same recording. ## Misses -The parts of a request are the method, the URL, each key header, each -top-level field of a JSON body, and each item of a top-level array, such as -`messages[3]`. The proxy adds each request without a recording to -`stats().misses`. `describeRecordingMisses()` in `report.ts` gives the +The proxy adds each request without a recording to `stats().misses`, with +its session and key. `describeRecordingMisses()` in `report.ts` gives the first miss of each test, which is the one to fix. -With the file store, the proxy also finds the recording with the most -equal parts, from the same session if it can, and logs the parts that -differ. This is only a hint to debug a miss. It never makes a replay. To -find it, the file store reads all recordings of a rule on the first miss of -that rule in a run, and then keeps them in memory. The GCS store does not -give this hint, because it would have to read every recording. - ## Service `cli.ts service ` starts the service. The config is @@ -353,21 +306,14 @@ set the secret again, and restart the VM. ```sh node src/cli.ts service -node src/cli.ts serve < config.json -node src/cli.ts prune ... ``` -`service` starts the shared service. `serve` starts a local proxy and prints -its address as one JSON line. `client.ts` runs it. `prune` deletes the -recordings that no used file lists. Give it the used files of every run -that shares the directory, or it deletes recordings that another run -needs. +It starts the service and prints its URL as one JSON line. ## Control API -The calls of a run are under `/__roach/runs/` on the service, and under -`/__roach` on a local proxy. They need `Authorization: Bearer `, with -the run token or the token of the local proxy. `client.ts` calls them. +The calls of a run are under `/__roach/runs/`. They need +`Authorization: Bearer `. `client.ts` calls them. - `POST /session` with `{"name": "..."}`: open a session. - `POST /session/end` with `{"name": "...", "passed": true}`: end it. @@ -392,7 +338,8 @@ pnpm check # format, lint, types, unused code, and tests, as CI runs them `AGENTS.md` has the conventions and the commands for one file. -- `tests/roach.test.ts` tests a local proxy. +- `tests/recording.test.ts` tests modes, sessions, changing values, and + misses. - `tests/service.test.ts` tests the service in the test process. - `tests/deployed.test.ts` runs the service as it runs in production: from the command line, behind a TLS server that does what the load balancer @@ -405,21 +352,20 @@ pnpm check # format, lint, types, unused code, and tests, as CI runs them The proxy is in `src/`. Tests are in `tests/`. The production setup is in `deploy/gcp/` and `Dockerfile`. -- `types.ts`: the configuration and the results of a local proxy. +- `types.ts`: rules, modes, and the results of a run. - `server.ts`: sockets, HTTPS interception, the allow list, and the control API. - `service.ts`: the shared service: tenants, runs, access, and metrics. - `recorder.ts`: modes, sessions, replay, and misses. -- `request-key.ts`: the key and the part hashes of a request. +- `request-key.ts`: the key of a request. - `store.ts`: the recording format and the store contract. -- `recordings.ts`: the file store and prune. +- `recordings.ts`: the file store, for tests and development. - `gcs.ts`: the GCS store. - `values.ts`: changing values and their placeholders. - `streams.ts`: merges the deltas of a recorded model stream. - `secrets.ts`: redaction of credentials. - `certificates.ts`: the certificate authority for HTTPS. -- `client.ts`: starts a remote run or a local proxy, and calls the control - API. +- `client.ts`: starts a run and calls the control API. - `report.ts`: text reports of a run. - `action.ts`: the GitHub Action (`action.yml` at the root). - `cli.ts`: the command line. diff --git a/package.json b/package.json index 7db7109..9a3ec3d 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@sentry/roach", "version": "0.0.0", "private": true, - "description": "A recording HTTPS proxy for tests, as a shared service or a local process.", + "description": "A recording HTTPS proxy for tests, as a shared service.", "license": "Apache-2.0", "repository": { "type": "git", @@ -15,7 +15,6 @@ "exports": { "./client": "./src/client.ts", "./report": "./src/report.ts", - "./server": "./src/server.ts", "./service": "./src/service.ts", "./types": "./src/types.ts", "./values": "./src/values.ts" diff --git a/src/certificates.ts b/src/certificates.ts index ef90a87..f10a5cc 100644 --- a/src/certificates.ts +++ b/src/certificates.ts @@ -3,9 +3,8 @@ * * The proxy intercepts HTTPS. It signs one certificate for each host when a * client first connects to that host, and signs it again before it - * expires. Clients must trust the authority certificate. A local proxy - * creates a new authority when it starts. The shared service loads a fixed - * one, so that clients can trust it across restarts. + * expires. Clients must trust the authority certificate. The service loads + * a fixed one, so that clients can trust it across restarts. */ import { execFile } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; diff --git a/src/cli.ts b/src/cli.ts index bd17c32..ed6ffaa 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,68 +1,35 @@ /** * The command line of Roach. * - * - `cli.ts service `: start the shared service (`service.ts`) - * with the configuration in the file. Print its URL as one JSON line. - * Stop on SIGINT or SIGTERM. The deployed container runs this command. - * - `cli.ts serve`: read the configuration of a local proxy as JSON from - * stdin, and start it. Print its address as one JSON line. Stop on SIGINT - * or SIGTERM. `client.ts` runs this command. - * - `cli.ts prune ...`: delete the recordings that no - * used file lists. + * `cli.ts service ` starts the shared service (`service.ts`) + * with the configuration in the file. It prints the URL of the service as + * one JSON line, and stops on SIGINT or SIGTERM. The deployed container + * runs this command. */ import { readFile } from "node:fs/promises"; -import { text } from "node:stream/consumers"; -import { pruneRecordings } from "./recordings.ts"; -import { startRoach } from "./server.ts"; -import type { RoachServiceConfig } from "./service.ts"; -import type { RoachAddress, RoachConfig } from "./types.ts"; - -const USAGE = `Usage: - cli.ts service - cli.ts serve < config.json - cli.ts prune ...`; +import { startRoachService, type RoachServiceConfig } from "./service.ts"; const [command, ...args] = process.argv.slice(2); - -/** Stop on SIGINT or SIGTERM. A failed close logs its error and exits 1. */ -function stopOnSignal(close: () => Promise): void { - const stop = () => { - close().then( - () => process.exit(0), - (error: unknown) => { - process.stderr.write(`[roach] Close failed: ${String(error)}\n`); - process.exit(1); - }, - ); - }; - process.once("SIGINT", stop); - process.once("SIGTERM", stop); -} - -if (command === "service" && args.length === 1) { - const config = JSON.parse( - await readFile(args[0]!, "utf8"), - ) as RoachServiceConfig; - // Only the service loads the Sentry SDK. - const { startRoachService } = await import("./service.ts"); - const service = await startRoachService(config); - process.stdout.write(`${JSON.stringify({ url: service.url })}\n`); - stopOnSignal(() => service.close()); -} else if (command === "serve") { - const config = JSON.parse(await text(process.stdin)) as RoachConfig; - const proxy = await startRoach(config); - const address: RoachAddress = { - url: proxy.url, - token: proxy.token, - caCert: proxy.caCert, - }; - process.stdout.write(`${JSON.stringify(address)}\n`); - stopOnSignal(() => proxy.close()); -} else if (command === "prune" && args.length >= 2) { - const [directory, ...usedFiles] = args as [string, ...string[]]; - const count = await pruneRecordings(directory, usedFiles); - process.stdout.write(`Deleted ${count} unused recordings\n`); -} else { - process.stderr.write(`${USAGE}\n`); +if (command !== "service" || args.length !== 1) { + process.stderr.write("Usage: cli.ts service \n"); process.exit(2); } + +const config = JSON.parse( + await readFile(args[0]!, "utf8"), +) as RoachServiceConfig; +const service = await startRoachService(config); +process.stdout.write(`${JSON.stringify({ url: service.url })}\n`); + +/** Stop on SIGINT or SIGTERM. A failed close logs its error and exits 1. */ +const stop = () => { + service.close().then( + () => process.exit(0), + (error: unknown) => { + process.stderr.write(`[roach] Close failed: ${String(error)}\n`); + process.exit(1); + }, + ); +}; +process.once("SIGINT", stop); +process.once("SIGTERM", stop); diff --git a/src/client.ts b/src/client.ts index 8f6e361..ffbe45a 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,22 +1,19 @@ /** * The client of Roach. See `README.md`. * - * `startRemoteRun()` starts a run on a Roach service (`service.ts`). - * `spawnRoach()` starts a local proxy in its own process. Both return - * `env`, the variables that send the traffic of a process through the - * proxy. `connectRoach()` controls a run or a local proxy from another - * process, such as a test worker. + * `startRemoteRun()` starts a run on a Roach service (`service.ts`). It + * returns `env`, the variables that send the traffic of a process through + * the run. `connectRoach()` controls a run from another process, such as a + * test worker. */ -import { spawn } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import http from "node:http"; import https from "node:https"; import { tmpdir } from "node:os"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { CONTROL_PATH } from "./server.ts"; import type { RemoteRun, RunConfig } from "./service.ts"; -import type { RoachAddress, RoachConfig, RecordingStats } from "./types.ts"; +import type { RecordingStats } from "./types.ts"; /** The open session of one test. */ export interface RecordingSession { @@ -28,7 +25,7 @@ export interface RecordingSession { end(passed: boolean): Promise<{ missed: number }>; } -/** The control API of a running proxy. */ +/** The control API of a run. */ export interface RoachControl { /** Open the session of one test. Only one session is open at a time. */ startSession(name: string): Promise; @@ -36,44 +33,18 @@ export interface RoachControl { stats(): Promise; } -/** A proxy that `spawnRoach()` started. */ -export interface Roach extends RoachAddress, RoachControl { - /** - * The variables that send the HTTP traffic of a process through the - * proxy: `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, - * and `NODE_EXTRA_CA_CERTS`. Give them to a process when it starts. Node - * reads the last two only at startup. - */ - env: Record; - /** Stop the proxy. It then writes `usedFile` of its config. */ - close(): Promise; -} - -const CLI = fileURLToPath(new URL("./cli.ts", import.meta.url)); - -/** Proxy variables. The proxy itself must not use a proxy. */ -const PROXY_VARIABLES = new Set([ - "all_proxy", - "http_proxy", - "https_proxy", - "no_proxy", - "node_use_env_proxy", -]); - -/** Where the control API is, and its token. */ -type ControlAddress = Pick & { - token?: string | undefined; -}; - -/** Call the control API, and return the JSON of the response. */ +/** + * Call the control API at `base`, and return the JSON of the response. + * `token` is the bearer token. + */ function callControl( - address: ControlAddress, + base: string, + token: string | undefined, method: "DELETE" | "GET" | "POST", route: string, body?: unknown, ): Promise { const payload = body === undefined ? undefined : JSON.stringify(body); - const base = address.controlUrl ?? `${address.url}${CONTROL_PATH}`; const url = route ? `${base}/${route}` : base; const client = url.startsWith("https:") ? https : http; return new Promise((resolve, reject) => { @@ -84,9 +55,7 @@ function callControl( agent: false, method, headers: { - ...(address.token - ? { authorization: `Bearer ${address.token}` } - : {}), + ...(token ? { authorization: `Bearer ${token}` } : {}), ...(payload ? { "content-type": "application/json" } : {}), }, }, @@ -114,92 +83,34 @@ function callControl( }); } -/** Control a run or a local proxy, for example from a test worker. */ -export function connectRoach( - address: Pick, -): RoachControl { +/** Control a run, for example from a test worker. */ +export function connectRoach({ + controlUrl, + token, +}: Pick): RoachControl { return { async startSession(name) { - await callControl(address, "POST", "session", { name }); + await callControl(controlUrl, token, "POST", "session", { name }); return { end: (passed) => - callControl(address, "POST", "session/end", { name, passed }), + callControl(controlUrl, token, "POST", "session/end", { + name, + passed, + }), }; }, - stats: () => callControl(address, "GET", "stats"), - }; -} - -/** Read the address line that `cli.ts serve` prints. */ -function readAddress(child: ReturnType): Promise { - return new Promise((resolve, reject) => { - let output = ""; - child.once("error", reject); - child.once("exit", (code) => - reject(new Error(`Roach exited with code ${code}`)), - ); - child.stdout!.on("data", (chunk: Buffer) => { - output += chunk.toString("utf8"); - const end = output.indexOf("\n"); - if (end >= 0) resolve(JSON.parse(output.slice(0, end))); - }); - }); -} - -/** - * Start the proxy in its own process. - * - * `launcher` is a command prefix that runs the proxy, such as `sudo`. Use it - * when the caller cannot reach the network, but the proxy must. `noProxy` - * lists the hosts that do not use the proxy, such as `localhost`. - */ -export async function spawnRoach( - config: RoachConfig, - { - launcher = [], - noProxy = "localhost,127.0.0.1,::1", - }: { launcher?: string[]; noProxy?: string } = {}, -): Promise { - const command = [ - ...launcher, - process.execPath, - "--experimental-strip-types", - "--disable-warning=ExperimentalWarning", - CLI, - "serve", - ]; - const child = spawn(command[0]!, command.slice(1), { - env: Object.fromEntries( - Object.entries(process.env).filter( - ([name]) => !PROXY_VARIABLES.has(name.toLowerCase()), - ), - ), - stdio: ["pipe", "pipe", "inherit"], - }); - const exited = new Promise((resolve) => - child.once("exit", () => resolve()), - ); - child.stdin.end(JSON.stringify(config)); - const address = await readAddress(child); - const proxyEnv = await createProxyEnv(address, noProxy); - - return { - ...address, - ...connectRoach(address), - env: proxyEnv.env, - async close() { - if (child.exitCode === null && child.signalCode === null) { - child.kill("SIGTERM"); - await exited; - } - await proxyEnv.remove(); - }, + stats: () => callControl(controlUrl, token, "GET", "stats"), }; } /** A run on a Roach service (`service.ts`). */ export interface RemoteRoach extends RemoteRun, RoachControl { - /** The same variables as `Roach.env`, for the proxy URL of the run. */ + /** + * The variables that send the HTTP traffic of a process through the run: + * `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, and + * `NODE_EXTRA_CA_CERTS`. Give them to a process when it starts. Node + * reads the last two only at startup. + */ env: Record; /** End the run. Returns its stats. */ close(): Promise; @@ -218,7 +129,8 @@ export async function startRemoteRun( { noProxy = "localhost,127.0.0.1,::1" }: { noProxy?: string } = {}, ): Promise { const run = await callControl( - { url: service.url.replace(/\/$/, ""), token: service.token }, + `${service.url.replace(/\/$/, "")}${CONTROL_PATH}`, + service.token, "POST", "runs", config, @@ -230,7 +142,12 @@ export async function startRemoteRun( env: proxyEnv.env, async close() { try { - return await callControl(run, "DELETE", ""); + return await callControl( + run.controlUrl, + run.token, + "DELETE", + "", + ); } finally { await proxyEnv.remove(); } @@ -244,7 +161,7 @@ export async function startRemoteRun( * so the process still trusts them. */ async function createProxyEnv( - address: Pick, + address: Pick, noProxy: string, ): Promise<{ env: Record; remove(): Promise }> { // `NODE_EXTRA_CA_CERTS` takes a file. diff --git a/src/recorder.ts b/src/recorder.ts index 4781479..d876c9e 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -14,13 +14,10 @@ * its session ended follows the result of that session. A request outside * a session is written at once. * - * The recorder never writes a known credential (`secrets.ts`). It redacts - * them in recordings and miss files. + * The recorder never writes a known credential (`secrets.ts`). */ -import { mkdir, writeFile } from "node:fs/promises"; import type { IncomingHttpHeaders } from "node:http"; -import path from "node:path"; -import { describeParts, keyRequest, type KeyedRequest } from "./request-key.ts"; +import { keyRequest, type KeyedRequest } from "./request-key.ts"; import { createSecrets } from "./secrets.ts"; import { recordingKey, @@ -35,7 +32,7 @@ import { } from "./streams.ts"; import type { RecordingMiss, - RoachConfig, + RecordingMode, RecordingRule, RecordingStats, } from "./types.ts"; @@ -63,8 +60,6 @@ interface Session { name: string; /** New recordings, by key. */ recorded: Map; - /** Keys that the session replayed. They stay used if it fails. */ - replayed: Set; /** Requests that `replay` mode failed. */ missed: number; /** @@ -123,11 +118,7 @@ function toRecording( } return { session, - request: { - method: request.method, - url: request.url.href, - parts: keyed.parts, - }, + request: { method: request.method, url: request.url.href }, response: { status: response.status, headers, body, bodyEncoding }, }; } @@ -145,26 +136,23 @@ function fromRecording( return { status: response.status, headers: response.headers, body }; } -/** The parts of a config that the recorder uses. */ -export type RecorderConfig = Pick< - RoachConfig, - "missDirectory" | "mode" | "rules" | "secrets" | "usedFile" ->; +/** The config of a recorder. */ +export interface RecorderConfig { + mode: RecordingMode; + rules: RecordingRule[]; + /** Credentials to redact, in addition to the ones that it learns. */ + secrets?: string[]; +} -/** Create the recorder of one proxy run, with the store of its recordings. */ +/** Create the recorder of one run, with the store of its recordings. */ export function createRecorder(config: RecorderConfig, store: RecordingStore) { for (const rule of config.rules) { if (!RULE_NAME.test(rule.name)) { throw new Error(`Roach rule name is not valid: ${rule.name}`); } } - const missDirectory = config.missDirectory - ? path.resolve(config.missDirectory) - : undefined; let session: Session | undefined; const secrets = createSecrets(config.secrets); - /** Recordings that passed sessions, or requests outside one, used. */ - const used = new Set(); const stats: RecordingStats = { counts: Object.fromEntries( config.rules.map((rule) => [ @@ -181,11 +169,6 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { stats.written += await store.write(recordings); }; - const markReplayed = (owner: Session | undefined, key: string) => { - if (owner && owner.passed === undefined) owner.replayed.add(key); - else used.add(key); - }; - const record = async ( owner: Session | undefined, key: string, @@ -196,41 +179,19 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } else if (owner?.passed === false) { stats.discarded += 1; } else { - used.add(key); await write([[key, recording]]); } }; - const reportMiss = async ( - rule: RecordingRule, - key: string, - keyed: KeyedRequest, - owner: Session | undefined, - ) => { - const closest = await store.closest?.(rule.name, keyed.parts, owner?.name); + const reportMiss = (rule: RecordingRule, key: string, owner?: Session) => { const miss: RecordingMiss = { rule: rule.name, session: owner?.name, file: key, - closest: closest?.key, - differs: closest?.differs ?? [], }; if (stats.misses.length < MAX_MISSES) stats.misses.push(miss); const where = owner ? ` in "${owner.name}"` : ""; - const why = closest - ? `closest is ${miss.closest}, which differs at ${describeParts(miss.differs)}` - : "no recording to compare"; - process.stderr.write(`[roach] No ${rule.name} recording${where}: ${why}\n`); - if (missDirectory) { - const target = path.join(missDirectory, miss.file); - await mkdir(path.dirname(target), { recursive: true }); - await writeFile( - target, - secrets.redact( - `${JSON.stringify({ ...miss, request: keyed.normalized }, null, 2)}\n`, - ), - ); - } + process.stderr.write(`[roach] No ${rule.name} recording${where}: ${key}\n`); }; /** End the open session. Returns its requests that `replay` mode failed. */ @@ -239,20 +200,8 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { if (!ended) return 0; session = undefined; ended.passed = passed; - // A failed test can still show that a recording is in use. - for (const key of ended.replayed) used.add(key); - if (passed) { - for (const key of ended.recorded.keys()) used.add(key); - await write([...ended.recorded]); - } else { - stats.discarded += ended.recorded.size; - // A failed test stops early, so it does not replay all of its - // recordings. Keep every recording that it made before, so that a - // prune does not delete them. - for (const key of (await store.keysOf?.(ended.name)) ?? []) { - used.add(key); - } - } + if (passed) await write([...ended.recorded]); + else stats.discarded += ended.recorded.size; return ended.missed; }; @@ -306,13 +255,12 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { const recording = await store.read(key); if (recording) { counts.replayed += 1; - markReplayed(owner, key); return { ...fromRecording(recording, keyed.values), source: "replayed", }; } - await reportMiss(rule, key, keyed, owner); + reportMiss(rule, key, owner); if (config.mode === "replay") { counts.missed += 1; if (owner) owner.missed += 1; @@ -354,7 +302,6 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { session = { name, recorded: new Map(), - replayed: new Set(), missed: 0, known: new Map(), }; @@ -375,19 +322,12 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { return stats; }, - /** Fail the open session, and write `usedFile` of the config. */ + /** Fail the open session. */ async close(): Promise { await finish(false); - if (config.usedFile) { - const keys = [...used].toSorted(); - await writeFile( - config.usedFile, - keys.map((key) => `${key}\n`).join(""), - ); - } }, }; } -/** The recorder of one proxy run. */ +/** The recorder of one run. */ export type Recorder = ReturnType; diff --git a/src/recordings.ts b/src/recordings.ts index 1d5341f..9e01f67 100644 --- a/src/recordings.ts +++ b/src/recordings.ts @@ -1,53 +1,24 @@ /** - * The file store, and prune. - * - * The file store keeps each recording (`store.ts`) in `/`, - * so recordings can be committed. + * The file store. It keeps each recording (`store.ts`) in + * `/`. The service uses it when its config has + * `directory`, such as in tests. */ -import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; -import { closestRequest } from "./request-key.ts"; import { formatRecording, type Recording, type RecordingStore, - type RequestParts, } from "./store.ts"; -const isMissing = (error: unknown) => - (error as NodeJS.ErrnoException).code === "ENOENT"; - const readText = (file: string) => readFile(file, "utf8").catch((error: unknown) => { - if (isMissing(error)) return undefined; + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; }); -/** The rule of a key, `/.json`. */ -const ruleOf = (key: string) => key.slice(0, key.indexOf("/")); - -interface IndexEntry { - key: string; - session?: string | undefined; - parts: RequestParts; -} - -/** - * The store of the recordings in `directory`. To find the closest - * recording of a miss, it reads all recordings of the rule on the first - * miss of that rule, and then keeps them in memory. - */ +/** The store of the recordings in `directory`. */ export function createFileStore(directory: string): RecordingStore { - const indexes = new Map>>(); - const indexOf = (rule: string) => { - let index = indexes.get(rule); - if (!index) { - index = loadIndex(directory, rule); - indexes.set(rule, index); - } - return index; - }; - return { async read(key) { const text = await readText(path.join(directory, key)); @@ -59,10 +30,6 @@ export function createFileStore(directory: string): RecordingStore { recordings.map(async ([key, recording]) => { const file = path.join(directory, key); const content = formatRecording(recording); - // Keep a loaded index current, so later misses compare with it. - if (indexes.has(ruleOf(key))) { - (await indexOf(ruleOf(key))).set(key, entryOf(key, recording)); - } if ((await readText(file)) === content) return false; await mkdir(path.dirname(file), { recursive: true }); await writeFile(file, content); @@ -71,87 +38,5 @@ export function createFileStore(directory: string): RecordingStore { ); return changed.filter(Boolean).length; }, - - async closest(rule, parts, session) { - const all = [...(await indexOf(rule)).values()]; - const same = all.filter((entry) => entry.session === session); - const found = closestRequest( - parts, - session !== undefined && same.length > 0 ? same : all, - ); - return found && { key: found.candidate.key, differs: found.differs }; - }, - - async keysOf(session) { - const entries = await readdir(directory, { withFileTypes: true }).catch( - (error: unknown) => { - if (isMissing(error)) return []; - throw error; - }, - ); - const keys: string[] = []; - // Each rule is a directory. Skip other files, such as `.DS_Store`. - for (const rule of entries.filter((entry) => entry.isDirectory())) { - for (const entry of (await indexOf(rule.name)).values()) { - if (entry.session === session) keys.push(entry.key); - } - } - return keys; - }, }; } - -const entryOf = (key: string, recording: Recording): IndexEntry => ({ - key, - session: recording.session, - parts: recording.request.parts, -}); - -/** Read every recording of one rule, by key. */ -async function loadIndex( - directory: string, - rule: string, -): Promise> { - const names = await readdir(path.join(directory, rule)).catch( - (error: unknown) => { - if (isMissing(error)) return []; - throw error; - }, - ); - const index = new Map(); - for (const name of names.filter((entry) => entry.endsWith(".json"))) { - const key = `${rule}/${name}`; - const text = await readText(path.join(directory, key)); - if (text !== undefined) { - index.set(key, entryOf(key, JSON.parse(text) as Recording)); - } - } - return index; -} - -/** - * Delete the recordings in `directory` that no used file lists. Each used - * file comes from `usedFile` of one proxy run. Returns how many it deleted. - * Give it the used files of every run that shares the directory, or it - * deletes recordings that another run needs. - */ -export async function pruneRecordings( - directory: string, - usedFiles: string[], -): Promise { - const used = new Set(); - for (const file of usedFiles) { - for (const line of (await readFile(file, "utf8")).split("\n")) { - if (line) used.add(line); - } - } - const recordings = (await readdir(directory, { recursive: true })).filter( - (file) => file.endsWith(".json"), - ); - // Used files list keys, which use `/`. On Windows, `readdir` uses `\`. - const unused = recordings.filter( - (file) => !used.has(file.split(path.sep).join("/")), - ); - await Promise.all(unused.map((file) => rm(path.join(directory, file)))); - return unused.length; -} diff --git a/src/report.ts b/src/report.ts index 52b8a78..1fd288b 100644 --- a/src/report.ts +++ b/src/report.ts @@ -1,7 +1,6 @@ /** * Text reports of a Roach run, for logs and CI summaries. */ -import { describeParts } from "./request-key.ts"; import type { RecordingMiss, RecordingStats } from "./types.ts"; /** Describe the totals of a run in one line. */ @@ -33,11 +32,5 @@ export function describeRecordingMisses(misses: RecordingMiss[]): string[] { seen.add(miss.session); return first; }) - .map((miss) => { - const where = miss.session ?? "outside a test"; - const why = miss.closest - ? `differs from ${miss.closest} at ${describeParts(miss.differs)}` - : "no recording to compare"; - return `${where}: ${miss.rule} ${miss.file} ${why}`; - }); + .map((miss) => `${miss.session ?? "outside a test"}: ${miss.file}`); } diff --git a/src/request-key.ts b/src/request-key.ts index 27e0587..6106296 100644 --- a/src/request-key.ts +++ b/src/request-key.ts @@ -1,23 +1,14 @@ /** - * The key of a recorded request, and the parts that miss diagnosis - * compares. + * The key of a recorded request. * * The key is the hash of the rule, the method, the URL, the key headers, * and the body. A JSON body has sorted object keys, so key order does not * matter. Each changing value of the rule is `<>` in the key * (`values.ts`). - * - * A recording also keeps a short hash of each part of its request: the - * method, the URL, each key header, each top-level field of a JSON body, - * and each item of a top-level array, such as `messages[3]`. When a request - * has no recording, the file store finds the recording with the most equal - * parts and reports the parts that differ. This only helps a person debug a - * miss. It never makes a replay. */ import { createHash } from "node:crypto"; import type { IncomingHttpHeaders } from "node:http"; import { THINKING_BLOCK_TYPES } from "./streams.ts"; -import type { RequestParts } from "./store.ts"; import type { RecordingRule } from "./types.ts"; import { extractValues, @@ -33,19 +24,11 @@ const KEY_VERSION = "http-v1"; export interface KeyedRequest { /** The name of the recording file, without `.json`. */ key: string; - parts: RequestParts; /** * The changing values of the request outside thinking blocks, in their * order. They fill the placeholders of a replayed response. */ values: RequestValues; - /** The request that the key hashes. Miss files show it. */ - normalized: { - method: string; - url: string; - headers: Record; - body: unknown; - }; } /** JSON with sorted object keys, so equal bodies give equal keys. */ @@ -115,36 +98,8 @@ function normalizeJson( const sha256 = (text: string) => createHash("sha256").update(text).digest("hex"); -const shortHash = (text: string) => sha256(text).slice(0, 12); - -function bodyParts( - json: unknown, - normalizedBody: string, - patterns: ValuePatterns | undefined, - known: KnownValues | undefined, -): RequestParts { - if (json === null || typeof json !== "object" || Array.isArray(json)) { - return normalizedBody ? { body: shortHash(normalizedBody) } : {}; - } - const part = (value: unknown) => - shortHash(normalizeJson(value, patterns, known).text); - const parts: RequestParts = {}; - for (const [field, value] of Object.entries(json)) { - if (value === undefined) continue; - if (Array.isArray(value)) { - parts[`${field}.length`] = shortHash(String(value.length)); - value.forEach((item, index) => { - parts[`${field}[${index}]`] = part(item); - }); - } else { - parts[field] = part(value); - } - } - return parts; -} - /** - * The key, the parts, and the changing values of a request. `known` holds + * The key and the changing values of a request. `known` holds * the values of earlier requests in the same session (`values.ts`). */ export function keyRequest( @@ -178,60 +133,5 @@ export function keyRequest( text, ].join("\n"), ); - const parts: RequestParts = { - method: shortHash(request.method), - url: shortHash(request.url), - }; - for (const [name, value] of Object.entries(headers)) { - parts[`header ${name}`] = shortHash(value); - } - Object.assign(parts, bodyParts(json, text, rule.values, known)); - return { - key, - parts, - values, - normalized: { - method: request.method, - url: request.url, - headers, - // A placeholder for a number makes the JSON invalid. Keep its text. - body: parseJson(text) ?? text, - }, - }; -} - -const partOrder = new Intl.Collator("en", { numeric: true }).compare; - -/** - * The candidate with the most equal parts, and the parts that differ from - * it, in a readable order. Returns `undefined` when there is no candidate. - */ -export function closestRequest( - parts: RequestParts, - candidates: Iterable, -): { candidate: T; differs: string[] } | undefined { - let best: T | undefined; - let bestEqual = -1; - for (const candidate of candidates) { - const equal = Object.keys(parts).filter( - (name) => candidate.parts[name] === parts[name], - ).length; - if (equal > bestEqual) { - bestEqual = equal; - best = candidate; - } - } - if (!best) return undefined; - const names = new Set([...Object.keys(parts), ...Object.keys(best.parts)]); - const differs = [...names] - .filter((name) => parts[name] !== best.parts[name]) - .toSorted(partOrder); - return { candidate: best, differs }; -} - -/** Name some parts, such as `messages[3], tools`. */ -export function describeParts(parts: string[]): string { - if (parts.length === 0) return "no part"; - const shown = parts.slice(0, 6).join(", "); - return parts.length > 6 ? `${shown} and ${parts.length - 6} more` : shown; + return { key, values }; } diff --git a/src/secrets.ts b/src/secrets.ts index 1f369e1..c5c4ae7 100644 --- a/src/secrets.ts +++ b/src/secrets.ts @@ -3,7 +3,7 @@ * * The proxy learns the values of credential headers from each request that * it sees, and the config can add more values. It replaces each known value - * with `<>` before it writes a recording or a miss file. + * with `<>` before it writes a recording. */ import type { IncomingHttpHeaders } from "node:http"; diff --git a/src/server.ts b/src/server.ts index e5a00cb..c46b12b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,5 +1,5 @@ /** - * The server of Roach. See `README.md`. + * The proxy sockets of the service (`service.ts`). * * The proxy is a forward proxy. It intercepts HTTPS with its own * certificate authority (`certificates.ts`). It gives each request that a @@ -14,33 +14,17 @@ * `missed` (`replay` mode had no recording), or `passthrough` (no rule * matched). */ -import { randomBytes, timingSafeEqual } from "node:crypto"; +import { timingSafeEqual } from "node:crypto"; import http from "node:http"; import https from "node:https"; import { isIP, type Socket } from "node:net"; import tls from "node:tls"; -import { - createCertificateAuthority, - type CertificateAuthority, -} from "./certificates.ts"; -import { - createRecorder, - type ProxyRequest, - type Recorder, -} from "./recorder.ts"; -import path from "node:path"; -import { createFileStore } from "./recordings.ts"; -import type { RoachAddress, RoachConfig } from "./types.ts"; +import type { CertificateAuthority } from "./certificates.ts"; +import type { ProxyRequest, Recorder } from "./recorder.ts"; /** The path prefix of the control API. */ export const CONTROL_PATH = "/__roach"; -/** A proxy that runs in this process. Most callers use `client.ts`. */ -export interface RoachServer extends RoachAddress { - /** Stop the proxy. A session that is still open fails. */ - close(): Promise; -} - /** Headers for one connection, which the proxy must not forward. */ const HOP_HEADERS = new Set([ "connection", @@ -294,29 +278,6 @@ export async function controlRecorder( } } -/** Answer one request to the control API of a local proxy. */ -async function control( - recorder: Recorder, - token: string, - incoming: http.IncomingMessage, - outgoing: http.ServerResponse, -): Promise { - if (!sameToken(incoming.headers.authorization, `Bearer ${token}`)) { - outgoing.writeHead(401).end(); - return; - } - const pathname = new URL(incoming.url ?? "/", "http://proxy").pathname; - const handled = - pathname.startsWith(`${CONTROL_PATH}/`) && - (await controlRecorder( - recorder, - `${incoming.method} ${pathname.slice(CONTROL_PATH.length)}`, - incoming, - outgoing, - )); - if (!handled) outgoing.writeHead(404).end(); -} - /** Where a proxied request goes: the allowed origins and the recorder. */ export interface ProxyTarget { origins: Map; @@ -399,8 +360,8 @@ function serveProxied( /** * Listen for proxied requests and control requests on one port. The - * caller decides the target of each proxied request, so one port can serve - * one run (`startRoach()`) or many runs (`service.ts`). + * caller decides the target of each proxied request, so one port serves + * many runs. */ export async function listenProxy( options: ListenOptions, @@ -529,44 +490,3 @@ export async function listenProxy( }, }; } - -/** Start the proxy in this process, on a free port of `127.0.0.1`. */ -export async function startRoach(config: RoachConfig): Promise { - if ( - config.missDirectory && - !path - .relative(config.directory, path.resolve(config.missDirectory)) - .startsWith("..") - ) { - // Miss files hold request bodies. Keep them out of the committed files. - throw new Error("missDirectory must not be inside directory"); - } - const target: ProxyTarget = { - origins: parseOrigins(config.allow), - recorder: createRecorder(config, createFileStore(config.directory)), - }; - const authority = await createCertificateAuthority(); - const token = randomBytes(24).toString("hex"); - // Only local processes reach this port, so proxied requests need no - // credentials. - const listening = await listenProxy({ - host: "127.0.0.1", - port: 0, - authority, - targetFor: () => target, - control: (incoming, outgoing) => - control(target.recorder, token, incoming, outgoing), - }); - - return { - url: `http://127.0.0.1:${listening.port}`, - token, - caCert: authority.caCert, - async close() { - target.closed = true; - await listening.close(); - await target.recorder.close(); - await authority.close(); - }, - }; -} diff --git a/src/service.ts b/src/service.ts index de1b141..d8be271 100644 --- a/src/service.ts +++ b/src/service.ts @@ -5,7 +5,7 @@ * GitHub repository, named `owner/repo`, such as `getsentry/junior`. The * service has no list of tenants: a run names its tenant. A run is one test * run of a tenant, such as one CI job. It has its own mode, rules, - * sessions, and stats, as one local proxy (`server.ts`) has. + * sessions, and stats. * * - Anyone can create a `replay` run of a tenant, so CI jobs of forks * replay recordings without a secret. Such a run never writes. diff --git a/src/store.ts b/src/store.ts index f287d74..9ad440c 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,23 +1,19 @@ /** * Recordings, and the contract of a store that keeps them. * - * A recording keeps the response of one request, the session (test) that - * recorded it, and the parts of the request (`request-key.ts`). It does not keep - * the request body, so prompts and other inputs are not stored. + * A recording keeps the response of one request, and the session (test) + * that recorded it. It does not keep the request body, so prompts and other + * inputs are not stored. * - * A store keeps recordings by key, `/.json`. The file store - * (`recordings.ts`) keeps them in a directory that can be committed. The - * GCS store (`gcs.ts`) keeps them in a bucket, for the shared service - * (`service.ts`). + * A store keeps recordings by key, `/.json`. The GCS store + * (`gcs.ts`) keeps them in a bucket. The file store (`recordings.ts`) keeps + * them in a directory, for tests and development. */ -/** A short hash of each part of a request, by part name. */ -export type RequestParts = Record; - /** One recorded response. */ export interface Recording { /** The session (test) that recorded the response. */ session?: string | undefined; - request: { method: string; url: string; parts: RequestParts }; + request: { method: string; url: string }; response: { status: number; headers: Record; @@ -31,34 +27,12 @@ export interface Recording { }; } -/** The recording with the most equal parts, and the parts that differ. */ -interface ClosestRecording { - key: string; - differs: string[]; -} - -/** Where the recordings of a proxy are. */ +/** Where the recordings of a tenant are. */ export interface RecordingStore { /** The recording of `key`, or `undefined` when there is none. */ read(key: string): Promise; /** Write recordings by key. Returns how many were new or changed. */ write(recordings: Array<[string, Recording]>): Promise; - /** - * The recording of `rule` with the most equal parts. Recordings of the - * same session come first, because a test usually sends the same - * requests as the last time it ran. Only the file store has this: it is a - * hint for debugging a miss, and a bucket has no index for it. - */ - closest?( - rule: string, - parts: RequestParts, - session: string | undefined, - ): Promise; - /** - * The keys that `session` recorded. Only the file store has this, - * because only it supports `usedFile` and prune. - */ - keysOf?(session: string): Promise; } /** A rule name that a store accepts. It is a directory name. */ diff --git a/src/types.ts b/src/types.ts index 8de368b..d46cd66 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,6 @@ /** - * The public types of Roach: its configuration and its - * results. See `README.md`. + * The public types of Roach: rules, modes, and the results of a run. See + * `README.md`. */ import type { ValuePatterns } from "./values.ts"; @@ -42,42 +42,6 @@ export interface RecordingRule { values?: ValuePatterns; } -/** - * The configuration of one local proxy (`server.ts`). The shared service - * (`service.ts`) has its own configuration. - */ -export interface RoachConfig { - /** The directory of the recordings. Each rule has a subdirectory. */ - directory: string; - mode: RecordingMode; - /** - * The only origins that the proxy sends requests to, such as - * `https://ai-gateway.vercel.sh`. The proxy refuses all other origins. - */ - allow: string[]; - rules: RecordingRule[]; - /** - * When the proxy stops, it lists here the recordings that sessions used. - * A failed session uses all recordings that it recorded before. Give these - * files to the `prune` command. - */ - usedFile?: string; - /** - * For each miss, the proxy writes the request as the key sees it to - * `//.json`. These files contain request - * bodies, such as prompts, so do not commit them. It must not be inside - * `directory`. - */ - missDirectory?: string; - /** - * Credentials that the proxy must never write, such as API keys. The - * proxy also learns the values of headers that can carry credentials, - * such as `authorization`, from each request. It redacts them in - * recordings and miss files. - */ - secrets?: string[]; -} - /** A request that a rule matched, but that had no recording. */ export interface RecordingMiss { rule: string; @@ -85,13 +49,9 @@ export interface RecordingMiss { session?: string | undefined; /** The key of the recording that the request needed. */ file: string; - /** The key of the recording with the most equal parts. */ - closest?: string | undefined; - /** The parts that differ from `closest`, such as `messages[3]`. */ - differs: string[]; } -/** The totals of a proxy run. */ +/** The totals of a run. */ export interface RecordingStats { /** * Requests by rule name. `missed` counts the requests that `replay` mode @@ -110,15 +70,3 @@ export interface RecordingStats { */ passthrough: Record; } - -/** The address of a running proxy. Give it to the processes that use it. */ -export interface RoachAddress { - /** The proxy URL, such as `http://127.0.0.1:1234`. */ - url: string; - /** The bearer token of the control API. */ - token: string; - /** The base URL of the control API. Default: `/__roach`. */ - controlUrl?: string; - /** The PEM certificate of the authority that signs intercepted hosts. */ - caCert: string; -} diff --git a/tests/deployed.test.ts b/tests/deployed.test.ts index 16a836a..1ba5b1a 100644 --- a/tests/deployed.test.ts +++ b/tests/deployed.test.ts @@ -353,7 +353,7 @@ describe("deployed roach", () => { const miss = await ciJob({ runId: "miss", prompt: "bye" }); expect(miss.response.status).toBe(412); expect(miss.exitCode).toBe(1); - expect(miss.summary).toMatch(/^- test: model model\/[0-9a-f]{64}\.json /m); + expect(miss.summary).toMatch(/^- test: model\/[0-9a-f]{64}\.json$/m); expect(liveRequests).toBe(1); // The service sends its metrics when it stops. diff --git a/tests/roach.test.ts b/tests/recording.test.ts similarity index 63% rename from tests/roach.test.ts rename to tests/recording.test.ts index 355530a..c085243 100644 --- a/tests/roach.test.ts +++ b/tests/recording.test.ts @@ -1,5 +1,10 @@ +/** + * Recording and replay: modes, sessions, changing values, and misses. Each + * test starts runs on a service with a local upstream. + */ +import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; -import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { ProxyAgent, request } from "undici"; @@ -11,12 +16,14 @@ import { it, onTestFinished, } from "vitest"; -import { connectRoach } from "../src/client.ts"; -import { pruneRecordings } from "../src/recordings.ts"; -import { startRoach, type RoachServer } from "../src/server.ts"; -import type { RecordingMode, RoachConfig } from "../src/types.ts"; +import { startRemoteRun, type RemoteRoach } from "../src/client.ts"; +import { startRoachService, type RoachService } from "../src/service.ts"; +import type { RecordingMode } from "../src/types.ts"; import { VALUE_PATTERNS } from "../src/values.ts"; +const TOKEN = "write-token"; +const TENANT = "acme/alpha"; + let upstream: Server; let origin: string; let liveRequests: number; @@ -25,33 +32,35 @@ let respond: (body: string) => string; /** The upstream answers when this settles. */ let upstreamGate: Promise; let directory: string; -let proxy: RoachServer | undefined; +let service: RoachService; +const runs: RemoteRoach[] = []; let agent: ProxyAgent | undefined; -async function start( - mode: RecordingMode, - options: Pick = {}, -): Promise { - proxy = await startRoach({ - directory, - mode, - allow: [origin], - rules: [ - { - name: "model", - match: { method: "POST", url: `${origin}/v1/` }, - values: { - uuid: VALUE_PATTERNS.uuid, - time: VALUE_PATTERNS.isoTime, - commit: VALUE_PATTERNS.gitCommit, +/** Start a run with the write token. Later requests go through it. */ +async function start(mode: RecordingMode): Promise { + const run = await startRemoteRun( + { url: service.url, token: TOKEN }, + { + tenant: TENANT, + mode, + rules: [ + { + name: "model", + match: { method: "POST", url: `${origin}/v1/` }, + values: { + uuid: VALUE_PATTERNS.uuid, + time: VALUE_PATTERNS.isoTime, + commit: VALUE_PATTERNS.gitCommit, + }, }, - }, - ], - ...options, - }); + ], + }, + ); + runs.push(run); + await agent?.close(); // Tunnel plain HTTP too, so the tests use `CONNECT` as HTTPS clients do. - agent = new ProxyAgent({ uri: proxy.url, proxyTunnel: true }); - return proxy; + agent = new ProxyAgent({ uri: run.url, proxyTunnel: true }); + return run; } /** One delta event of an Anthropic Messages stream. */ @@ -76,15 +85,15 @@ async function send(bodies: unknown[], target = `${origin}/v1/messages`) { } /** Send the requests of one test session, then end it. */ -async function session(running: RoachServer, bodies: unknown[], passed = true) { - const opened = await connectRoach(running).startSession("test"); +async function session(running: RemoteRoach, bodies: unknown[], passed = true) { + const opened = await running.startSession("test"); const responses = await send(bodies); const { missed } = await opened.end(passed); return Object.assign(responses, { missed }); } async function files(): Promise { - return readdir(path.join(directory, "model")).catch(() => []); + return readdir(path.join(directory, TENANT, "model")).catch(() => []); } beforeEach(async () => { @@ -108,24 +117,28 @@ beforeEach(async () => { if (!address || typeof address === "string") throw new Error("No port"); origin = `http://127.0.0.1:${address.port}`; directory = await mkdtemp(path.join(tmpdir(), "roach-")); + service = await startRoachService({ + directory, + allow: [origin], + writeTokenHash: createHash("sha256").update(TOKEN).digest("hex"), + }); }); afterEach(async () => { await agent?.close(); agent = undefined; - await proxy?.close(); - proxy = undefined; + await Promise.all(runs.splice(0).map((run) => run.close().catch(() => {}))); + await service.close(); await new Promise((resolve) => upstream.close(() => resolve())); await rm(directory, { recursive: true, force: true }); }); -describe("roach", () => { +describe("recording", () => { it("replays a passed session for the same requests in auto mode", async () => { const running = await start("auto"); await session(running, [ { model: "m", messages: [{ content: "hi", at: "2026-10-07T03:18:03Z" }] }, ]); - const [first] = await files(); // Same request with other key order and another clock time. const replay = await session(running, [ @@ -141,18 +154,14 @@ describe("roach", () => { { body: "data: 2\n\n", source: "live" }, ]); expect(liveRequests).toBe(2); - await expect(connectRoach(running).stats()).resolves.toEqual({ + await expect(running.stats()).resolves.toEqual({ counts: { model: { live: 2, missed: 0, replayed: 1 } }, - // The miss names the closest recording of the test and the part - // of the request that differs from it. misses: [ expect.objectContaining({ session: "test" }), { rule: "model", session: "test", file: expect.stringMatching(/^model\/[0-9a-f]{64}\.json$/), - closest: `model/${first}`, - differs: ["messages[0]"], }, ], written: 2, @@ -208,7 +217,7 @@ describe("roach", () => { ]); const [file] = await files(); const recording = await readFile( - path.join(directory, "model", file!), + path.join(directory, TENANT, "model", file!), "utf8", ); expect(recording).toContain("<>"); @@ -261,50 +270,12 @@ describe("roach", () => { expect(replay.missed).toBe(1); expect(liveRequests).toBe(1); await expect(files()).resolves.toEqual(recorded); - await expect(connectRoach(running).stats()).resolves.toMatchObject({ + await expect(running.stats()).resolves.toMatchObject({ counts: { model: { live: 0, missed: 1, replayed: 1 } }, - misses: [{ closest: `model/${recorded[0]}`, differs: ["model"] }], + misses: [{ rule: "model", session: "test" }], }); }); - it("redacts credentials in recordings and miss files", async () => { - // Miss files must not be inside the recordings directory. - const missDirectory = `${directory}-misses`; - onTestFinished(() => rm(missDirectory, { recursive: true, force: true })); - const running = await start("auto", { - missDirectory, - // Each part of this value is short, so only the whole value matches. - secrets: ["cfg-secret=012345"], - }); - // The upstream repeats the credential of the request header. - respond = () => `data: key-from-header-0123456789\n\n`; - const opened = await connectRoach(running).startSession("test"); - const response = await request(`${origin}/v1/messages`, { - body: JSON.stringify({ model: "m", note: "cfg-secret=012345" }), - dispatcher: agent!, - // Any header whose name can mean a credential is learned. - headers: { "x-custom-auth": "Bearer key-from-header-0123456789" }, - method: "POST", - }); - await opened.end(true); - - expect(await response.body.text()).toBe("data: <>\n\n"); - const [recorded] = await files(); - const recording = await readFile( - path.join(directory, "model", recorded!), - "utf8", - ); - expect(recording).toContain("<>"); - expect(recording).not.toContain("key-from-header-0123456789"); - const [miss] = await readdir(path.join(missDirectory, "model")); - const text = await readFile( - path.join(missDirectory, "model", miss!), - "utf8", - ); - expect(text).toContain("<>"); - expect(text).not.toContain("cfg-secret=012345"); - }); - it("sends no request to another origin", async () => { await start("auto"); @@ -312,7 +283,7 @@ describe("roach", () => { // refuses it in a tunnel and as an absolute-form request. const other = `${origin.replace("127.0.0.1", "localhost")}/v1/messages`; await expect(send([{}], other)).rejects.toThrow("403"); - const absolute = new ProxyAgent({ uri: proxy!.url, proxyTunnel: false }); + const absolute = new ProxyAgent({ uri: runs[0]!.url, proxyTunnel: false }); onTestFinished(() => absolute.close()); const response = await request(other, { dispatcher: absolute }); await response.body.dump(); @@ -320,30 +291,11 @@ describe("roach", () => { expect(liveRequests).toBe(0); }); - it("refuses control requests without the token", async () => { - const running = await start("auto"); - - await expect( - connectRoach({ url: running.url, token: "wrong" }).stats(), - ).rejects.toThrow("HTTP 401"); - }); - - it("writes nothing for a failed session", async () => { - // A file next to the rule directories is not a rule. - await writeFile(path.join(directory, ".DS_Store"), ""); - const running = await start("auto"); - await session(running, [{ model: "m" }], false); - await session(running, [{ model: "m" }], false); - - expect(liveRequests).toBe(2); - await expect(files()).resolves.toEqual([]); - }); - it("writes a request that ends after its passed session", async () => { const running = await start("auto"); let open!: () => void; upstreamGate = new Promise((resolve) => (open = resolve)); - const opened = await connectRoach(running).startSession("test"); + const opened = await running.startSession("test"); const pending = send([{ model: "m" }]); // Wait until the request reaches the upstream, then end the test. // The upstream server changes `liveRequests`, not this loop. @@ -358,46 +310,6 @@ describe("roach", () => { expect(liveRequests).toBe(1); }); - it("lists used recordings and prunes the others", async () => { - const usedFile = path.join( - directory, - "..", - `${path.basename(directory)}.used`, - ); - const running = await start("auto", { usedFile }); - await session(running, [{ model: "m" }, { model: "n" }]); - const recorded = (await files()).toSorted(); - const contents = await Promise.all( - recorded.map((file) => readFile(path.join(directory, "model", file))), - ); - const stale = JSON.parse(contents[0]!.toString("utf8")); - await writeFile( - path.join(directory, "model", "stale.json"), - JSON.stringify({ ...stale, session: "removed test" }), - ); - - // A replay neither writes a file again nor makes it unused. The test - // fails before its second request, and its recordings stay in use. - await session(running, [{ model: "m" }], false); - await running.close(); - proxy = undefined; - - await expect(readFile(usedFile, "utf8")).resolves.toBe( - recorded.map((file) => `model/${file}\n`).join(""), - ); - await expect(pruneRecordings(directory, [usedFile])).resolves.toBe(1); - await expect(files().then((names) => names.toSorted())).resolves.toEqual( - recorded, - ); - await expect( - Promise.all( - recorded.map((file) => readFile(path.join(directory, "model", file))), - ), - ).resolves.toEqual(contents); - expect(liveRequests).toBe(2); - await rm(usedFile); - }); - it("records again in record mode", async () => { const running = await start("record"); await session(running, [{ model: "m" }]); diff --git a/tests/service.test.ts b/tests/service.test.ts index 3fbb54c..71a3573 100644 --- a/tests/service.test.ts +++ b/tests/service.test.ts @@ -5,7 +5,11 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { ProxyAgent, request } from "undici"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { startRemoteRun, type RemoteRoach } from "../src/client.ts"; +import { + connectRoach, + startRemoteRun, + type RemoteRoach, +} from "../src/client.ts"; import { startRoachService, type RoachService, @@ -203,6 +207,11 @@ describe("roach service", () => { const wrong = new URL(run.url); wrong.password = "not-the-token"; await expect(send(wrong.href, { prompt: "hi" })).rejects.toThrow(/407/); + // The control API of a run needs the run token. A wrong one gets the + // answer of a run that does not exist. + await expect( + connectRoach({ controlUrl: run.controlUrl, token: "wrong" }).stats(), + ).rejects.toThrow(/404/); // The tenant is a path in the store, so it must be owner/repo. for (const tenant of ["junior", "acme/..", "../acme", "a/b/c", "acme/"]) { From 231e32c621fd59c8543f38cb622276ebbf1d0859 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:09:24 +0000 Subject: [PATCH 3/5] build: Release the action and the image with Craft The Release workflow runs the Craft workflow, as getsentry/warden does. A release makes the GitHub release v, moves the v tag for the action, and copies the image : to :. CI and the Image workflow now run on release branches, so Craft can check them and find the image. The action needs no build, so unlike warden there is no dist commit and no -src tag. package.json has no version: the git tag is the version. Co-Authored-By: Junior --- .craft.yml | 18 ++++++++++++++++++ .github/workflows/ci.yml | 3 ++- .github/workflows/image.yml | 14 ++++++++------ .github/workflows/release.yml | 32 ++++++++++++++++++++++++++++++++ README.md | 16 +++++++++++++++- package.json | 1 - 6 files changed, 75 insertions(+), 9 deletions(-) create mode 100644 .craft.yml create mode 100644 .github/workflows/release.yml diff --git a/.craft.yml b/.craft.yml new file mode 100644 index 0000000..db78443 --- /dev/null +++ b/.craft.yml @@ -0,0 +1,18 @@ +minVersion: "2.21.0" +changelog: + policy: auto +# Nothing reads a version from a file. The git tag is the version. +preReleaseCommand: "" +artifactProvider: + name: none +targets: + # The service image. The Image workflow pushes `:` for the release + # branch, and Craft copies it to `:`. + - name: docker + source: ghcr.io/getsentry/roach + target: ghcr.io/getsentry/roach + # The action. Repositories use `getsentry/roach@v`. + - name: github + tagPrefix: v + floatingTags: + - "v{major}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4d5f99..12d1435 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,8 @@ name: CI on: push: - branches: [main] + # Craft publishes a release only when the checks of its branch pass. + branches: [main, "release/**"] pull_request: permissions: diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 5fd5439..7ef553b 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -1,10 +1,12 @@ name: Image -# Build the service image on each pull request. On main, also push it to -# ghcr.io/getsentry/roach, where deploy/gcp pulls it from. +# Build the service image on each pull request. On main and release +# branches, also push it to ghcr.io/getsentry/roach as `:`. On main, +# also push `:main`, which deploy/gcp pulls. A release copies `:` to +# `:` (`.craft.yml`). on: push: - branches: [main] + branches: [main, "release/**"] pull_request: permissions: @@ -27,7 +29,7 @@ jobs: with: persist-credentials: false - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - if: github.ref == 'refs/heads/main' + - if: github.event_name == 'push' uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io @@ -36,10 +38,10 @@ jobs: - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . - push: ${{ github.ref == 'refs/heads/main' }} + push: ${{ github.event_name == 'push' }} tags: | - ghcr.io/getsentry/roach:main ghcr.io/getsentry/roach:${{ github.sha }} + ${{ github.ref == 'refs/heads/main' && 'ghcr.io/getsentry/roach:main' || '' }} terraform: name: terraform diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..765d32d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: Release + +# Craft makes a release branch, and a release manager accepts it in +# getsentry/publish. See "Release" in README.md. +on: + workflow_dispatch: + inputs: + version: + description: Version bump type + required: true + type: choice + default: minor + options: + - minor + - patch + - major + force: + description: Force release (bypass blockers) + required: false + type: boolean + default: false + +jobs: + release: + # The Craft workflow pushes the release branch. + permissions: + contents: write + uses: getsentry/craft/.github/workflows/release.yml@3f2abdc5703191d12ef2cc013f792f1bd220f44d # v2 + with: + version: ${{ inputs.version }} + force: ${{ inputs.force && 'true' || 'false' }} + secrets: inherit diff --git a/README.md b/README.md index 669aaf1..92e897e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ A client that does not use `client.ts` can do the same with plain HTTP. See A repository adds one step and a `roach.json` file: ```yaml -- uses: getsentry/roach@main +- uses: getsentry/roach@v0 with: token: ${{ secrets.ROACH_TOKEN }} # without it, the run can only replay run: pnpm test @@ -329,6 +329,20 @@ The service also has: - `DELETE /__roach/runs/`: end the run. Returns its stats. - `GET /__roach/ca.pem`: the CA certificate. No token. +## Release + +Run the `Release` workflow, and choose `minor`, `patch`, or `major`. Craft +(`.craft.yml`) makes a `release/` branch and asks for approval in +[getsentry/publish](https://github.com/getsentry/publish). When a release +manager accepts it, Craft: + +- copies the image `ghcr.io/getsentry/roach:` to `:`. +- makes the GitHub release `v`, with the changelog, and moves the + `v` tag. Repositories use the action as `getsentry/roach@v`. + +The action runs `src/action.ts` from the tag, so a release has no build +step. + ## Development ```sh diff --git a/package.json b/package.json index 9a3ec3d..99b941f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,5 @@ { "name": "@sentry/roach", - "version": "0.0.0", "private": true, "description": "A recording HTTPS proxy for tests, as a shared service.", "license": "Apache-2.0", From a9c993d261255a8ad7db7a4397a30e04ae04908b Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:18:29 +0000 Subject: [PATCH 4/5] fix(action): Stop all command processes and close runs that fail to start Send cancel signals to the process group of the command, because bash does not pass them to its children. Also stop what the command leaves in the background before the run ends. If startRemoteRun cannot write the CA file, end the run on the service before it throws. Before, the run stayed open until the 6-hour timeout. --- src/action.ts | 23 +++++++++++++++++++---- src/client.ts | 11 ++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/action.ts b/src/action.ts index 9968e8c..65e605a 100644 --- a/src/action.ts +++ b/src/action.ts @@ -17,16 +17,31 @@ const HIDDEN_VARIABLE = /^(INPUT_.*|https?_proxy|no_proxy|all_proxy)$/i; /** Run the command in bash, and return its exit code. */ function runCommand(command: string, env: NodeJS.ProcessEnv): Promise { + // `detached` puts bash and its children in a new process group. const child = spawn("bash", ["-e", "-o", "pipefail", "-c", command], { + detached: true, env, stdio: "inherit", }); - // A canceled job sends a signal. Stop the command, then end the run. - process.on("SIGINT", () => child.kill("SIGINT")); - process.on("SIGTERM", () => child.kill("SIGTERM")); + // A canceled job sends a signal. Bash does not pass it to its children, so + // send it to the whole process group. Then end the run. + const stop = (signal: NodeJS.Signals) => { + try { + process.kill(-child.pid!, signal); + } catch { + // The process group has already stopped. + } + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); return new Promise((resolve, reject) => { child.once("error", reject); - child.once("exit", (code) => resolve(code ?? 1)); + child.once("exit", (code) => { + // Stop what the command left in the background, such as `cmd &`. Bash + // makes it ignore `SIGINT`, and the run ends next. + stop("SIGTERM"); + resolve(code ?? 1); + }); }); } diff --git a/src/client.ts b/src/client.ts index ffbe45a..00ade8c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -135,7 +135,16 @@ export async function startRemoteRun( "runs", config, ); - const proxyEnv = await createProxyEnv(run, noProxy); + let proxyEnv: Awaited>; + try { + proxyEnv = await createProxyEnv(run, noProxy); + } catch (error) { + // Without this, the run stays open until the service times it out. + await callControl(run.controlUrl, run.token, "DELETE", "").catch( + () => undefined, + ); + throw error; + } return { ...run, ...connectRoach(run), From 2646379ad1e3cc414ac6c002277da3a98528b03f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:30:23 +0000 Subject: [PATCH 5/5] ref: Simplify how startRemoteRun ends a run that fails to start Co-Authored-By: David Cramer --- src/action.ts | 5 ++++- src/client.ts | 13 ++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/action.ts b/src/action.ts index 65e605a..f86a961 100644 --- a/src/action.ts +++ b/src/action.ts @@ -12,7 +12,10 @@ import { startRemoteRun } from "./client.ts"; import { describeRecordingMisses, describeRecordingStats } from "./report.ts"; import type { RunConfig } from "./service.ts"; -/** Variables that the command must not get: the inputs, with the token. */ +/** + * Variables that the command does not get: the inputs, which have the token, + * and the proxy variables of the job. + */ const HIDDEN_VARIABLE = /^(INPUT_.*|https?_proxy|no_proxy|all_proxy)$/i; /** Run the command in bash, and return its exit code. */ diff --git a/src/client.ts b/src/client.ts index 00ade8c..cc921f9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -135,16 +135,11 @@ export async function startRemoteRun( "runs", config, ); - let proxyEnv: Awaited>; - try { - proxyEnv = await createProxyEnv(run, noProxy); - } catch (error) { - // Without this, the run stays open until the service times it out. - await callControl(run.controlUrl, run.token, "DELETE", "").catch( - () => undefined, - ); + const proxyEnv = await createProxyEnv(run, noProxy).catch(async (error) => { + // End the run, or it stays open until the service times it out. + await callControl(run.controlUrl, run.token, "DELETE", "").catch(() => {}); throw error; - } + }); return { ...run, ...connectRoach(run),