diff --git a/.craft.yml b/.craft.yml new file mode 100644 index 0000000..db78443 --- /dev/null +++ b/.craft.yml @@ -0,0 +1,18 @@ +minVersion: "2.21.0" +changelog: + policy: auto +# Nothing reads a version from a file. The git tag is the version. +preReleaseCommand: "" +artifactProvider: + name: none +targets: + # The service image. The Image workflow pushes `:` for the release + # branch, and Craft copies it to `:`. + - name: docker + source: ghcr.io/getsentry/roach + target: ghcr.io/getsentry/roach + # The action. Repositories use `getsentry/roach@v`. + - name: github + tagPrefix: v + floatingTags: + - "v{major}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4d5f99..12d1435 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,8 @@ name: CI on: push: - branches: [main] + # Craft publishes a release only when the checks of its branch pass. + branches: [main, "release/**"] pull_request: permissions: diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 5fd5439..7ef553b 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -1,10 +1,12 @@ name: Image -# Build the service image on each pull request. On main, also push it to -# ghcr.io/getsentry/roach, where deploy/gcp pulls it from. +# Build the service image on each pull request. On main and release +# branches, also push it to ghcr.io/getsentry/roach as `:`. On main, +# also push `:main`, which deploy/gcp pulls. A release copies `:` to +# `:` (`.craft.yml`). on: push: - branches: [main] + branches: [main, "release/**"] pull_request: permissions: @@ -27,7 +29,7 @@ jobs: with: persist-credentials: false - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - if: github.ref == 'refs/heads/main' + - if: github.event_name == 'push' uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io @@ -36,10 +38,10 @@ jobs: - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . - push: ${{ github.ref == 'refs/heads/main' }} + push: ${{ github.event_name == 'push' }} tags: | - ghcr.io/getsentry/roach:main ghcr.io/getsentry/roach:${{ github.sha }} + ${{ github.ref == 'refs/heads/main' && 'ghcr.io/getsentry/roach:main' || '' }} terraform: name: terraform diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..765d32d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: Release + +# Craft makes a release branch, and a release manager accepts it in +# getsentry/publish. See "Release" in README.md. +on: + workflow_dispatch: + inputs: + version: + description: Version bump type + required: true + type: choice + default: minor + options: + - minor + - patch + - major + force: + description: Force release (bypass blockers) + required: false + type: boolean + default: false + +jobs: + release: + # The Craft workflow pushes the release branch. + permissions: + contents: write + uses: getsentry/craft/.github/workflows/release.yml@3f2abdc5703191d12ef2cc013f792f1bd220f44d # v2 + with: + version: ${{ inputs.version }} + force: ${{ inputs.force && 'true' || 'false' }} + secrets: inherit diff --git a/AGENTS.md b/AGENTS.md index d7d7f46..31e2123 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,16 +7,16 @@ ## Commands -| Task | Command | -| ---------------- | ------------------------------------------------- | -| Test file | `pnpm exec vitest run tests/roach.test.ts` | -| Test case | `pnpm exec vitest run tests/roach.test.ts -t "…"` | -| Lint file | `pnpm exec oxlint --deny-warnings src/server.ts` | -| Format | `pnpm format` | -| Typecheck | `pnpm typecheck` | -| Deployed shape | `pnpm exec vitest run tests/deployed.test.ts` | -| Unused code/deps | `pnpm knip` | -| Everything (CI) | `pnpm check` | +| Task | Command | +| ---------------- | ----------------------------------------------------- | +| Test file | `pnpm exec vitest run tests/recording.test.ts` | +| Test case | `pnpm exec vitest run tests/recording.test.ts -t "…"` | +| Lint file | `pnpm exec oxlint --deny-warnings src/server.ts` | +| Format | `pnpm format` | +| Typecheck | `pnpm typecheck` | +| Deployed shape | `pnpm exec vitest run tests/deployed.test.ts` | +| Unused code/deps | `pnpm knip` | +| Everything (CI) | `pnpm check` | ## External References @@ -29,14 +29,14 @@ ## Key Conventions -- Use Node built-ins and the `openssl` command. The only runtime dependency is `@sentry/node`, and only `src/service.ts` imports it. The local proxy and the client must not load it. +- Use Node built-ins and the `openssl` command. The only runtime dependency is `@sentry/node`, and only `src/service.ts` imports it. The client and the action must not load it. - `deploy/gcp/` is the production setup (Terraform). Change it in the same change when the service config (`RoachServiceConfig`) changes. CI runs `terraform validate` on it. - Node runs `src/` as TypeScript with type stripping. Use only erasable syntax, and import local files with the `.ts` extension. - Use functions and plain objects, not classes. - Start each source file with a comment that says what the file owns. Give each export a short JSDoc. - Update `README.md` in the same change when behavior, config, or the control API changes. - Test through a real proxy and a local upstream server in `tests/`. Do not mock modules. Do not reach the internet. -- The proxy decrypts HTTPS. Never write a credential: every recording and miss file goes through `src/secrets.ts`. +- The proxy decrypts HTTPS. Never write a credential: every recording goes through `src/secrets.ts`. - The upstream host always comes from `allow`, never from the client. - Write docs in ASD-STE100 English: common words, active voice, short sentences. diff --git a/README.md b/README.md index aefe809..92e897e 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,11 @@ replays them, so a second run of a test suite makes no live calls. The proxy owns every read and write of the recordings. A test runner only gives it rules and tells it when each test starts and ends. -Roach runs in one of two ways: - -- **A shared service** (`service.ts`): one deployed proxy for many - projects and CI runs. A client points `HTTPS_PROXY` at it and trusts its - certificate authority. It installs nothing else. The recordings are in a - GCS bucket, which deletes each one 30 days after it was written. See - [Service](#service) and [Deploy](#deploy). -- **A local proxy** (`server.ts`): a process in the process tree of the - test run, with the recordings in JSON files that you commit to git. +Roach is a shared service (`service.ts`): one deployed proxy for many +projects and CI runs. A client points `HTTPS_PROXY` at it and trusts its +certificate authority. It installs nothing else. The recordings are in a +GCS bucket, which deletes each one 30 days after it was written. See +[Service](#service) and [Deploy](#deploy). Roach started in [`getsentry/junior`](https://github.com/getsentry/junior/tree/main/packages/junior-evals/src/roach). @@ -46,9 +42,9 @@ const run = await startRemoteRun( ); spawn("vitest", { env: { ...process.env, ...run.env } }); -// Once per test, in a worker that has the `url`, `controlUrl`, and `token` -// of the run. -const session = await connectRoach({ url, controlUrl, token }).startSession( +// Once per test, in a worker that has the `controlUrl` and `token` of the +// run. +const session = await connectRoach({ controlUrl, token }).startSession( testName, ); const { missed } = await session.end(passed); @@ -58,40 +54,56 @@ const { missed } = await session.end(passed); console.log(describeRecordingStats(await run.close())); ``` +`run.env` has `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, +and `NODE_EXTRA_CA_CERTS`. Node 24 reads the last two only at startup. The +CA file also has the certificates of `NODE_EXTRA_CA_CERTS` of the caller. A client that does not use `client.ts` can do the same with plain HTTP. See [Control API](#control-api). -## Use a local proxy +## Use the GitHub Action -```ts -import { spawnRoach } from "@sentry/roach/client"; - -const proxy = await spawnRoach({ - directory: "recordings", - mode: "auto", - allow: ["https://ai-gateway.vercel.sh"], - rules: [/* as above */], -}); -spawn("vitest", { env: { ...process.env, ...proxy.env } }); -// Sessions as above, with connectRoach({ url: proxy.url, token: proxy.token }). -console.log(describeRecordingStats(await proxy.stats())); -await proxy.close(); +A repository adds one step and a `roach.json` file: + +```yaml +- uses: getsentry/roach@v0 + with: + token: ${{ secrets.ROACH_TOKEN }} # without it, the run can only replay + run: pnpm test ``` -- `env` has `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, - and `NODE_EXTRA_CA_CERTS`. Node 24 reads the last two only at startup. A - process that is already running must set its own agents. -- `spawnRoach(config, { launcher })` runs the proxy with a command - prefix, such as `sudo`. Use it when the caller cannot reach the network, - but the proxy must. +`roach.json` is at the root of the repository. It has the `rules` of the +run, and can have `allow` (`RunConfig` in `src/service.ts`): + +```json +{ + "rules": [ + { + "name": "model", + "match": { "method": "POST", "url": "https://ai-gateway.vercel.sh/" }, + "keyHeaders": ["ai-model-id"] + } + ] +} +``` + +The action (`src/action.ts`) starts a run of the tenant +`GITHUB_REPOSITORY`, in `auto` mode with the token or `replay` mode without +it. It runs `run` in bash with the proxy variables of the run, then ends +the run and writes its stats to the job summary. + +- Only `run` uses the proxy. Later steps, such as cache and artifact + uploads, do not. `run` does not get the write token. +- `run` is one session. When it fails, the run drops its new recordings. A + miss in `replay` mode fails the step. The summary names the job, not the + test, of each miss. +- `NODE_EXTRA_CA_CERTS` works only for Node. Python and `curl` need a CA + bundle that also has the system certificates. ## Config -A local proxy takes `RoachConfig` (`src/types.ts`). A run on the service -takes `RunConfig` (`src/service.ts`): `tenant`, `mode`, `rules`, `allow`, -and `name`. The fields below mean the same in both. +A run takes `RunConfig` (`src/service.ts`): `tenant`, `mode`, `rules`, +`allow`, and `name`. -- `directory`: where the recordings are. Each rule has a subdirectory. - `mode`: `auto` replays recordings and records misses. `replay` replays recordings and fails a miss with HTTP 412, so nothing goes live. `record` sends every request live and records it again. `off` records and replays @@ -103,20 +115,11 @@ and `name`. The fields below mean the same in both. and `-`. `match` has `method`, `url` (a prefix), and `headers`. `keyHeaders` adds request headers to the key. `values` names the values that change from run to run. -- `usedFile`: when the proxy stops, it lists here the recordings that - sessions used. A failed session uses all recordings that it recorded - before, because it stops early. `cli.ts prune` takes these files. Only - for `directory`. -- `missDirectory`: for each miss, the proxy writes the request as the key - sees it, and its diagnosis, to `//.json`. These - files contain request bodies, but no request headers except `keyHeaders`. - Do not commit them. The proxy refuses a `missDirectory` inside - `directory`. -- `secrets`: credentials that the proxy must never write. The proxy also - learns the value of each request header whose name can mean a credential, +- Credentials: the proxy never writes the `secrets` of the service config. + It also learns the value of each request header whose name can mean a credential, such as one with `auth`, `token`, `key`, `secret`, `cookie`, or `session`. - The match is broad on purpose. Recordings and miss files show - `<>` in place of each known value. + The match is broad on purpose. Recordings show `<>` in place of + each known value. Requests that match no rule go live without a change, and their responses stream. Each response has an `x-roach` header: `replayed`, `live`, @@ -124,11 +127,10 @@ stream. Each response has an `x-roach` header: `replayed`, `live`, ## Recordings -The key of a recording is `/.json`. The file store keeps it at -`/`. The hash is the hash of the -rule, the method, the URL, the key headers, and the body. A JSON body has -sorted keys. A recording keeps the response, the session that recorded it, -and a short hash of each part of the request. It does not keep the request +The key of a recording is `/.json`, under `/` in the +store. The hash is the hash of the rule, the method, the URL, the key +headers, and the body. A JSON body has sorted keys. A recording keeps the +response and the session that recorded it. It does not keep the request body. - One session is open at a time. The proxy keeps the new recordings of a @@ -180,19 +182,10 @@ real way use the same recording. ## Misses -The parts of a request are the method, the URL, each key header, each -top-level field of a JSON body, and each item of a top-level array, such as -`messages[3]`. The proxy adds each request without a recording to -`stats().misses`. `describeRecordingMisses()` in `report.ts` gives the +The proxy adds each request without a recording to `stats().misses`, with +its session and key. `describeRecordingMisses()` in `report.ts` gives the first miss of each test, which is the one to fix. -With the file store, the proxy also finds the recording with the most -equal parts, from the same session if it can, and logs the parts that -differ. This is only a hint to debug a miss. It never makes a replay. To -find it, the file store reads all recordings of a rule on the first miss of -that rule in a run, and then keeps them in memory. The GCS store does not -give this hint, because it would have to read every recording. - ## Service `cli.ts service ` starts the service. The config is @@ -313,21 +306,14 @@ set the secret again, and restart the VM. ```sh node src/cli.ts service -node src/cli.ts serve < config.json -node src/cli.ts prune ... ``` -`service` starts the shared service. `serve` starts a local proxy and prints -its address as one JSON line. `client.ts` runs it. `prune` deletes the -recordings that no used file lists. Give it the used files of every run -that shares the directory, or it deletes recordings that another run -needs. +It starts the service and prints its URL as one JSON line. ## Control API -The calls of a run are under `/__roach/runs/` on the service, and under -`/__roach` on a local proxy. They need `Authorization: Bearer `, with -the run token or the token of the local proxy. `client.ts` calls them. +The calls of a run are under `/__roach/runs/`. They need +`Authorization: Bearer `. `client.ts` calls them. - `POST /session` with `{"name": "..."}`: open a session. - `POST /session/end` with `{"name": "...", "passed": true}`: end it. @@ -343,6 +329,20 @@ The service also has: - `DELETE /__roach/runs/`: end the run. Returns its stats. - `GET /__roach/ca.pem`: the CA certificate. No token. +## Release + +Run the `Release` workflow, and choose `minor`, `patch`, or `major`. Craft +(`.craft.yml`) makes a `release/` branch and asks for approval in +[getsentry/publish](https://github.com/getsentry/publish). When a release +manager accepts it, Craft: + +- copies the image `ghcr.io/getsentry/roach:` to `:`. +- makes the GitHub release `v`, with the changelog, and moves the + `v` tag. Repositories use the action as `getsentry/roach@v`. + +The action runs `src/action.ts` from the tag, so a release has no build +step. + ## Development ```sh @@ -352,32 +352,34 @@ pnpm check # format, lint, types, unused code, and tests, as CI runs them `AGENTS.md` has the conventions and the commands for one file. -- `tests/roach.test.ts` tests a local proxy. +- `tests/recording.test.ts` tests modes, sessions, changing values, and + misses. - `tests/service.test.ts` tests the service in the test process. - `tests/deployed.test.ts` runs the service as it runs in production: from the command line, behind a TLS server that does what the load balancer does, with a local GCS server, metadata server, and Sentry server. Each - CI job (`tests/ci-job.ts`) sets only the proxy variables of its run. + CI job runs the GitHub Action, and its command has only the proxy + variables of its run. ## Files The proxy is in `src/`. Tests are in `tests/`. The production setup is in `deploy/gcp/` and `Dockerfile`. -- `types.ts`: the configuration and the results of a local proxy. +- `types.ts`: rules, modes, and the results of a run. - `server.ts`: sockets, HTTPS interception, the allow list, and the control API. - `service.ts`: the shared service: tenants, runs, access, and metrics. - `recorder.ts`: modes, sessions, replay, and misses. -- `request-key.ts`: the key and the part hashes of a request. +- `request-key.ts`: the key of a request. - `store.ts`: the recording format and the store contract. -- `recordings.ts`: the file store and prune. +- `recordings.ts`: the file store, for tests and development. - `gcs.ts`: the GCS store. - `values.ts`: changing values and their placeholders. - `streams.ts`: merges the deltas of a recorded model stream. - `secrets.ts`: redaction of credentials. - `certificates.ts`: the certificate authority for HTTPS. -- `client.ts`: starts a remote run or a local proxy, and calls the control - API. +- `client.ts`: starts a run and calls the control API. - `report.ts`: text reports of a run. +- `action.ts`: the GitHub Action (`action.yml` at the root). - `cli.ts`: the command line. diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..bb2c735 --- /dev/null +++ b/action.yml @@ -0,0 +1,20 @@ +name: Roach +description: Run a command with its HTTPS traffic recorded and replayed by the Roach service. + +inputs: + run: + description: The command to run, in bash. Only this command uses the proxy. + required: true + token: + description: >- + The write token of the service. Without it, the run can only replay, + as in jobs of forks. Give it only to workflows that you trust. + required: false + url: + description: The URL of the Roach service. + required: false + default: https://roach-proxy.getsentry.net + +runs: + using: node24 + main: src/action.ts diff --git a/knip.json b/knip.json index 5742288..3a2363c 100644 --- a/knip.json +++ b/knip.json @@ -2,5 +2,5 @@ "$schema": "https://unpkg.com/knip@6/schema.json", "project": ["src/**/*.ts", "tests/**/*.ts"], "ignoreBinaries": ["openssl"], - "entry": ["tests/ci-job.ts"] + "entry": ["src/action.ts"] } diff --git a/package.json b/package.json index 7db7109..99b941f 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,7 @@ { "name": "@sentry/roach", - "version": "0.0.0", "private": true, - "description": "A recording HTTPS proxy for tests, as a shared service or a local process.", + "description": "A recording HTTPS proxy for tests, as a shared service.", "license": "Apache-2.0", "repository": { "type": "git", @@ -15,7 +14,6 @@ "exports": { "./client": "./src/client.ts", "./report": "./src/report.ts", - "./server": "./src/server.ts", "./service": "./src/service.ts", "./types": "./src/types.ts", "./values": "./src/values.ts" diff --git a/src/action.ts b/src/action.ts new file mode 100644 index 0000000..f86a961 --- /dev/null +++ b/src/action.ts @@ -0,0 +1,98 @@ +/** + * The GitHub Action of Roach (`action.yml`). See "Use the GitHub Action" in + * `README.md`. + * + * It starts a run, runs the command of the step with the proxy variables of + * the run, and ends the run. The rules of the run come from `roach.json`. + * The runner gives each input as `INPUT_`. + */ +import { spawn } from "node:child_process"; +import { appendFile, readFile } from "node:fs/promises"; +import { startRemoteRun } from "./client.ts"; +import { describeRecordingMisses, describeRecordingStats } from "./report.ts"; +import type { RunConfig } from "./service.ts"; + +/** + * Variables that the command does not get: the inputs, which have the token, + * and the proxy variables of the job. + */ +const HIDDEN_VARIABLE = /^(INPUT_.*|https?_proxy|no_proxy|all_proxy)$/i; + +/** Run the command in bash, and return its exit code. */ +function runCommand(command: string, env: NodeJS.ProcessEnv): Promise { + // `detached` puts bash and its children in a new process group. + const child = spawn("bash", ["-e", "-o", "pipefail", "-c", command], { + detached: true, + env, + stdio: "inherit", + }); + // A canceled job sends a signal. Bash does not pass it to its children, so + // send it to the whole process group. Then end the run. + const stop = (signal: NodeJS.Signals) => { + try { + process.kill(-child.pid!, signal); + } catch { + // The process group has already stopped. + } + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + return new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => { + // Stop what the command left in the background, such as `cmd &`. Bash + // makes it ignore `SIGINT`, and the run ends next. + stop("SIGTERM"); + resolve(code ?? 1); + }); + }); +} + +const command = process.env.INPUT_RUN; +if (!command) throw new Error("The run input is required"); +// The runner gives an empty string for an input that the step does not set. +const token = + process.env.INPUT_TOKEN === "" ? undefined : process.env.INPUT_TOKEN; +// Without the token, the service only allows `replay`, as for forks. +const mode = token ? "auto" : "replay"; +const config = JSON.parse(await readFile("roach.json", "utf8")) as Pick< + RunConfig, + "allow" | "rules" +>; +const run = await startRemoteRun( + { url: process.env.INPUT_URL!, token }, + { + ...config, + tenant: process.env.GITHUB_REPOSITORY!, + mode, + name: `${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT}`, + }, +); +// The proxy URL has the run token. Keep it out of the log. +process.stdout.write(`::add-mask::${run.token}\n`); + +let exitCode = 1; +try { + const session = await run.startSession(process.env.GITHUB_JOB ?? "command"); + exitCode = await runCommand(command, { + ...Object.fromEntries( + Object.entries(process.env).filter( + ([name]) => !HIDDEN_VARIABLE.test(name), + ), + ), + ...run.env, + }); + const { missed } = await session.end(exitCode === 0); + if (missed > 0 && exitCode === 0) exitCode = 1; +} finally { + const stats = await run.close(); + const report = [ + `Roach (${mode}): ${describeRecordingStats(stats)}`, + ...describeRecordingMisses(stats.misses).map((miss) => `- ${miss}`), + ].join("\n"); + process.stdout.write(`${report}\n`); + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile(process.env.GITHUB_STEP_SUMMARY, `${report}\n`); + } +} +process.exitCode = exitCode; diff --git a/src/certificates.ts b/src/certificates.ts index ef90a87..f10a5cc 100644 --- a/src/certificates.ts +++ b/src/certificates.ts @@ -3,9 +3,8 @@ * * The proxy intercepts HTTPS. It signs one certificate for each host when a * client first connects to that host, and signs it again before it - * expires. Clients must trust the authority certificate. A local proxy - * creates a new authority when it starts. The shared service loads a fixed - * one, so that clients can trust it across restarts. + * expires. Clients must trust the authority certificate. The service loads + * a fixed one, so that clients can trust it across restarts. */ import { execFile } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; diff --git a/src/cli.ts b/src/cli.ts index bd17c32..ed6ffaa 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,68 +1,35 @@ /** * The command line of Roach. * - * - `cli.ts service `: start the shared service (`service.ts`) - * with the configuration in the file. Print its URL as one JSON line. - * Stop on SIGINT or SIGTERM. The deployed container runs this command. - * - `cli.ts serve`: read the configuration of a local proxy as JSON from - * stdin, and start it. Print its address as one JSON line. Stop on SIGINT - * or SIGTERM. `client.ts` runs this command. - * - `cli.ts prune ...`: delete the recordings that no - * used file lists. + * `cli.ts service ` starts the shared service (`service.ts`) + * with the configuration in the file. It prints the URL of the service as + * one JSON line, and stops on SIGINT or SIGTERM. The deployed container + * runs this command. */ import { readFile } from "node:fs/promises"; -import { text } from "node:stream/consumers"; -import { pruneRecordings } from "./recordings.ts"; -import { startRoach } from "./server.ts"; -import type { RoachServiceConfig } from "./service.ts"; -import type { RoachAddress, RoachConfig } from "./types.ts"; - -const USAGE = `Usage: - cli.ts service - cli.ts serve < config.json - cli.ts prune ...`; +import { startRoachService, type RoachServiceConfig } from "./service.ts"; const [command, ...args] = process.argv.slice(2); - -/** Stop on SIGINT or SIGTERM. A failed close logs its error and exits 1. */ -function stopOnSignal(close: () => Promise): void { - const stop = () => { - close().then( - () => process.exit(0), - (error: unknown) => { - process.stderr.write(`[roach] Close failed: ${String(error)}\n`); - process.exit(1); - }, - ); - }; - process.once("SIGINT", stop); - process.once("SIGTERM", stop); -} - -if (command === "service" && args.length === 1) { - const config = JSON.parse( - await readFile(args[0]!, "utf8"), - ) as RoachServiceConfig; - // Only the service loads the Sentry SDK. - const { startRoachService } = await import("./service.ts"); - const service = await startRoachService(config); - process.stdout.write(`${JSON.stringify({ url: service.url })}\n`); - stopOnSignal(() => service.close()); -} else if (command === "serve") { - const config = JSON.parse(await text(process.stdin)) as RoachConfig; - const proxy = await startRoach(config); - const address: RoachAddress = { - url: proxy.url, - token: proxy.token, - caCert: proxy.caCert, - }; - process.stdout.write(`${JSON.stringify(address)}\n`); - stopOnSignal(() => proxy.close()); -} else if (command === "prune" && args.length >= 2) { - const [directory, ...usedFiles] = args as [string, ...string[]]; - const count = await pruneRecordings(directory, usedFiles); - process.stdout.write(`Deleted ${count} unused recordings\n`); -} else { - process.stderr.write(`${USAGE}\n`); +if (command !== "service" || args.length !== 1) { + process.stderr.write("Usage: cli.ts service \n"); process.exit(2); } + +const config = JSON.parse( + await readFile(args[0]!, "utf8"), +) as RoachServiceConfig; +const service = await startRoachService(config); +process.stdout.write(`${JSON.stringify({ url: service.url })}\n`); + +/** Stop on SIGINT or SIGTERM. A failed close logs its error and exits 1. */ +const stop = () => { + service.close().then( + () => process.exit(0), + (error: unknown) => { + process.stderr.write(`[roach] Close failed: ${String(error)}\n`); + process.exit(1); + }, + ); +}; +process.once("SIGINT", stop); +process.once("SIGTERM", stop); diff --git a/src/client.ts b/src/client.ts index 07c7ef7..cc921f9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,22 +1,19 @@ /** * The client of Roach. See `README.md`. * - * `startRemoteRun()` starts a run on a Roach service (`service.ts`). - * `spawnRoach()` starts a local proxy in its own process. Both return - * `env`, the variables that send the traffic of a process through the - * proxy. `connectRoach()` controls a run or a local proxy from another - * process, such as a test worker. + * `startRemoteRun()` starts a run on a Roach service (`service.ts`). It + * returns `env`, the variables that send the traffic of a process through + * the run. `connectRoach()` controls a run from another process, such as a + * test worker. */ -import { spawn } from "node:child_process"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import http from "node:http"; import https from "node:https"; import { tmpdir } from "node:os"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { CONTROL_PATH } from "./server.ts"; import type { RemoteRun, RunConfig } from "./service.ts"; -import type { RoachAddress, RoachConfig, RecordingStats } from "./types.ts"; +import type { RecordingStats } from "./types.ts"; /** The open session of one test. */ export interface RecordingSession { @@ -28,7 +25,7 @@ export interface RecordingSession { end(passed: boolean): Promise<{ missed: number }>; } -/** The control API of a running proxy. */ +/** The control API of a run. */ export interface RoachControl { /** Open the session of one test. Only one session is open at a time. */ startSession(name: string): Promise; @@ -36,44 +33,18 @@ export interface RoachControl { stats(): Promise; } -/** A proxy that `spawnRoach()` started. */ -export interface Roach extends RoachAddress, RoachControl { - /** - * The variables that send the HTTP traffic of a process through the - * proxy: `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, - * and `NODE_EXTRA_CA_CERTS`. Give them to a process when it starts. Node - * reads the last two only at startup. - */ - env: Record; - /** Stop the proxy. It then writes `usedFile` of its config. */ - close(): Promise; -} - -const CLI = fileURLToPath(new URL("./cli.ts", import.meta.url)); - -/** Proxy variables. The proxy itself must not use a proxy. */ -const PROXY_VARIABLES = new Set([ - "all_proxy", - "http_proxy", - "https_proxy", - "no_proxy", - "node_use_env_proxy", -]); - -/** Where the control API is, and its token. */ -type ControlAddress = Pick & { - token?: string | undefined; -}; - -/** Call the control API, and return the JSON of the response. */ +/** + * Call the control API at `base`, and return the JSON of the response. + * `token` is the bearer token. + */ function callControl( - address: ControlAddress, + base: string, + token: string | undefined, method: "DELETE" | "GET" | "POST", route: string, body?: unknown, ): Promise { const payload = body === undefined ? undefined : JSON.stringify(body); - const base = address.controlUrl ?? `${address.url}${CONTROL_PATH}`; const url = route ? `${base}/${route}` : base; const client = url.startsWith("https:") ? https : http; return new Promise((resolve, reject) => { @@ -84,9 +55,7 @@ function callControl( agent: false, method, headers: { - ...(address.token - ? { authorization: `Bearer ${address.token}` } - : {}), + ...(token ? { authorization: `Bearer ${token}` } : {}), ...(payload ? { "content-type": "application/json" } : {}), }, }, @@ -114,92 +83,34 @@ function callControl( }); } -/** Control a run or a local proxy, for example from a test worker. */ -export function connectRoach( - address: Pick, -): RoachControl { +/** Control a run, for example from a test worker. */ +export function connectRoach({ + controlUrl, + token, +}: Pick): RoachControl { return { async startSession(name) { - await callControl(address, "POST", "session", { name }); + await callControl(controlUrl, token, "POST", "session", { name }); return { end: (passed) => - callControl(address, "POST", "session/end", { name, passed }), + callControl(controlUrl, token, "POST", "session/end", { + name, + passed, + }), }; }, - stats: () => callControl(address, "GET", "stats"), - }; -} - -/** Read the address line that `cli.ts serve` prints. */ -function readAddress(child: ReturnType): Promise { - return new Promise((resolve, reject) => { - let output = ""; - child.once("error", reject); - child.once("exit", (code) => - reject(new Error(`Roach exited with code ${code}`)), - ); - child.stdout!.on("data", (chunk: Buffer) => { - output += chunk.toString("utf8"); - const end = output.indexOf("\n"); - if (end >= 0) resolve(JSON.parse(output.slice(0, end))); - }); - }); -} - -/** - * Start the proxy in its own process. - * - * `launcher` is a command prefix that runs the proxy, such as `sudo`. Use it - * when the caller cannot reach the network, but the proxy must. `noProxy` - * lists the hosts that do not use the proxy, such as `localhost`. - */ -export async function spawnRoach( - config: RoachConfig, - { - launcher = [], - noProxy = "localhost,127.0.0.1,::1", - }: { launcher?: string[]; noProxy?: string } = {}, -): Promise { - const command = [ - ...launcher, - process.execPath, - "--experimental-strip-types", - "--disable-warning=ExperimentalWarning", - CLI, - "serve", - ]; - const child = spawn(command[0]!, command.slice(1), { - env: Object.fromEntries( - Object.entries(process.env).filter( - ([name]) => !PROXY_VARIABLES.has(name.toLowerCase()), - ), - ), - stdio: ["pipe", "pipe", "inherit"], - }); - const exited = new Promise((resolve) => - child.once("exit", () => resolve()), - ); - child.stdin.end(JSON.stringify(config)); - const address = await readAddress(child); - const proxyEnv = await createProxyEnv(address, noProxy); - - return { - ...address, - ...connectRoach(address), - env: proxyEnv.env, - async close() { - if (child.exitCode === null && child.signalCode === null) { - child.kill("SIGTERM"); - await exited; - } - await proxyEnv.remove(); - }, + stats: () => callControl(controlUrl, token, "GET", "stats"), }; } /** A run on a Roach service (`service.ts`). */ export interface RemoteRoach extends RemoteRun, RoachControl { - /** The same variables as `Roach.env`, for the proxy URL of the run. */ + /** + * The variables that send the HTTP traffic of a process through the run: + * `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, and + * `NODE_EXTRA_CA_CERTS`. Give them to a process when it starts. Node + * reads the last two only at startup. + */ env: Record; /** End the run. Returns its stats. */ close(): Promise; @@ -218,19 +129,29 @@ export async function startRemoteRun( { noProxy = "localhost,127.0.0.1,::1" }: { noProxy?: string } = {}, ): Promise { const run = await callControl( - { url: service.url.replace(/\/$/, ""), token: service.token }, + `${service.url.replace(/\/$/, "")}${CONTROL_PATH}`, + service.token, "POST", "runs", config, ); - const proxyEnv = await createProxyEnv(run, noProxy); + const proxyEnv = await createProxyEnv(run, noProxy).catch(async (error) => { + // End the run, or it stays open until the service times it out. + await callControl(run.controlUrl, run.token, "DELETE", "").catch(() => {}); + throw error; + }); return { ...run, ...connectRoach(run), env: proxyEnv.env, async close() { try { - return await callControl(run, "DELETE", ""); + return await callControl( + run.controlUrl, + run.token, + "DELETE", + "", + ); } finally { await proxyEnv.remove(); } @@ -238,15 +159,25 @@ export async function startRemoteRun( }; } -/** The proxy variables of a process, with the CA certificate in a file. */ +/** + * The proxy variables of a process, with the CA certificate in a file. The + * file also has the certificates of `NODE_EXTRA_CA_CERTS` of this process, + * so the process still trusts them. + */ async function createProxyEnv( - address: Pick, + address: Pick, noProxy: string, ): Promise<{ env: Record; remove(): Promise }> { // `NODE_EXTRA_CA_CERTS` takes a file. const caDirectory = await mkdtemp(path.join(tmpdir(), "roach-")); const caFile = path.join(caDirectory, "ca.pem"); - await writeFile(caFile, address.caCert); + const extra = process.env.NODE_EXTRA_CA_CERTS; + await writeFile( + caFile, + extra + ? `${address.caCert.trimEnd()}\n${await readFile(extra, "utf8")}` + : address.caCert, + ); return { env: { HTTP_PROXY: address.url, diff --git a/src/recorder.ts b/src/recorder.ts index 4781479..d876c9e 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -14,13 +14,10 @@ * its session ended follows the result of that session. A request outside * a session is written at once. * - * The recorder never writes a known credential (`secrets.ts`). It redacts - * them in recordings and miss files. + * The recorder never writes a known credential (`secrets.ts`). */ -import { mkdir, writeFile } from "node:fs/promises"; import type { IncomingHttpHeaders } from "node:http"; -import path from "node:path"; -import { describeParts, keyRequest, type KeyedRequest } from "./request-key.ts"; +import { keyRequest, type KeyedRequest } from "./request-key.ts"; import { createSecrets } from "./secrets.ts"; import { recordingKey, @@ -35,7 +32,7 @@ import { } from "./streams.ts"; import type { RecordingMiss, - RoachConfig, + RecordingMode, RecordingRule, RecordingStats, } from "./types.ts"; @@ -63,8 +60,6 @@ interface Session { name: string; /** New recordings, by key. */ recorded: Map; - /** Keys that the session replayed. They stay used if it fails. */ - replayed: Set; /** Requests that `replay` mode failed. */ missed: number; /** @@ -123,11 +118,7 @@ function toRecording( } return { session, - request: { - method: request.method, - url: request.url.href, - parts: keyed.parts, - }, + request: { method: request.method, url: request.url.href }, response: { status: response.status, headers, body, bodyEncoding }, }; } @@ -145,26 +136,23 @@ function fromRecording( return { status: response.status, headers: response.headers, body }; } -/** The parts of a config that the recorder uses. */ -export type RecorderConfig = Pick< - RoachConfig, - "missDirectory" | "mode" | "rules" | "secrets" | "usedFile" ->; +/** The config of a recorder. */ +export interface RecorderConfig { + mode: RecordingMode; + rules: RecordingRule[]; + /** Credentials to redact, in addition to the ones that it learns. */ + secrets?: string[]; +} -/** Create the recorder of one proxy run, with the store of its recordings. */ +/** Create the recorder of one run, with the store of its recordings. */ export function createRecorder(config: RecorderConfig, store: RecordingStore) { for (const rule of config.rules) { if (!RULE_NAME.test(rule.name)) { throw new Error(`Roach rule name is not valid: ${rule.name}`); } } - const missDirectory = config.missDirectory - ? path.resolve(config.missDirectory) - : undefined; let session: Session | undefined; const secrets = createSecrets(config.secrets); - /** Recordings that passed sessions, or requests outside one, used. */ - const used = new Set(); const stats: RecordingStats = { counts: Object.fromEntries( config.rules.map((rule) => [ @@ -181,11 +169,6 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { stats.written += await store.write(recordings); }; - const markReplayed = (owner: Session | undefined, key: string) => { - if (owner && owner.passed === undefined) owner.replayed.add(key); - else used.add(key); - }; - const record = async ( owner: Session | undefined, key: string, @@ -196,41 +179,19 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { } else if (owner?.passed === false) { stats.discarded += 1; } else { - used.add(key); await write([[key, recording]]); } }; - const reportMiss = async ( - rule: RecordingRule, - key: string, - keyed: KeyedRequest, - owner: Session | undefined, - ) => { - const closest = await store.closest?.(rule.name, keyed.parts, owner?.name); + const reportMiss = (rule: RecordingRule, key: string, owner?: Session) => { const miss: RecordingMiss = { rule: rule.name, session: owner?.name, file: key, - closest: closest?.key, - differs: closest?.differs ?? [], }; if (stats.misses.length < MAX_MISSES) stats.misses.push(miss); const where = owner ? ` in "${owner.name}"` : ""; - const why = closest - ? `closest is ${miss.closest}, which differs at ${describeParts(miss.differs)}` - : "no recording to compare"; - process.stderr.write(`[roach] No ${rule.name} recording${where}: ${why}\n`); - if (missDirectory) { - const target = path.join(missDirectory, miss.file); - await mkdir(path.dirname(target), { recursive: true }); - await writeFile( - target, - secrets.redact( - `${JSON.stringify({ ...miss, request: keyed.normalized }, null, 2)}\n`, - ), - ); - } + process.stderr.write(`[roach] No ${rule.name} recording${where}: ${key}\n`); }; /** End the open session. Returns its requests that `replay` mode failed. */ @@ -239,20 +200,8 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { if (!ended) return 0; session = undefined; ended.passed = passed; - // A failed test can still show that a recording is in use. - for (const key of ended.replayed) used.add(key); - if (passed) { - for (const key of ended.recorded.keys()) used.add(key); - await write([...ended.recorded]); - } else { - stats.discarded += ended.recorded.size; - // A failed test stops early, so it does not replay all of its - // recordings. Keep every recording that it made before, so that a - // prune does not delete them. - for (const key of (await store.keysOf?.(ended.name)) ?? []) { - used.add(key); - } - } + if (passed) await write([...ended.recorded]); + else stats.discarded += ended.recorded.size; return ended.missed; }; @@ -306,13 +255,12 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { const recording = await store.read(key); if (recording) { counts.replayed += 1; - markReplayed(owner, key); return { ...fromRecording(recording, keyed.values), source: "replayed", }; } - await reportMiss(rule, key, keyed, owner); + reportMiss(rule, key, owner); if (config.mode === "replay") { counts.missed += 1; if (owner) owner.missed += 1; @@ -354,7 +302,6 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { session = { name, recorded: new Map(), - replayed: new Set(), missed: 0, known: new Map(), }; @@ -375,19 +322,12 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { return stats; }, - /** Fail the open session, and write `usedFile` of the config. */ + /** Fail the open session. */ async close(): Promise { await finish(false); - if (config.usedFile) { - const keys = [...used].toSorted(); - await writeFile( - config.usedFile, - keys.map((key) => `${key}\n`).join(""), - ); - } }, }; } -/** The recorder of one proxy run. */ +/** The recorder of one run. */ export type Recorder = ReturnType; diff --git a/src/recordings.ts b/src/recordings.ts index 1d5341f..9e01f67 100644 --- a/src/recordings.ts +++ b/src/recordings.ts @@ -1,53 +1,24 @@ /** - * The file store, and prune. - * - * The file store keeps each recording (`store.ts`) in `/`, - * so recordings can be committed. + * The file store. It keeps each recording (`store.ts`) in + * `/`. The service uses it when its config has + * `directory`, such as in tests. */ -import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; -import { closestRequest } from "./request-key.ts"; import { formatRecording, type Recording, type RecordingStore, - type RequestParts, } from "./store.ts"; -const isMissing = (error: unknown) => - (error as NodeJS.ErrnoException).code === "ENOENT"; - const readText = (file: string) => readFile(file, "utf8").catch((error: unknown) => { - if (isMissing(error)) return undefined; + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; }); -/** The rule of a key, `/.json`. */ -const ruleOf = (key: string) => key.slice(0, key.indexOf("/")); - -interface IndexEntry { - key: string; - session?: string | undefined; - parts: RequestParts; -} - -/** - * The store of the recordings in `directory`. To find the closest - * recording of a miss, it reads all recordings of the rule on the first - * miss of that rule, and then keeps them in memory. - */ +/** The store of the recordings in `directory`. */ export function createFileStore(directory: string): RecordingStore { - const indexes = new Map>>(); - const indexOf = (rule: string) => { - let index = indexes.get(rule); - if (!index) { - index = loadIndex(directory, rule); - indexes.set(rule, index); - } - return index; - }; - return { async read(key) { const text = await readText(path.join(directory, key)); @@ -59,10 +30,6 @@ export function createFileStore(directory: string): RecordingStore { recordings.map(async ([key, recording]) => { const file = path.join(directory, key); const content = formatRecording(recording); - // Keep a loaded index current, so later misses compare with it. - if (indexes.has(ruleOf(key))) { - (await indexOf(ruleOf(key))).set(key, entryOf(key, recording)); - } if ((await readText(file)) === content) return false; await mkdir(path.dirname(file), { recursive: true }); await writeFile(file, content); @@ -71,87 +38,5 @@ export function createFileStore(directory: string): RecordingStore { ); return changed.filter(Boolean).length; }, - - async closest(rule, parts, session) { - const all = [...(await indexOf(rule)).values()]; - const same = all.filter((entry) => entry.session === session); - const found = closestRequest( - parts, - session !== undefined && same.length > 0 ? same : all, - ); - return found && { key: found.candidate.key, differs: found.differs }; - }, - - async keysOf(session) { - const entries = await readdir(directory, { withFileTypes: true }).catch( - (error: unknown) => { - if (isMissing(error)) return []; - throw error; - }, - ); - const keys: string[] = []; - // Each rule is a directory. Skip other files, such as `.DS_Store`. - for (const rule of entries.filter((entry) => entry.isDirectory())) { - for (const entry of (await indexOf(rule.name)).values()) { - if (entry.session === session) keys.push(entry.key); - } - } - return keys; - }, }; } - -const entryOf = (key: string, recording: Recording): IndexEntry => ({ - key, - session: recording.session, - parts: recording.request.parts, -}); - -/** Read every recording of one rule, by key. */ -async function loadIndex( - directory: string, - rule: string, -): Promise> { - const names = await readdir(path.join(directory, rule)).catch( - (error: unknown) => { - if (isMissing(error)) return []; - throw error; - }, - ); - const index = new Map(); - for (const name of names.filter((entry) => entry.endsWith(".json"))) { - const key = `${rule}/${name}`; - const text = await readText(path.join(directory, key)); - if (text !== undefined) { - index.set(key, entryOf(key, JSON.parse(text) as Recording)); - } - } - return index; -} - -/** - * Delete the recordings in `directory` that no used file lists. Each used - * file comes from `usedFile` of one proxy run. Returns how many it deleted. - * Give it the used files of every run that shares the directory, or it - * deletes recordings that another run needs. - */ -export async function pruneRecordings( - directory: string, - usedFiles: string[], -): Promise { - const used = new Set(); - for (const file of usedFiles) { - for (const line of (await readFile(file, "utf8")).split("\n")) { - if (line) used.add(line); - } - } - const recordings = (await readdir(directory, { recursive: true })).filter( - (file) => file.endsWith(".json"), - ); - // Used files list keys, which use `/`. On Windows, `readdir` uses `\`. - const unused = recordings.filter( - (file) => !used.has(file.split(path.sep).join("/")), - ); - await Promise.all(unused.map((file) => rm(path.join(directory, file)))); - return unused.length; -} diff --git a/src/report.ts b/src/report.ts index 52b8a78..1fd288b 100644 --- a/src/report.ts +++ b/src/report.ts @@ -1,7 +1,6 @@ /** * Text reports of a Roach run, for logs and CI summaries. */ -import { describeParts } from "./request-key.ts"; import type { RecordingMiss, RecordingStats } from "./types.ts"; /** Describe the totals of a run in one line. */ @@ -33,11 +32,5 @@ export function describeRecordingMisses(misses: RecordingMiss[]): string[] { seen.add(miss.session); return first; }) - .map((miss) => { - const where = miss.session ?? "outside a test"; - const why = miss.closest - ? `differs from ${miss.closest} at ${describeParts(miss.differs)}` - : "no recording to compare"; - return `${where}: ${miss.rule} ${miss.file} ${why}`; - }); + .map((miss) => `${miss.session ?? "outside a test"}: ${miss.file}`); } diff --git a/src/request-key.ts b/src/request-key.ts index 27e0587..6106296 100644 --- a/src/request-key.ts +++ b/src/request-key.ts @@ -1,23 +1,14 @@ /** - * The key of a recorded request, and the parts that miss diagnosis - * compares. + * The key of a recorded request. * * The key is the hash of the rule, the method, the URL, the key headers, * and the body. A JSON body has sorted object keys, so key order does not * matter. Each changing value of the rule is `<>` in the key * (`values.ts`). - * - * A recording also keeps a short hash of each part of its request: the - * method, the URL, each key header, each top-level field of a JSON body, - * and each item of a top-level array, such as `messages[3]`. When a request - * has no recording, the file store finds the recording with the most equal - * parts and reports the parts that differ. This only helps a person debug a - * miss. It never makes a replay. */ import { createHash } from "node:crypto"; import type { IncomingHttpHeaders } from "node:http"; import { THINKING_BLOCK_TYPES } from "./streams.ts"; -import type { RequestParts } from "./store.ts"; import type { RecordingRule } from "./types.ts"; import { extractValues, @@ -33,19 +24,11 @@ const KEY_VERSION = "http-v1"; export interface KeyedRequest { /** The name of the recording file, without `.json`. */ key: string; - parts: RequestParts; /** * The changing values of the request outside thinking blocks, in their * order. They fill the placeholders of a replayed response. */ values: RequestValues; - /** The request that the key hashes. Miss files show it. */ - normalized: { - method: string; - url: string; - headers: Record; - body: unknown; - }; } /** JSON with sorted object keys, so equal bodies give equal keys. */ @@ -115,36 +98,8 @@ function normalizeJson( const sha256 = (text: string) => createHash("sha256").update(text).digest("hex"); -const shortHash = (text: string) => sha256(text).slice(0, 12); - -function bodyParts( - json: unknown, - normalizedBody: string, - patterns: ValuePatterns | undefined, - known: KnownValues | undefined, -): RequestParts { - if (json === null || typeof json !== "object" || Array.isArray(json)) { - return normalizedBody ? { body: shortHash(normalizedBody) } : {}; - } - const part = (value: unknown) => - shortHash(normalizeJson(value, patterns, known).text); - const parts: RequestParts = {}; - for (const [field, value] of Object.entries(json)) { - if (value === undefined) continue; - if (Array.isArray(value)) { - parts[`${field}.length`] = shortHash(String(value.length)); - value.forEach((item, index) => { - parts[`${field}[${index}]`] = part(item); - }); - } else { - parts[field] = part(value); - } - } - return parts; -} - /** - * The key, the parts, and the changing values of a request. `known` holds + * The key and the changing values of a request. `known` holds * the values of earlier requests in the same session (`values.ts`). */ export function keyRequest( @@ -178,60 +133,5 @@ export function keyRequest( text, ].join("\n"), ); - const parts: RequestParts = { - method: shortHash(request.method), - url: shortHash(request.url), - }; - for (const [name, value] of Object.entries(headers)) { - parts[`header ${name}`] = shortHash(value); - } - Object.assign(parts, bodyParts(json, text, rule.values, known)); - return { - key, - parts, - values, - normalized: { - method: request.method, - url: request.url, - headers, - // A placeholder for a number makes the JSON invalid. Keep its text. - body: parseJson(text) ?? text, - }, - }; -} - -const partOrder = new Intl.Collator("en", { numeric: true }).compare; - -/** - * The candidate with the most equal parts, and the parts that differ from - * it, in a readable order. Returns `undefined` when there is no candidate. - */ -export function closestRequest( - parts: RequestParts, - candidates: Iterable, -): { candidate: T; differs: string[] } | undefined { - let best: T | undefined; - let bestEqual = -1; - for (const candidate of candidates) { - const equal = Object.keys(parts).filter( - (name) => candidate.parts[name] === parts[name], - ).length; - if (equal > bestEqual) { - bestEqual = equal; - best = candidate; - } - } - if (!best) return undefined; - const names = new Set([...Object.keys(parts), ...Object.keys(best.parts)]); - const differs = [...names] - .filter((name) => parts[name] !== best.parts[name]) - .toSorted(partOrder); - return { candidate: best, differs }; -} - -/** Name some parts, such as `messages[3], tools`. */ -export function describeParts(parts: string[]): string { - if (parts.length === 0) return "no part"; - const shown = parts.slice(0, 6).join(", "); - return parts.length > 6 ? `${shown} and ${parts.length - 6} more` : shown; + return { key, values }; } diff --git a/src/secrets.ts b/src/secrets.ts index 1f369e1..c5c4ae7 100644 --- a/src/secrets.ts +++ b/src/secrets.ts @@ -3,7 +3,7 @@ * * The proxy learns the values of credential headers from each request that * it sees, and the config can add more values. It replaces each known value - * with `<>` before it writes a recording or a miss file. + * with `<>` before it writes a recording. */ import type { IncomingHttpHeaders } from "node:http"; diff --git a/src/server.ts b/src/server.ts index e5a00cb..c46b12b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,5 +1,5 @@ /** - * The server of Roach. See `README.md`. + * The proxy sockets of the service (`service.ts`). * * The proxy is a forward proxy. It intercepts HTTPS with its own * certificate authority (`certificates.ts`). It gives each request that a @@ -14,33 +14,17 @@ * `missed` (`replay` mode had no recording), or `passthrough` (no rule * matched). */ -import { randomBytes, timingSafeEqual } from "node:crypto"; +import { timingSafeEqual } from "node:crypto"; import http from "node:http"; import https from "node:https"; import { isIP, type Socket } from "node:net"; import tls from "node:tls"; -import { - createCertificateAuthority, - type CertificateAuthority, -} from "./certificates.ts"; -import { - createRecorder, - type ProxyRequest, - type Recorder, -} from "./recorder.ts"; -import path from "node:path"; -import { createFileStore } from "./recordings.ts"; -import type { RoachAddress, RoachConfig } from "./types.ts"; +import type { CertificateAuthority } from "./certificates.ts"; +import type { ProxyRequest, Recorder } from "./recorder.ts"; /** The path prefix of the control API. */ export const CONTROL_PATH = "/__roach"; -/** A proxy that runs in this process. Most callers use `client.ts`. */ -export interface RoachServer extends RoachAddress { - /** Stop the proxy. A session that is still open fails. */ - close(): Promise; -} - /** Headers for one connection, which the proxy must not forward. */ const HOP_HEADERS = new Set([ "connection", @@ -294,29 +278,6 @@ export async function controlRecorder( } } -/** Answer one request to the control API of a local proxy. */ -async function control( - recorder: Recorder, - token: string, - incoming: http.IncomingMessage, - outgoing: http.ServerResponse, -): Promise { - if (!sameToken(incoming.headers.authorization, `Bearer ${token}`)) { - outgoing.writeHead(401).end(); - return; - } - const pathname = new URL(incoming.url ?? "/", "http://proxy").pathname; - const handled = - pathname.startsWith(`${CONTROL_PATH}/`) && - (await controlRecorder( - recorder, - `${incoming.method} ${pathname.slice(CONTROL_PATH.length)}`, - incoming, - outgoing, - )); - if (!handled) outgoing.writeHead(404).end(); -} - /** Where a proxied request goes: the allowed origins and the recorder. */ export interface ProxyTarget { origins: Map; @@ -399,8 +360,8 @@ function serveProxied( /** * Listen for proxied requests and control requests on one port. The - * caller decides the target of each proxied request, so one port can serve - * one run (`startRoach()`) or many runs (`service.ts`). + * caller decides the target of each proxied request, so one port serves + * many runs. */ export async function listenProxy( options: ListenOptions, @@ -529,44 +490,3 @@ export async function listenProxy( }, }; } - -/** Start the proxy in this process, on a free port of `127.0.0.1`. */ -export async function startRoach(config: RoachConfig): Promise { - if ( - config.missDirectory && - !path - .relative(config.directory, path.resolve(config.missDirectory)) - .startsWith("..") - ) { - // Miss files hold request bodies. Keep them out of the committed files. - throw new Error("missDirectory must not be inside directory"); - } - const target: ProxyTarget = { - origins: parseOrigins(config.allow), - recorder: createRecorder(config, createFileStore(config.directory)), - }; - const authority = await createCertificateAuthority(); - const token = randomBytes(24).toString("hex"); - // Only local processes reach this port, so proxied requests need no - // credentials. - const listening = await listenProxy({ - host: "127.0.0.1", - port: 0, - authority, - targetFor: () => target, - control: (incoming, outgoing) => - control(target.recorder, token, incoming, outgoing), - }); - - return { - url: `http://127.0.0.1:${listening.port}`, - token, - caCert: authority.caCert, - async close() { - target.closed = true; - await listening.close(); - await target.recorder.close(); - await authority.close(); - }, - }; -} diff --git a/src/service.ts b/src/service.ts index de1b141..d8be271 100644 --- a/src/service.ts +++ b/src/service.ts @@ -5,7 +5,7 @@ * GitHub repository, named `owner/repo`, such as `getsentry/junior`. The * service has no list of tenants: a run names its tenant. A run is one test * run of a tenant, such as one CI job. It has its own mode, rules, - * sessions, and stats, as one local proxy (`server.ts`) has. + * sessions, and stats. * * - Anyone can create a `replay` run of a tenant, so CI jobs of forks * replay recordings without a secret. Such a run never writes. diff --git a/src/store.ts b/src/store.ts index f287d74..9ad440c 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,23 +1,19 @@ /** * Recordings, and the contract of a store that keeps them. * - * A recording keeps the response of one request, the session (test) that - * recorded it, and the parts of the request (`request-key.ts`). It does not keep - * the request body, so prompts and other inputs are not stored. + * A recording keeps the response of one request, and the session (test) + * that recorded it. It does not keep the request body, so prompts and other + * inputs are not stored. * - * A store keeps recordings by key, `/.json`. The file store - * (`recordings.ts`) keeps them in a directory that can be committed. The - * GCS store (`gcs.ts`) keeps them in a bucket, for the shared service - * (`service.ts`). + * A store keeps recordings by key, `/.json`. The GCS store + * (`gcs.ts`) keeps them in a bucket. The file store (`recordings.ts`) keeps + * them in a directory, for tests and development. */ -/** A short hash of each part of a request, by part name. */ -export type RequestParts = Record; - /** One recorded response. */ export interface Recording { /** The session (test) that recorded the response. */ session?: string | undefined; - request: { method: string; url: string; parts: RequestParts }; + request: { method: string; url: string }; response: { status: number; headers: Record; @@ -31,34 +27,12 @@ export interface Recording { }; } -/** The recording with the most equal parts, and the parts that differ. */ -interface ClosestRecording { - key: string; - differs: string[]; -} - -/** Where the recordings of a proxy are. */ +/** Where the recordings of a tenant are. */ export interface RecordingStore { /** The recording of `key`, or `undefined` when there is none. */ read(key: string): Promise; /** Write recordings by key. Returns how many were new or changed. */ write(recordings: Array<[string, Recording]>): Promise; - /** - * The recording of `rule` with the most equal parts. Recordings of the - * same session come first, because a test usually sends the same - * requests as the last time it ran. Only the file store has this: it is a - * hint for debugging a miss, and a bucket has no index for it. - */ - closest?( - rule: string, - parts: RequestParts, - session: string | undefined, - ): Promise; - /** - * The keys that `session` recorded. Only the file store has this, - * because only it supports `usedFile` and prune. - */ - keysOf?(session: string): Promise; } /** A rule name that a store accepts. It is a directory name. */ diff --git a/src/types.ts b/src/types.ts index 8de368b..d46cd66 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,6 @@ /** - * The public types of Roach: its configuration and its - * results. See `README.md`. + * The public types of Roach: rules, modes, and the results of a run. See + * `README.md`. */ import type { ValuePatterns } from "./values.ts"; @@ -42,42 +42,6 @@ export interface RecordingRule { values?: ValuePatterns; } -/** - * The configuration of one local proxy (`server.ts`). The shared service - * (`service.ts`) has its own configuration. - */ -export interface RoachConfig { - /** The directory of the recordings. Each rule has a subdirectory. */ - directory: string; - mode: RecordingMode; - /** - * The only origins that the proxy sends requests to, such as - * `https://ai-gateway.vercel.sh`. The proxy refuses all other origins. - */ - allow: string[]; - rules: RecordingRule[]; - /** - * When the proxy stops, it lists here the recordings that sessions used. - * A failed session uses all recordings that it recorded before. Give these - * files to the `prune` command. - */ - usedFile?: string; - /** - * For each miss, the proxy writes the request as the key sees it to - * `//.json`. These files contain request - * bodies, such as prompts, so do not commit them. It must not be inside - * `directory`. - */ - missDirectory?: string; - /** - * Credentials that the proxy must never write, such as API keys. The - * proxy also learns the values of headers that can carry credentials, - * such as `authorization`, from each request. It redacts them in - * recordings and miss files. - */ - secrets?: string[]; -} - /** A request that a rule matched, but that had no recording. */ export interface RecordingMiss { rule: string; @@ -85,13 +49,9 @@ export interface RecordingMiss { session?: string | undefined; /** The key of the recording that the request needed. */ file: string; - /** The key of the recording with the most equal parts. */ - closest?: string | undefined; - /** The parts that differ from `closest`, such as `messages[3]`. */ - differs: string[]; } -/** The totals of a proxy run. */ +/** The totals of a run. */ export interface RecordingStats { /** * Requests by rule name. `missed` counts the requests that `replay` mode @@ -110,15 +70,3 @@ export interface RecordingStats { */ passthrough: Record; } - -/** The address of a running proxy. Give it to the processes that use it. */ -export interface RoachAddress { - /** The proxy URL, such as `http://127.0.0.1:1234`. */ - url: string; - /** The bearer token of the control API. */ - token: string; - /** The base URL of the control API. Default: `/__roach`. */ - controlUrl?: string; - /** The PEM certificate of the authority that signs intercepted hosts. */ - caCert: string; -} diff --git a/tests/ci-job.ts b/tests/ci-job.ts deleted file mode 100644 index f40d11c..0000000 --- a/tests/ci-job.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * One CI job that uses a deployed Roach service. `deployed.test.ts` runs it - * in its own process, because it must trust the TLS certificate of the - * service at startup, as a CI job trusts a public one. - * - * It starts a run, then runs a test process that has only the proxy - * variables of the run. That process sends one HTTPS request with plain - * `fetch`. The job prints the response, the stats, and the CA certificate - * as one JSON line. - * - * Argument: JSON `{ service, token?, config, url, body }`. - */ -import { execFile } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; -import { promisify } from "node:util"; -import { startRemoteRun } from "../src/client.ts"; -import type { RunConfig } from "../src/service.ts"; - -const { service, token, config, url, body } = JSON.parse(process.argv[2]!) as { - service: string; - token?: string; - config: RunConfig; - url: string; - body: string; -}; - -const run = await startRemoteRun({ url: service, token }, config); -// The test process trusts the CA of Roach and the TLS certificate of the -// service, as this job does. -const caFile = run.env.NODE_EXTRA_CA_CERTS!; -await writeFile( - caFile, - `${await readFile(caFile, "utf8")}${await readFile(process.env.NODE_EXTRA_CA_CERTS!, "utf8")}`, -); -const session = await run.startSession("test"); -const script = ` -const response = await fetch(${JSON.stringify(url)}, { method: "POST", body: ${JSON.stringify(body)} }); -console.log(JSON.stringify({ status: response.status, source: response.headers.get("x-roach"), body: await response.text() })); -`; -const { stdout } = await promisify(execFile)( - process.execPath, - ["--input-type=module", "--eval", script], - { env: { ...process.env, ...run.env } }, -); -await session.end(true); -const stats = await run.close(); -process.stdout.write( - `${JSON.stringify({ response: JSON.parse(stdout), stats, caCert: run.caCert })}\n`, -); diff --git a/tests/deployed.test.ts b/tests/deployed.test.ts index d20b98a..1ba5b1a 100644 --- a/tests/deployed.test.ts +++ b/tests/deployed.test.ts @@ -7,8 +7,8 @@ * TLS and passes the bytes to the service. * - A local server plays the GCS JSON API and the metadata server of the * VM. Another one plays Sentry. - * - Each CI job (`ci-job.ts`) sends HTTPS through the service with only the - * proxy variables of its run. + * - Each CI job runs the GitHub Action (`src/action.ts`). Its command sends + * HTTPS through the service with only the proxy variables of its run. */ import { execFile, spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; @@ -22,7 +22,7 @@ import path from "node:path"; import tls from "node:tls"; import { promisify } from "node:util"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import type { RoachServiceConfig, RunConfig } from "../src/service.ts"; +import type { RoachServiceConfig } from "../src/service.ts"; const execFileAsync = promisify(execFile); const ROOT = path.join(import.meta.dirname, ".."); @@ -76,34 +76,73 @@ function readEnvelope(envelope: string) { } } -/** Run one CI job, and return what it printed. */ -async function ciJob(token: string | undefined, mode: RunConfig["mode"]) { - const config: RunConfig = { - tenant: TENANT, - mode, - name: `${mode}-run`, - rules: [{ name: "model", match: { method: "POST" } }], - }; - const { stdout } = await execFileAsync( +/** + * Run one CI job: the action (`src/action.ts`) with a command that sends + * one HTTPS request with plain `fetch`. Returns the exit code of the + * action, what the command saw, and the job summary. + */ +async function ciJob(job: { + token?: string; + runId: string; + prompt: string; +}): Promise<{ + exitCode: number; + response: { status: number; source: string | null; body: string }; + token: string | null; + trusted: string; + summary: string; +}> { + const workspace = await mkdtemp(path.join(directory, "job-")); + const config = { rules: [{ name: "model", match: { method: "POST" } }] }; + await writeFile(path.join(workspace, "roach.json"), JSON.stringify(config)); + await writeFile( + path.join(workspace, "job.mjs"), + `import { readFileSync } from "node:fs"; +const response = await fetch(${JSON.stringify(`${upstreamOrigin}/v1/messages`)}, { + method: "POST", + body: JSON.stringify({ prompt: ${JSON.stringify(job.prompt)} }), +}); +console.log(JSON.stringify({ + response: { status: response.status, source: response.headers.get("x-roach"), body: await response.text() }, + token: process.env.INPUT_TOKEN ?? null, + trusted: readFileSync(process.env.NODE_EXTRA_CA_CERTS, "utf8"), +})); +`, + ); + const summary = path.join(workspace, "summary.md"); + const action = spawn( process.execPath, [ "--experimental-strip-types", "--disable-warning=ExperimentalWarning", - path.join(ROOT, "tests/ci-job.ts"), - JSON.stringify({ - service: frontUrl, - token, - config, - url: `${upstreamOrigin}/v1/messages`, - body: JSON.stringify({ prompt: "hi" }), - }), + path.join(ROOT, "src/action.ts"), ], - { env: childEnv({ NODE_EXTRA_CA_CERTS: hostCertFile }) }, + { + cwd: workspace, + env: childEnv({ + // The action trusts the TLS certificate of the service, as a CI job + // trusts a public one. + NODE_EXTRA_CA_CERTS: hostCertFile, + GITHUB_REPOSITORY: TENANT, + GITHUB_RUN_ID: job.runId, + GITHUB_RUN_ATTEMPT: "1", + GITHUB_JOB: "test", + GITHUB_STEP_SUMMARY: summary, + INPUT_RUN: "node job.mjs > result.json", + INPUT_URL: frontUrl, + INPUT_TOKEN: job.token ?? "", + }), + stdio: ["ignore", "ignore", "inherit"], + }, + ); + const [exitCode] = (await once(action, "exit")) as [number]; + const result = JSON.parse( + await readFile(path.join(workspace, "result.json"), "utf8"), ); - return JSON.parse(stdout) as { - response: { status: number; source: string; body: string }; - stats: { written: number }; - caCert: string; + return { + exitCode, + ...result, + summary: await readFile(summary, "utf8"), }; } @@ -280,20 +319,29 @@ afterAll(async () => { describe("deployed roach", () => { it("records with the token, replays for a fork, and counts in Sentry", async () => { - const writer = await ciJob(TOKEN, "auto"); + const writer = await ciJob({ token: TOKEN, runId: "writer", prompt: "hi" }); + expect(writer.exitCode).toBe(0); expect(writer.response).toEqual({ status: 200, source: "live", body: JSON.stringify({ n: 1 }), }); - expect(writer.stats.written).toBe(1); - // A restart keeps the authority that clients trust. - expect(writer.caCert).toBe(caCert); + expect(writer.summary).toContain("1 recordings new or changed"); + // The command never sees the write token. + expect(writer.token).toBeNull(); + // It trusts the authority of the service, which a restart keeps, and + // still trusts the certificates that the job trusted. + expect(writer.trusted).toContain(caCert.trim()); + expect(writer.trusted).toContain( + (await readFile(hostCertFile, "utf8")).trim(), + ); expect([...objects.keys()]).toEqual([ expect.stringMatching(/^getsentry\/junior\/model\/[0-9a-f]{64}\.json$/), ]); - const fork = await ciJob(undefined, "replay"); + // Without the token, the action only replays. + const fork = await ciJob({ runId: "fork", prompt: "hi" }); + expect(fork.exitCode).toBe(0); expect(fork.response).toEqual({ status: 200, source: "replayed", @@ -301,10 +349,18 @@ describe("deployed roach", () => { }); expect(liveRequests).toBe(1); + // A request without a recording fails the step, and the summary names it. + const miss = await ciJob({ runId: "miss", prompt: "bye" }); + expect(miss.response.status).toBe(412); + expect(miss.exitCode).toBe(1); + expect(miss.summary).toMatch(/^- test: model\/[0-9a-f]{64}\.json$/m); + expect(liveRequests).toBe(1); + // The service sends its metrics when it stops. service.kill("SIGTERM"); expect(await once(service, "exit")).toEqual([0, null]); const [key] = objects.keys(); + const relativeKey = key!.slice(`${TENANT}/`.length); expect( metrics.map(({ tenant, run, result, key: metricKey }) => ({ tenant, @@ -313,23 +369,14 @@ describe("deployed roach", () => { key: metricKey, })), ).toEqual([ + { tenant: TENANT, run: "writer-1", result: "missed", key: relativeKey }, + { tenant: TENANT, run: "writer-1", result: "written", key: relativeKey }, + { tenant: TENANT, run: "fork-1", result: "replayed", key: relativeKey }, { tenant: TENANT, - run: "auto-run", + run: "miss-1", result: "missed", - key: key!.slice(`${TENANT}/`.length), - }, - { - tenant: TENANT, - run: "auto-run", - result: "written", - key: key!.slice(`${TENANT}/`.length), - }, - { - tenant: TENANT, - run: "replay-run", - result: "replayed", - key: key!.slice(`${TENANT}/`.length), + key: expect.stringMatching(/^model\//), }, ]); }); diff --git a/tests/roach.test.ts b/tests/recording.test.ts similarity index 63% rename from tests/roach.test.ts rename to tests/recording.test.ts index 355530a..c085243 100644 --- a/tests/roach.test.ts +++ b/tests/recording.test.ts @@ -1,5 +1,10 @@ +/** + * Recording and replay: modes, sessions, changing values, and misses. Each + * test starts runs on a service with a local upstream. + */ +import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; -import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { ProxyAgent, request } from "undici"; @@ -11,12 +16,14 @@ import { it, onTestFinished, } from "vitest"; -import { connectRoach } from "../src/client.ts"; -import { pruneRecordings } from "../src/recordings.ts"; -import { startRoach, type RoachServer } from "../src/server.ts"; -import type { RecordingMode, RoachConfig } from "../src/types.ts"; +import { startRemoteRun, type RemoteRoach } from "../src/client.ts"; +import { startRoachService, type RoachService } from "../src/service.ts"; +import type { RecordingMode } from "../src/types.ts"; import { VALUE_PATTERNS } from "../src/values.ts"; +const TOKEN = "write-token"; +const TENANT = "acme/alpha"; + let upstream: Server; let origin: string; let liveRequests: number; @@ -25,33 +32,35 @@ let respond: (body: string) => string; /** The upstream answers when this settles. */ let upstreamGate: Promise; let directory: string; -let proxy: RoachServer | undefined; +let service: RoachService; +const runs: RemoteRoach[] = []; let agent: ProxyAgent | undefined; -async function start( - mode: RecordingMode, - options: Pick = {}, -): Promise { - proxy = await startRoach({ - directory, - mode, - allow: [origin], - rules: [ - { - name: "model", - match: { method: "POST", url: `${origin}/v1/` }, - values: { - uuid: VALUE_PATTERNS.uuid, - time: VALUE_PATTERNS.isoTime, - commit: VALUE_PATTERNS.gitCommit, +/** Start a run with the write token. Later requests go through it. */ +async function start(mode: RecordingMode): Promise { + const run = await startRemoteRun( + { url: service.url, token: TOKEN }, + { + tenant: TENANT, + mode, + rules: [ + { + name: "model", + match: { method: "POST", url: `${origin}/v1/` }, + values: { + uuid: VALUE_PATTERNS.uuid, + time: VALUE_PATTERNS.isoTime, + commit: VALUE_PATTERNS.gitCommit, + }, }, - }, - ], - ...options, - }); + ], + }, + ); + runs.push(run); + await agent?.close(); // Tunnel plain HTTP too, so the tests use `CONNECT` as HTTPS clients do. - agent = new ProxyAgent({ uri: proxy.url, proxyTunnel: true }); - return proxy; + agent = new ProxyAgent({ uri: run.url, proxyTunnel: true }); + return run; } /** One delta event of an Anthropic Messages stream. */ @@ -76,15 +85,15 @@ async function send(bodies: unknown[], target = `${origin}/v1/messages`) { } /** Send the requests of one test session, then end it. */ -async function session(running: RoachServer, bodies: unknown[], passed = true) { - const opened = await connectRoach(running).startSession("test"); +async function session(running: RemoteRoach, bodies: unknown[], passed = true) { + const opened = await running.startSession("test"); const responses = await send(bodies); const { missed } = await opened.end(passed); return Object.assign(responses, { missed }); } async function files(): Promise { - return readdir(path.join(directory, "model")).catch(() => []); + return readdir(path.join(directory, TENANT, "model")).catch(() => []); } beforeEach(async () => { @@ -108,24 +117,28 @@ beforeEach(async () => { if (!address || typeof address === "string") throw new Error("No port"); origin = `http://127.0.0.1:${address.port}`; directory = await mkdtemp(path.join(tmpdir(), "roach-")); + service = await startRoachService({ + directory, + allow: [origin], + writeTokenHash: createHash("sha256").update(TOKEN).digest("hex"), + }); }); afterEach(async () => { await agent?.close(); agent = undefined; - await proxy?.close(); - proxy = undefined; + await Promise.all(runs.splice(0).map((run) => run.close().catch(() => {}))); + await service.close(); await new Promise((resolve) => upstream.close(() => resolve())); await rm(directory, { recursive: true, force: true }); }); -describe("roach", () => { +describe("recording", () => { it("replays a passed session for the same requests in auto mode", async () => { const running = await start("auto"); await session(running, [ { model: "m", messages: [{ content: "hi", at: "2026-10-07T03:18:03Z" }] }, ]); - const [first] = await files(); // Same request with other key order and another clock time. const replay = await session(running, [ @@ -141,18 +154,14 @@ describe("roach", () => { { body: "data: 2\n\n", source: "live" }, ]); expect(liveRequests).toBe(2); - await expect(connectRoach(running).stats()).resolves.toEqual({ + await expect(running.stats()).resolves.toEqual({ counts: { model: { live: 2, missed: 0, replayed: 1 } }, - // The miss names the closest recording of the test and the part - // of the request that differs from it. misses: [ expect.objectContaining({ session: "test" }), { rule: "model", session: "test", file: expect.stringMatching(/^model\/[0-9a-f]{64}\.json$/), - closest: `model/${first}`, - differs: ["messages[0]"], }, ], written: 2, @@ -208,7 +217,7 @@ describe("roach", () => { ]); const [file] = await files(); const recording = await readFile( - path.join(directory, "model", file!), + path.join(directory, TENANT, "model", file!), "utf8", ); expect(recording).toContain("<>"); @@ -261,50 +270,12 @@ describe("roach", () => { expect(replay.missed).toBe(1); expect(liveRequests).toBe(1); await expect(files()).resolves.toEqual(recorded); - await expect(connectRoach(running).stats()).resolves.toMatchObject({ + await expect(running.stats()).resolves.toMatchObject({ counts: { model: { live: 0, missed: 1, replayed: 1 } }, - misses: [{ closest: `model/${recorded[0]}`, differs: ["model"] }], + misses: [{ rule: "model", session: "test" }], }); }); - it("redacts credentials in recordings and miss files", async () => { - // Miss files must not be inside the recordings directory. - const missDirectory = `${directory}-misses`; - onTestFinished(() => rm(missDirectory, { recursive: true, force: true })); - const running = await start("auto", { - missDirectory, - // Each part of this value is short, so only the whole value matches. - secrets: ["cfg-secret=012345"], - }); - // The upstream repeats the credential of the request header. - respond = () => `data: key-from-header-0123456789\n\n`; - const opened = await connectRoach(running).startSession("test"); - const response = await request(`${origin}/v1/messages`, { - body: JSON.stringify({ model: "m", note: "cfg-secret=012345" }), - dispatcher: agent!, - // Any header whose name can mean a credential is learned. - headers: { "x-custom-auth": "Bearer key-from-header-0123456789" }, - method: "POST", - }); - await opened.end(true); - - expect(await response.body.text()).toBe("data: <>\n\n"); - const [recorded] = await files(); - const recording = await readFile( - path.join(directory, "model", recorded!), - "utf8", - ); - expect(recording).toContain("<>"); - expect(recording).not.toContain("key-from-header-0123456789"); - const [miss] = await readdir(path.join(missDirectory, "model")); - const text = await readFile( - path.join(missDirectory, "model", miss!), - "utf8", - ); - expect(text).toContain("<>"); - expect(text).not.toContain("cfg-secret=012345"); - }); - it("sends no request to another origin", async () => { await start("auto"); @@ -312,7 +283,7 @@ describe("roach", () => { // refuses it in a tunnel and as an absolute-form request. const other = `${origin.replace("127.0.0.1", "localhost")}/v1/messages`; await expect(send([{}], other)).rejects.toThrow("403"); - const absolute = new ProxyAgent({ uri: proxy!.url, proxyTunnel: false }); + const absolute = new ProxyAgent({ uri: runs[0]!.url, proxyTunnel: false }); onTestFinished(() => absolute.close()); const response = await request(other, { dispatcher: absolute }); await response.body.dump(); @@ -320,30 +291,11 @@ describe("roach", () => { expect(liveRequests).toBe(0); }); - it("refuses control requests without the token", async () => { - const running = await start("auto"); - - await expect( - connectRoach({ url: running.url, token: "wrong" }).stats(), - ).rejects.toThrow("HTTP 401"); - }); - - it("writes nothing for a failed session", async () => { - // A file next to the rule directories is not a rule. - await writeFile(path.join(directory, ".DS_Store"), ""); - const running = await start("auto"); - await session(running, [{ model: "m" }], false); - await session(running, [{ model: "m" }], false); - - expect(liveRequests).toBe(2); - await expect(files()).resolves.toEqual([]); - }); - it("writes a request that ends after its passed session", async () => { const running = await start("auto"); let open!: () => void; upstreamGate = new Promise((resolve) => (open = resolve)); - const opened = await connectRoach(running).startSession("test"); + const opened = await running.startSession("test"); const pending = send([{ model: "m" }]); // Wait until the request reaches the upstream, then end the test. // The upstream server changes `liveRequests`, not this loop. @@ -358,46 +310,6 @@ describe("roach", () => { expect(liveRequests).toBe(1); }); - it("lists used recordings and prunes the others", async () => { - const usedFile = path.join( - directory, - "..", - `${path.basename(directory)}.used`, - ); - const running = await start("auto", { usedFile }); - await session(running, [{ model: "m" }, { model: "n" }]); - const recorded = (await files()).toSorted(); - const contents = await Promise.all( - recorded.map((file) => readFile(path.join(directory, "model", file))), - ); - const stale = JSON.parse(contents[0]!.toString("utf8")); - await writeFile( - path.join(directory, "model", "stale.json"), - JSON.stringify({ ...stale, session: "removed test" }), - ); - - // A replay neither writes a file again nor makes it unused. The test - // fails before its second request, and its recordings stay in use. - await session(running, [{ model: "m" }], false); - await running.close(); - proxy = undefined; - - await expect(readFile(usedFile, "utf8")).resolves.toBe( - recorded.map((file) => `model/${file}\n`).join(""), - ); - await expect(pruneRecordings(directory, [usedFile])).resolves.toBe(1); - await expect(files().then((names) => names.toSorted())).resolves.toEqual( - recorded, - ); - await expect( - Promise.all( - recorded.map((file) => readFile(path.join(directory, "model", file))), - ), - ).resolves.toEqual(contents); - expect(liveRequests).toBe(2); - await rm(usedFile); - }); - it("records again in record mode", async () => { const running = await start("record"); await session(running, [{ model: "m" }]); diff --git a/tests/service.test.ts b/tests/service.test.ts index 3fbb54c..71a3573 100644 --- a/tests/service.test.ts +++ b/tests/service.test.ts @@ -5,7 +5,11 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { ProxyAgent, request } from "undici"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { startRemoteRun, type RemoteRoach } from "../src/client.ts"; +import { + connectRoach, + startRemoteRun, + type RemoteRoach, +} from "../src/client.ts"; import { startRoachService, type RoachService, @@ -203,6 +207,11 @@ describe("roach service", () => { const wrong = new URL(run.url); wrong.password = "not-the-token"; await expect(send(wrong.href, { prompt: "hi" })).rejects.toThrow(/407/); + // The control API of a run needs the run token. A wrong one gets the + // answer of a run that does not exist. + await expect( + connectRoach({ controlUrl: run.controlUrl, token: "wrong" }).stats(), + ).rejects.toThrow(/404/); // The tenant is a path in the store, so it must be owner/repo. for (const tenant of ["junior", "acme/..", "../acme", "a/b/c", "acme/"]) {