diff --git a/README.md b/README.md index 6ec6841..a91c048 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ import { VALUE_PATTERNS } from "@sentry/roach/values"; const run = await startRemoteRun( { url: "https://roach.example.com", token: process.env.ROACH_TOKEN }, { - tenant: "junior", + tenant: process.env.GITHUB_REPOSITORY!, // such as getsentry/junior mode: "auto", name: `${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT}`, rules: [ @@ -198,14 +198,20 @@ give this hint, because it would have to read every recording. `cli.ts service ` starts the service. The config is `RoachServiceConfig` in `src/service.ts`. `deploy/gcp/` writes it for you. -- **Tenants.** A tenant is a project, such as `junior`. Each tenant has a - write token. The config keeps only the SHA-256 of each token. The - recordings of a tenant are under `/` in the bucket. Tenants never - share recordings. +- **Tenants.** A tenant is a GitHub repository, as `owner/repo`, such as + `getsentry/junior`. In GitHub Actions, use `GITHUB_REPOSITORY`. The + service has no list of tenants, so a new project needs no change to the + service. The recordings of a tenant are under `//` in the + bucket. Tenants never share recordings. +- **One write token.** All tenants share one write token. The config keeps + only its SHA-256, in `writeTokenHash`. The tenant name is only a label, + not proof of identity: a job with the token can write the recordings of + any tenant. So give the token only to repositories and workflows that + you trust, and never to a workflow that runs code from a fork. - **Reads are public.** Anyone can start a `replay` run of a tenant, so CI jobs of forks replay without a secret. Such a run never writes, and it refuses a request that no rule records, so nothing goes live. Every - other mode needs the tenant token. So treat each recording as readable by + other mode needs the write token. So treat each recording as readable by anyone who can make its request. Do not record responses that must stay private. - **Runs.** Each run has its own id, token, mode, rules, sessions, and @@ -229,8 +235,9 @@ give this hint, because it would have to read every recording. - Runs live in memory in one process. A restart ends the open runs, and their CI jobs fail. A run that is open for 6 hours ends as failed. -- A tenant can have 50 open runs without a token. Another one gets HTTP - 429 until one ends. Runs with the token have no cap. +- The service can have 100 open runs without a token, for all tenants + together. Another one gets HTTP 429 until one ends. Runs with the token + have no cap. - A request body over 64 MiB gets HTTP 413. - GCS deletes each recording 30 days after it was written, also when runs still replay it. Then `auto` mode records it again, and `replay` mode @@ -242,7 +249,7 @@ give this hint, because it would have to read every recording. `deploy/gcp/` is the production setup, in Terraform. It makes: - a GCS bucket that deletes each recording after `recording_days` (30), -- the certificate authority and one write token for each tenant, +- the certificate authority and the write token, - the service config, in Secret Manager, - one Container-Optimized OS VM that runs the image `ghcr.io/getsentry/roach:main`, with a service account that can only use @@ -260,7 +267,7 @@ The `Image` workflow builds the image on each pull request and pushes it on 2. Copy `deploy/gcp/roach.tfvars.example` to `roach.tfvars` and fill it in. `allow` and `value_patterns` must cover the rules of every tenant. 3. Keep the Terraform state in a private bucket. It holds the CA key and the - tenant tokens. See the `backend "gcs"` comment in `versions.tf`. + write token. See the `backend "gcs"` comment in `versions.tf`. 4. Apply: ```sh @@ -271,15 +278,32 @@ The `Image` workflow builds the image on each pull request and pushes it on 5. Point an A record of `domain` at the `ip_address` output. The certificate works some minutes after DNS does. -6. Give each tenant its token as a CI secret, such as `ROACH_TOKEN`: - `terraform output -json tenant_tokens`. +6. Put the write token in one GitHub organization secret, `ROACH_TOKEN`. + Give it access only to the repositories that you trust. This command + does not show the token: + + ```sh + terraform output -raw write_token \ + | gh secret set ROACH_TOKEN --org getsentry --visibility selected \ + --repos getsentry/junior + ``` + 7. Check it: `curl https:///__roach/ca.pem` returns the CA certificate. To deploy a new image, restart the VM: -`gcloud compute instances reset roach --zone `. To add a tenant, -add it to `tenants` and apply again. Terraform writes a new config version, -and the VM reads it at its next start. +`gcloud compute instances reset roach --zone `. + +To add a project, add its repository to the repositories of the +`ROACH_TOKEN` organization secret, in the GitHub settings of the +organization. Its CI jobs set `tenant` to `GITHUB_REPOSITORY`. When the +project calls origins or uses value patterns that are not in +`roach.tfvars` yet, add them and apply again. Terraform writes a new config version, and the +VM reads it at its next start. + +To change the write token, run +`terraform apply -var-file=roach.tfvars -replace=random_password.write_token`, +set the secret again, and restart the VM. ## Command line @@ -310,7 +334,7 @@ the run token or the token of the local proxy. `client.ts` calls them. The service also has: - `POST /__roach/runs` with a `RunConfig`: start a run. Send - `Authorization: Bearer ` for any mode but `replay`. Returns + `Authorization: Bearer ` for any mode but `replay`. Returns the `id`, `token`, proxy `url`, `controlUrl`, and `caCert` of the run. - `DELETE /__roach/runs/`: end the run. Returns its stats. - `GET /__roach/ca.pem`: the CA certificate. No token. diff --git a/deploy/gcp/main.tf b/deploy/gcp/main.tf index 7cd97b3..0b2c309 100644 --- a/deploy/gcp/main.tf +++ b/deploy/gcp/main.tf @@ -58,11 +58,12 @@ resource "tls_self_signed_cert" "ca" { } } -# One write token for each tenant. A run without a token can only replay. -resource "random_password" "tenant" { - for_each = var.tenants - length = 48 - special = false +# The write token. All tenants share it, so a new project needs no change +# here. A run without it can only replay. To change it: +# terraform apply -replace=random_password.write_token +resource "random_password" "write_token" { + length = 48 + special = false } # The service config (`RoachServiceConfig` in src/service.ts). The VM reads @@ -79,13 +80,13 @@ resource "google_secret_manager_secret_version" "config" { secret = google_secret_manager_secret.config.id secret_data = jsonencode(merge( { - host = "0.0.0.0" - port = local.port - publicUrl = "https://${var.domain}" - bucket = google_storage_bucket.recordings.name - allow = var.allow - tenants = { for name, token in random_password.tenant : name => sha256(token.result) } - valuePatterns = var.value_patterns + host = "0.0.0.0" + port = local.port + publicUrl = "https://${var.domain}" + bucket = google_storage_bucket.recordings.name + allow = var.allow + writeTokenHash = sha256(random_password.write_token.result) + valuePatterns = var.value_patterns ca = { cert = tls_self_signed_cert.ca.cert_pem key = tls_private_key.ca.private_key_pem diff --git a/deploy/gcp/outputs.tf b/deploy/gcp/outputs.tf index 8038c04..c8ec12f 100644 --- a/deploy/gcp/outputs.tf +++ b/deploy/gcp/outputs.tf @@ -13,8 +13,8 @@ output "ca_cert" { value = tls_self_signed_cert.ca.cert_pem } -output "tenant_tokens" { - description = "The write token of each tenant. Keep each one in a CI secret of its project, such as ROACH_TOKEN." - value = { for name, token in random_password.tenant : name => token.result } +output "write_token" { + description = "The write token of all tenants. Keep it in one organization secret, ROACH_TOKEN, for the repositories that you trust." + value = random_password.write_token.result sensitive = true } diff --git a/deploy/gcp/roach.tfvars.example b/deploy/gcp/roach.tfvars.example index 9935360..84fc894 100644 --- a/deploy/gcp/roach.tfvars.example +++ b/deploy/gcp/roach.tfvars.example @@ -3,8 +3,6 @@ project = "my-gcp-project" domain = "roach.example.com" -tenants = ["junior"] - # The origins of Junior's evals (packages/junior-evals/src/recording-rules.ts). allow = [ "https://ai-gateway.vercel.sh", diff --git a/deploy/gcp/variables.tf b/deploy/gcp/variables.tf index 87ee166..aa8ba70 100644 --- a/deploy/gcp/variables.tf +++ b/deploy/gcp/variables.tf @@ -32,11 +32,6 @@ variable "machine_type" { default = "e2-small" } -variable "tenants" { - description = "The names of the projects that use Roach, such as junior. Terraform makes a write token for each one." - type = set(string) -} - variable "allow" { description = "The only origins that runs can reach, such as https://ai-gateway.vercel.sh." type = list(string) diff --git a/deploy/gcp/versions.tf b/deploy/gcp/versions.tf index e7bf85d..573c32c 100644 --- a/deploy/gcp/versions.tf +++ b/deploy/gcp/versions.tf @@ -14,7 +14,7 @@ terraform { version = "~> 4.4" } } - # The state holds the CA key and the tenant tokens. Keep it in a private + # The state holds the CA key and the write token. Keep it in a private # bucket, for example: # backend "gcs" { # bucket = "my-terraform-state" diff --git a/src/client.ts b/src/client.ts index e5b1193..07c7ef7 100644 --- a/src/client.ts +++ b/src/client.ts @@ -208,7 +208,7 @@ export interface RemoteRoach extends RemoteRun, RoachControl { /** * Start a run on a Roach service, such as `https://roach.example.com`. * - * `token` is the token of the tenant. Only this call uses it. Without it, + * `token` is the write token of the service. Only this call uses it. Without it, * the service only allows `replay` mode, so CI jobs of forks can replay * without a secret. The run and its workers use the run token. */ diff --git a/src/server.ts b/src/server.ts index be8b5cc..e5a00cb 100644 --- a/src/server.ts +++ b/src/server.ts @@ -323,7 +323,7 @@ export interface ProxyTarget { recorder: Recorder; /** * Refuse requests that no rule matches, so nothing goes live. The - * service sets this for a run without the tenant token. + * service sets this for a run without the write token. */ replayOnly?: boolean; /** Set when the target stops. Its tunnels then refuse new requests. */ diff --git a/src/service.ts b/src/service.ts index eb44f9b..de1b141 100644 --- a/src/service.ts +++ b/src/service.ts @@ -2,14 +2,17 @@ * Roach as a shared service. See "Service" in `README.md`. * * One process serves the runs of many tenants on one port. A tenant is a - * project, such as Junior. A run is one test run of a tenant, such as one - * CI job. It has its own mode, rules, sessions, and stats, as one local - * proxy (`server.ts`) has. + * GitHub repository, named `owner/repo`, such as `getsentry/junior`. The + * service has no list of tenants: a run names its tenant. A run is one test + * run of a tenant, such as one CI job. It has its own mode, rules, + * sessions, and stats, as one local proxy (`server.ts`) has. * * - Anyone can create a `replay` run of a tenant, so CI jobs of forks * replay recordings without a secret. Such a run never writes. - * - A run in another mode needs the token of the tenant, because it - * writes recordings. The service keeps only the SHA-256 of each token. + * - A run in another mode needs the write token, because it writes + * recordings. All tenants share one write token, and the service keeps + * only its SHA-256. The tenant name is a label, not proof of identity: + * a holder of the token can write the recordings of any tenant. * - The service gives each run its own id and token. A proxied request * names its run in `Proxy-Authorization`, as `Basic base64(:)`. * The proxy URL of the run has them, so `HTTPS_PROXY` sends them. @@ -63,10 +66,10 @@ export interface RoachServiceConfig { /** The only origins that runs can reach. A run can allow fewer. */ allow: string[]; /** - * The tenants, by name. Each value is the SHA-256 of the token of the - * tenant, in hex. + * The SHA-256 of the write token, in hex. All tenants share the token. + * Give it only to CI jobs that you trust. */ - tenants: Record; + writeTokenHash: string; /** * More regular expression sources that rules can use in `values`, in * addition to `VALUE_PATTERNS`. A slow pattern blocks all runs, so check @@ -87,7 +90,10 @@ export interface RoachServiceConfig { /** The configuration of one run. The client sends it. */ export interface RunConfig { - /** The tenant of the run. */ + /** + * The tenant of the run, as `owner/repo`, such as + * `process.env.GITHUB_REPOSITORY`. + */ tenant: string; mode: RecordingMode; rules: RecordingRule[]; @@ -124,7 +130,6 @@ export interface RoachService { interface Run extends ProxyTarget { id: string; - tenant: string; token: string; started: number; } @@ -133,27 +138,28 @@ const MODES = new Set(["auto", "off", "record", "replay"]); /** A run that is open longer than this ends as failed, such as a dead job. */ const RUN_TIMEOUT_MS = 6 * 60 * 60 * 1000; /** - * The open runs without a token that one tenant can have. Anyone can start - * them, so the cap keeps memory bounded. Runs with the token have no cap. + * The open runs without a token that the service can have, for all tenants. + * Anyone can start them with any tenant name, so the cap is not per tenant. + * It keeps memory bounded. Runs with the token have no cap. */ -const MAX_PUBLIC_RUNS = 50; +const MAX_PUBLIC_RUNS = 100; const MAX_RUN_NAME = 128; +/** + * A tenant name: `owner/repo`, with the characters of GitHub names. It is a + * path in the store, so each part starts with a letter, digit, or `_`, and + * there are always two parts. Then no tenant is in the directory of another. + */ +const TENANT_NAME = /^[A-Za-z0-9_][\w.-]{0,99}\/[A-Za-z0-9_][\w.-]{0,99}$/; const sha256 = (value: string) => createHash("sha256").update(value).digest(); -/** Check that `token` is the token of `tenant`, in constant time. */ -function isTenantToken( - tenants: Record, - tenant: string, +/** Check that `authorization` has the write token, in constant time. */ +function isWriteToken( + expected: Buffer, authorization: string | undefined, ): boolean { const token = /^Bearer (\S+)$/.exec(authorization ?? "")?.[1]; - const expected = Object.hasOwn(tenants, tenant) - ? Buffer.from(tenants[tenant]!, "hex") - : undefined; - if (!token || !expected) return false; - const actual = sha256(token); - return expected.length === actual.length && timingSafeEqual(expected, actual); + return token !== undefined && timingSafeEqual(expected, sha256(token)); } /** The run id and token in a `Proxy-Authorization` header. */ @@ -263,14 +269,10 @@ export async function startRoachService( if ((config.directory === undefined) === (config.bucket === undefined)) { throw new Error("Roach service config must set directory or bucket"); } - for (const [tenant, hash] of Object.entries(config.tenants)) { - if (!RULE_NAME.test(tenant)) { - throw new Error(`Roach tenant name must match ${RULE_NAME}: ${tenant}`); - } - if (!/^[0-9a-f]{64}$/i.test(hash)) { - throw new Error(`Roach tenant ${tenant} needs a SHA-256 token hash`); - } + if (!/^[0-9a-f]{64}$/i.test(config.writeTokenHash ?? "")) { + throw new Error("Roach service config needs writeTokenHash, a SHA-256"); } + const writeTokenHash = Buffer.from(config.writeTokenHash, "hex"); if (config.sentryDsn) { Sentry.init({ dsn: config.sentryDsn, tracesSampleRate: 0 }); } @@ -305,7 +307,6 @@ export async function startRoachService( ); const run: Run = { id, - tenant: runConfig.tenant, token, started: Date.now(), origins: parseOrigins(allow.map((origin) => origin.replace(/\/$/, ""))), @@ -361,11 +362,8 @@ export async function startRoachService( if (incoming.method === "POST" && pathname === `${CONTROL_PATH}/runs`) { const runConfig = await readJson>(incoming); const tenant = runConfig.tenant; - if ( - typeof tenant !== "string" || - !Object.hasOwn(config.tenants, tenant) - ) { - sendJson(outgoing, 400, { error: "tenant is not known" }); + if (typeof tenant !== "string" || !TENANT_NAME.test(tenant)) { + sendJson(outgoing, 400, { error: "tenant must be owner/repo" }); return; } const error = invalidRunConfig(runConfig, serviceOrigins, valuePatterns); @@ -373,26 +371,25 @@ export async function startRoachService( sendJson(outgoing, 400, { error }); return; } - const canWrite = isTenantToken( - config.tenants, - tenant, + const canWrite = isWriteToken( + writeTokenHash, incoming.headers.authorization, ); // Reads are public. Every other mode can write, so it needs the token. if (!canWrite && runConfig.mode !== "replay") { sendJson(outgoing, 401, { - error: `${runConfig.mode} mode needs the token of ${tenant}`, + error: `${runConfig.mode} mode needs the write token`, }); return; } if (!canWrite) { let open = 0; for (const other of runs.values()) { - if (other.replayOnly && other.tenant === tenant) open += 1; + if (other.replayOnly) open += 1; } if (open >= MAX_PUBLIC_RUNS) { sendJson(outgoing, 429, { - error: `${tenant} has ${MAX_PUBLIC_RUNS} open runs without a token`, + error: `The service has ${MAX_PUBLIC_RUNS} open runs without a token`, }); return; } diff --git a/tests/deployed.test.ts b/tests/deployed.test.ts index 2198cfa..d20b98a 100644 --- a/tests/deployed.test.ts +++ b/tests/deployed.test.ts @@ -26,7 +26,8 @@ import type { RoachServiceConfig, RunConfig } from "../src/service.ts"; const execFileAsync = promisify(execFile); const ROOT = path.join(import.meta.dirname, ".."); -const TOKEN = "junior-write-token"; +const TOKEN = "roach-write-token"; +const TENANT = "getsentry/junior"; const BUCKET = "roach-recordings"; const GCS_TOKEN = "vm-access-token"; @@ -78,7 +79,7 @@ function readEnvelope(envelope: string) { /** Run one CI job, and return what it printed. */ async function ciJob(token: string | undefined, mode: RunConfig["mode"]) { const config: RunConfig = { - tenant: "junior", + tenant: TENANT, mode, name: `${mode}-run`, rules: [{ name: "model", match: { method: "POST" } }], @@ -234,7 +235,7 @@ beforeAll(async () => { publicUrl: frontUrl, bucket: BUCKET, allow: [upstreamOrigin], - tenants: { junior: createHash("sha256").update(TOKEN).digest("hex") }, + writeTokenHash: createHash("sha256").update(TOKEN).digest("hex"), ca: { cert: caCert, key: await readFile(caKeyFile, "utf8") }, sentryDsn: `http://public@127.0.0.1:${sentryPort}/1`, }; @@ -289,7 +290,7 @@ describe("deployed roach", () => { // A restart keeps the authority that clients trust. expect(writer.caCert).toBe(caCert); expect([...objects.keys()]).toEqual([ - expect.stringMatching(/^junior\/model\/[0-9a-f]{64}\.json$/), + expect.stringMatching(/^getsentry\/junior\/model\/[0-9a-f]{64}\.json$/), ]); const fork = await ciJob(undefined, "replay"); @@ -313,22 +314,22 @@ describe("deployed roach", () => { })), ).toEqual([ { - tenant: "junior", + tenant: TENANT, run: "auto-run", result: "missed", - key: key!.slice(7), + key: key!.slice(`${TENANT}/`.length), }, { - tenant: "junior", + tenant: TENANT, run: "auto-run", result: "written", - key: key!.slice(7), + key: key!.slice(`${TENANT}/`.length), }, { - tenant: "junior", + tenant: TENANT, run: "replay-run", result: "replayed", - key: key!.slice(7), + key: key!.slice(`${TENANT}/`.length), }, ]); }); diff --git a/tests/service.test.ts b/tests/service.test.ts index ce82641..3fbb54c 100644 --- a/tests/service.test.ts +++ b/tests/service.test.ts @@ -33,7 +33,7 @@ async function startRun( const run = await startRemoteRun( { url: service.url, token }, { - tenant: "alpha", + tenant: "acme/alpha", mode, rules: [ { name: "model", match: { method: "POST", url: `${origin}/v1/` } }, @@ -99,7 +99,7 @@ beforeEach(async () => { service = await startRoachService({ directory, allow: [origin], - tenants: { alpha: hash("alpha-token"), beta: hash("beta-token") }, + writeTokenHash: hash("write-token"), secrets: ["service-secret-value-123"], }); }); @@ -113,8 +113,8 @@ afterEach(async () => { }); describe("roach service", () => { - it("writes with the tenant token and replays for anyone", async () => { - const writer = await startRun("alpha-token", "auto"); + it("writes with the write token and replays for anyone", async () => { + const writer = await startRun("write-token", "auto"); expect(await send(writer.url, { prompt: "hi" })).toMatchObject({ source: "live", }); @@ -140,22 +140,31 @@ describe("roach service", () => { for (const mode of ["auto", "record", "off"] as const) { await expect(startRun(undefined, mode)).rejects.toThrow(/401/); } - // The token of another tenant does not count. - await expect(startRun("beta-token", "auto")).rejects.toThrow(/401/); + // A wrong token does not count. + await expect(startRun("not-the-token", "auto")).rejects.toThrow(/401/); - // Another tenant has its own recordings. - const beta = await startRun(undefined, "replay", { tenant: "beta" }); + // Another tenant has its own recordings. The service has no list of + // tenants, so the write token works for a new one too. + const beta = await startRun(undefined, "replay", { tenant: "acme/beta" }); expect(await send(beta.url, { prompt: "hi" })).toMatchObject({ status: 412, }); - expect(liveRequests).toBe(1); - expect(await recordings("alpha")).toHaveLength(1); - expect(await recordings("beta")).toEqual([]); + const newTenant = await startRun("write-token", "auto", { + tenant: "other-org/new.repo", + }); + expect(await send(newTenant.url, { prompt: "hi" })).toMatchObject({ + source: "live", + }); + expect(await newTenant.close()).toMatchObject({ written: 1 }); + expect(liveRequests).toBe(2); + expect(await recordings("acme/alpha")).toHaveLength(1); + expect(await recordings("acme/beta")).toEqual([]); + expect(await recordings("other-org/new.repo")).toHaveLength(1); }); it("keeps the sessions of concurrent runs apart", async () => { - const first = await startRun("alpha-token", "auto"); - const second = await startRun("alpha-token", "auto"); + const first = await startRun("write-token", "auto"); + const second = await startRun("write-token", "auto"); const failing = await first.startSession("test"); const passing = await second.startSession("test"); @@ -165,10 +174,13 @@ describe("roach service", () => { await failing.end(false); await passing.end(true); - expect(await recordings("alpha")).toHaveLength(1); - const [file] = await recordings("alpha"); + expect(await recordings("acme/alpha")).toHaveLength(1); + const [file] = await recordings("acme/alpha"); const recording = JSON.parse( - await readFile(path.join(directory, "alpha", "model", file!), "utf8"), + await readFile( + path.join(directory, "acme/alpha", "model", file!), + "utf8", + ), ); expect(recording.session).toBe("test"); expect(JSON.parse(recording.response.body).n).toBe(2); @@ -177,7 +189,7 @@ describe("roach service", () => { }); it("refuses callers without the right credentials", async () => { - const run = await startRun("alpha-token", "auto"); + const run = await startRun("write-token", "auto"); const bare = new URL(run.url); bare.username = ""; bare.password = ""; @@ -192,24 +204,27 @@ describe("roach service", () => { wrong.password = "not-the-token"; await expect(send(wrong.href, { prompt: "hi" })).rejects.toThrow(/407/); + // The tenant is a path in the store, so it must be owner/repo. + for (const tenant of ["junior", "acme/..", "../acme", "a/b/c", "acme/"]) { + await expect(startRun("write-token", "auto", { tenant })).rejects.toThrow( + /tenant must be owner\/repo/, + ); + } await expect( - startRun("alpha-token", "auto", { tenant: "nobody" }), - ).rejects.toThrow(/tenant is not known/); - await expect( - startRun("alpha-token", "auto", { allow: ["https://example.com"] }), + startRun("write-token", "auto", { allow: ["https://example.com"] }), ).rejects.toThrow(/does not allow/); await expect( - startRun("alpha-token", "auto", { + startRun("write-token", "auto", { rules: [{ name: "../escape", match: {} }], }), ).rejects.toThrow(/rule name/); // A pattern runs for all runs, so only known patterns are allowed. await expect( - startRun("alpha-token", "auto", { + startRun("write-token", "auto", { rules: [{ name: "model", match: {}, values: { slow: "(a+)+$" } }], }), ).rejects.toThrow(/not a pattern that the service allows/); - await startRun("alpha-token", "auto", { + await startRun("write-token", "auto", { rules: [ { name: "model", match: {}, values: { id: VALUE_PATTERNS.uuid } }, ], @@ -227,14 +242,17 @@ describe("roach service", () => { }); expect(tooLarge.status).toBe(413); - // Anyone can start a run without a token, so a tenant has a cap on them. + // Anyone can start a run without a token, so the service has a cap on + // them. Another tenant name does not get around it. const open = await Promise.all( - Array.from({ length: 50 }, () => startRun(undefined, "replay")), + Array.from({ length: 100 }, () => startRun(undefined, "replay")), ); await expect(startRun(undefined, "replay")).rejects.toThrow(/429/); - // The cap is per tenant, and runs with the token have none. - await startRun(undefined, "replay", { tenant: "beta" }); - await startRun("alpha-token", "auto"); + await expect( + startRun(undefined, "replay", { tenant: "acme/beta" }), + ).rejects.toThrow(/429/); + // Runs with the token have no cap. + await startRun("write-token", "auto"); // An ended run frees its place. await open[0]!.close(); await startRun(undefined, "replay"); @@ -255,7 +273,7 @@ describe("roach service", () => { }); it("redacts request credentials and the secrets of the service", async () => { - const run = await startRun("alpha-token", "auto"); + const run = await startRun("write-token", "auto"); const response = await send( run.url, { prompt: "hi" }, @@ -268,9 +286,9 @@ describe("roach service", () => { key: "<>", serviceSecret: "<>", }); - const [file] = await recordings("alpha"); + const [file] = await recordings("acme/alpha"); const text = await readFile( - path.join(directory, "alpha", "model", file!), + path.join(directory, "acme/alpha", "model", file!), "utf8", ); expect(text).not.toContain("sk-live-request-key-456");