From 44138831cffe15e1e88b9cfe43ef59871f77f70e Mon Sep 17 00:00:00 2001 From: Shaz Date: Wed, 28 Oct 2026 08:53:34 -0400 Subject: [PATCH 1/3] fix(redact): cover standalone apikey credentials --- src/redact.js | 1 + test/analyze-reuse.test.js | 22 ++++++++++++++++++++-- test/distill.test.js | 21 +++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/redact.js b/src/redact.js index d67910b1..1cbb5a74 100644 --- a/src/redact.js +++ b/src/redact.js @@ -10,6 +10,7 @@ const PATTERNS = [ [/\b(sk-proj-[A-Za-z0-9_-]{16,})/g, "OPENAI_KEY"], [/\b(sk-or-v1-[A-Za-z0-9_-]{16,})/g, "OPENROUTER_KEY"], [/\b(sk-[A-Za-z0-9]{32,})/g, "API_KEY"], + [/\b(apikey_[A-Za-z0-9_-]{16,})/gi, "API_KEY"], [/\b(gh[pousr]_[A-Za-z0-9]{16,})/g, "GITHUB_TOKEN"], [/\b(xox[abposr]-[A-Za-z0-9-]{10,})/g, "SLACK_TOKEN"], [/\b(AKIA[0-9A-Z]{16})\b/g, "AWS_ACCESS_KEY_ID"], diff --git a/test/analyze-reuse.test.js b/test/analyze-reuse.test.js index f75bd67f..49aada91 100644 --- a/test/analyze-reuse.test.js +++ b/test/analyze-reuse.test.js @@ -77,12 +77,12 @@ function initRepo(memory) { } /** A minimal Pi session, non-trivial enough to clear the triviality filter. */ -function writeSession(home, id, cwd) { +function writeSession(home, id, cwd, userText = "Please build the project.") { const dir = path.join(home, ".pi", "agent", "sessions", id); fs.mkdirSync(dir, { recursive: true }); const entries = [ { type: "session", version: 3, id, timestamp: new Date().toISOString(), cwd }, - { type: "message", message: { role: "user", content: "Please build the project." } }, + { type: "message", message: { role: "user", content: userText } }, { type: "message", message: { role: "assistant", content: "Ran make build as instructed." } }, { type: "message", message: { role: "user", content: "Now run the tests too." } }, { type: "message", message: { role: "assistant", content: "Tests pass." } }, @@ -125,6 +125,24 @@ function runAnalyze(dir, home, extraArgs = []) { const MEMORY = "# Agent instructions\n\n- Run `make build` before every push.\n"; const MEMORY_EDITED = "# Agent instructions\n\n- Run `make build` before every push.\n- Never skip the changelog.\n"; +test("standalone apikey credentials never reach the cached analysis prompt", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-redact-home-")); + const dir = initRepo(MEMORY); + const tokens = [`apikey_${"a".repeat(64)}`, `APIKEY_${"B".repeat(64)}`]; + writeSession(home, "redacted-session", dir, `Please build the project using (${tokens[0]}).\n${tokens[1]}`); + + const result = runAnalyze(dir, home); + assert.equal(result.status, 0, result.output); + assert.equal(result.summary.analyzed, 1); + const promptDir = path.join(dir, ".backpass", "prompts"); + const prompts = fs.readdirSync(promptDir); + assert.equal(prompts.length, 1); + const prompt = fs.readFileSync(path.join(promptDir, prompts[0]), "utf8"); + assert.match(prompt, /Please build the project using \(\[redacted:API_KEY\]\)\.\n\[redacted:API_KEY\]/); + for (const token of tokens) + assert.ok(!prompt.includes(token), "the cached model prompt must not contain a full credential"); +}); + test("the first analysis performs work, an unchanged second reuses it, and --force reanalyzes anyway", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-reuse-home-")); const dir = initRepo(MEMORY); diff --git a/test/distill.test.js b/test/distill.test.js index 39dab9ae..52cd080f 100644 --- a/test/distill.test.js +++ b/test/distill.test.js @@ -124,6 +124,27 @@ test("redaction leaves benign token-count arguments alone", () => { assert.equal(redact("token=abcdefgh1234,next=1"), "token=[redacted],next=1"); }); +test("standalone apikey credentials are redacted case-insensitively with a 16-character floor", () => { + const lower = `apikey_${"a".repeat(64)}`; + const upper = `APIKEY_${"B".repeat(64)}`; + const mixed = "ApiKey_aB3_-aB3_-aB3_-x"; + const label = "[redacted:API_KEY]"; + for (const token of [lower, upper, mixed, `apikey_${"c".repeat(16)}`]) { + assert.equal(redact(token), label); + assert.equal(redact(`(${token}), next.`), `(${label}), next.`); + } + assert.equal(redact(`before\n${lower}\n${upper}\nafter`), `before\n${label}\n${label}\nafter`); + for (const text of [ + "apikey", + "apikey_documentation", + `apikey_${"a".repeat(15)}`, + `APIKEY_${"B".repeat(15)}`, + `not${lower}`, + ]) { + assert.equal(redact(text), text); + } +}); + test("redaction runs on tool input and output inside the trace", () => { const { trace } = distill( [ From fc07de1cf4e96fb549fef0d1ecaf3f7615a600e1 Mon Sep 17 00:00:00 2001 From: Shaz Date: Wed, 28 Oct 2026 09:06:28 -0400 Subject: [PATCH 2/3] no-mistakes(review): Prevent standalone API key matching from exposing assignment secrets --- src/redact.js | 2 +- test/analyze-reuse.test.js | 21 ++++++++++++++++++++- test/distill.test.js | 27 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/redact.js b/src/redact.js index 1cbb5a74..d276fbea 100644 --- a/src/redact.js +++ b/src/redact.js @@ -10,7 +10,6 @@ const PATTERNS = [ [/\b(sk-proj-[A-Za-z0-9_-]{16,})/g, "OPENAI_KEY"], [/\b(sk-or-v1-[A-Za-z0-9_-]{16,})/g, "OPENROUTER_KEY"], [/\b(sk-[A-Za-z0-9]{32,})/g, "API_KEY"], - [/\b(apikey_[A-Za-z0-9_-]{16,})/gi, "API_KEY"], [/\b(gh[pousr]_[A-Za-z0-9]{16,})/g, "GITHUB_TOKEN"], [/\b(xox[abposr]-[A-Za-z0-9-]{10,})/g, "SLACK_TOKEN"], [/\b(AKIA[0-9A-Z]{16})\b/g, "AWS_ACCESS_KEY_ID"], @@ -26,6 +25,7 @@ const PATTERNS = [ /\b([A-Za-z0-9_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|API_?KEY|ACCESS_?KEY)[A-Za-z0-9_]*)\s*[=:]\s*(?:"([^"]{8,})"(?=[ \t]*(?:$|[,;})\]]))|'([^']{8,})'(?=[ \t]*(?:$|[,;})\]]))|"([^"\n]{8,})"|'([^'\n]{8,})'|["']?([^\s"',]{8,}))/gim, "ASSIGNMENT", ], + [/\b(apikey_[A-Za-z0-9_-]{16,})/gi, "API_KEY"], ]; export function redact(text) { diff --git a/test/analyze-reuse.test.js b/test/analyze-reuse.test.js index 49aada91..6f9e732b 100644 --- a/test/analyze-reuse.test.js +++ b/test/analyze-reuse.test.js @@ -129,7 +129,23 @@ test("standalone apikey credentials never reach the cached analysis prompt", () const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-redact-home-")); const dir = initRepo(MEMORY); const tokens = [`apikey_${"a".repeat(64)}`, `APIKEY_${"B".repeat(64)}`]; - writeSession(home, "redacted-session", dir, `Please build the project using (${tokens[0]}).\n${tokens[1]}`); + const assignments = []; + const secrets = []; + for (const name of ["apikey_openrouter_production", "APIKEY_OPENROUTER_PRODUCTION"]) { + for (const separator of ["=", ":"]) { + for (const quote of ["", '"', "'"]) { + const secret = `synthetic-secret-${secrets.length}`; + secrets.push(secret); + assignments.push(`${name}${separator}${quote}${secret}${quote}`); + } + } + } + writeSession( + home, + "redacted-session", + dir, + `Please build the project using (${tokens[0]}).\n${tokens[1]}\n${assignments.join("\n")}`, + ); const result = runAnalyze(dir, home); assert.equal(result.status, 0, result.output); @@ -141,6 +157,9 @@ test("standalone apikey credentials never reach the cached analysis prompt", () assert.match(prompt, /Please build the project using \(\[redacted:API_KEY\]\)\.\n\[redacted:API_KEY\]/); for (const token of tokens) assert.ok(!prompt.includes(token), "the cached model prompt must not contain a full credential"); + for (const secret of secrets) + assert.ok(!prompt.includes(secret), "the cached model prompt must not contain an assignment secret"); + assert.equal(prompt.match(/\[redacted:API_KEY\]=\[redacted\]/g)?.length, assignments.length); }); test("the first analysis performs work, an unchanged second reuses it, and --force reanalyzes anyway", () => { diff --git a/test/distill.test.js b/test/distill.test.js index 52cd080f..00687d53 100644 --- a/test/distill.test.js +++ b/test/distill.test.js @@ -145,6 +145,33 @@ test("standalone apikey credentials are redacted case-insensitively with a 16-ch } }); +test("standalone apikey matching cannot expose secret assignment values", () => { + for (const name of ["apikey_openrouter_production", "APIKEY_OPENROUTER_PRODUCTION"]) { + for (const separator of ["=", ":"]) { + for (const value of [ + "hunter2hunter2", + '"hunter2hunter2,secret"', + "'hunter2hunter2,secret'", + '"hunter2\nhunter2"', + ]) { + const assignment = `${name} ${separator} ${value}`; + assert.equal(redact(assignment), "[redacted:API_KEY]=[redacted]"); + const { trace } = distill( + [ + { kind: "message", role: "user", text: assignment }, + { kind: "message", role: "assistant", text: assignment }, + { kind: "tool", name: "Bash", input: assignment, result: assignment }, + { kind: "tool", name: "Bash", input: { command: assignment }, result: assignment }, + ], + META, + ); + assert.ok(!trace.includes("hunter2"), "assignment values must not survive in any trace field"); + assert.equal(trace.match(/\[redacted:API_KEY\]=\[redacted\]/g)?.length, 6); + } + } + } +}); + test("redaction runs on tool input and output inside the trace", () => { const { trace } = distill( [ From f9558255479a47c4d16012b80a20dbda901cc940 Mon Sep 17 00:00:00 2001 From: Shaz Date: Wed, 28 Oct 2026 09:14:34 -0400 Subject: [PATCH 3/3] no-mistakes(document): Document redaction ordering invariant --- src/redact.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redact.js b/src/redact.js index d276fbea..1b57afb6 100644 --- a/src/redact.js +++ b/src/redact.js @@ -25,6 +25,7 @@ const PATTERNS = [ /\b([A-Za-z0-9_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|API_?KEY|ACCESS_?KEY)[A-Za-z0-9_]*)\s*[=:]\s*(?:"([^"]{8,})"(?=[ \t]*(?:$|[,;})\]]))|'([^']{8,})'(?=[ \t]*(?:$|[,;})\]]))|"([^"\n]{8,})"|'([^'\n]{8,})'|["']?([^\s"',]{8,}))/gim, "ASSIGNMENT", ], + // Run after assignments so a matching variable name cannot hide an unredacted value. [/\b(apikey_[A-Za-z0-9_-]{16,})/gi, "API_KEY"], ];