Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/redact.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ const PATTERNS = [
/\b([A-Za-z0-9_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|API_?KEY|ACCESS_?KEY)[A-Za-z0-9_]*)\s*[=:]\s*(?:"([^"]{8,})"(?=[ \t]*(?:$|[,;})\]]))|'([^']{8,})'(?=[ \t]*(?:$|[,;})\]]))|"([^"\n]{8,})"|'([^'\n]{8,})'|["']?([^\s"',]{8,}))/gim,
"ASSIGNMENT",
],
// Run after assignments so a matching variable name cannot hide an unredacted value.
[/\b(apikey_[A-Za-z0-9_-]{16,})/gi, "API_KEY"],
];

export function redact(text) {
Expand Down
41 changes: 39 additions & 2 deletions test/analyze-reuse.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,12 @@ function initRepo(memory) {
}

/** A minimal Pi session, non-trivial enough to clear the triviality filter. */
function writeSession(home, id, cwd) {
function writeSession(home, id, cwd, userText = "Please build the project.") {
const dir = path.join(home, ".pi", "agent", "sessions", id);
fs.mkdirSync(dir, { recursive: true });
const entries = [
{ type: "session", version: 3, id, timestamp: new Date().toISOString(), cwd },
{ type: "message", message: { role: "user", content: "Please build the project." } },
{ type: "message", message: { role: "user", content: userText } },
{ type: "message", message: { role: "assistant", content: "Ran make build as instructed." } },
{ type: "message", message: { role: "user", content: "Now run the tests too." } },
{ type: "message", message: { role: "assistant", content: "Tests pass." } },
Expand Down Expand Up @@ -125,6 +125,43 @@ function runAnalyze(dir, home, extraArgs = []) {
const MEMORY = "# Agent instructions\n\n- Run `make build` before every push.\n";
const MEMORY_EDITED = "# Agent instructions\n\n- Run `make build` before every push.\n- Never skip the changelog.\n";

test("standalone apikey credentials never reach the cached analysis prompt", () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-redact-home-"));
const dir = initRepo(MEMORY);
const tokens = [`apikey_${"a".repeat(64)}`, `APIKEY_${"B".repeat(64)}`];
const assignments = [];
const secrets = [];
for (const name of ["apikey_openrouter_production", "APIKEY_OPENROUTER_PRODUCTION"]) {
for (const separator of ["=", ":"]) {
for (const quote of ["", '"', "'"]) {
const secret = `synthetic-secret-${secrets.length}`;
secrets.push(secret);
assignments.push(`${name}${separator}${quote}${secret}${quote}`);
}
}
}
writeSession(
home,
"redacted-session",
dir,
`Please build the project using (${tokens[0]}).\n${tokens[1]}\n${assignments.join("\n")}`,
);

const result = runAnalyze(dir, home);
assert.equal(result.status, 0, result.output);
assert.equal(result.summary.analyzed, 1);
const promptDir = path.join(dir, ".backpass", "prompts");
const prompts = fs.readdirSync(promptDir);
assert.equal(prompts.length, 1);
const prompt = fs.readFileSync(path.join(promptDir, prompts[0]), "utf8");
assert.match(prompt, /Please build the project using \(\[redacted:API_KEY\]\)\.\n\[redacted:API_KEY\]/);
for (const token of tokens)
assert.ok(!prompt.includes(token), "the cached model prompt must not contain a full credential");
for (const secret of secrets)
assert.ok(!prompt.includes(secret), "the cached model prompt must not contain an assignment secret");
assert.equal(prompt.match(/\[redacted:API_KEY\]=\[redacted\]/g)?.length, assignments.length);
});

test("the first analysis performs work, an unchanged second reuses it, and --force reanalyzes anyway", () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-reuse-home-"));
const dir = initRepo(MEMORY);
Expand Down
48 changes: 48 additions & 0 deletions test/distill.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,54 @@ test("redaction leaves benign token-count arguments alone", () => {
assert.equal(redact("token=abcdefgh1234,next=1"), "token=[redacted],next=1");
});

test("standalone apikey credentials are redacted case-insensitively with a 16-character floor", () => {
const lower = `apikey_${"a".repeat(64)}`;
const upper = `APIKEY_${"B".repeat(64)}`;
const mixed = "ApiKey_aB3_-aB3_-aB3_-x";
const label = "[redacted:API_KEY]";
for (const token of [lower, upper, mixed, `apikey_${"c".repeat(16)}`]) {
assert.equal(redact(token), label);
assert.equal(redact(`(${token}), next.`), `(${label}), next.`);
}
assert.equal(redact(`before\n${lower}\n${upper}\nafter`), `before\n${label}\n${label}\nafter`);
for (const text of [
"apikey",
"apikey_documentation",
`apikey_${"a".repeat(15)}`,
`APIKEY_${"B".repeat(15)}`,
`not${lower}`,
]) {
assert.equal(redact(text), text);
}
});

test("standalone apikey matching cannot expose secret assignment values", () => {
for (const name of ["apikey_openrouter_production", "APIKEY_OPENROUTER_PRODUCTION"]) {
for (const separator of ["=", ":"]) {
for (const value of [
"hunter2hunter2",
'"hunter2hunter2,secret"',
"'hunter2hunter2,secret'",
'"hunter2\nhunter2"',
]) {
const assignment = `${name} ${separator} ${value}`;
assert.equal(redact(assignment), "[redacted:API_KEY]=[redacted]");
const { trace } = distill(
[
{ kind: "message", role: "user", text: assignment },
{ kind: "message", role: "assistant", text: assignment },
{ kind: "tool", name: "Bash", input: assignment, result: assignment },
{ kind: "tool", name: "Bash", input: { command: assignment }, result: assignment },
],
META,
);
assert.ok(!trace.includes("hunter2"), "assignment values must not survive in any trace field");
assert.equal(trace.match(/\[redacted:API_KEY\]=\[redacted\]/g)?.length, 6);
}
}
}
});

test("redaction runs on tool input and output inside the trace", () => {
const { trace } = distill(
[
Expand Down
Loading