diff --git a/.release-please-manifest.json b/.release-please-manifest.json index b6f6e2a5..a31ca47e 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.1.36" + ".": "0.1.42" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 342f2214..49b5c65a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,49 @@ # Changelog +## [0.1.42](https://github.com/kunchenguid/backpass/compare/backpass-v0.1.41...backpass-v0.1.42) (2026-10-29) + + +### Bug Fixes + +* recognize CLAUDE.md import pointers with leading headings ([#240](https://github.com/kunchenguid/backpass/issues/240)) ([df9cdce](https://github.com/kunchenguid/backpass/commit/df9cdcebab3fd4520a2a0d09bc696d9438f3cb24)) +* redact standalone apikey_ credentials ([#147](https://github.com/kunchenguid/backpass/issues/147)) ([#242](https://github.com/kunchenguid/backpass/issues/242)) ([24f8283](https://github.com/kunchenguid/backpass/commit/24f8283930119250593dd70e6d62d67f4c8801bc)) +* report prompt size when synthesis fails ([#243](https://github.com/kunchenguid/backpass/issues/243)) ([a2d710b](https://github.com/kunchenguid/backpass/commit/a2d710b5c6fe9ba529ebd5c2356a73689a38f28e)) + +## [0.1.41](https://github.com/kunchenguid/backpass/compare/backpass-v0.1.40...backpass-v0.1.41) (2026-10-26) + + +### Bug Fixes + +* **synthesize:** report a mid-run change to a watched repo file as observed ([#224](https://github.com/kunchenguid/backpass/issues/224)) ([1a1a175](https://github.com/kunchenguid/backpass/commit/1a1a17531de333d8095a944baa884a0896d5672d)) + +## [0.1.40](https://github.com/kunchenguid/backpass/compare/backpass-v0.1.39...backpass-v0.1.40) (2026-10-25) + + +### Bug Fixes + +* deduplicate symlinked skills in the analysis index ([#231](https://github.com/kunchenguid/backpass/issues/231)) ([9ded619](https://github.com/kunchenguid/backpass/commit/9ded6192cd7eae8c63e40559b7832774b3bac2fb)) + +## [0.1.39](https://github.com/kunchenguid/backpass/compare/backpass-v0.1.38...backpass-v0.1.39) (2026-10-23) + + +### Bug Fixes + +* run first Claude analysis call alone before fan-out ([#226](https://github.com/kunchenguid/backpass/issues/226)) ([3882b86](https://github.com/kunchenguid/backpass/commit/3882b86d84017bcdbf5e2cbe2e8c1fa9abeb3126)) + +## [0.1.38](https://github.com/kunchenguid/backpass/compare/backpass-v0.1.37...backpass-v0.1.38) (2026-10-21) + + +### Bug Fixes + +* **workspace:** keep VCS internals and binary files out of the staging copy ([#220](https://github.com/kunchenguid/backpass/issues/220)) ([46a2169](https://github.com/kunchenguid/backpass/commit/46a21696be63197815346d01f6d4e116c16b4c10)) + +## [0.1.37](https://github.com/kunchenguid/backpass/compare/backpass-v0.1.36...backpass-v0.1.37) (2026-10-20) + + +### Bug Fixes + +* **acpx:** treat a model the adapter turns down as model-unavailable ([#219](https://github.com/kunchenguid/backpass/issues/219)) ([3986087](https://github.com/kunchenguid/backpass/commit/398608764b6a7d40de142eafbf2e568921406fa9)) + ## [0.1.36](https://github.com/kunchenguid/backpass/compare/backpass-v0.1.35...backpass-v0.1.36) (2026-10-16) diff --git a/README.md b/README.md index 5b1a1a14..a3e62dbe 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ one command only; neither flag changes a config file or harness default. Harness load paths, verified for v1: - **Claude Code** loads `CLAUDE.md` from `CLAUDE_CONFIG_DIR` (default `~/.claude`) - and inlines `@` imports, including `~/` and absolute paths. A CLAUDE.md containing - only an import that resolves to the canonical user memory is a valid pointer, such + and inlines `@` imports, including `~/` and absolute paths. A CLAUDE.md + [import pointer](#9-which-file-is-the-weights) must resolve to the canonical user memory, such as `@~/.agents/AGENTS.md` with the default paths. - **Codex** loads `AGENTS.md` from `CODEX_HOME` (default `~/.codex`). It follows the AGENTS.md convention; `@` import is not assumed. @@ -213,7 +213,7 @@ whole-surface run (see [Nested memory files](#10-nested-memory-files-in-a-monore Nothing else resolves - not a basename, a directory, a path to a SKILL.md, or an existing file the config does not name - and an unknown name fails, listing the valid ones, instead of falling back to the whole surface. A -configured file that contains only an `@` import is rejected rather than rewritten or silently +configured file using the [import pointer form](#9-which-file-is-the-weights) is rejected rather than rewritten or silently mapped to its import; the error names the imported memory file, which must itself be configured to be targeted. A correctly named skill whose file backpass cannot write - one resolving into a location nothing may write, or in project scope one resolving outside the repository - is refused @@ -429,6 +429,9 @@ orchestrating tool) - and the analysis is shown the ledger's open gaps so it can cite an existing gap id instead of coining a paraphrase of it. +With `jobs` greater than one, later Claude analysis calls wait for the first actual Claude +invocation to finish, even when analysis falls back to Claude from another agent. + **Every claim must carry a verbatim quote, and the quote must be in the trace.** Quoteless items are discarded - the single most important defence against a model confabulating influence - and so are quotes that do not actually appear in the distilled trace they claim @@ -520,7 +523,9 @@ A high-reasoning synthesis run turns the aggregated gradients into concrete edit REMOVE, REWRITE, EXTRACT→SKILL, or MOVE. The agent does not describe edits for backpass to splice in - it makes them, with its harness's own file tools, in a **staging copy** of the memory file and project skills under `.backpass/synthesis/` (the repo itself is read-only -to it, for grounding). Named nested files use their own staging copies and do not stage skills +to it, for grounding). Version-control internals (`.git`, `.hg`, `.svn`) and binary files +under a skill directory stay out of the copy and are read-only. Named nested files use +their own staging copies and do not stage skills (see [Nested memory files](#10-nested-memory-files-in-a-monorepo)). backpass then diffs against the originals and shows the agent the measured changes by id; the agent annotates each one with a title, rationale, and the verbatim evidence behind it. Nothing textual is ever taken from the model: every hunk's @@ -569,8 +574,10 @@ keeps the merged change rather than guessing. A malformed answer or gate violation triggers a re-prompt naming the exact breach (at most two). If those also fail, backpass **fails loudly** and preserves the latest parseable -rejected proposal, if one was produced. It never silently truncates. A harness that writes -past the staging copy into the repo is an error, never an apply. +rejected proposal, if one was produced. It never silently truncates. If a watched repository +file changes during synthesis, backpass stops without a proposal and reports the change. +When the repository file now matches the staging copy, it reports a likely direct harness +write; otherwise, the change may have come from a concurrent editor. Not every annotate turn is an answer, and the three cases are kept apart because they call for different things: @@ -586,7 +593,9 @@ for different things: When a run does fail, the advice it prints comes from the condition it ended on. If the rejected proposal included notes, the failure prints those too instead of hiding useful -diagnostic context. Run `backpass propose` again to start a fresh synthesis. +diagnostic context. A failed synthesis prompt call reports the assembled prompt's measured +UTF-8 byte size without printing its contents. That size is diagnostic context, not a +claim that the prompt was too large. Run `backpass propose` again to start a fresh synthesis. Token deltas shown to you are measured by backpass from the actual text - never taken from the model's own arithmetic. @@ -639,6 +648,9 @@ every skill's name and trigger line, stamps a gap it judges covered by an existi with `coveredBySkill`, and the gap ledger corroborates those citations across observers before synthesis is told to fix the trigger. The same coverage check retires a ledger gap once a skill's content answers it, so a gap resolved by extraction stops resurfacing. +The analysis index lists each skill file once by realpath, keeping its first loaded path +when symlinks across harness roots share a file. Distinct files remain separate even when +their names match. This does not change [per-copy budget accounting](#6-the-budget---model-size). Skills are protected by the same floors as the memory file: text deleted from a skill file is a removal, and since no evidence can attribute to skill text, no deletion there @@ -676,6 +688,12 @@ It cannot be combined with `--no-ui`. The flag is repeatable: all supplied vectors are checked together as one vector, each must name at least one edit, and naming an edit twice is refused, even across flags. Edits it does not name stay untouched, and every gate below still applies. +After a successful non-dry-run apply with at least one accepted edit, the saved proposal +records `appliedAt` and `appliedBy: "apply"`. A second apply refuses it before opening +review or writing anything. Run `backpass` again to produce a fresh proposal, including +for edits skipped in the completed review. Dry runs and reviews with no accepted edits +leave the proposal unstamped and available to review again. + The live budget gauge is not just a readout. Apply rechecks the accepted subset against the same budget gate as synthesis: stay under the cap, or shrink if the file is already over. An incompatible set writes nothing and does not record rejections, so you can pick @@ -697,8 +715,9 @@ refuses the whole write if any created skill target already exists or two accept resolve to the same file. Skills and non-memory files are written only after every edit has composed, with the memory -file committed last. A later write failure rolls back files, skills, and loading-layout -entries created earlier in that round. If another process changes a committed file before +file committed last. A later write failure, including failure to save the proposal's +applied stamp, leaves the proposal unstamped and rolls back files, skills, and loading-layout +entries created earlier in that round so the apply can be retried. If another process changes a committed file before rollback reaches it, apply leaves that change untouched and reports the rollback conflict. For compatibility, proposals created by older backpass versions that do not contain a @@ -717,9 +736,11 @@ backpass apply --decisions "e1=accepted e2=rejected:disagree" # decided elsewh `memoryFiles` is an ordered list (default `["AGENTS.md", "CLAUDE.md"]`); the first one that exists is the root file a run optimizes, so **AGENTS.md is canonical**. Named [nested files](#10-nested-memory-files-in-a-monorepo) are additional weights, not -separate root files. A `CLAUDE.md` containing only the `@AGENTS.md` import (apart from -blank lines and HTML comments), or a symlink resolving to the same file, is treated as -a pointer: +separate root files. A `CLAUDE.md` is a pointer when it contains exactly one import +resolving to the primary file, such as `@AGENTS.md` or `@./AGENTS.md`, with optional +leading ATX headings (`#` through `######`), blank lines and HTML comments. Prose, +additional imports, and headings after the import make it a separate file. A symlink +resolving to the primary file is also a pointer: - Optimizing AGENTS.md covers both harness families and the pointer stays valid. Nothing to report. @@ -800,8 +821,8 @@ weight of its own; a missing file is reported, never created: `sampledSessions`, `minGapEvidence`, and `belowMinGapEvidence`. The count is report-only and never gates a run. -Each nested directory keeps the pointer model above: name its canonical file, and a -sibling `AGENTS.md` or `CLAUDE.md` that only imports it needs nothing, while a sibling +Each nested directory keeps the [pointer model above](#9-which-file-is-the-weights): name its canonical file, and a +sibling `CLAUDE.md` that points to it needs nothing, while a sibling with content of its own is warned about. An entry that is only a pointer is refused. A file listed in both `memoryFiles` and `nestedMemoryFiles` is nested. `--target` and `--memory-file` name exactly the files a run trains, so neither trains a nested file. diff --git a/package.json b/package.json index cf8f69af..ad797fba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "backpass", - "version": "0.1.36", + "version": "0.1.42", "packageManager": "pnpm@11.5.0", "description": "Gradient descent for your agent memory - analyzes past agent session transcripts and proposes evidence-backed edits to AGENTS.md / CLAUDE.md", "type": "module", diff --git a/src/acpx.js b/src/acpx.js index 50197339..28d602d3 100644 --- a/src/acpx.js +++ b/src/acpx.js @@ -164,6 +164,10 @@ function sessionCreateTimeoutError({ agent, acpxAgentArgs, timeoutMs, result }) ); } +/** What acpx prints when the adapter turns down the requested model after `session/new`. */ +const MODEL_REJECTED = + /did not advertise that model|does not support this model|session\/set_(?:config_option|model) for model/i; + /** * Availability verdicts for a failed acpx call. Only a *classifiable* failure is a * reason to drop a candidate and fall through to the next one; anything else (a @@ -184,7 +188,7 @@ export function classifyAcpxFailure(failure) { if (failure.spawnError?.code === "ENOENT") return "unreachable"; const text = failure.stderr || ""; if (/AUTH_REQUIRED|authentication required/i.test(text)) return "unauthenticated"; - if (/did not advertise that model|does not support this model/i.test(text)) return "model-unavailable"; + if (MODEL_REJECTED.test(text)) return "model-unavailable"; if (/\b(ENOENT|command not found|not found on PATH|failed to spawn|spawn .* ENOENT)\b/i.test(text)) { return "unreachable"; } diff --git a/src/agents.js b/src/agents.js index 1ffe8765..d01be56a 100644 --- a/src/agents.js +++ b/src/agents.js @@ -592,7 +592,7 @@ function pinnedFailureError(role, pick, verdict, err) { } else if (verdict) { hint = pin; } - const detail = err?.message ? ` (${String(err.message).split("\n")[0]})` : ""; + const detail = err?.message ? ` (${String(err.message)})` : ""; return new UserError(`pinned ${role} agent ${pick.agent}${model} ${label}${detail}`, hint); } diff --git a/src/analyze.js b/src/analyze.js index b1adfdc4..14644cd2 100644 --- a/src/analyze.js +++ b/src/analyze.js @@ -324,6 +324,7 @@ async function analyzeOne({ config, repo, modelCwd = null, + invokeAnalysis, slot = 0, openGapIndex = "(none yet)", skillIndex = "(this repo has no skills)", @@ -406,9 +407,11 @@ async function analyzeOne({ }; // Route effortful calls through a fresh per-transcript session so each harness's // invocation-scoped overlay or safe fallback is applied; otherwise one-shot is cheaper. - return runModelCall(call, pick, { - sessionName: () => `backpass-analysis-${process.pid}-${slot}-${++callCounter}`, - }); + return invokeAnalysis(pick, () => + runModelCall(call, pick, { + sessionName: () => `backpass-analysis-${process.pid}-${slot}-${++callCounter}`, + }), + ); }); for (const note of result.notes || []) noteOnce(note); @@ -599,11 +602,24 @@ export async function analyzeTranscripts({ path: path.isAbsolute(skill.path) ? skill.path : path.join(repo.root, skill.path), })) : skills, + repo.root, ); + let claudeWarmup = null; + const invokeAnalysis = (current, invoke) => { + if (current.agent !== "claude" || config.jobs <= 1) return invoke(); + if (claudeWarmup) return claudeWarmup.then(invoke); + const first = invoke(); + claudeWarmup = first.then( + () => {}, + () => {}, + ); + return first; + }; + let done = 0; const evidenceTotals = { positive: 0, negative: 0, gaps: 0 }; - await pool(pending, config.jobs, async (transcript, _index, slot) => { + const analyzePending = async (transcript, _index, slot) => { const base = { transcript: transcriptMetadata(transcript), memoryHash, @@ -627,25 +643,26 @@ export async function analyzeTranscripts({ config, repo, modelCwd, + invokeAnalysis, slot, openGapIndex, skillIndex, alsoLoaded, }); if (result.status === "skipped") { - summary.skipped += 1; state.writeEvidence(transcript, { ...base, status: "skipped", reason: result.reason }); + summary.skipped += 1; } else { base.key = evidenceKey(transcript, memoryHash, result.route); base.route = result.route; - summary.analyzed += 1; - summary.usage.push(result.usage); state.writeEvidence(transcript, { ...base, status: "ok", stats: result.distilled.stats, ...result.evidence, }); + summary.analyzed += 1; + summary.usage.push(result.usage); evidenceTotals.positive += result.evidence.positive.length; evidenceTotals.negative += result.evidence.negative.length; evidenceTotals.gaps += result.evidence.gaps.length; @@ -671,7 +688,9 @@ export async function analyzeTranscripts({ info(`${color.dim(` ${done}/${pending.length} analyzed`)}`); } } - }); + }; + + await pool(pending, config.jobs, analyzePending); if (summary.quotesNotInTrace) { warn( diff --git a/src/apply/writer.js b/src/apply/writer.js index b9451677..cb48f5bd 100644 --- a/src/apply/writer.js +++ b/src/apply/writer.js @@ -301,7 +301,16 @@ function removeEmptyDirectories(directories) { * A file is therefore applied all at once or not at all. Skills are written only after * every accepted edit has composed, and before the files that reference them. */ -export function applyDecisions({ proposal, decisions, repo, state, config, dryRun = false, rejectReasons = {} }) { +export function applyDecisions({ + proposal, + decisions, + repo, + state, + config, + dryRun = false, + rejectReasons = {}, + appliedBy = null, +}) { const accepted = proposal.edits.filter((e) => decisions[e.id] === "accepted"); const rejected = proposal.edits.filter((e) => decisions[e.id] === "rejected"); @@ -711,7 +720,7 @@ export function applyDecisions({ proposal, decisions, repo, state, config, dryRu if (!dryRun && canonical) { try { - const layout = ensureSkillsLayout(repo.root, claudeSkillsLink); + const layout = ensureSkillsLayout(repo.root, claudeSkillsLink, ownedSkillPaths); const result = results.skills.find(({ path: skillPath }) => skillPath === canonical.skill.path); result.created = [...new Set([...result.created, ...layout.created])]; for (const w of layout.warnings) if (!results.warnings.includes(w)) results.warnings.push(w); @@ -724,12 +733,24 @@ export function applyDecisions({ proposal, decisions, repo, state, config, dryRu } // Rejections are remembered so the same edit is not re-proposed without new evidence. - if (!dryRun && rejected.length) { - const rejections = state.readRejections(); - for (const edit of rejected) - recordRejection(edit, rejections, new Date().toISOString(), rejectReasons[edit.id], proposal.provenance); - state.writeRejections(rejections); - results.rejectionsRecorded = true; + if (!dryRun) { + try { + if (rejected.length) { + const rejections = state.readRejections(); + for (const edit of rejected) + recordRejection(edit, rejections, new Date().toISOString(), rejectReasons[edit.id], proposal.provenance); + state.writeRejections(rejections); + results.rejectionsRecorded = true; + } + if (accepted.length && appliedBy) { + state.writeProposal({ ...proposal, appliedAt: new Date().toISOString(), appliedBy }, proposal); + } + } catch (err) { + results.failed.push({ error: err.message }); + rollbackCommitted(); + rollbackSkills(); + return results; + } } return results; diff --git a/src/commands/apply.js b/src/commands/apply.js index 912336e8..c534e3c8 100644 --- a/src/commands/apply.js +++ b/src/commands/apply.js @@ -165,6 +165,7 @@ export async function cmdApply(ctx) { config, dryRun: Boolean(ctx.flags["dry-run"]), rejectReasons, + appliedBy: "apply", }); if (surfaceFile) await closeApplySurface(surfaceFile); diff --git a/src/memory.js b/src/memory.js index b395d446..25c44b64 100644 --- a/src/memory.js +++ b/src/memory.js @@ -504,23 +504,20 @@ export function reanchor(reference, file, threshold = 0.6) { } /** - * Pointer detection. The convention for covering both harness families without - * duplicating content is a canonical AGENTS.md plus a CLAUDE.md that only contains the - * `@AGENTS.md` import (Claude Code inlines it at load time). Such a file carries no - * instructions of its own, so optimizing the target is fully correct and the pointer - * stays valid afterwards. + * Pointer detection excludes files with instructions of their own, so optimizing the + * imported target covers the importing file without leaving instructions unoptimized. * - * A file is a pointer to `target` when, ignoring blank lines and HTML comments, its - * only content is the import line (`@AGENTS.md` or `@./AGENTS.md`). + * A file is a pointer when, ignoring blank lines, HTML comments and leading + * ATX headings, its only content is the import line (`@AGENTS.md` or `@./AGENTS.md`). */ export function pointerImportPath(text, options = {}) { const lines = text .replace(//g, "") .split("\n") - .map((l) => l.trim()) - .filter(Boolean); + .filter((l) => l.trim()); + while (lines.length && /^ {0,3}#{1,6}(?:\s|$)/.test(lines[0])) lines.shift(); if (lines.length !== 1) return null; - const imported = lines[0].replace(/^@\.\//, "@"); + const imported = lines[0].trim().replace(/^@\.\//, "@"); if (!imported.startsWith("@")) return null; const spec = imported.slice(1); diff --git a/src/redact.js b/src/redact.js index d67910b1..1b57afb6 100644 --- a/src/redact.js +++ b/src/redact.js @@ -25,6 +25,8 @@ const PATTERNS = [ /\b([A-Za-z0-9_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|API_?KEY|ACCESS_?KEY)[A-Za-z0-9_]*)\s*[=:]\s*(?:"([^"]{8,})"(?=[ \t]*(?:$|[,;})\]]))|'([^']{8,})'(?=[ \t]*(?:$|[,;})\]]))|"([^"\n]{8,})"|'([^'\n]{8,})'|["']?([^\s"',]{8,}))/gim, "ASSIGNMENT", ], + // Run after assignments so a matching variable name cannot hide an unredacted value. + [/\b(apikey_[A-Za-z0-9_-]{16,})/gi, "API_KEY"], ]; export function redact(text) { diff --git a/src/skills.js b/src/skills.js index f7be3d65..ebe83ef9 100644 --- a/src/skills.js +++ b/src/skills.js @@ -77,23 +77,26 @@ export function loadSkills(repoRoot, skillsDir) { } catch { continue; } - const frontmatter = parseFrontmatter(text); - skills.push({ - name: frontmatter.name || entry.name.replace(/\.md$/, ""), - description: frontmatter.description || "", - path: (() => { - const relative = path.relative(repoRoot, file); - return relative.startsWith("..") || path.isAbsolute(relative) ? file : relative.split(path.sep).join("/"); - })(), - body: skillBody(text), - bodyTokens: estimateTokens(text), - descriptionTokens: estimateTokens(frontmatter.description || ""), - }); + const relative = path.relative(repoRoot, file); + const logical = relative.startsWith("..") || path.isAbsolute(relative) ? file : relative.split(path.sep).join("/"); + skills.push(describeSkill(logical, text, entry.name.replace(/\.md$/, ""))); } return skills.sort((a, b) => a.name.localeCompare(b.name) || a.path.localeCompare(b.path)); } +export function describeSkill(file, text, fallbackName) { + const frontmatter = parseFrontmatter(text); + return { + name: frontmatter.name || fallbackName, + description: frontmatter.description || "", + path: file, + body: skillBody(text), + bodyTokens: estimateTokens(text), + descriptionTokens: estimateTokens(frontmatter.description || ""), + }; +} + /** * Resolve supported skill roots without double-counting directories reached by symlink. * Project scope includes the conventional roots; exact mode uses only the configured @@ -250,11 +253,28 @@ export function renderSkillIndex(skills) { * The skill index the analysis prompt sees: name, path, and trigger description only. * Bodies are deliberately absent - the analysis turn opens a SKILL.md by path when a * description looks relevant to a mistake, instead of paying for every body on every - * transcript. + * transcript. Each real file is listed once, with its first loaded path; the original + * entries stay intact for per-copy description billing. Relative paths belong to repoRoot, + * not necessarily the process cwd. If a file disappeared, keep its entry rather than + * treating all unresolved paths as the same skill. */ -export function renderSkillIndexForAnalysis(skills) { +export function renderSkillIndexForAnalysis(skills, repoRoot = process.cwd()) { if (!skills.length) return "(this repo has no skills)"; - return skills.map((s) => `- ${s.name} (${s.path}) :: ${s.description || "(no description)"}`).join("\n"); + const seen = new Set(); + return skills + .filter((skill) => { + let identity; + try { + identity = fs.realpathSync(path.resolve(repoRoot, skill.path)); + } catch { + return true; + } + if (seen.has(identity)) return false; + seen.add(identity); + return true; + }) + .map((s) => `- ${s.name} (${s.path}) :: ${s.description || "(no description)"}`) + .join("\n"); } /** Serialize a skill draft to the common SKILL.md shape. */ @@ -520,7 +540,7 @@ export function removeOwnedSkillPaths(paths) { * `.claude/skills` is never clobbered: a symlink (to anywhere) is left as is, and a real * directory is reported so the user can merge it by hand. */ -export function ensureSkillsLayout(repoRoot, claudeSkillsDir = CLAUDE_SKILLS_LINK) { +export function ensureSkillsLayout(repoRoot, claudeSkillsDir = CLAUDE_SKILLS_LINK, ownership = []) { const created = []; const warnings = []; const canonical = path.join(repoRoot, CANONICAL_SKILLS_DIR); @@ -535,6 +555,7 @@ export function ensureSkillsLayout(repoRoot, claudeSkillsDir = CLAUDE_SKILLS_LIN if (claude.state === "missing") { fs.mkdirSync(path.dirname(link), { recursive: true }); fs.symlinkSync(target, link, "dir"); + ownership.push({ absolute: link, identity: pathIdentity(fs.lstatSync(link)), relative: claudeSkillsDir }); created.push(`${claudeSkillsDir} -> ${target}`); } else if (claude.state === "dir") { warnings.push(claudeSkillsDirWarning(claudeSkillsDir, target)); diff --git a/src/state.js b/src/state.js index 7280a419..3e281b0e 100644 --- a/src/state.js +++ b/src/state.js @@ -2,6 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import crypto from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; import { STATE_DIRNAME } from "./config.js"; import { UserError, warn } from "./logger.js"; @@ -219,7 +220,10 @@ export class State { return this.readJsonFile(this.proposalPath, null); } - writeProposal(proposal) { + writeProposal(proposal, reviewed = undefined) { + if (reviewed !== undefined && !isDeepStrictEqual(this.readProposal(), reviewed)) { + throw new UserError("the saved proposal changed during review; review the current proposal again"); + } this.writeJsonFile(this.proposalPath, proposal); } diff --git a/src/synthesize.js b/src/synthesize.js index 350a41fd..f91f01a0 100644 --- a/src/synthesize.js +++ b/src/synthesize.js @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; -import { extractJson, isBlankOutput, openSession, usageRecord } from "./acpx.js"; +import { AcpxError, extractJson, isBlankOutput, openSession, usageRecord } from "./acpx.js"; import { expandHomePath, userClaudeSkillsDir } from "./config.js"; import { renderEvidenceForPrompt } from "./fold.js"; import { renderInstructionIndex, resolveMemoryPath } from "./memory.js"; @@ -9,6 +9,7 @@ import { renderPrompt, render, loadPrompt } from "./prompts.js"; import { buildProposal, effectiveMaxEdits, ProposalViolation, renderChangesForPrompt } from "./proposal.js"; import { routeForPick } from "./provenance.js"; import { + describeSkill, loadProjectSkills, renderSkillIndex, resolveOverflowTarget, @@ -20,8 +21,11 @@ import { SURFACE_TARGET } from "./target.js"; import { emitProgress } from "./progress.js"; import { READ_ONLY_SEARCH_PATH, + isSkillFilePath, measureWorkspace, prepareWorkspace, + rebaseFingerprint, + repoFilePath, repoFingerprint, workspacePathFor, } from "./workspace.js"; @@ -168,11 +172,13 @@ function harnessCountsOf(transcripts) { * reaches that decision. An ordinary repository skill stays fingerprinted either way. A * fingerprinted path can still resolve outside the repository - that is the ordinary * user-scope layout - so a change there is reported for what it is rather than as a direct - * repository edit. A configured `skillSearchPaths` root is the one withheld reason that - * stays fingerprinted anyway: its read-only promise is not "backpass will never write - * this," it is "nothing may ever write this," so a change there must still fail the run. + * repository edit. A change inside the repository is reported as what was observed, since + * a concurrent editor in a shared workspace looks the same as a harness (#199). A + * configured `skillSearchPaths` root is the one withheld reason that stays fingerprinted + * anyway: its read-only promise is not "backpass will never write this," it is "nothing + * may ever write this," so a change there must still fail the run. */ -function assertRepoUntouched(repo, before, workspaceRoot) { +function assertRepoUntouched(repo, before, workspace) { const after = repoFingerprint(repo, Object.keys(before)); const moved = Object.keys(before).filter((file) => before[file] !== after[file]); if (!moved.length) return; @@ -184,12 +190,29 @@ function assertRepoUntouched(repo, before, workspaceRoot) { return false; } }; + // The fingerprint only proves a file changed, not who changed it: in a shared workspace + // another session or a cron job can edit it mid-run. The harness is named only when the + // repository file now equals the staging copy, which a bystander has no reason to produce. + const matchesStaging = (file) => { + const staged = workspace.stagedPaths?.get(file) ?? workspacePathFor(file); + try { + return ( + fs.readFileSync(path.join(workspace.root, staged), "utf8") === fs.readFileSync(repoFilePath(repo, file), "utf8") + ); + } catch { + return false; + } + }; const inside = moved.filter(insideRepo); const outside = moved.filter((file) => !inside.includes(file)); + const wroteThrough = inside.filter(matchesStaging); const claims = []; if (inside.length) { + claims.push(`${inside.join(", ")} changed in the repository during synthesis`); + } + if (wroteThrough.length) { claims.push( - `synthesis changed ${inside.join(", ")} in the repository directly instead of the staging copy (${workspaceRoot})`, + `${wroteThrough.join(", ")} now matches the staging copy (${workspace.root}), so the synthesis harness likely wrote it directly`, ); } if (outside.length) { @@ -200,9 +223,11 @@ function assertRepoUntouched(repo, before, workspaceRoot) { } throw new UserError( `${claims.join("; also ")}; nothing was proposed`, - inside.length + wroteThrough.length ? `inspect the change with \`git diff\`, restore the file, and re-run - a harness that edits outside its cwd cannot be trusted with the synthesis role` - : `inspect the file and re-run - either the synthesis harness wrote through the link to a skill it was told is read-only, or another process changed the shared library mid-run`, + : inside.length + ? `inspect the change with \`git diff\` and re-run - another process (a concurrent agent session or a scheduled job) may have edited the file mid-run; run from a worktree snapshot if other writers share this directory, and if nothing else was writing, suspect the synthesis harness` + : `inspect the file and re-run - either the synthesis harness wrote through the link to a skill it was told is read-only, or another process changed the shared library mid-run`, ); } @@ -423,7 +448,7 @@ async function annotateLoop({ let editMadeNoChanges = false; for (let turn = 1; ; turn += 1) { - assertRepoUntouched(repo, fingerprint, workspace.root); + assertRepoUntouched(repo, fingerprint, workspace); const measured = measureWorkspace(workspace); if (turn === 1) editMadeNoChanges = measured.changes.length === 0 && !(measured.stray || []).length; @@ -452,7 +477,7 @@ async function annotateLoop({ // The agent may keep editing during an annotate turn; the ids it was answering about // are then stale, so the answer is dropped and the fresh measurement shown instead. // That is a measurement problem, not a failed annotation: it costs no attempt. - assertRepoUntouched(repo, fingerprint, workspace.root); + assertRepoUntouched(repo, fingerprint, workspace); if (measureWorkspace(workspace).signature !== measured.signature) { remeasures += 1; justRemeasured = true; @@ -556,21 +581,7 @@ export async function synthesizeProposal({ }) { config.state.clearProposal(); const harnessCounts = harnessCountsOf(transcripts); - const { - state, - rejections, - overflow, - skillDirs, - skillFiles, - searchPathRoots, - target, - nested, - descriptionTokens, - maxEdits, - common, - context, - promptDir, - } = synthesisSetup({ + const setup = synthesisSetup({ memoryFile, summary, config, @@ -580,6 +591,8 @@ export async function synthesizeProposal({ provenance, routing, }); + const { state, rejections, overflow, skillDirs, searchPathRoots, target, nested, common, context, promptDir } = setup; + let { skillFiles, descriptionTokens, maxEdits } = setup; // Staging holds only the write surface: every skill on a surface run, none of them on // a memory-file target (new extracts are still measured), just the one on a skill target. @@ -603,46 +616,48 @@ export async function synthesizeProposal({ // confined out are also marked, so a run that narrows nothing still says so. const readOnlyReason = (file) => workspace.unstageable.find((entry) => file === entry.path || file.startsWith(`${entry.path}/`))?.reason || null; - const stagedSkillFiles = skillFiles.map((skill) => ({ - ...skill, - path: workspace.stagedPaths.get(skill.path) || skill.path, - readOnly: readOnlyReason(skill.path), - })); - - const editValues = { - ...common, - TARGET_RULE: targetRule( - target, - workspace.memoryWorkspacePath, - stagedSkillsDir, - target.kind === "skill" ? workspace.stagedPaths.get(target.path) || workspacePathFor(target.path) : null, - stagedSkillFiles.filter((skill) => skill.readOnly), - ), - REPO_NAME: repo.name, - REPO_ROOT: repo.root, - TRANSCRIPT_COUNT: String(summary.analyzedSessions), - RUN_NOTE: runNote, - HARNESS_SUMMARY: - Object.entries(harnessCounts) - .map(([h, n]) => `${h} ${n}`) - .join(" · ") || "none", - CURRENT_TOKENS: String(memoryFile.tokens + descriptionTokens), - BUDGET_TOKENS: String(config.budgetTokens), - BUDGET_STATE: budgetState(memoryFile, config, descriptionTokens), - INSTRUCTION_INDEX: renderInstructionIndex(memoryFile), - SKILLS_DIR: stagedSkillsDir, - SKILL_INDEX: nested - ? "(skills belong to the root surface; a nested memory file run does not edit them)" - : renderSkillIndex(stagedSkillFiles), - BUDGET_COUNT: nested - ? "The count is this file alone: it loads on top of the root memory\nfile, which has a budget of its own." - : "The count is this file PLUS every skill's `description:` line;\nskill bodies are free until triggered.", - EVIDENCE: renderEvidenceForPrompt(summary), - REJECTIONS: renderRejections(rejections), - }; - const editPromptFile = path.join(promptDir, "synthesis-edit.md"); - fs.writeFileSync(editPromptFile, renderPrompt("synthesis", editValues)); + const writeEditPrompt = () => { + const stagedSkillFiles = skillFiles.map((skill) => ({ + ...skill, + path: workspace.stagedPaths.get(skill.path) || skill.path, + readOnly: readOnlyReason(skill.path), + })); + + const editValues = { + ...common, + TARGET_RULE: targetRule( + target, + workspace.memoryWorkspacePath, + stagedSkillsDir, + target.kind === "skill" ? workspace.stagedPaths.get(target.path) || workspacePathFor(target.path) : null, + stagedSkillFiles.filter((skill) => skill.readOnly), + ), + REPO_NAME: repo.name, + REPO_ROOT: repo.root, + TRANSCRIPT_COUNT: String(summary.analyzedSessions), + RUN_NOTE: runNote, + HARNESS_SUMMARY: + Object.entries(harnessCounts) + .map(([h, n]) => `${h} ${n}`) + .join(" · ") || "none", + CURRENT_TOKENS: String(memoryFile.tokens + descriptionTokens), + BUDGET_TOKENS: String(config.budgetTokens), + BUDGET_STATE: budgetState(memoryFile, config, descriptionTokens), + INSTRUCTION_INDEX: renderInstructionIndex(memoryFile), + SKILLS_DIR: stagedSkillsDir, + SKILL_INDEX: nested + ? "(skills belong to the root surface; a nested memory file run does not edit them)" + : renderSkillIndex(stagedSkillFiles), + BUDGET_COUNT: nested + ? "The count is this file alone: it loads on top of the root memory\nfile, which has a budget of its own." + : "The count is this file PLUS every skill's `description:` line;\nskill bodies are free until triggered.", + EVIDENCE: renderEvidenceForPrompt(summary), + REJECTIONS: renderRejections(rejections), + }; + + fs.writeFileSync(editPromptFile, renderPrompt("synthesis", editValues)); + }; // A search-path skill is unstageable like any other read-only skill, but unlike the // rest of them the read-only promise it carries must be enforceable: a direct write to @@ -701,11 +716,20 @@ export async function synthesizeProposal({ const holder = { session: null, ranWith, - prompt(args) { + async prompt(args) { if (!this.session) throw new Error("synthesis session is not open"); - return this.session.prompt(args); + // Measure the assembled UTF-8 artifact before handing it to the harness. Size is + // diagnostic context for any failed turn, not a verdict about why it failed. + const promptBytes = fs.statSync(args.promptFile).size; + try { + return await this.session.prompt(args); + } catch (err) { + if (err instanceof AcpxError) err.message += `\nSynthesis prompt size: ${promptBytes} UTF-8 bytes.`; + throw err; + } }, }; + let attempted = false; const editResult = await config.agents.withFallthrough("synthesis", async (current) => { ranWith = current.agent; holder.ranWith = current.agent; @@ -715,11 +739,39 @@ export async function synthesizeProposal({ const currentRoute = routeForPick(config, current); if (JSON.stringify(currentRoute) !== JSON.stringify(context.provenance.routeProfile.synthesisResolved)) { rejections.entries = {}; - fs.writeFileSync(editPromptFile, renderPrompt("synthesis", { ...editValues, REJECTIONS: "(none)" })); } context.provenance.routeProfile.synthesisResolved = currentRoute; } + // A candidate after the first runs behind a harness that may already have written to the + // repository; rebasing over that write would hide it, so it is reported before the rebuild. + if (attempted) assertRepoUntouched(repo, fingerprint, workspace); workspace = prepareWorkspace(workspaceOptions); + const snapshotPaths = new Set([ + ...skillFiles.map((skill) => skill.path), + ...[...workspace.originals.keys()].filter((file) => file !== memoryFile.path && isSkillFilePath(file, skillDirs)), + ]); + const previousSkills = new Map(skillFiles.map((skill) => [skill.path, skill])); + skillFiles = [...snapshotPaths] + .map((file) => { + const identity = workspace.unstageable.find((entry) => entry.path === file)?.identity; + const owner = workspace.stagedIdentities.get(identity); + const text = workspace.originals.get(file) ?? workspace.originals.get(owner); + const dir = skillDirs.find((dir) => isSkillFilePath(file, dir)); + const fallbackName = path.relative(dir, file).split(path.sep)[0].replace(/\.md$/, ""); + return text === undefined ? previousSkills.get(file) : describeSkill(file, text, fallbackName); + }) + .sort((a, b) => a.name.localeCompare(b.name) || a.path.localeCompare(b.path)); + for (const skill of skillFiles) { + if (workspace.originals.has(skill.path) && !(skill.path in fingerprint)) fingerprint[skill.path] = null; + } + rebaseFingerprint(fingerprint, workspace); + context.skillFiles = skillFiles; + descriptionTokens = skillDescriptionTokens(skillFiles); + maxEdits = effectiveMaxEdits(memoryFile, config, descriptionTokens); + common.MAX_EDITS = String(maxEdits); + common.BUDGET_RULE = budgetRule(memoryFile, config, maxEdits, descriptionTokens, { extraction: !nested }); + writeEditPrompt(); + attempted = true; progress("edit", { attempt: 1 }); holder.session = await openSession({ agent: current.agent, @@ -730,7 +782,7 @@ export async function synthesizeProposal({ writeAccess: true, }); try { - return await holder.session.prompt({ + return await holder.prompt({ promptFile: editPromptFile, approveAll: true, timeoutSeconds, diff --git a/src/workspace.js b/src/workspace.js index 7cce147b..68aa282b 100644 --- a/src/workspace.js +++ b/src/workspace.js @@ -19,6 +19,10 @@ import { sha256 } from "./state.js"; * The agent reads the real repository by absolute path for grounding; what it changes in * the copy is measured here (`measureWorkspace`) and becomes the proposal the human * reviews. The copy is wiped and rebuilt on every synthesis. + * + * Version-control internals and binary files are not the memory surface, so they stay out + * of the copy and each is named in `unstageable` with its reason (#75): a skill that ships + * its own `.git` or compiled tools must not turn staging into a copy of the whole store. */ export const WORKSPACE_DIRNAME = "synthesis"; @@ -75,6 +79,7 @@ export function prepareWorkspace({ staged: workspacePathFor(logical), source: path.isAbsolute(logical) ? logical : path.join(repo.root, logical), })); + /** @type {Array<{ path: string, reason: string, identity?: string | null }>} */ const unstageable = []; const stagedIdentities = new Map(); for (const { logical: sourceDir, staged: stagedDir, source: skillsSource } of skillMappings) { @@ -104,7 +109,14 @@ export function prepareWorkspace({ // one stays behind the narrowing: only a staged name can own anything. const owner = identity && stagedIdentities.get(identity); if (owner) { - unstageable.push({ path: logical, reason: `the same file is already staged as ${owner}` }); + unstageable.push({ path: logical, reason: `the same file is already staged as ${owner}`, identity }); + continue; + } + // `originals` holds every staged file as utf-8 text, and a binary does not survive that + // decode: it can never be a faithful original, no text edit applies to it, and a store + // of compiled tools held this way exhausts the heap. It stays out, named. + if (isBinaryFile(from)) { + unstageable.push({ path: logical, reason: READ_ONLY_BINARY }); continue; } const staged = path.posix.join(stagedDir, relative); @@ -120,7 +132,7 @@ export function prepareWorkspace({ // touched; the source keeps its own mode, and apply takes the mode it writes from // the repository file rather than from here. fs.chmodSync(to, (fs.statSync(from).mode & 0o777) | 0o600); - originals.set(logical, fs.readFileSync(from, "utf8")); + originals.set(logical, fs.readFileSync(to, "utf8")); } catch (err) { fs.rmSync(to, { force: true }); originals.delete(logical); @@ -174,6 +186,27 @@ function entryKind(dir, entry) { const SKILL_FILENAME = "SKILL.md"; +/** Version-control internals: never skill content, so never walked, copied, or read. */ +const VCS_ENTRIES = new Set([".git", ".hg", ".svn"]); + +/** Bytes sniffed for a NUL: git's own test for "not text" (xdiff `buffer_is_binary`). */ +const BINARY_SNIFF_BYTES = 8000; + +/** True when the file's first bytes hold a NUL; a file that cannot be read is left to the copy's own handling. */ +function isBinaryFile(file) { + let fd = null; + try { + fd = fs.openSync(file, "r"); + const head = Buffer.alloc(BINARY_SNIFF_BYTES); + const read = fs.readSync(fd, head, 0, BINARY_SNIFF_BYTES, 0); + return head.subarray(0, read).includes(0); + } catch { + return false; + } finally { + if (fd !== null) fs.closeSync(fd); + } +} + function isFile(file) { try { return fs.statSync(file).isFile(); @@ -209,6 +242,12 @@ function walkFiles(dir, prefix = "", confineTo = null, confined = [], searchPath }; for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { const relative = prefix ? path.posix.join(prefix, entry.name) : entry.name; + // A nested object store is pruned, never descended into: it is not the memory surface, + // and walking it is how staging copied a whole repository's history (#75). + if (VCS_ENTRIES.has(entry.name)) { + confined.push({ relative, reason: READ_ONLY_VCS }); + continue; + } // Follow symlinks: a skills directory is commonly a set of links into a shared // library, and those are the files the harness loads. Staging copies what it finds, // so an edit lands in the staging copy and never writes through a link. @@ -251,6 +290,9 @@ function walkFiles(dir, prefix = "", confineTo = null, confined = [], searchPath const READ_ONLY_OUTSIDE_REPO = "resolves outside the repository"; const READ_ONLY_UNREADABLE = "could not be read when the staging copy was built"; const READ_ONLY_UNWRITABLE = "resolves to a location that cannot be written"; +/** Not the memory surface, so staging leaves it out: a nested object store, or a binary file. */ +export const READ_ONLY_VCS = "is version-control internals, not skill content"; +export const READ_ONLY_BINARY = "is a binary file, not text an edit can apply to"; /** A configured `skillSearchPaths` root: read-only in every scope, never subject to `allowExternal`. */ export const READ_ONLY_SEARCH_PATH = "resolves inside a configured skillSearchPaths root"; @@ -311,6 +353,7 @@ export function skillStagingRefusal( /** Why measurement dropped a file the model wrote: the note the human reads must say which. */ export const STRAY_OUTSIDE_SURFACE = "synthesis wrote it outside the memory file and skills"; export const STRAY_OUTSIDE_REPO = "it resolves outside the repository, which project scope cannot write"; +export const STRAY_VCS = "synthesis wrote version-control internals, which are never carried or applied"; export const strayAliasReason = (owner) => `it is the same file already staged as ${owner}`; export const STRAY_UNWRITABLE = "it resolves to a location that cannot be written, so staging withheld it from the copy"; @@ -507,7 +550,9 @@ export function measureWorkspace(workspace) { const stray = []; const texts = new Map(); - const present = new Set(walkFiles(root).map((p) => p.split(path.sep).join("/"))); + // Staging never copies a VCS entry, so any the walk prunes here was written by synthesis. + const skipped = []; + const present = new Set(walkFiles(root, "", null, skipped).map((p) => p.split(path.sep).join("/"))); const ordered = [memoryPath, ...[...originals.keys()].filter((f) => f !== memoryPath).sort()]; for (const logical of ordered) { @@ -533,20 +578,31 @@ export function measureWorkspace(workspace) { { logical: path.dirname(memoryPath), staged: path.posix.dirname(memoryStaged) }, ]; + const mappingFor = (staged) => dirMappings.find(({ staged: dir }) => staged === dir || staged.startsWith(`${dir}/`)); + // Every note names the path the reader knows - the repository path, or the real one a + // user-scope entry resolves to - never the `.external/` the staging copy uses. + const logicalFor = (mapping, staged) => { + const inside = staged.slice(mapping.staged.length).replace(/^\//, ""); + return path.isAbsolute(mapping.logical) + ? path.join(mapping.logical, inside) + : path.posix.join(mapping.logical, inside); + }; + + for (const { relative } of skipped.sort((a, b) => a.relative.localeCompare(b.relative))) { + const mapping = mappingFor(relative); + stray.push({ file: mapping ? logicalFor(mapping, relative) : relative, reason: STRAY_VCS }); + } + const knownStaged = new Set(stagedPaths.values()); for (const staged of [...present].sort()) { if (knownStaged.has(staged)) continue; - const mapping = dirMappings.find(({ staged: dir }) => staged === dir || staged.startsWith(`${dir}/`)); + const mapping = mappingFor(staged); if (!mapping) { stray.push({ file: staged, reason: STRAY_OUTSIDE_SURFACE }); continue; } const inside = staged.slice(mapping.staged.length).replace(/^\//, ""); - // Every note names the path the reader knows - the repository path, or the real one a - // user-scope entry resolves to - never the `.external/` the staging copy uses. - const logical = path.isAbsolute(mapping.logical) - ? path.join(mapping.logical, inside) - : path.posix.join(mapping.logical, inside); + const logical = logicalFor(mapping, staged); if (!isSkillFilePath(logical, skillDirs)) { stray.push({ file: logical, reason: STRAY_OUTSIDE_SURFACE }); continue; @@ -652,12 +708,30 @@ function signatureOf(changes) { ).slice(0, 16); } +/** Where a fingerprinted key lives on disk: configured memory paths may be absolute, skills are repo-relative. */ +export function repoFilePath(repo, file) { + return path.isAbsolute(file) ? file : path.join(repo.root, file); +} + /** The files backpass must find untouched in the repo after synthesis: a moved write is a bug, loudly. */ export function repoFingerprint(repo, files) { const out = {}; for (const relative of files) { - const absolute = path.isAbsolute(relative) ? relative : path.join(repo.root, relative); + const absolute = repoFilePath(repo, relative); out[relative] = fs.existsSync(absolute) ? sha256(fs.readFileSync(absolute, "utf8")) : null; } return out; } + +/** + * Point the fingerprint of each staged skill at the text its staging copy was built from. + * The workspace is rebuilt after the fingerprint is taken, and a skill another process edits + * in between is copied as the new text; leaving the old hash would later read that edit as + * a write-through that happens to equal the staging copy. The memory file is not rebased: + * it is staged from the text already loaded, not re-read from disk. + */ +export function rebaseFingerprint(fingerprint, workspace) { + for (const [file, text] of workspace.originals) { + if (file !== workspace.memoryPath && file in fingerprint) fingerprint[file] = sha256(text); + } +} diff --git a/test/acpx-session-create-timeout.test.js b/test/acpx-session-create-timeout.test.js index 50ed2022..f921b6d3 100644 --- a/test/acpx-session-create-timeout.test.js +++ b/test/acpx-session-create-timeout.test.js @@ -24,6 +24,7 @@ fs.writeFileSync(promptFile, "analyze this\n"); * One fake acpx, two behaviours picked at spawn time by `FAKE_ACPX_MODE`: * hang `sessions new` never answers, then exits 130 on SIGTERM as acpx does * no-sessions `sessions new` is rejected outright, the genuine unsupported case + * model-rejected `sessions new` fails after the adapter turned down `--model` * `ACPX_BIN` is bound when `src/acpx.js` is imported, so the mode cannot be a second path. */ const fakeAcpx = path.join(fixtureDir, "acpx"); @@ -47,6 +48,10 @@ if (creating && process.env.FAKE_ACPX_MODE === "no-sessions") { process.stderr.write("error: unknown command 'sessions'\\n"); process.exit(2); } +if (creating && process.env.FAKE_ACPX_MODE === "model-rejected") { + process.stderr.write(process.env.FAKE_ACPX_STDERR || ""); + process.exit(1); +} if (creating && process.env.FAKE_ACPX_MODE === "scoped-timeouts") { setTimeout(() => process.exit(0), 300); } else if ((status || closing) && process.env.FAKE_ACPX_MODE === "scoped-timeouts") { @@ -64,7 +69,7 @@ fs.chmodSync(fakeAcpx, 0o755); process.env.BACKPASS_ACPX_BIN = fakeAcpx; process.env.FAKE_ACPX_MODE = "hang"; -const { AcpxError, openSession, probeSession, sessionPrompt } = await import("../src/acpx.js"); +const { AcpxError, classifyAcpxFailure, openSession, probeSession, sessionPrompt } = await import("../src/acpx.js"); const { UserError } = await import("../src/logger.js"); test.after(() => { @@ -166,6 +171,41 @@ test("an adapter that really rejects sessions still reports missing session supp } }); +/** + * Regression for a real run with acpx 0.19.6 and claude-agent-acp 0.81.2. A Claude Code + * build that predates the pinned model leaves it out of the advertised list, the adapter + * turns down `--model` after `session/new`, and acpx exits 1. Read as unsupported, the + * pinned analysis pass stopped with "its acpx adapter does not support sessions - upgrade + * acpx or omit the effort override", although sessions worked and acpx was current. + */ +test("an adapter that turns down the pinned model is named as a model failure, not missing session support", async () => { + process.env.FAKE_ACPX_MODE = "model-rejected"; + process.env.FAKE_ACPX_STDERR = + '[acpx] warning: requested model "claude-haiku-5-5" was not in the Claude ACP advertised model list (default, sonnet); forwarding it to Claude Code so the adapter can accept or reject it.\n' + + 'Failed session/set_config_option for model "claude-haiku-5-5": Invalid value for config option model: claude-haiku-5-5 (ACP -32603, adapter reported "Internal error")\n'; + try { + const options = { + agent: "claude", + model: "claude-haiku-5-5", + sessionName: "backpass-model-rejected", + cwd: fixtureDir, + createTimeoutMs: 1500, + }; + const named = (err) => { + assert.ok(err instanceof AcpxError, String(err)); + assert.notEqual(err.unsupported, true); + assert.match(err.message, /^acpx claude session create failed: /); + assert.equal(classifyAcpxFailure(err), "model-unavailable"); + return true; + }; + await assert.rejects(() => openSession(options), named); + await assert.rejects(() => sessionPrompt({ ...options, effort: "medium", promptFile }), named); + } finally { + process.env.FAKE_ACPX_MODE = "hang"; + delete process.env.FAKE_ACPX_STDERR; + } +}); + test("the availability probe scopes the cold-start budget to session creation", async () => { process.env.FAKE_ACPX_MODE = "scoped-timeouts"; const startedAt = Date.now(); diff --git a/test/analyze-claude-warmup.test.js b/test/analyze-claude-warmup.test.js new file mode 100644 index 00000000..6c9c77ea --- /dev/null +++ b/test/analyze-claude-warmup.test.js @@ -0,0 +1,174 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; + +/** + * Claude's OAuth refresh lock (#203): when the token is due, whichever Claude process + * starts first takes `.oauth_refresh.lock` and must stay alive until the refresh is done. + * backpass opens each analysis call with a short-lived `sessions new` helper, so four of + * them racing at `--jobs 4` can abandon the lock and every session then fails. The first + * analysis call with Claude therefore runs alone, start to finish, before the fan-out. + * This drives the real CLI against a fake acpx that logs when each call begins and ends. + */ + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const CLI = path.join(ROOT, "bin", "backpass.js"); + +const binDir = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-warmup-bin-")); +const callLog = path.join(binDir, "calls.log"); +const fakeAcpx = path.join(binDir, "acpx"); +const fakeClaude = path.join(binDir, "claude"); +const fakePi = path.join(binDir, "pi"); + +fs.writeFileSync(fakeClaude, `#!${process.execPath}\nprocess.stdout.write('{"loggedIn":true}\\n');\n`); +fs.writeFileSync(fakePi, `#!${process.execPath}\nprocess.exit(0);\n`); +// Every call is logged as it begins and ends: `new`/`end` per `sessions new`, `prompt`/`done` per prompt. +fs.writeFileSync( + fakeAcpx, + `#!${process.execPath} +const fs = require("node:fs"); +const argv = process.argv.slice(2); +const log = (what) => fs.appendFileSync(${JSON.stringify(callLog)}, what + "\\n"); +if (argv.includes("config") && argv.includes("show")) { + process.stdout.write(JSON.stringify({ agents: {} }) + "\\n"); + process.exit(0); +} +if (argv.includes("sessions") && argv.includes("new")) { + log("new"); + setTimeout(() => { process.stdout.write("fake-session-id\\n"); process.exit(0); }, 200); +} else if (argv.includes("--file")) { + const prompt = fs.readFileSync(argv[argv.indexOf("--file") + 1], "utf8"); + const session = /Do the (\\S+) work\\./.exec(prompt)[1]; + log("prompt " + session); + setTimeout(() => { + // A session id starting with "fail" returns unparseable text: the model call itself + // completes (so it is a genuine call attempt), but analysis fails afterward. + if (session.startsWith("fail")) { + log("failed " + session); + process.stdout.write("not json\\n"); + process.exit(0); + } + log("done " + session); + process.stdout.write(JSON.stringify({ positive: [], negative: [], gaps: [] }) + "\\n"); + process.exit(0); + }, 500); +} else { + process.exit(0); +} +`, +); +for (const file of [fakeAcpx, fakeClaude, fakePi]) fs.chmodSync(file, 0o755); + +function git(args, cwd) { + spawnSync("git", args, { cwd, stdio: "ignore" }); +} + +function initRepo() { + const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "backpass-warmup-repo-"))); + git(["init", "--quiet", "-b", "main"], dir); + git(["config", "user.email", "test@example.com"], dir); + git(["config", "user.name", "test"], dir); + fs.writeFileSync(path.join(dir, "AGENTS.md"), "# Agent instructions\n\n- Run `make build` before every push.\n"); + git(["add", "-A"], dir); + git(["commit", "-q", "-m", "memory"], dir); + return dir; +} + +function writeSession(home, id, cwd, ageSeconds, trivial = false) { + const dir = path.join(home, ".pi", "agent", "sessions", id); + fs.mkdirSync(dir, { recursive: true }); + const entries = [ + { type: "session", version: 3, id, timestamp: new Date().toISOString(), cwd }, + { type: "message", message: { role: "user", content: `Do the ${id} work.` } }, + ...(trivial + ? [] + : [ + { type: "message", message: { role: "assistant", content: "Working on it." } }, + { type: "message", message: { role: "user", content: "Now run the tests too." } }, + { type: "message", message: { role: "assistant", content: "Tests pass." } }, + ]), + ]; + const file = path.join(dir, `${id}.jsonl`); + fs.writeFileSync(file, `${entries.map((e) => JSON.stringify(e)).join("\n")}\n`); + const at = new Date(Date.now() - ageSeconds * 1000); + fs.utimesSync(file, at, at); +} + +function analyze(agent, jobs, sessions) { + fs.rmSync(callLog, { force: true }); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-warmup-home-")); + const dir = initRepo(); + sessions.forEach((session, index) => writeSession(home, session.id, dir, 60 * (index + 1), session.trivial)); + const result = spawnSync( + process.execPath, + [CLI, "analyze", "--harness", "pi", "--since", "all", "--analysis-agent", agent, "--jobs", String(jobs)], + { + cwd: dir, + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + PATH: `${binDir}${path.delimiter}${process.env.PATH}`, + BACKPASS_ACPX_BIN: fakeAcpx, + NO_COLOR: "1", + }, + encoding: "utf8", + timeout: 60000, + }, + ); + const output = `${result.stdout}${result.stderr}`; + assert.equal(result.status, 0, output); + return fs.readFileSync(callLog, "utf8").trim().split("\n"); +} + +const ids = (...names) => names.map((id) => ({ id })); + +test("with claude at jobs above one, the first analysis call finishes before any other call starts", () => { + const calls = analyze("claude", 4, ids("a1", "a2", "a3", "a4", "a5")); + const firstDone = calls.findIndex((line) => line.startsWith("done ")); + assert.ok(firstDone > 0, calls.join("\n")); + assert.deepEqual( + calls.slice(0, firstDone + 1).map((line) => line.split(" ")[0]), + ["new", "prompt", "done"], + `nothing else may start while the first call runs:\n${calls.join("\n")}`, + ); + assert.equal(calls.filter((line) => line.startsWith("done ")).length, 5, "the rest still run"); +}); + +test("a trivial session that makes no model call does not count as the warm-up", () => { + const calls = analyze("claude", 4, [{ id: "t1", trivial: true }, ...ids("b1", "b2", "b3")]); + const firstDone = calls.findIndex((line) => line.startsWith("done ")); + assert.deepEqual( + calls.slice(0, firstDone + 1).map((line) => line.split(" ")[0]), + ["new", "prompt", "done"], + calls.join("\n"), + ); +}); + +test("when the first analysis call fails, it still finishes alone and the rest run together", () => { + const calls = analyze("claude", 4, [{ id: "failing1" }, ...ids("e1", "e2", "e3", "e4")]); + const firstEnd = calls.findIndex((line) => line.startsWith("failed ") || line.startsWith("done ")); + assert.ok(firstEnd > 0 && calls[firstEnd].startsWith("failed "), calls.join("\n")); + assert.deepEqual( + calls.slice(0, firstEnd + 1).map((line) => line.split(" ")[0]), + ["new", "prompt", "failed"], + `nothing else may start while the first (failing) call runs:\n${calls.join("\n")}`, + ); + const afterFirst = calls.slice(firstEnd + 1); + assert.ok( + afterFirst.filter((line) => line.startsWith("prompt ")).length > 1, + `the remaining calls should overlap, not run one at a time:\n${calls.join("\n")}`, + ); + assert.equal(afterFirst.filter((line) => line.startsWith("done ")).length, 4, "the rest still run and succeed"); +}); + +test("other agents keep the full fan-out from the first call", () => { + const calls = analyze("pi", 4, ids("c1", "c2", "c3", "c4", "c5")); + const firstDone = calls.findIndex((line) => line.startsWith("done ")); + const startedBefore = calls.slice(0, firstDone).filter((line) => line.startsWith("prompt ")).length; + assert.ok(startedBefore > 1, `calls overlap without the warm-up:\n${calls.join("\n")}`); +}); diff --git a/test/analyze-reuse.test.js b/test/analyze-reuse.test.js index fd52e702..c3235002 100644 --- a/test/analyze-reuse.test.js +++ b/test/analyze-reuse.test.js @@ -78,12 +78,12 @@ function initRepo(memory) { } /** A minimal Pi session, non-trivial enough to clear the triviality filter. */ -function writeSession(home, id, cwd) { +function writeSession(home, id, cwd, userText = "Please build the project.") { const dir = path.join(home, ".pi", "agent", "sessions", id); fs.mkdirSync(dir, { recursive: true }); const entries = [ { type: "session", version: 3, id, timestamp: new Date().toISOString(), cwd }, - { type: "message", message: { role: "user", content: "Please build the project." } }, + { type: "message", message: { role: "user", content: userText } }, { type: "message", message: { role: "assistant", content: "Ran make build as instructed." } }, { type: "message", message: { role: "user", content: "Now run the tests too." } }, { type: "message", message: { role: "assistant", content: "Tests pass." } }, @@ -126,6 +126,43 @@ function runAnalyze(dir, home, extraArgs = []) { const MEMORY = "# Agent instructions\n\n- Run `make build` before every push.\n"; const MEMORY_EDITED = "# Agent instructions\n\n- Run `make build` before every push.\n- Never skip the changelog.\n"; +test("standalone apikey credentials never reach the cached analysis prompt", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-redact-home-")); + const dir = initRepo(MEMORY); + const tokens = [`apikey_${"a".repeat(64)}`, `APIKEY_${"B".repeat(64)}`]; + const assignments = []; + const secrets = []; + for (const name of ["apikey_openrouter_production", "APIKEY_OPENROUTER_PRODUCTION"]) { + for (const separator of ["=", ":"]) { + for (const quote of ["", '"', "'"]) { + const secret = `synthetic-secret-${secrets.length}`; + secrets.push(secret); + assignments.push(`${name}${separator}${quote}${secret}${quote}`); + } + } + } + writeSession( + home, + "redacted-session", + dir, + `Please build the project using (${tokens[0]}).\n${tokens[1]}\n${assignments.join("\n")}`, + ); + + const result = runAnalyze(dir, home); + assert.equal(result.status, 0, result.output); + assert.equal(result.summary.analyzed, 1); + const promptDir = path.join(dir, ".backpass", "prompts"); + const prompts = fs.readdirSync(promptDir); + assert.equal(prompts.length, 1); + const prompt = fs.readFileSync(path.join(promptDir, prompts[0]), "utf8"); + assert.match(prompt, /Please build the project using \(\[redacted:API_KEY\]\)\.\n\[redacted:API_KEY\]/); + for (const token of tokens) + assert.ok(!prompt.includes(token), "the cached model prompt must not contain a full credential"); + for (const secret of secrets) + assert.ok(!prompt.includes(secret), "the cached model prompt must not contain an assignment secret"); + assert.equal(prompt.match(/\[redacted:API_KEY\]=\[redacted\]/g)?.length, assignments.length); +}); + test("the first analysis performs work, an unchanged second reuses it, and --force reanalyzes anyway", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-reuse-home-")); const dir = initRepo(MEMORY); diff --git a/test/analyze-warmup-write-fault.test.js b/test/analyze-warmup-write-fault.test.js new file mode 100644 index 00000000..4b027f08 --- /dev/null +++ b/test/analyze-warmup-write-fault.test.js @@ -0,0 +1,135 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const binDir = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-warmup-fault-bin-")); +const fakeAcpx = path.join(binDir, "acpx"); +const fakePi = path.join(binDir, "pi"); +fs.writeFileSync(fakePi, `#!${process.execPath}\nprocess.exit(0);\n`); +fs.chmodSync(fakePi, 0o755); +process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH}`; +fs.writeFileSync( + fakeAcpx, + `#!${process.execPath} +const fs = require("node:fs"); +const argv = process.argv.slice(2); +if (argv.includes("config")) { + process.stdout.write('{"agents":{}}\\n'); +} else if (argv.includes("sessions")) { + process.stdout.write('fake-session\\n'); +} else if (argv.includes("--file")) { + const agent = argv.includes("pi") ? "pi" : "claude"; + const prompt = fs.readFileSync(argv[argv.indexOf("--file") + 1], "utf8"); + const id = /Do the (\\S+) work\\./.exec(prompt)[1]; + const log = (event) => fs.appendFileSync(process.env.WARMUP_CALL_LOG, JSON.stringify({ agent, id, event }) + "\\n"); + log("start"); + setTimeout(() => { + log("end"); + if (agent === "pi") { + process.stderr.write("[acpx] error: RUNTIME AUTH_REQUIRED Authentication required\\n"); + process.exit(1); + } + process.stdout.write('{"positive":[],"negative":[],"gaps":[]}\\n'); + }, 300); +} +`, +); +fs.chmodSync(fakeAcpx, 0o755); +process.env.BACKPASS_ACPX_BIN = fakeAcpx; + +const { analyzeTranscripts } = await import("../src/analyze.js"); +const { State } = await import("../src/state.js"); +const { AgentResolver } = await import("../src/agents.js"); +const { loadConfig } = await import("../src/config.js"); + +function writeSession(root, id) { + const file = path.join(root, `${id}.jsonl`); + const entries = [ + { type: "session", version: 3, id, timestamp: new Date().toISOString(), cwd: root }, + { type: "message", message: { role: "user", content: `Do the ${id} work.` } }, + { type: "message", message: { role: "assistant", content: "Working on it." } }, + { type: "message", message: { role: "user", content: "Now run the tests too." } }, + { type: "message", message: { role: "assistant", content: "Tests pass." } }, + ]; + fs.writeFileSync(file, `${entries.map((e) => JSON.stringify(e)).join("\n")}\n`); + return file; +} + +for (const fault of ["evidence", "prompt", "fallthrough"]) { + test(`Claude warm-up survives ${fault} failures and then permits fan-out`, async (t) => { + const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "backpass-warmup-fault-"))); + const callLog = path.join(dir, "calls.jsonl"); + process.env.WARMUP_CALL_LOG = callLog; + const transcripts = ["t1", "t2", "t3", "t4", "t5"].map((id) => ({ + harness: "pi", + id: `pi-${id}`, + nativeId: id, + path: writeSession(dir, id), + mtimeMs: Date.now(), + bytes: 200, + })); + class FaultingState extends State { + writeEvidence(transcript, evidence) { + if (fault === "evidence" && evidence.status === "ok" && transcript.nativeId === "t1") { + throw new Error("ENOSPC: evidence write failed"); + } + return super.writeEvidence(transcript, evidence); + } + } + const state = new FaultingState(dir, { exclude: false }).ensure(); + if (fault === "prompt") { + const write = fs.writeFileSync; + t.mock.method(fs, "writeFileSync", (file, data, options) => { + if (String(file).includes(`${path.sep}prompts${path.sep}`) && String(data).includes("Do the t1 work.")) { + throw new Error("ENOSPC: prompt write failed"); + } + return write(file, data, options); + }); + } + const config = { ...loadConfig(dir), state, jobs: 4 }; + if (fault === "fallthrough") { + config.ladders.analysis = [ + { model: "pi-test", agents: ["pi"] }, + { model: "claude-test", agents: ["claude"] }, + ]; + config.enforceEvidenceRoute = true; + config.agents = new AgentResolver(config, { + probeCandidate: async ({ model }) => ({ verdict: "ok", resolvedModel: model }), + acpxVersion: async () => "0.13.0", + providerAuthState: () => "test-seat", + }); + } else { + const pick = { agent: "claude" }; + config.agents = { resolve: async () => pick, withFallthrough: async (_role, fn) => fn(pick) }; + } + const summary = await analyzeTranscripts({ + transcripts, + memoryFile: { path: "AGENTS.md", units: [] }, + memoryHash: "test-memory", + repo: { root: dir }, + config, + }); + assert.equal(summary.failed, fault === "fallthrough" ? 0 : 1); + assert.equal(summary.analyzed, fault === "fallthrough" ? 5 : 4); + const records = state.listEvidence(); + if (fault !== "fallthrough") { + assert.equal(records.find((record) => record.transcript.id === "pi-t1").status, "failed"); + } else { + assert.ok(records.every((record) => record.route.agent === "claude")); + } + const events = fs + .readFileSync(callLog, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const claude = events.filter((event) => event.agent === "claude"); + assert.equal(claude[0].event, "start"); + assert.deepEqual(claude[1], { ...claude[0], event: "end" }); + assert.equal(claude.filter((event) => event.event === "start").length, fault === "prompt" ? 4 : 5); + const remaining = claude.slice(2); + assert.ok(remaining.findIndex((event) => event.event === "end") > 1, "calls after warm-up must overlap"); + if (fault === "fallthrough") assert.ok(events.some((event) => event.agent === "pi" && event.event === "end")); + }); +} diff --git a/test/apply-cli.test.js b/test/apply-cli.test.js index cdd23f48..0f3944dc 100644 --- a/test/apply-cli.test.js +++ b/test/apply-cli.test.js @@ -714,9 +714,203 @@ function rejectionsOf(dir) { return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")).entries : {}; } -test("--decisions writes accepted edits and remembers a rejection with its reason", () => { +test("successful normal apply stamps its proposal and refuses replay before review", () => { + for (const json of [false, true]) { + const dir = initRepo(); + const proposal = proposeExtractions(dir); + const proposalFile = path.join(dir, ".backpass", "proposal.json"); + const original = JSON.parse(fs.readFileSync(proposalFile, "utf8")); + const ids = proposal.edits.map((edit) => edit.id); + const beforeApply = Date.now(); + const applied = json + ? runDecided(dir, ids.map((id) => `${id}=accepted`).join(" "), ["--json"]) + : runApply(dir, ids); + + assert.equal(applied.status, 0, applied.output); + const saved = JSON.parse(fs.readFileSync(proposalFile, "utf8")); + assert.equal(saved.appliedBy, "apply"); + assert.ok(Date.parse(saved.appliedAt) >= beforeApply); + assert.ok(Date.parse(saved.appliedAt) <= Date.now()); + assert.deepEqual(saved, { ...original, appliedAt: saved.appliedAt, appliedBy: "apply" }); + let expected = MEMORY_TEXT; + for (const [heading, skill] of [ + ["CI", "ci-details"], + ["Release", "release-details"], + ]) { + expected = expected.replace(sectionBody(expected, heading), `- Load \`${skill}\` for this topic.`); + } + const files = ["AGENTS.md", ".agents/skills/ci-details/SKILL.md", ".agents/skills/release-details/SKILL.md"]; + assert.equal(fs.readFileSync(path.join(dir, "AGENTS.md"), "utf8"), expected); + const bytes = files.map((file) => fs.readFileSync(path.join(dir, file))); + const stamped = fs.readFileSync(proposalFile); + + const replay = runApply(dir, ids); + assert.equal(replay.status, 1, replay.output); + assert.match(replay.output, /already applied by apply/); + assert.doesNotMatch(replay.output, /review surface:/); + const invalid = runDecided(dir, "invalid-decision"); + assert.match(invalid.output, /already applied by apply/); + assert.deepEqual(fs.readFileSync(proposalFile), stamped); + files.forEach((file, i) => assert.deepEqual(fs.readFileSync(path.join(dir, file)), bytes[i])); + } +}); + +test("dry-run and rejected-only apply leave the proposal unstamped and retryable", () => { + for (const dryRun of [false, true]) { + const dir = initRepo(); + const proposal = proposeExtractions(dir); + const proposalFile = path.join(dir, ".backpass", "proposal.json"); + const before = fs.readFileSync(proposalFile); + const vector = proposal.edits.map((edit) => `${edit.id}=${dryRun ? "accepted" : "rejected"}`).join(" "); + const result = runDecided(dir, vector, dryRun ? ["--dry-run"] : []); + + assert.equal(result.status, 0, result.output); + assert.deepEqual(fs.readFileSync(proposalFile), before); + assert.equal(fs.readFileSync(path.join(dir, "AGENTS.md"), "utf8"), MEMORY_TEXT); + assert.equal(fs.existsSync(path.join(dir, ".agents")), false); + const retry = runDecided(dir, proposal.edits.map((edit) => `${edit.id}=accepted`).join(" ")); + assert.equal(retry.status, 0, retry.output); + assert.equal(JSON.parse(fs.readFileSync(proposalFile, "utf8")).appliedBy, "apply"); + } +}); + +test("an older review preserves a replacement or cleared proposal and rolls back its edits", () => { + for (const replaced of [true, false]) { + const dir = initRepo(); + proposeExtractions(dir); + const state = new State(dir).ensure(); + const proposal = state.readProposal(); + if (replaced) state.writeProposal({ ...proposal, notes: ["A fresh proposal saved during review"] }); + else state.clearProposal(); + const fresh = replaced ? fs.readFileSync(state.proposalPath) : null; + + const results = applyDecisions({ + proposal, + decisions: Object.fromEntries(proposal.edits.map((edit) => [edit.id, "accepted"])), + repo: { root: dir, name: path.basename(dir) }, + state, + config: { budgetTokens: 5000, skillsDir: ".agents/skills" }, + appliedBy: "apply", + }); + + assert.deepEqual(fs.existsSync(state.proposalPath) ? fs.readFileSync(state.proposalPath) : null, fresh); + assert.match(results.failed[0]?.error || "", /proposal changed/); + assert.deepEqual(results.written, []); + assert.deepEqual(results.skills, []); + assert.equal(fs.readFileSync(path.join(dir, "AGENTS.md"), "utf8"), MEMORY_TEXT); + assert.equal(fs.existsSync(path.join(dir, ".agents")), false); + assert.equal(fs.existsSync(path.join(dir, ".claude")), false); + if (!replaced) state.writeProposal(proposal); + const retry = runDecided(dir, proposal.edits.map((edit) => `${edit.id}=accepted`).join(" ")); + assert.equal(retry.status, 0, retry.output); + assert.equal(state.readProposal().appliedBy, "apply"); + } +}); + +test("an interrupted proposal save does not block later saves, apply, or clearing", () => { const dir = initRepo(); - const [accepted, rejected] = proposeExtractions(dir).edits; + proposeExtractions(dir); + const state = new State(dir).ensure(); + const proposal = state.readProposal(); + const before = fs.readFileSync(state.proposalPath); + const interrupted = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { State } from "./src/state.js"; + const state = new State(process.argv[1]); + state.writeJsonFile = () => process.exit(1); + state.writeProposal({});`, + dir, + ], + { cwd: ROOT, encoding: "utf8" }, + ); + + assert.equal(interrupted.status, 1, interrupted.stderr); + assert.equal(interrupted.stderr, ""); + assert.deepEqual(fs.readFileSync(state.proposalPath), before); + const fresh = { ...proposal, notes: ["Saved after interruption"] }; + state.writeProposal(fresh); + assert.deepEqual(state.readProposal(), fresh); + const applied = runDecided(dir, proposal.edits.map((edit) => `${edit.id}=accepted`).join(" ")); + assert.equal(applied.status, 0, applied.output); + assert.equal(state.readProposal().appliedBy, "apply"); + state.clearProposal(); + assert.equal(state.readProposal(), null); +}); + +test("a write failure leaves the proposal unstamped and retryable", () => { + const dir = initRepo(); + const proposal = proposeExtractions(dir); + const proposalFile = path.join(dir, ".backpass", "proposal.json"); + const before = fs.readFileSync(proposalFile); + const vector = proposal.edits.map((edit) => `${edit.id}=accepted`).join(" "); + fs.chmodSync(dir, 0o555); + let failed; + try { + failed = runDecided(dir, vector); + } finally { + fs.chmodSync(dir, 0o755); + } + assert.equal(failed.status, 1, failed.output); + assert.match(failed.output, /EACCES|EPERM/); + assert.deepEqual(fs.readFileSync(proposalFile), before); + assert.equal(fs.readFileSync(path.join(dir, "AGENTS.md"), "utf8"), MEMORY_TEXT); + assert.equal(fs.existsSync(path.join(dir, ".agents")), false); + const retry = runDecided(dir, vector); + assert.equal(retry.status, 0, retry.output); + assert.equal(JSON.parse(fs.readFileSync(proposalFile, "utf8")).appliedBy, "apply"); +}); + +test("an unwritable state directory rolls back accepted edits before retry", () => { + for (const json of [false, true]) { + for (const rejectOne of [false, true]) { + const dir = initRepo(); + const proposal = proposeExtractions(dir); + const stateDir = path.join(dir, ".backpass"); + const proposalFile = path.join(stateDir, "proposal.json"); + const before = fs.readFileSync(proposalFile); + const vector = proposal.edits + .map((edit, i) => `${edit.id}=${rejectOne && i ? "rejected" : "accepted"}`) + .join(" "); + fs.chmodSync(stateDir, 0o555); + let failed; + try { + failed = runDecided(dir, vector, json ? ["--json"] : []); + } finally { + fs.chmodSync(stateDir, 0o755); + } + + assert.equal(failed.status, 1, failed.output); + assert.match(failed.output, /EACCES|EPERM/); + if (json) { + const { results } = JSON.parse(failed.stdout); + assert.deepEqual(results.written, []); + assert.deepEqual(results.skills, []); + } + assert.deepEqual(fs.readFileSync(proposalFile), before); + assert.equal(fs.readFileSync(path.join(dir, "AGENTS.md"), "utf8"), MEMORY_TEXT); + assert.equal(fs.existsSync(path.join(dir, ".agents")), false); + assert.equal(fs.existsSync(path.join(dir, ".claude")), false); + assert.deepEqual(rejectionsOf(dir), {}); + + const retry = runDecided(dir, vector, json ? ["--json"] : []); + assert.equal(retry.status, 0, retry.output); + assert.equal(JSON.parse(fs.readFileSync(proposalFile, "utf8")).appliedBy, "apply"); + assert.notEqual(fs.readFileSync(path.join(dir, "AGENTS.md"), "utf8"), MEMORY_TEXT); + assert.equal(fs.existsSync(path.join(dir, ".agents/skills/ci-details/SKILL.md")), true); + } + } +}); + +test("--decisions stamps accepted edits and preserves rejection reason and provenance", () => { + const dir = initRepo(); + const proposal = proposeExtractions(dir); + const provenance = { source: { kind: "native" } }; + const state = new State(dir).ensure(); + state.writeProposal({ ...proposal, provenance }); + const [accepted, rejected] = proposal.edits; const applied = runDecided(dir, `${accepted.id}=accepted ${rejected.id}=rejected:too-narrow`); @@ -727,9 +921,11 @@ test("--decisions writes accepted edits and remembers a rejection with its reaso assert.doesNotMatch(memory, /release-details/); assert.equal(fs.existsSync(path.join(dir, ".agents/skills/release-details")), false); assert.deepEqual( - Object.values(rejectionsOf(dir)).map((entry) => [entry.title, entry.reason]), - [[rejected.title, "too-narrow"]], + Object.values(rejectionsOf(dir)).map((entry) => [entry.title, entry.reason, entry.provenance]), + [[rejected.title, "too-narrow", provenance]], ); + assert.equal(state.readProposal().appliedBy, "apply"); + assert.deepEqual(state.readProposal().provenance, provenance); }); test("repeated --decisions apply one combined vector and preserve reject reasons", () => { diff --git a/test/distill.test.js b/test/distill.test.js index 39dab9ae..00687d53 100644 --- a/test/distill.test.js +++ b/test/distill.test.js @@ -124,6 +124,54 @@ test("redaction leaves benign token-count arguments alone", () => { assert.equal(redact("token=abcdefgh1234,next=1"), "token=[redacted],next=1"); }); +test("standalone apikey credentials are redacted case-insensitively with a 16-character floor", () => { + const lower = `apikey_${"a".repeat(64)}`; + const upper = `APIKEY_${"B".repeat(64)}`; + const mixed = "ApiKey_aB3_-aB3_-aB3_-x"; + const label = "[redacted:API_KEY]"; + for (const token of [lower, upper, mixed, `apikey_${"c".repeat(16)}`]) { + assert.equal(redact(token), label); + assert.equal(redact(`(${token}), next.`), `(${label}), next.`); + } + assert.equal(redact(`before\n${lower}\n${upper}\nafter`), `before\n${label}\n${label}\nafter`); + for (const text of [ + "apikey", + "apikey_documentation", + `apikey_${"a".repeat(15)}`, + `APIKEY_${"B".repeat(15)}`, + `not${lower}`, + ]) { + assert.equal(redact(text), text); + } +}); + +test("standalone apikey matching cannot expose secret assignment values", () => { + for (const name of ["apikey_openrouter_production", "APIKEY_OPENROUTER_PRODUCTION"]) { + for (const separator of ["=", ":"]) { + for (const value of [ + "hunter2hunter2", + '"hunter2hunter2,secret"', + "'hunter2hunter2,secret'", + '"hunter2\nhunter2"', + ]) { + const assignment = `${name} ${separator} ${value}`; + assert.equal(redact(assignment), "[redacted:API_KEY]=[redacted]"); + const { trace } = distill( + [ + { kind: "message", role: "user", text: assignment }, + { kind: "message", role: "assistant", text: assignment }, + { kind: "tool", name: "Bash", input: assignment, result: assignment }, + { kind: "tool", name: "Bash", input: { command: assignment }, result: assignment }, + ], + META, + ); + assert.ok(!trace.includes("hunter2"), "assignment values must not survive in any trace field"); + assert.equal(trace.match(/\[redacted:API_KEY\]=\[redacted\]/g)?.length, 6); + } + } + } +}); + test("redaction runs on tool input and output inside the trace", () => { const { trace } = distill( [ diff --git a/test/memory-resolution.test.js b/test/memory-resolution.test.js index 5e944239..c6236bd3 100644 --- a/test/memory-resolution.test.js +++ b/test/memory-resolution.test.js @@ -3,6 +3,8 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { execFileSync, spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; import { primaryMemoryFile } from "../src/commands/analyze.js"; import { applyDecisions } from "../src/apply/writer.js"; @@ -85,6 +87,47 @@ test("isPointerTo accepts the @AGENTS.md import forms and nothing else", () => { assert.equal(isPointerTo("", "AGENTS.md"), false); }); +test("pointer imports allow leading Markdown headings, blanks and HTML comments", () => { + for (const text of [ + "# my-repo\n\n@AGENTS.md\n", + "\n\n# my-repo\n\n\n@./AGENTS.md\n\n", + "# Title\n## Subtitle\n### Context\n#### More\n##### Detail\n###### Last\n@AGENTS.md\n", + "#\n\n@AGENTS.md\n", + "# my-repo\r\n\r\n@AGENTS.md\r\n", + ]) { + assert.equal(isPointerTo(text, "AGENTS.md"), true, text); + } +}); + +for (const indent of ["", " ", " ", " ", " ", "\t"]) { + const isHeading = indent.length < 4 && indent !== "\t"; + test(`pointer heading indentation ${JSON.stringify(indent)} is ${isHeading ? "accepted" : "content"}`, () => { + for (let level = 1; level <= 6; level += 1) { + for (const newline of ["\n", "\r\n"]) { + const text = `${indent}${"#".repeat(level)} Never deploy without approval${newline}${newline}@AGENTS.md${newline}`; + assert.equal(isPointerTo(text, "AGENTS.md"), isHeading, text); + } + } + }); +} + +test("headings do not hide prose, multiple imports or other instruction content", () => { + for (const text of [ + "# my-repo\nReal prose.\n@AGENTS.md\n", + "# my-repo\n@AGENTS.md\nReal prose.\n", + "# my-repo\n@AGENTS.md\n@AGENTS.md\n", + "# my-repo\n@AGENTS.md\n@OTHER.md\n", + "# my-repo\n@AGENTS.md\n- Follow this rule.\n", + "# my-repo\n```\n@AGENTS.md\n```\n", + "#not-a-heading\n@AGENTS.md\n", + "####### not-a-heading\n@AGENTS.md\n", + "@AGENTS.md\n# trailing heading\n", + "# heading without an import\n", + ]) { + assert.equal(isPointerTo(text, "AGENTS.md"), false, text); + } +}); + test("a relative pointer resolves from the importing file's directory", () => { const repo = repoWith({ ".agents/AGENTS.md": AGENTS, @@ -121,7 +164,8 @@ test("only CLAUDE.md can cover a secondary memory file by import", () => { }); test("CLAUDE.md as a pointer resolves AGENTS.md silently and only AGENTS.md is written", () => { - const repo = repoWith({ "AGENTS.md": AGENTS, "CLAUDE.md": renderPointer("AGENTS.md") }); + const pointer = `# my-repo\n\n${renderPointer("AGENTS.md")}`; + const repo = repoWith({ "AGENTS.md": AGENTS, "CLAUDE.md": pointer }); const config = loadConfig(repo.root); const { result, warnings } = captureWarnings(() => primaryMemoryFile(repo, config)); @@ -136,9 +180,47 @@ test("CLAUDE.md as a pointer resolves AGENTS.md silently and only AGENTS.md is w ["AGENTS.md"], ); assert.match(fs.readFileSync(path.join(repo.root, "AGENTS.md"), "utf8"), /full test suite/); - assert.equal(fs.readFileSync(path.join(repo.root, "CLAUDE.md"), "utf8"), renderPointer("AGENTS.md")); + assert.equal(fs.readFileSync(path.join(repo.root, "CLAUDE.md"), "utf8"), pointer); }); +for (const indent of ["", " ", " ", " ", " ", "\t"]) { + const isHeading = indent.length < 4 && indent !== "\t"; + test(`CLI ${isHeading ? "recognizes a pointer" : "warns about separate content"} with indentation ${JSON.stringify(indent)}`, () => { + const pointer = `${indent}# my-repo\n\n\n@AGENTS.md\n`; + const repo = repoWith({ + "AGENTS.md": AGENTS, + "CLAUDE.md": pointer, + ".backpassrc.json": JSON.stringify({ discovery: { harnesses: [] } }), + }); + execFileSync("git", ["init", "-q"], { cwd: repo.root }); + const cli = fileURLToPath(new URL("../bin/backpass.js", import.meta.url)); + for (const command of ["status", "analyze"]) { + const result = spawnSync(process.execPath, [cli, command], { + cwd: repo.root, + encoding: "utf8", + env: { ...process.env, NO_COLOR: "1", CI: "1" }, + timeout: 10_000, + }); + const output = `${result.stdout}${result.stderr}`; + assert.equal(result.status, 0, output); + if (isHeading) { + assert.doesNotMatch(output, /separate memory file|separate - not optimized/); + if (command === "status") assert.match(output, /CLAUDE\.md\s+pointer to AGENTS\.md/); + } else if (command === "status") { + assert.match(output, /CLAUDE\.md[^\n]*separate - not optimized/); + } else { + assert.match( + output, + /CLAUDE\.md is a separate memory file and will NOT be updated - only AGENTS\.md is optimized/, + ); + } + if (command === "analyze") assert.match(output, /no transcripts associated/); + } + assert.equal(fs.readFileSync(path.join(repo.root, "AGENTS.md"), "utf8"), AGENTS); + assert.equal(fs.readFileSync(path.join(repo.root, "CLAUDE.md"), "utf8"), pointer); + }); +} + test("CLAUDE.md symlink to AGENTS.md resolves silently and only AGENTS.md is written", () => { const repo = repoWith({ "AGENTS.md": AGENTS }); fs.symlinkSync("AGENTS.md", path.join(repo.root, "CLAUDE.md")); diff --git a/test/skills.test.js b/test/skills.test.js index bf502b0e..a0be33a2 100644 --- a/test/skills.test.js +++ b/test/skills.test.js @@ -11,6 +11,9 @@ import { ensureSkillsLayout, loadProjectSkills, loadSkills, + loadedCopies, + skillDescriptionTokens, + renderSkillIndexForAnalysis, parseFrontmatter, renderSkillFile, resolveOverflowTarget, @@ -221,6 +224,62 @@ test("project skill discovery includes separate canonical and Claude roots witho ); }); +test("analysis lists a shared user skill once while billing every harness copy", (t) => { + const root = tmpRepo(); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const dirs = [".agents/skills", ".claude/skills", ".codex/skills"]; + const file = path.join(root, dirs[0], "release-signing", "SKILL.md"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, renderSkillFile(SKILL)); + for (const dir of dirs.slice(1)) { + fs.mkdirSync(path.join(root, dir), { recursive: true }); + fs.symlinkSync(path.dirname(file), path.join(root, dir, "release-signing"), "dir"); + } + + const skills = loadProjectSkills(root, dirs[0], dirs, { exact: true }); + const before = structuredClone(skills); + assert.equal(skills.length, 3); + assert.equal(loadedCopies(root, skills, file), 3); + assert.equal(skillDescriptionTokens(skills), 3 * skills[0].descriptionTokens); + const expected = `- ${skills[0].name} (${skills[0].path}) :: ${skills[0].description}`; + assert.equal(renderSkillIndexForAnalysis(skills, root), expected); + assert.equal( + renderSkillIndexForAnalysis(skills.map((skill) => ({ ...skill, path: path.join(root, skill.path) }))), + `- ${skills[0].name} (${path.join(root, skills[0].path)}) :: ${skills[0].description}`, + ); + assert.deepEqual(skills, before, "rendering must not dedupe the accounting input"); + assert.equal(loadedCopies(root, skills, file), 3); + assert.equal(skillDescriptionTokens(skills), 3 * skills[0].descriptionTokens); +}); + +test("analysis retains distinct files with identical names and descriptions", (t) => { + const root = tmpRepo(); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const dirs = [".agents/skills", ".claude/skills"]; + for (const dir of dirs) { + const file = path.join(root, dir, "release-signing", "SKILL.md"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, renderSkillFile(SKILL)); + } + const skills = loadProjectSkills(root); + assert.equal(skills.length, 2); + assert.equal(renderSkillIndexForAnalysis(skills, root).split("\n").length, 2); + for (const skill of skills) assert.ok(renderSkillIndexForAnalysis(skills, root).includes(skill.path)); +}); + +test("analysis preserves unresolved skill paths and the empty-index message", () => { + const skills = ["missing-one/SKILL.md", "missing-two/SKILL.md"].map((file) => ({ + name: "missing", + path: file, + description: "", + })); + assert.equal(renderSkillIndexForAnalysis([]), "(this repo has no skills)"); + assert.equal( + renderSkillIndexForAnalysis(skills), + "- missing (missing-one/SKILL.md) :: (no description)\n- missing (missing-two/SKILL.md) :: (no description)", + ); +}); + test("ensureSkillsLayout creates the dir and symlink when none exists and is idempotent", () => { const root = tmpRepo(); const first = ensureSkillsLayout(root); diff --git a/test/synthesize.test.js b/test/synthesize.test.js index d23d2a43..00710e64 100644 --- a/test/synthesize.test.js +++ b/test/synthesize.test.js @@ -31,10 +31,15 @@ if (argv.includes("sessions")) { if (argv.includes("new")) process.stdout.write("fake-session-id\\n"); process.exit(0); } -if (argv.includes("set")) process.exit(0); +if (argv.includes("set") || argv.includes("set-mode")) process.exit(0); const fileAt = argv.indexOf("--file"); if (fileAt < 0) process.exit(2); const prompt = fs.readFileSync(argv[fileAt + 1], "utf8"); +const phase = prompt.includes("## Measured changes") ? "annotate" : "edit"; +if (script.fail === phase) { + process.stderr.write("synthetic adapter failure"); + process.exit(1); +} const statePath = process.env.FAKE_ACPX_STATE; const state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, "utf8")) : { annotate: 0 }; function applyEdits(edits) { @@ -79,6 +84,8 @@ const { parseMemoryUnits, readMemoryFile } = await import("../src/memory.js"); const { foldEvidence } = await import("../src/fold.js"); const { ProposalViolation } = await import("../src/proposal.js"); const { State } = await import("../src/state.js"); +const { AcpxError } = await import("../src/acpx.js"); +const { AgentResolver } = await import("../src/agents.js"); const { UserError, setLoggerSink } = await import("../src/logger.js"); const { workspacePathFor } = await import("../src/workspace.js"); const { makeRepo } = await import("./helpers/staging.js"); @@ -223,6 +230,57 @@ function setup( return { repo, config, memoryFile, externalSkillsDir, run, calls }; } +test("failed synthesis reports the failed prompt's UTF-8 bytes without its contents", async () => { + for (const { phase, repeats } of [ + { phase: "edit", repeats: 1 }, + { phase: "edit", repeats: 180_000 }, + { phase: "annotate", repeats: 1 }, + ]) { + const privateText = `SYNTH_PRIVATE_${"密🙂".repeat(repeats)}`; + const { config, run, calls } = setup( + { fail: phase, edit: {}, annotations: [] }, + { + text: `# Memory\n\n- ${privateText}\n`, + overrides: { synthesis: { agent: "codex", model: null } }, + }, + ); + config.agents = new AgentResolver(config); + await assert.rejects(run, (err) => { + if (phase === "edit") { + assert.ok(err instanceof UserError); + assert.match(err.message, /pinned synthesis agent codex/); + assert.match(err.hint, /check the codex\/acpx failure above and retry/); + } else { + assert.ok(err instanceof AcpxError); + assert.equal(err.code, 1); + assert.equal(err.stderr, "synthetic adapter failure"); + } + assert.equal(err.cause, undefined); + assert.match(err.message, /synthetic adapter failure/); + const invocation = calls() + .filter((call) => call.argv.includes("--file")) + .at(-1); + const promptFile = invocation.argv[invocation.argv.indexOf("--file") + 1]; + const prompt = fs.readFileSync(promptFile, "utf8"); + const bytes = Buffer.byteLength(prompt, "utf8"); + assert.match(err.message, new RegExp(`synthesis prompt size: ${bytes} UTF-8 bytes`, "i")); + assert.match(err.stack, new RegExp(`synthesis prompt size: ${bytes} UTF-8 bytes`, "i")); + if (phase === "edit") { + assert.ok(bytes > prompt.length, "non-ASCII bytes must not be counted as characters"); + assert.equal(bytes > 1_000_000, repeats > 1); + } + assert.ok(!err.message.includes("SYNTH_PRIVATE_")); + assert.ok(!err.stack.includes("SYNTH_PRIVATE_")); + assert.ok(!err.message.includes("too large"), "size is context, not an inferred cause"); + return true; + }); + assert.ok( + calls().some((call) => call.argv.includes("close")), + "failed sessions are closed", + ); + } +}); + test("root and nested synthesis both refuse direct writes to the other memory file", async () => { for (const pass of ["root", "nested"]) { const { repo, config, run } = setup( @@ -256,7 +314,7 @@ test("root and nested synthesis both refuse direct writes to the other memory fi (err) => err instanceof UserError && err.message.includes( - `synthesis changed ${pass === "root" ? nestedPath : "AGENTS.md"} in the repository directly`, + `${pass === "root" ? nestedPath : "AGENTS.md"} changed in the repository during synthesis`, ), ); assert.equal(fs.readFileSync(target, "utf8"), "# Direct write\n"); @@ -531,7 +589,49 @@ test("a harness that writes to the repository instead of the staging copy is ref ); await assert.rejects(run(), (err) => { assert.ok(err instanceof UserError); - assert.match(err.message, /synthesis changed AGENTS\.md in the repository directly/); + assert.match(err.message, /AGENTS\.md changed in the repository during synthesis/); + return true; + }); +}); + +test("a change to a watched repo file during synthesis is reported as observed, not blamed on the harness", async () => { + const { repo, run } = setup({ edit: {} }); + // The fake cannot tell a concurrent editor from a harness: it just writes the repo file. + fs.writeFileSync( + process.env.FAKE_ACPX_SCRIPT, + JSON.stringify({ + edit: { [path.join(repo.root, "AGENTS.md")]: { replace: [[TWO_ITEMS, ""]] } }, + annotations: [{ reply: { edits: [] } }], + }), + ); + await assert.rejects(run(), (err) => { + assert.ok(err instanceof UserError); + assert.match(err.message, /AGENTS\.md changed in the repository during synthesis; nothing was proposed/); + assert.doesNotMatch(err.message, /harness|staging copy|directly/); + assert.match(err.hint, /another process \(a concurrent agent session or a scheduled job\)/); + assert.doesNotMatch(err.hint, /cannot be trusted/); + return true; + }); +}); + +test("a repo file that now equals the staging copy is still attributed to the harness", async () => { + const { repo, run } = setup({ edit: {} }); + const edit = { replace: [[TWO_ITEMS, ""]] }; + fs.writeFileSync( + process.env.FAKE_ACPX_SCRIPT, + JSON.stringify({ + edit: { "AGENTS.md": edit, [path.join(repo.root, "AGENTS.md")]: edit }, + annotations: [{ reply: { edits: [] } }], + }), + ); + await assert.rejects(run(), (err) => { + assert.ok(err instanceof UserError); + assert.match(err.message, /AGENTS\.md changed in the repository during synthesis/); + assert.match( + err.message, + /AGENTS\.md now matches the staging copy .*so the synthesis harness likely wrote it directly/, + ); + assert.match(err.hint, /cannot be trusted with the synthesis role/); return true; }); }); @@ -580,7 +680,11 @@ test("a skill target in a relative external directory is staged and uses its act }); test("an agent that changes nothing yields an empty proposal, never an invented edit", async () => { - const { run } = setup({ edit: {}, annotations: [{ reply: { edits: [], notes: ["the evidence is too thin"] } }] }); + const { config, run } = setup( + { edit: {}, annotations: [{ reply: { edits: [], notes: ["the evidence is too thin"] } }] }, + { overrides: { synthesis: { agent: "codex", model: null } } }, + ); + config.agents = new AgentResolver(config); const { proposal, violations } = await run(); assert.deepEqual(violations, []); assert.equal(proposal.edits.length, 0); @@ -815,7 +919,7 @@ test("a skill target stages that skill alone; a staged write to AGENTS.md is ref annotations: [{ reply: { edits: [] } }], }), ); - await assert.rejects(direct.run(), /synthesis changed AGENTS\.md in the repository directly/); + await assert.rejects(direct.run(), /AGENTS\.md changed in the repository during synthesis/); }); test("an ordinary in-repo skill stays fingerprinted, whether or not the run narrows to it", async () => { @@ -838,12 +942,116 @@ test("an ordinary in-repo skill stays fingerprinted, whether or not the run narr await assert.rejects(guarded.run(), (err) => { assert.ok(err instanceof UserError, `${target ? "targeted" : "surface"} run: ${err}`); - assert.match(err.message, /synthesis changed \.agents\/skills\/db\/SKILL\.md in the repository directly/); + assert.match(err.message, /\.agents\/skills\/db\/SKILL\.md changed in the repository during synthesis/); return true; }); } }); +test("a skill another process edits after the fingerprint but before staging is rebuilt is not blamed on the harness", async () => { + const { repo, config, run } = setup({ + edit: { "AGENTS.md": { replace: [[TWO_ITEMS, ""]] } }, + annotations: [{ reply: { edits: [removal(["H1"])] } }], + }); + const skill = path.join(repo.root, ".agents/skills/db/SKILL.md"); + fs.mkdirSync(path.dirname(skill), { recursive: true }); + fs.writeFileSync(skill, DB_SKILL); + // `resolve` runs after the fingerprint is taken and before the staging copy is rebuilt. + config.agents = { + ...agents, + resolve: async () => { + fs.writeFileSync(skill, DB_SKILL.replace("Keep transactions short.", "Keep every transaction short.")); + return pick; + }, + }; + + const { proposal, violations } = await run(); + assert.deepEqual(violations, []); + assert.equal(proposal.edits.length, 1); + const staged = path.join(config.state.root, "synthesis/.agents/skills/db/SKILL.md"); + assert.match(fs.readFileSync(staged, "utf8"), /Keep every transaction short\./); +}); + +test("an edit after the staging rebuild is still reported even when an earlier one was absorbed", async () => { + const { repo, config, run } = setup({ edit: {} }); + const skill = path.join(repo.root, ".agents/skills/db/SKILL.md"); + fs.mkdirSync(path.dirname(skill), { recursive: true }); + fs.writeFileSync(skill, DB_SKILL); + config.agents = { + ...agents, + resolve: async () => { + fs.writeFileSync(skill, DB_SKILL.replace("Keep transactions short.", "Keep every transaction short.")); + return pick; + }, + }; + fs.writeFileSync( + process.env.FAKE_ACPX_SCRIPT, + JSON.stringify({ + edit: { [skill]: { replace: [["Keep every transaction short.", "Keep transactions under a second."]] } }, + annotations: [{ reply: { edits: [] } }], + }), + ); + + await assert.rejects(run(), (err) => { + assert.ok(err instanceof UserError); + assert.match(err.message, /\.agents\/skills\/db\/SKILL\.md changed in the repository during synthesis/); + assert.doesNotMatch(err.message, /staging copy/); + return true; + }); +}); + +test("a direct write by a candidate that then falls through is reported, not absorbed by the next rebuild", async () => { + const { repo, config, run } = setup({ edit: {} }); + const skill = path.join(repo.root, ".agents/skills/db/SKILL.md"); + fs.mkdirSync(path.dirname(skill), { recursive: true }); + fs.writeFileSync(skill, DB_SKILL); + // The first candidate edits the repository skill and is then given up on (an expired login, + // say); the second candidate must not start from a staging copy that includes that write. + config.agents = { + resolve: async () => pick, + withFallthrough: async (_role, fn) => { + await fn(pick); + return fn(pick); + }, + }; + fs.writeFileSync( + process.env.FAKE_ACPX_SCRIPT, + JSON.stringify({ + edit: { [skill]: DB_SKILL.replace("Keep transactions short.", "Keep every transaction short.") }, + annotations: [{ reply: { edits: [] } }], + }), + ); + + await assert.rejects(run(), (err) => { + assert.ok(err instanceof UserError); + assert.match(err.message, /\.agents\/skills\/db\/SKILL\.md changed in the repository during synthesis/); + return true; + }); +}); + +test("a write-through to a memory file configured by absolute path is still attributed to the harness", async () => { + const { repo, run } = setup({ edit: {} }); + const absolute = path.join(repo.root, "AGENTS.md"); + const edit = { replace: [[TWO_ITEMS, ""]] }; + fs.writeFileSync( + process.env.FAKE_ACPX_SCRIPT, + JSON.stringify({ + edit: { [workspacePathFor(absolute)]: edit, [absolute]: edit }, + annotations: [{ reply: { edits: [] } }], + }), + ); + + await assert.rejects(run({ memoryFile: readMemoryFile(repo.root, absolute) }), (err) => { + assert.ok(err instanceof UserError); + assert.match( + err.message, + new RegExp(`${absolute.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} now matches the staging copy`), + ); + assert.match(err.hint, /cannot be trusted with the synthesis role/); + return true; + }); +}); + test("a staged skill that resolves outside the repository is reported as such, not as a direct repo edit", async () => { const outside = setup({ edit: {} }, { scope: { kind: "user" }, externalSkills: true }); const skillPath = path.join(outside.externalSkillsDir, "db/SKILL.md"); @@ -865,7 +1073,7 @@ test("a staged skill that resolves outside the repository is reported as such, n err.message.includes(`${skillPath} changed during synthesis; that path resolves outside the repository`), err.message, ); - assert.doesNotMatch(err.message, /in the repository directly/); + assert.doesNotMatch(err.message, /in the repository during synthesis/); assert.match(err.hint, /another process changed the shared library mid-run/); return true; }); @@ -979,3 +1187,40 @@ test("the synthesis prompt lists stored rejection identities and reasons", () => assert.match(text, /already-covered/); assert.doesNotMatch(text, /\(none\)/); }); + +for (const alias of [false, true]) { + test(`rebuilt skill descriptions refresh prompt and proposal accounting${alias ? " across aliases" : ""}`, async () => { + const { repo, config, memoryFile, run } = setup({ + edit: { "AGENTS.md": { replace: [[TWO_ITEMS, ""]] } }, + annotations: [{ reply: "" }, { reply: { edits: [removal(["H1"])] } }], + }); + const skill = path.join(repo.root, ".agents/skills/db/SKILL.md"); + fs.mkdirSync(path.dirname(skill), { recursive: true }); + fs.writeFileSync(skill, DB_SKILL); + if (alias) fs.symlinkSync(path.dirname(skill), path.join(repo.root, ".agents/skills/db-alias")); + const description = "x".repeat(2400); + const descriptions = alias ? 1200 : 600; + config.budgetTokens = memoryFile.tokens + descriptions - 320; + config.agents = { + ...agents, + resolve: async () => { + fs.writeFileSync(skill, DB_SKILL.replace("Load for database work.", description)); + return pick; + }, + }; + const { proposal, violations } = await run(); + assert.deepEqual(violations, []); + assert.equal(proposal.budget.descriptionTokens, descriptions); + assert.equal(proposal.budget.current, memoryFile.tokens + descriptions); + assert.equal(proposal.config.maxEditsPerRun, 8); + const prompt = fs.readFileSync(path.join(config.state.root, "prompts/synthesis-edit.md"), "utf8"); + assert.ok(prompt.includes(`Budget: ${memoryFile.tokens + descriptions} / ${config.budgetTokens}`)); + assert.ok(prompt.includes("At most 8 edits.")); + assert.equal(prompt.split(` :: ${description}`).length - 1, alias ? 2 : 1); + const retry = fs.readFileSync(path.join(config.state.root, "prompts/synthesis-annotate-2.md"), "utf8"); + assert.ok(retry.includes(`${memoryFile.tokens + descriptions} tokens`)); + const applied = applyDecisions({ proposal, decisions: { e1: "accepted" }, repo, state: config.state, config }); + assert.deepEqual(applied.failed, []); + assert.equal(fs.readFileSync(path.join(repo.root, "AGENTS.md"), "utf8"), AGENTS.replace(TWO_ITEMS, "")); + }); +} diff --git a/test/workspace.test.js b/test/workspace.test.js index d1862933..bf774199 100644 --- a/test/workspace.test.js +++ b/test/workspace.test.js @@ -9,7 +9,9 @@ import { loadProjectSkills, loadSkills, resolveProjectSkillDirs, skillDescriptio import { estimateTokens } from "../src/tokens.js"; import { State } from "../src/state.js"; import { + READ_ONLY_BINARY, READ_ONLY_SEARCH_PATH, + READ_ONLY_VCS, STRAY_OUTSIDE_SURFACE, STRAY_READ_ONLY_SEARCH_PATH, STRAY_UNWRITABLE, @@ -19,6 +21,7 @@ import { parseSkillFile, prepareWorkspace, repoFingerprint, + rebaseFingerprint, recoveredLineCounts, skillStagingRefusal, splitRemovalHunk, @@ -81,6 +84,27 @@ test("the staging copy holds exactly the memory file and the skills directory, u assert.equal(fs.readFileSync(path.join(again.root, "AGENTS.md"), "utf8"), AGENTS); }); +test("a skill's own object store and compiled tools stay out of the staging copy, named (#75)", () => { + // The store that ran a real synthesis out of heap: a skill checked out with its .git and + // shipping compiled binaries beside its instructions. + const { workspace } = stage({ + ".agents/skills/db/SKILL.md": SKILL, + ".agents/skills/db/notes.txt": "n", + ".agents/skills/db/.git/objects/pack/pack-1.pack": "PACK\u0000\u0000\u0000\u0002", + ".agents/skills/db/bin/tool.exe": "MZ\u0090\u0000\u0003\u0000", + }); + assert.deepEqual(walkStaged(path.join(workspace.root, ".agents/skills")), ["db/SKILL.md", "db/notes.txt"]); + assert.deepEqual( + [...workspace.originals.keys()].sort(), + ["AGENTS.md", ".agents/skills/db/SKILL.md", ".agents/skills/db/notes.txt"].sort(), + ); + assert.deepEqual(workspace.unstageable, [ + { path: ".agents/skills/db/bin/tool.exe", reason: READ_ONLY_BINARY }, + { path: ".agents/skills/db/.git", reason: READ_ONLY_VCS }, + ]); + assert.deepEqual(measureWorkspace(workspace).changes, [], "a file left out is never measured as deleted"); +}); + test("an untouched workspace measures as no change, and ids are stable across re-measurement", () => { const { workspace } = stage(); const first = measureWorkspace(workspace); @@ -355,6 +379,7 @@ test("two links to one shared library are both billed, and exactly one of them i { path: ".agents/skills/db/SKILL.md", reason: "the same file is already staged as .agents/skills/database/SKILL.md", + identity: path.join(library, "SKILL.md"), }, ]); }); @@ -973,3 +998,27 @@ test("a thematic break between a heading and its section never lets an extractio null, ); }); + +for (const relative of [".agents/skills/db/SKILL.md", ".agents/skills/db/notes.txt"]) { + test(`a concurrent source write after copying ${relative} cannot become a staged edit`, (t) => { + const repo = makeRepo({ "AGENTS.md": AGENTS, [relative]: SKILL }); + const state = new State(repo.root).ensure(); + const fingerprint = repoFingerprint(repo, ["AGENTS.md", relative]); + const copy = fs.copyFileSync; + t.mock.method(fs, "copyFileSync", (from, to, mode) => { + copy(from, to, mode); + if (from === path.join(repo.root, relative)) fs.writeFileSync(from, `${SKILL}Concurrent edit.\n`); + }); + const workspace = prepareWorkspace({ + state, + repo, + memoryFile: readMemoryFile(repo.root, "AGENTS.md"), + skillsDir: ".agents/skills", + }); + rebaseFingerprint(fingerprint, workspace); + assert.equal(workspace.originals.get(relative), SKILL); + assert.equal(fs.readFileSync(path.join(workspace.root, relative), "utf8"), SKILL); + assert.deepEqual(measureWorkspace(workspace).changes, []); + assert.notEqual(repoFingerprint(repo, [relative])[relative], fingerprint[relative]); + }); +}