From de460d707debb1f049c3103cc85a87dee9a66575 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 07:58:04 +0900 Subject: [PATCH] feat(report): a completion claim must show the runs behind it; same-diff docs are not evidence (AGT-4900) A worker report that says tests or lint pass now has to carry one Ran: line per check (command, exit code, and passed/failed counts for test runners). Without one, or when its only evidence is a document written in the same diff, the worker result is not a success. Where the adapter observed execution, reported runs must match an observed command and its exit code. The rejection is fed back to the retry with the exact line format, and the worker prompt (en/ko) asks for these lines. --- src/agents/agentPair.ts | 3 + src/agents/completionReportEvidence.test.ts | 120 ++++++++++++ src/agents/completionReportEvidence.ts | 185 ++++++++++++++++++ src/agents/pairIterationLoop.ts | 11 ++ .../pairPipeline.completionEvidence.test.ts | 65 ++++++ src/agents/worker.completionEvidence.test.ts | 93 +++++++++ src/agents/worker.ts | 15 ++ src/agents/workerValidationEvidence.ts | 5 + src/locale/prompts/en.ts | 11 ++ src/locale/prompts/ko.ts | 10 + 10 files changed, 518 insertions(+) create mode 100644 src/agents/completionReportEvidence.test.ts create mode 100644 src/agents/completionReportEvidence.ts create mode 100644 src/agents/pairPipeline.completionEvidence.test.ts create mode 100644 src/agents/worker.completionEvidence.test.ts diff --git a/src/agents/agentPair.ts b/src/agents/agentPair.ts index 7824e8eb..298f2f24 100644 --- a/src/agents/agentPair.ts +++ b/src/agents/agentPair.ts @@ -56,6 +56,9 @@ export interface WorkerResult { /** Claimed success, changed nothing, and gave no noChangesReason — a contract * violation the caller cannot tell apart from "nothing to do" (AX-868). */ zeroDiffWithoutReason?: boolean; + /** Claimed a check passed without execution evidence (command, exit code, + * counts), or cited a document from the same diff as the proof (AGT-4900). */ + unverifiedCompletionClaim?: boolean; uncertaintySignals?: string[]; // Detected uncertainty phrases costInfo?: CostInfo; } diff --git a/src/agents/completionReportEvidence.test.ts b/src/agents/completionReportEvidence.test.ts new file mode 100644 index 00000000..e01209d3 --- /dev/null +++ b/src/agents/completionReportEvidence.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from 'vitest'; +import { + completionReportEvidenceIssues, + parseReportedRuns, + verificationClaims, +} from './completionReportEvidence.js'; + +function report(output: string, filesChanged: string[] = ['src/a.ts'], executedCommands?: string[]) { + return { output, summary: '', filesChanged, executedCommands }; +} + +describe('verificationClaims', () => { + it('finds assertions that a check passed', () => { + expect(verificationClaims('Codename: Atlas\nAll tests pass.\nLint is clean')).toEqual([ + 'All tests pass.', + 'Lint is clean', + ]); + expect(verificationClaims('테스트 모두 통과했습니다')).toHaveLength(1); + }); + + it('does not treat stated limits or ordinary prose as claims', () => { + expect(verificationClaims('Tests were not run: no node_modules in this worktree.')).toEqual([]); + expect(verificationClaims('Could not verify that the build passes on Windows.')).toEqual([]); + expect(verificationClaims('Fixed the step where the build cleans the cache.')).toEqual([]); + expect(verificationClaims('테스트는 실행하지 못했습니다')).toEqual([]); + expect(verificationClaims('{"type":"tool_result","content":"Tests 46 passed"}')).toEqual([]); + }); +}); + +describe('parseReportedRuns', () => { + it('reads command, exit code and counts', () => { + const { runs, malformed } = parseReportedRuns( + 'Ran: `npm test` -> exit 0, 412 passed, 0 failed\n- Ran: npm run lint → exit 0\nRan: pytest', + ); + expect(runs).toEqual([ + expect.objectContaining({ command: 'npm test', exitCode: 0, passed: 412, failed: 0 }), + expect.objectContaining({ command: 'npm run lint', exitCode: 0, passed: undefined }), + ]); + expect(malformed).toEqual(['Ran: pytest']); + }); +}); + +describe('completionReportEvidenceIssues', () => { + it('rejects a "tests pass" report with no execution evidence', () => { + const issues = completionReportEvidenceIssues(report('Codename: Atlas\nImplemented the fix. All tests pass.')); + expect(issues[0]).toMatch(/claims a check passed but shows no execution evidence/); + // The rejection quotes the format so the next attempt can comply. + expect(issues.at(-1)).toMatch(/Ran: -> exit /); + }); + + it('rejects a report whose only evidence is a document written in the same diff', () => { + const issues = completionReportEvidenceIssues(report( + 'No credentials are committed; the readiness check passes.\nEvidence: docs/a1-readiness.md', + ['src/a.ts', 'docs/a1-readiness.md'], + )); + expect(issues[0]).toMatch(/cites documents written in this same change as evidence \(docs\/a1-readiness\.md\)/); + }); + + it('does not count a `Ran:` line that points at a self-authored doc', () => { + const issues = completionReportEvidenceIssues(report( + 'Tests pass.\nRan: cat docs/results.md -> exit 0', + ['src/a.ts', 'docs/results.md'], + )); + expect(issues[0]).toMatch(/cites documents written in this same change/); + }); + + it('accepts a claim backed by command, exit code and counts', () => { + expect(completionReportEvidenceIssues(report( + 'All tests pass.\nRan: npm test -> exit 0, 412 passed, 0 failed\nRan: npm run lint -> exit 0', + ))).toEqual([]); + }); + + it('accepts evidence even when an unchanged doc is also cited', () => { + expect(completionReportEvidenceIssues(report( + 'Tests pass.\nRan: pytest -> exit 0, 10 passed, 0 failed\nEvidence: docs/existing.md', + ['src/a.ts'], + ))).toEqual([]); + }); + + it('requires passed/failed counts from a test runner', () => { + const issues = completionReportEvidenceIssues(report('Tests pass.\nRan: npx vitest run -> exit 0')); + expect(issues[0]).toMatch(/without passed\/failed counts/); + }); + + it('rejects a pass claim whose own evidence shows a failing run', () => { + expect(completionReportEvidenceIssues(report('Tests pass.\nRan: npm test -> exit 1, 40 passed, 2 failed'))[0]) + .toMatch(/evidence shows a failing run/); + }); + + it('rejects an evidence line without an exit code', () => { + expect(completionReportEvidenceIssues(report('Tests pass.\nRan: npm test (looked fine)')).join(' ')) + .toMatch(/without an exit code/); + }); + + it('checks reported runs against what the adapter observed', () => { + const neverRan = completionReportEvidenceIssues(report( + 'Tests pass.\nRan: npm test -> exit 0, 3 passed, 0 failed', + ['src/a.ts'], + ['git diff'], + )); + expect(neverRan.join(' ')).toMatch(/never observed executing: npm test/); + + const wrongExit = completionReportEvidenceIssues(report( + 'Tests pass.\nRan: npm test -> exit 0, 3 passed, 0 failed', + ['src/a.ts'], + ['npm test [exit 1]'], + )); + expect(wrongExit.join(' ')).toMatch(/exit code does not match the observed run/); + + expect(completionReportEvidenceIssues(report( + 'Tests pass.\nRan: npm test -> exit 0, 3 passed, 0 failed', + ['src/a.ts'], + ['cd pkg && npm test'], + ))).toEqual([]); + }); + + it('leaves a report that claims nothing to the missing-validation nudge', () => { + expect(completionReportEvidenceIssues(report('Codename: Atlas\nRenamed the helper and updated callers.'))).toEqual([]); + }); +}); diff --git a/src/agents/completionReportEvidence.ts b/src/agents/completionReportEvidence.ts new file mode 100644 index 00000000..4df40faa --- /dev/null +++ b/src/agents/completionReportEvidence.ts @@ -0,0 +1,185 @@ +/** + * Completion-report evidence gate (AGT-4900, defences 4 and 5 of AGT-4190). + * + * A worker that finishes with "all tests pass" has made a claim, not shown a + * result. Two shortcuts were observed in practice: + * + * - "Tests pass" with nothing that ran behind it. Green-looking reports shipped + * while the project's test suite had never been executed. + * - Citing a document written in the same change as the proof (cgf-portal #215: + * a readiness doc claimed no credentials were committed, in the commit that + * committed them). A document is part of the output, not evidence about it. + * + * So a report that claims a check passed must carry one `Ran:` line per check: + * + * Ran: npm test -> exit 0, 412 passed, 0 failed + * Ran: npm run lint -> exit 0 + * + * Test runners must state passed/failed counts; lint, typecheck and build + * commands have no such counts, so an exit code is enough for them. Evidence + * lines that point at a doc file changed in this same diff are dropped. A claim + * left with no execution evidence is not a success. + * + * A report that makes no claim is not judged here: the missing-validation nudge + * (workerValidationEvidence.ts) covers silence. This gate covers assertion. + */ +import type { WorkerResult } from './agentPair.js'; +import { claimedButNotObserved, isDocOnlyFile } from './workerValidationEvidence.js'; + +/** One parsed `Ran:` line. */ +export interface ReportedRun { + command: string; + exitCode: number; + passed?: number; + failed?: number; + /** The line as written, for messages. */ + line: string; +} + +/** The format every rejection message quotes, so the next attempt can comply. */ +export const EVIDENCE_LINE_FORMAT = + 'Ran: -> exit , passed, failed (test runners) or Ran: -> exit (lint/typecheck/build)'; + +const RAN_LINE = /^\s*(?:[-*]\s*)?(?:\*\*)?Ran:(?:\*\*)?\s*(.+?)\s*$/i; +// `->`, `→` or `=>` separates the command from its outcome. +const RAN_SPLIT = /\s*(?:->|→|=>)\s*/; +const EXIT_CODE = /\bexit(?:\s*code)?\s*[:=]?\s*(-?\d+)\b/i; +const PASSED_COUNT = /(\d+)\s+pass(?:ed|ing)?\b/i; +const FAILED_COUNT = /(\d+)\s+fail(?:ed|ing|ures?)?\b/i; +const EVIDENCE_LINE = /^\s*(?:[-*]\s*)?(?:\*\*)?(?:Evidence|근거)\s*:(?:\*\*)?\s*(.+?)\s*$/i; + +/** Commands whose output has pass/fail counts, so the report must state them. */ +const TEST_RUNNER_COMMAND = /\b(?:npm\s+(?:run\s+)?test|pnpm\s+(?:run\s+)?test|yarn\s+(?:run\s+)?test|bun\s+(?:run\s+)?test|vitest|jest|mocha|pytest|cargo\s+test|go\s+test|swift\s+test|gradle\s+test|mvn\s+test|unittest)\b/i; + +/** + * A line asserting that a check passed. Deliberately narrow: it names a kind + * of check and a positive outcome. Lines that also negate or hedge ("tests + * were not run", "could not verify the build passes") are statements of a + * limit, which the worker prompt asks for, and are not claims. + */ +const CLAIM_LINE = new RegExp([ + String.raw`\b(?:tests?|test\s+suite|specs?)\b[^.\n]{0,40}?\b(?:pass(?:es|ed|ing)?|green)\b`, + String.raw`\b(?:lint(?:ing|er)?|type-?check(?:ing|s)?|tsc|oxlint|eslint|ruff|mypy|pyright|clippy)\b[^.\n]{0,40}?\b(?:pass(?:es|ed)?|clean|green|no\s+(?:errors|warnings))\b`, + String.raw`\b(?:build|ci|checks?)\b[^.\n]{0,40}?\b(?:pass(?:es|ed)?|succeed(?:s|ed)?|green)\b`, + String.raw`테스트[^.\n]{0,20}?통과`, + String.raw`(?:린트|타입\s*체크|빌드)[^.\n]{0,20}?(?:통과|성공)`, +].join('|'), 'i'); +const HEDGE_LINE = + /\b(?:not|n['’]t|never|unable|cannot|could\s+not|without|skipped|unverified|pending|todo)\b|못\s*했|않았|미실행|확인\s*못/i; + +/** Parse every `Ran:` line in a report. Lines without an exit code are returned as malformed. */ +export function parseReportedRuns(text: string): { runs: ReportedRun[]; malformed: string[] } { + const runs: ReportedRun[] = []; + const malformed: string[] = []; + for (const raw of text.split('\n')) { + const match = raw.match(RAN_LINE); + if (!match) continue; + const body = match[1].replace(/`/g, ''); + const [command, ...rest] = body.split(RAN_SPLIT); + const outcome = rest.join(' '); + const exit = outcome.match(EXIT_CODE); + if (!command.trim() || !exit) { + malformed.push(raw.trim()); + continue; + } + const passed = outcome.match(PASSED_COUNT); + const failed = outcome.match(FAILED_COUNT); + runs.push({ + command: command.trim(), + exitCode: Number(exit[1]), + passed: passed ? Number(passed[1]) : undefined, + failed: failed ? Number(failed[1]) : undefined, + line: raw.trim(), + }); + } + return { runs, malformed }; +} + +/** Lines of the report that assert a check passed. */ +export function verificationClaims(text: string): string[] { + return text + .split('\n') + .map((line) => line.trim()) + // JSON-looking lines are raw stream events (a parse fallback), not the report. + .filter((line) => line && !/^[{[]/.test(line) && !RAN_LINE.test(line)) + .filter((line) => CLAIM_LINE.test(line) && !HEDGE_LINE.test(line)); +} + +function normalizePath(path: string): string { + return path.replace(/\\/g, '/').replace(/^\.\//, '').trim(); +} + +/** Doc files changed in this diff that an evidence line cites. */ +export function selfAuthoredDocCitations(line: string, filesChanged: readonly string[]): string[] { + const text = line.replace(/\\/g, '/'); + return filesChanged + .map(normalizePath) + .filter((file) => file && isDocOnlyFile(file) && text.includes(file)); +} + +/** + * Why this completion report is not acceptable as a success, or [] when it is. + * Only judged when the report claims a check passed. + */ +export function completionReportEvidenceIssues( + result: Pick, +): string[] { + const text = result.output || result.summary || ''; + const claims = verificationClaims(text); + if (claims.length === 0) return []; + + const filesChanged = result.filesChanged ?? []; + const issues: string[] = []; + const { runs, malformed } = parseReportedRuns(text); + + // Evidence lines that cite a doc this same diff wrote are not evidence. + const citedDocs = new Set(); + for (const raw of text.split('\n')) { + if (!RAN_LINE.test(raw) && !EVIDENCE_LINE.test(raw)) continue; + for (const doc of selfAuthoredDocCitations(raw, filesChanged)) citedDocs.add(doc); + } + const usableRuns = runs.filter((run) => selfAuthoredDocCitations(run.line, filesChanged).length === 0); + + if (malformed.length > 0) { + issues.push(`Evidence lines without an exit code are not evidence: ${malformed.slice(0, 3).join(' | ')}`); + } + + for (const run of usableRuns) { + if (TEST_RUNNER_COMMAND.test(run.command) && (run.passed === undefined || run.failed === undefined)) { + issues.push(`Test run reported without passed/failed counts: ${run.line}`); + } + if (run.exitCode !== 0 || (run.failed ?? 0) > 0) { + issues.push(`Report claims success but its own evidence shows a failing run: ${run.line}`); + } + } + + // Where the adapter observed execution, a reported run must have happened. + const observed = result.executedCommands; + if (observed) { + const neverRan = claimedButNotObserved(usableRuns.map((run) => run.command), observed); + if (neverRan.length > 0) { + issues.push(`Reported runs that OpenSwarm never observed executing: ${neverRan.slice(0, 3).join('; ')}`); + } + for (const run of usableRuns) { + const exitMatch = observed + .map((line) => line.match(/^(.*?)(?: \[exit (-?\d+)\])?$/)) + .filter((m): m is RegExpMatchArray => m !== null) + .filter((m) => claimedButNotObserved([run.command], [m[1]]).length === 0); + if (exitMatch.length > 0 && !exitMatch.some((m) => Number(m[2] ?? 0) === run.exitCode)) { + issues.push(`Reported exit code does not match the observed run: ${run.line}`); + } + } + } + + if (usableRuns.length === 0) { + const docs = [...citedDocs]; + issues.unshift( + docs.length > 0 + ? `Report cites documents written in this same change as evidence (${docs.slice(0, 3).join(', ')}); a document you wrote is output, not evidence. Claim: "${claims[0].slice(0, 160)}"` + : `Report claims a check passed but shows no execution evidence. Claim: "${claims[0].slice(0, 160)}"`, + ); + } + + if (issues.length === 0) return []; + return [...issues, `Report each check you ran as one line: ${EVIDENCE_LINE_FORMAT}. If you did not run a check, say so instead of claiming it passed.`]; +} diff --git a/src/agents/pairIterationLoop.ts b/src/agents/pairIterationLoop.ts index 37b0acf5..8d66b43a 100644 --- a/src/agents/pairIterationLoop.ts +++ b/src/agents/pairIterationLoop.ts @@ -181,6 +181,17 @@ export async function runFullIterationLoop(host: PipelineLoopHost, context: Pipe return { success: false }; } if (detail?.startsWith('worker-scope:')) context.repeatedScopeRejection = detail; + // An unbacked "tests pass" is rejected in the worker, but a bare + // failure never reaches the next attempt's prompt. Record it so the + // retry sees exactly what was missing and the line format to use + // instead of repeating the same claim blind (AGT-4900). + if (failedWorker.unverifiedCompletionClaim && failedWorker.error) { + recordReflection(context.reflection, { + iteration: context.currentIteration, + source: 'validation', + errors: [failedWorker.error.replace(/^worker-evidence:\s*/, '')], + }); + } // A no-edit stop with a stated reason is a claim about the task: put it // to the reviewer once instead of retrying blind (AGT-4535). const blocker = await reviewWorkerBlocker(host.config, context, failedWorker, context.abortSignal); diff --git a/src/agents/pairPipeline.completionEvidence.test.ts b/src/agents/pairPipeline.completionEvidence.test.ts new file mode 100644 index 00000000..87c0505d --- /dev/null +++ b/src/agents/pairPipeline.completionEvidence.test.ts @@ -0,0 +1,65 @@ +// A worker rejected for claiming "tests pass" without evidence (AGT-4900) +// must be told so on the retry. A bare worker failure does not reach the next +// prompt, so without this the retry repeats the same unbacked claim. +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { TaskItem } from '../orchestration/decisionEngine.js'; + +const runWorker = vi.fn(); + +vi.mock('./worker.js', async () => ({ + ...(await vi.importActual('./worker.js')), + runWorker, +})); +vi.mock('../knowledge/index.js', () => ({ + hasRepoSnapshot: () => true, + scanAndCache: vi.fn(), + analyzeIssue: vi.fn().mockResolvedValue(undefined), +})); +vi.mock('../memory/repoKnowledge.js', () => ({ recallRepoKnowledge: vi.fn().mockResolvedValue([]) })); +vi.mock('../core/eventHub.js', () => ({ broadcastEvent: vi.fn() })); +vi.mock('../adapters/index.js', async () => ({ + ...(await vi.importActual('../adapters/index.js')), + getAdapter: () => ({ getDefaultModel: vi.fn().mockResolvedValue('default-model') }), +})); + +const task: TaskItem = { + id: 'completion-evidence-task', + source: 'linear', + title: 'fix projection', + description: 'exercise the completion evidence feedback', + priority: 1, + createdAt: Date.now(), +}; + +describe('retry after an unbacked completion claim (AGT-4900)', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + }); + + it('puts the missing-evidence issue and the line format into the next worker prompt', async () => { + runWorker + .mockResolvedValueOnce({ + success: false, summary: 'done', filesChanged: ['src/a.ts'], commands: [], output: 'All tests pass.', + unverifiedCompletionClaim: true, + error: 'worker-evidence: Report claims a check passed but shows no execution evidence. Claim: "All tests pass." Report each check you ran as one line: Ran: -> exit , passed, failed', + }) + .mockResolvedValue({ + success: false, summary: 'stopped', filesChanged: [], commands: [], output: '', error: 'stop', + }); + const { PairPipeline } = await import('./pairPipeline.js'); + const pipeline = new PairPipeline({ + stages: ['worker'], maxIterations: 2, + roles: { worker: { enabled: true, model: 'w', timeoutMs: 0 } }, + } as unknown as ConstructorParameters[0]); + + await pipeline.run(task, process.cwd()); + + expect(runWorker).toHaveBeenCalledTimes(2); + const retry = runWorker.mock.calls[1][0] as { previousFeedback?: string }; + expect(retry.previousFeedback).toContain('shows no execution evidence'); + expect(retry.previousFeedback).toContain('Ran: -> exit '); + expect(retry.previousFeedback).not.toContain('worker-evidence:'); + }); +}); diff --git a/src/agents/worker.completionEvidence.test.ts b/src/agents/worker.completionEvidence.test.ts new file mode 100644 index 00000000..def5d0ed --- /dev/null +++ b/src/agents/worker.completionEvidence.test.ts @@ -0,0 +1,93 @@ +// A worker report that claims a check passed must show what ran; a document +// written in the same diff is not evidence (AGT-4900). These run through +// runWorker so the gate is checked against the git-derived file list and after +// the promote-to-success step, which must not flip it back. +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const spawnCli = vi.fn(async () => ({ stdout: 'raw' })); +const parseWorkerOutput = vi.fn(); +const getChangedFilesSinceSnapshot = vi.fn(); + +vi.mock('../adapters/index.js', () => ({ + getAdapter: () => ({ parseWorkerOutput }), + getDefaultAdapterName: () => 'gpt', + spawnCli: (...args: unknown[]) => spawnCli(...(args as [])), +})); +vi.mock('../support/gitTracker.js', () => ({ + isGitRepo: vi.fn(async () => true), + takeSnapshot: vi.fn(async () => 'snapshot-tree'), + getChangedFilesSinceSnapshot, +})); + +const { runWorker } = await import('./worker.js'); + +const run = () => runWorker({ + taskTitle: 'fix projection', taskDescription: 'AX-1', projectPath: '/repo', adapterName: 'gpt', +}); + +describe('runWorker completion-report evidence (AGT-4900)', () => { + beforeEach(() => { + vi.clearAllMocks(); + getChangedFilesSinceSnapshot.mockResolvedValue(['src/projection.py', 'docs/readiness.md']); + }); + + it('does not accept "tests pass" with no execution evidence as success', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'Codename: Atlas\nFixed the projection. All tests pass.', + }); + + const result = await run(); + + expect(result.success).toBe(false); + expect(result.unverifiedCompletionClaim).toBe(true); + expect(result.error).toMatch(/^worker-evidence: Report claims a check passed but shows no execution evidence/); + }); + + it('does not accept a document from the same diff as the evidence', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'No credentials are committed; the readiness check passes.\nEvidence: docs/readiness.md', + }); + + const result = await run(); + + expect(result.success).toBe(false); + expect(result.error).toContain('docs/readiness.md'); + }); + + it('is not undone by git promotion when the model reported no JSON success', async () => { + // Text-only reports start as success=false and are promoted by git changes; + // the evidence gate must run after that promotion. + parseWorkerOutput.mockReturnValue({ + success: false, summary: 'Done', filesChanged: [], commands: [], + output: 'All tests pass.', + }); + + const result = await run(); + + expect(result.success).toBe(false); + expect(result.unverifiedCompletionClaim).toBe(true); + }); + + it('accepts the same claim when the report shows the run', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'All tests pass.\nRan: uv run pytest -> exit 0, 2385 passed, 0 failed', + }); + + const result = await run(); + + expect(result.success).toBe(true); + expect(result.unverifiedCompletionClaim).toBeUndefined(); + }); + + it('leaves a report without any claim alone', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'Codename: Atlas\nRenamed the projection helper.', + }); + + expect((await run()).success).toBe(true); + }); +}); diff --git a/src/agents/worker.ts b/src/agents/worker.ts index 48e5136f..52cdec60 100644 --- a/src/agents/worker.ts +++ b/src/agents/worker.ts @@ -30,6 +30,7 @@ import { getCoordinationStore } from '../coordination/coordinationStore.js'; import { filesOutsideWriteScope } from '../orchestration/writeScope.js'; import { acceptWorkerPaths, noteRejectedWorkerPaths, rejectedWorkerPaths } from '../support/rejectedWorkerPaths.js'; import { formatProjectGoalSection } from '../support/projectGoal.js'; +import { completionReportEvidenceIssues } from './completionReportEvidence.js'; // Types @@ -549,6 +550,20 @@ export async function runWorker(options: WorkerOptions): Promise { } } + // A report that claims a check passed must show what ran: command, exit + // code and, for test runners, counts. Documents written in this same diff + // are not evidence. Runs after git promotion so it cannot be flipped back, + // and against the git-derived filesChanged, not the model's list. (AGT-4900) + if (parsedResult.success) { + const evidenceIssues = completionReportEvidenceIssues(parsedResult); + if (evidenceIssues.length > 0) { + parsedResult.success = false; + parsedResult.unverifiedCompletionClaim = true; + parsedResult.error = `worker-evidence: ${evidenceIssues.join(' ')}`; + emitWorkerStatus(options, `[Worker] Completion claim without execution evidence: ${evidenceIssues[0].slice(0, 300)}`); + } + } + // The agent introduces itself with a `Codename:` line (AGT-4019); the name // is its display identity on the coordination board. const codenameMatch = (parsedResult.output || '').match(/^\s*Codename:\s*(.{1,60}?)\s*$/mi); diff --git a/src/agents/workerValidationEvidence.ts b/src/agents/workerValidationEvidence.ts index a78d04b2..ac15e822 100644 --- a/src/agents/workerValidationEvidence.ts +++ b/src/agents/workerValidationEvidence.ts @@ -23,6 +23,11 @@ const INSPECTION_ONLY_COMMAND_RE = /^\s*(?:rg|grep|sed|cat|ls|find|pwd|git\s+(?: const SCRIPT_SMOKE_COMMAND_RE = /^\s*(?:python3?|node|tsx|ts-node|uv\s+run|npx|bunx|bash|sh)\b.*\b(?:scripts?\/|tests?\/|\.py|\.js|\.ts|\.sh|--help|--dry-run|smoke|verify|validate|check|test|build|compile)\b/i; const TESTER_CODE_FILE_RE = /\.(ts|tsx|mts|cts|js|jsx|mjs|cjs|py|rs|go|java|rb|c|cpp|h|hpp)$/; +/** README/CHANGELOG/LICENSE/NOTICE, anything under docs/, and prose formats. */ +export function isDocOnlyFile(file: string): boolean { + return DOC_ONLY_FILE_RE.test(file); +} + export function isValidationRelevantFile(file: string): boolean { if (/(^|\/)docs?\//i.test(file)) return false; if (VALIDATION_RELEVANT_BASENAME_RE.test(file)) return true; diff --git a/src/locale/prompts/en.ts b/src/locale/prompts/en.ts index c93461f8..f59661b7 100644 --- a/src/locale/prompts/en.ts +++ b/src/locale/prompts/en.ts @@ -386,6 +386,17 @@ That name is how you appear on the team board and in conversation with your reviewer; keep the same one across iterations. Then write the summary itself as if you were telling your reviewer what you did and what they should look at. +Report what you ran, not whether a gate passed. For every check you ran, write +one line in this exact form (test runners give counts; lint/typecheck/build +give the exit code): + +Ran: npm test -> exit 0, 412 passed, 0 failed +Ran: npm run lint -> exit 0 + +A report that says tests or lint pass without such a line is not accepted as a +success. A document you wrote or edited in this change is not evidence. If you +could not run a check, say so instead of claiming it passed. + If (and only if) you want to flag low confidence or a blocker, end with this JSON: \`\`\`json { "success": false, "confidencePercent": 40, "haltReason": "why you're stuck" } diff --git a/src/locale/prompts/ko.ts b/src/locale/prompts/ko.ts index daf679b2..09871299 100644 --- a/src/locale/prompts/ko.ts +++ b/src/locale/prompts/ko.ts @@ -374,6 +374,16 @@ Codename: <스스로 지은 표시 이름 — 스타일·언어 자유> 바뀌어도 같은 이름을 유지하라. 요약 본문은 리뷰어에게 말하듯 — 무엇을 했고 어디를 봐 달라는지 — 써라. +게이트 통과 여부가 아니라 무엇을 실행했는지를 보고하라. 실행한 검사마다 정확히 이 +형식으로 한 줄씩 적어라(테스트 러너는 통과/실패 수, lint·typecheck·build는 종료코드): + +Ran: npm test -> exit 0, 412 passed, 0 failed +Ran: npm run lint -> exit 0 + +이런 줄 없이 테스트·린트가 통과했다고 쓴 보고는 성공으로 인정되지 않는다. 이번 +변경에서 네가 쓰거나 고친 문서는 근거가 아니다. 검사를 실행하지 못했다면 통과했다고 +쓰지 말고 실행하지 못했다고 적어라. + 낮은 확신이나 블로커를 알릴 때만(그럴 때만) 마지막에 이 JSON을 붙여라: \`\`\`json { "success": false, "confidencePercent": 40, "haltReason": "막힌 이유" }