diff --git a/src/agents/agentPair.ts b/src/agents/agentPair.ts index 7824e8eb..298f2f24 100644 --- a/src/agents/agentPair.ts +++ b/src/agents/agentPair.ts @@ -56,6 +56,9 @@ export interface WorkerResult { /** Claimed success, changed nothing, and gave no noChangesReason — a contract * violation the caller cannot tell apart from "nothing to do" (AX-868). */ zeroDiffWithoutReason?: boolean; + /** Claimed a check passed without execution evidence (command, exit code, + * counts), or cited a document from the same diff as the proof (AGT-4900). */ + unverifiedCompletionClaim?: boolean; uncertaintySignals?: string[]; // Detected uncertainty phrases costInfo?: CostInfo; } diff --git a/src/agents/completionReportEvidence.test.ts b/src/agents/completionReportEvidence.test.ts new file mode 100644 index 00000000..e01209d3 --- /dev/null +++ b/src/agents/completionReportEvidence.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from 'vitest'; +import { + completionReportEvidenceIssues, + parseReportedRuns, + verificationClaims, +} from './completionReportEvidence.js'; + +function report(output: string, filesChanged: string[] = ['src/a.ts'], executedCommands?: string[]) { + return { output, summary: '', filesChanged, executedCommands }; +} + +describe('verificationClaims', () => { + it('finds assertions that a check passed', () => { + expect(verificationClaims('Codename: Atlas\nAll tests pass.\nLint is clean')).toEqual([ + 'All tests pass.', + 'Lint is clean', + ]); + expect(verificationClaims('테스트 모두 통과했습니다')).toHaveLength(1); + }); + + it('does not treat stated limits or ordinary prose as claims', () => { + expect(verificationClaims('Tests were not run: no node_modules in this worktree.')).toEqual([]); + expect(verificationClaims('Could not verify that the build passes on Windows.')).toEqual([]); + expect(verificationClaims('Fixed the step where the build cleans the cache.')).toEqual([]); + expect(verificationClaims('테스트는 실행하지 못했습니다')).toEqual([]); + expect(verificationClaims('{"type":"tool_result","content":"Tests 46 passed"}')).toEqual([]); + }); +}); + +describe('parseReportedRuns', () => { + it('reads command, exit code and counts', () => { + const { runs, malformed } = parseReportedRuns( + 'Ran: `npm test` -> exit 0, 412 passed, 0 failed\n- Ran: npm run lint → exit 0\nRan: pytest', + ); + expect(runs).toEqual([ + expect.objectContaining({ command: 'npm test', exitCode: 0, passed: 412, failed: 0 }), + expect.objectContaining({ command: 'npm run lint', exitCode: 0, passed: undefined }), + ]); + expect(malformed).toEqual(['Ran: pytest']); + }); +}); + +describe('completionReportEvidenceIssues', () => { + it('rejects a "tests pass" report with no execution evidence', () => { + const issues = completionReportEvidenceIssues(report('Codename: Atlas\nImplemented the fix. All tests pass.')); + expect(issues[0]).toMatch(/claims a check passed but shows no execution evidence/); + // The rejection quotes the format so the next attempt can comply. + expect(issues.at(-1)).toMatch(/Ran: -> exit /); + }); + + it('rejects a report whose only evidence is a document written in the same diff', () => { + const issues = completionReportEvidenceIssues(report( + 'No credentials are committed; the readiness check passes.\nEvidence: docs/a1-readiness.md', + ['src/a.ts', 'docs/a1-readiness.md'], + )); + expect(issues[0]).toMatch(/cites documents written in this same change as evidence \(docs\/a1-readiness\.md\)/); + }); + + it('does not count a `Ran:` line that points at a self-authored doc', () => { + const issues = completionReportEvidenceIssues(report( + 'Tests pass.\nRan: cat docs/results.md -> exit 0', + ['src/a.ts', 'docs/results.md'], + )); + expect(issues[0]).toMatch(/cites documents written in this same change/); + }); + + it('accepts a claim backed by command, exit code and counts', () => { + expect(completionReportEvidenceIssues(report( + 'All tests pass.\nRan: npm test -> exit 0, 412 passed, 0 failed\nRan: npm run lint -> exit 0', + ))).toEqual([]); + }); + + it('accepts evidence even when an unchanged doc is also cited', () => { + expect(completionReportEvidenceIssues(report( + 'Tests pass.\nRan: pytest -> exit 0, 10 passed, 0 failed\nEvidence: docs/existing.md', + ['src/a.ts'], + ))).toEqual([]); + }); + + it('requires passed/failed counts from a test runner', () => { + const issues = completionReportEvidenceIssues(report('Tests pass.\nRan: npx vitest run -> exit 0')); + expect(issues[0]).toMatch(/without passed\/failed counts/); + }); + + it('rejects a pass claim whose own evidence shows a failing run', () => { + expect(completionReportEvidenceIssues(report('Tests pass.\nRan: npm test -> exit 1, 40 passed, 2 failed'))[0]) + .toMatch(/evidence shows a failing run/); + }); + + it('rejects an evidence line without an exit code', () => { + expect(completionReportEvidenceIssues(report('Tests pass.\nRan: npm test (looked fine)')).join(' ')) + .toMatch(/without an exit code/); + }); + + it('checks reported runs against what the adapter observed', () => { + const neverRan = completionReportEvidenceIssues(report( + 'Tests pass.\nRan: npm test -> exit 0, 3 passed, 0 failed', + ['src/a.ts'], + ['git diff'], + )); + expect(neverRan.join(' ')).toMatch(/never observed executing: npm test/); + + const wrongExit = completionReportEvidenceIssues(report( + 'Tests pass.\nRan: npm test -> exit 0, 3 passed, 0 failed', + ['src/a.ts'], + ['npm test [exit 1]'], + )); + expect(wrongExit.join(' ')).toMatch(/exit code does not match the observed run/); + + expect(completionReportEvidenceIssues(report( + 'Tests pass.\nRan: npm test -> exit 0, 3 passed, 0 failed', + ['src/a.ts'], + ['cd pkg && npm test'], + ))).toEqual([]); + }); + + it('leaves a report that claims nothing to the missing-validation nudge', () => { + expect(completionReportEvidenceIssues(report('Codename: Atlas\nRenamed the helper and updated callers.'))).toEqual([]); + }); +}); diff --git a/src/agents/completionReportEvidence.ts b/src/agents/completionReportEvidence.ts new file mode 100644 index 00000000..4df40faa --- /dev/null +++ b/src/agents/completionReportEvidence.ts @@ -0,0 +1,185 @@ +/** + * Completion-report evidence gate (AGT-4900, defences 4 and 5 of AGT-4190). + * + * A worker that finishes with "all tests pass" has made a claim, not shown a + * result. Two shortcuts were observed in practice: + * + * - "Tests pass" with nothing that ran behind it. Green-looking reports shipped + * while the project's test suite had never been executed. + * - Citing a document written in the same change as the proof (cgf-portal #215: + * a readiness doc claimed no credentials were committed, in the commit that + * committed them). A document is part of the output, not evidence about it. + * + * So a report that claims a check passed must carry one `Ran:` line per check: + * + * Ran: npm test -> exit 0, 412 passed, 0 failed + * Ran: npm run lint -> exit 0 + * + * Test runners must state passed/failed counts; lint, typecheck and build + * commands have no such counts, so an exit code is enough for them. Evidence + * lines that point at a doc file changed in this same diff are dropped. A claim + * left with no execution evidence is not a success. + * + * A report that makes no claim is not judged here: the missing-validation nudge + * (workerValidationEvidence.ts) covers silence. This gate covers assertion. + */ +import type { WorkerResult } from './agentPair.js'; +import { claimedButNotObserved, isDocOnlyFile } from './workerValidationEvidence.js'; + +/** One parsed `Ran:` line. */ +export interface ReportedRun { + command: string; + exitCode: number; + passed?: number; + failed?: number; + /** The line as written, for messages. */ + line: string; +} + +/** The format every rejection message quotes, so the next attempt can comply. */ +export const EVIDENCE_LINE_FORMAT = + 'Ran: -> exit , passed, failed (test runners) or Ran: -> exit (lint/typecheck/build)'; + +const RAN_LINE = /^\s*(?:[-*]\s*)?(?:\*\*)?Ran:(?:\*\*)?\s*(.+?)\s*$/i; +// `->`, `→` or `=>` separates the command from its outcome. +const RAN_SPLIT = /\s*(?:->|→|=>)\s*/; +const EXIT_CODE = /\bexit(?:\s*code)?\s*[:=]?\s*(-?\d+)\b/i; +const PASSED_COUNT = /(\d+)\s+pass(?:ed|ing)?\b/i; +const FAILED_COUNT = /(\d+)\s+fail(?:ed|ing|ures?)?\b/i; +const EVIDENCE_LINE = /^\s*(?:[-*]\s*)?(?:\*\*)?(?:Evidence|근거)\s*:(?:\*\*)?\s*(.+?)\s*$/i; + +/** Commands whose output has pass/fail counts, so the report must state them. */ +const TEST_RUNNER_COMMAND = /\b(?:npm\s+(?:run\s+)?test|pnpm\s+(?:run\s+)?test|yarn\s+(?:run\s+)?test|bun\s+(?:run\s+)?test|vitest|jest|mocha|pytest|cargo\s+test|go\s+test|swift\s+test|gradle\s+test|mvn\s+test|unittest)\b/i; + +/** + * A line asserting that a check passed. Deliberately narrow: it names a kind + * of check and a positive outcome. Lines that also negate or hedge ("tests + * were not run", "could not verify the build passes") are statements of a + * limit, which the worker prompt asks for, and are not claims. + */ +const CLAIM_LINE = new RegExp([ + String.raw`\b(?:tests?|test\s+suite|specs?)\b[^.\n]{0,40}?\b(?:pass(?:es|ed|ing)?|green)\b`, + String.raw`\b(?:lint(?:ing|er)?|type-?check(?:ing|s)?|tsc|oxlint|eslint|ruff|mypy|pyright|clippy)\b[^.\n]{0,40}?\b(?:pass(?:es|ed)?|clean|green|no\s+(?:errors|warnings))\b`, + String.raw`\b(?:build|ci|checks?)\b[^.\n]{0,40}?\b(?:pass(?:es|ed)?|succeed(?:s|ed)?|green)\b`, + String.raw`테스트[^.\n]{0,20}?통과`, + String.raw`(?:린트|타입\s*체크|빌드)[^.\n]{0,20}?(?:통과|성공)`, +].join('|'), 'i'); +const HEDGE_LINE = + /\b(?:not|n['’]t|never|unable|cannot|could\s+not|without|skipped|unverified|pending|todo)\b|못\s*했|않았|미실행|확인\s*못/i; + +/** Parse every `Ran:` line in a report. Lines without an exit code are returned as malformed. */ +export function parseReportedRuns(text: string): { runs: ReportedRun[]; malformed: string[] } { + const runs: ReportedRun[] = []; + const malformed: string[] = []; + for (const raw of text.split('\n')) { + const match = raw.match(RAN_LINE); + if (!match) continue; + const body = match[1].replace(/`/g, ''); + const [command, ...rest] = body.split(RAN_SPLIT); + const outcome = rest.join(' '); + const exit = outcome.match(EXIT_CODE); + if (!command.trim() || !exit) { + malformed.push(raw.trim()); + continue; + } + const passed = outcome.match(PASSED_COUNT); + const failed = outcome.match(FAILED_COUNT); + runs.push({ + command: command.trim(), + exitCode: Number(exit[1]), + passed: passed ? Number(passed[1]) : undefined, + failed: failed ? Number(failed[1]) : undefined, + line: raw.trim(), + }); + } + return { runs, malformed }; +} + +/** Lines of the report that assert a check passed. */ +export function verificationClaims(text: string): string[] { + return text + .split('\n') + .map((line) => line.trim()) + // JSON-looking lines are raw stream events (a parse fallback), not the report. + .filter((line) => line && !/^[{[]/.test(line) && !RAN_LINE.test(line)) + .filter((line) => CLAIM_LINE.test(line) && !HEDGE_LINE.test(line)); +} + +function normalizePath(path: string): string { + return path.replace(/\\/g, '/').replace(/^\.\//, '').trim(); +} + +/** Doc files changed in this diff that an evidence line cites. */ +export function selfAuthoredDocCitations(line: string, filesChanged: readonly string[]): string[] { + const text = line.replace(/\\/g, '/'); + return filesChanged + .map(normalizePath) + .filter((file) => file && isDocOnlyFile(file) && text.includes(file)); +} + +/** + * Why this completion report is not acceptable as a success, or [] when it is. + * Only judged when the report claims a check passed. + */ +export function completionReportEvidenceIssues( + result: Pick, +): string[] { + const text = result.output || result.summary || ''; + const claims = verificationClaims(text); + if (claims.length === 0) return []; + + const filesChanged = result.filesChanged ?? []; + const issues: string[] = []; + const { runs, malformed } = parseReportedRuns(text); + + // Evidence lines that cite a doc this same diff wrote are not evidence. + const citedDocs = new Set(); + for (const raw of text.split('\n')) { + if (!RAN_LINE.test(raw) && !EVIDENCE_LINE.test(raw)) continue; + for (const doc of selfAuthoredDocCitations(raw, filesChanged)) citedDocs.add(doc); + } + const usableRuns = runs.filter((run) => selfAuthoredDocCitations(run.line, filesChanged).length === 0); + + if (malformed.length > 0) { + issues.push(`Evidence lines without an exit code are not evidence: ${malformed.slice(0, 3).join(' | ')}`); + } + + for (const run of usableRuns) { + if (TEST_RUNNER_COMMAND.test(run.command) && (run.passed === undefined || run.failed === undefined)) { + issues.push(`Test run reported without passed/failed counts: ${run.line}`); + } + if (run.exitCode !== 0 || (run.failed ?? 0) > 0) { + issues.push(`Report claims success but its own evidence shows a failing run: ${run.line}`); + } + } + + // Where the adapter observed execution, a reported run must have happened. + const observed = result.executedCommands; + if (observed) { + const neverRan = claimedButNotObserved(usableRuns.map((run) => run.command), observed); + if (neverRan.length > 0) { + issues.push(`Reported runs that OpenSwarm never observed executing: ${neverRan.slice(0, 3).join('; ')}`); + } + for (const run of usableRuns) { + const exitMatch = observed + .map((line) => line.match(/^(.*?)(?: \[exit (-?\d+)\])?$/)) + .filter((m): m is RegExpMatchArray => m !== null) + .filter((m) => claimedButNotObserved([run.command], [m[1]]).length === 0); + if (exitMatch.length > 0 && !exitMatch.some((m) => Number(m[2] ?? 0) === run.exitCode)) { + issues.push(`Reported exit code does not match the observed run: ${run.line}`); + } + } + } + + if (usableRuns.length === 0) { + const docs = [...citedDocs]; + issues.unshift( + docs.length > 0 + ? `Report cites documents written in this same change as evidence (${docs.slice(0, 3).join(', ')}); a document you wrote is output, not evidence. Claim: "${claims[0].slice(0, 160)}"` + : `Report claims a check passed but shows no execution evidence. Claim: "${claims[0].slice(0, 160)}"`, + ); + } + + if (issues.length === 0) return []; + return [...issues, `Report each check you ran as one line: ${EVIDENCE_LINE_FORMAT}. If you did not run a check, say so instead of claiming it passed.`]; +} diff --git a/src/agents/pairIterationLoop.ts b/src/agents/pairIterationLoop.ts index 37b0acf5..8d66b43a 100644 --- a/src/agents/pairIterationLoop.ts +++ b/src/agents/pairIterationLoop.ts @@ -181,6 +181,17 @@ export async function runFullIterationLoop(host: PipelineLoopHost, context: Pipe return { success: false }; } if (detail?.startsWith('worker-scope:')) context.repeatedScopeRejection = detail; + // An unbacked "tests pass" is rejected in the worker, but a bare + // failure never reaches the next attempt's prompt. Record it so the + // retry sees exactly what was missing and the line format to use + // instead of repeating the same claim blind (AGT-4900). + if (failedWorker.unverifiedCompletionClaim && failedWorker.error) { + recordReflection(context.reflection, { + iteration: context.currentIteration, + source: 'validation', + errors: [failedWorker.error.replace(/^worker-evidence:\s*/, '')], + }); + } // A no-edit stop with a stated reason is a claim about the task: put it // to the reviewer once instead of retrying blind (AGT-4535). const blocker = await reviewWorkerBlocker(host.config, context, failedWorker, context.abortSignal); diff --git a/src/agents/pairPipeline.completionEvidence.test.ts b/src/agents/pairPipeline.completionEvidence.test.ts new file mode 100644 index 00000000..87c0505d --- /dev/null +++ b/src/agents/pairPipeline.completionEvidence.test.ts @@ -0,0 +1,65 @@ +// A worker rejected for claiming "tests pass" without evidence (AGT-4900) +// must be told so on the retry. A bare worker failure does not reach the next +// prompt, so without this the retry repeats the same unbacked claim. +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { TaskItem } from '../orchestration/decisionEngine.js'; + +const runWorker = vi.fn(); + +vi.mock('./worker.js', async () => ({ + ...(await vi.importActual('./worker.js')), + runWorker, +})); +vi.mock('../knowledge/index.js', () => ({ + hasRepoSnapshot: () => true, + scanAndCache: vi.fn(), + analyzeIssue: vi.fn().mockResolvedValue(undefined), +})); +vi.mock('../memory/repoKnowledge.js', () => ({ recallRepoKnowledge: vi.fn().mockResolvedValue([]) })); +vi.mock('../core/eventHub.js', () => ({ broadcastEvent: vi.fn() })); +vi.mock('../adapters/index.js', async () => ({ + ...(await vi.importActual('../adapters/index.js')), + getAdapter: () => ({ getDefaultModel: vi.fn().mockResolvedValue('default-model') }), +})); + +const task: TaskItem = { + id: 'completion-evidence-task', + source: 'linear', + title: 'fix projection', + description: 'exercise the completion evidence feedback', + priority: 1, + createdAt: Date.now(), +}; + +describe('retry after an unbacked completion claim (AGT-4900)', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + }); + + it('puts the missing-evidence issue and the line format into the next worker prompt', async () => { + runWorker + .mockResolvedValueOnce({ + success: false, summary: 'done', filesChanged: ['src/a.ts'], commands: [], output: 'All tests pass.', + unverifiedCompletionClaim: true, + error: 'worker-evidence: Report claims a check passed but shows no execution evidence. Claim: "All tests pass." Report each check you ran as one line: Ran: -> exit , passed, failed', + }) + .mockResolvedValue({ + success: false, summary: 'stopped', filesChanged: [], commands: [], output: '', error: 'stop', + }); + const { PairPipeline } = await import('./pairPipeline.js'); + const pipeline = new PairPipeline({ + stages: ['worker'], maxIterations: 2, + roles: { worker: { enabled: true, model: 'w', timeoutMs: 0 } }, + } as unknown as ConstructorParameters[0]); + + await pipeline.run(task, process.cwd()); + + expect(runWorker).toHaveBeenCalledTimes(2); + const retry = runWorker.mock.calls[1][0] as { previousFeedback?: string }; + expect(retry.previousFeedback).toContain('shows no execution evidence'); + expect(retry.previousFeedback).toContain('Ran: -> exit '); + expect(retry.previousFeedback).not.toContain('worker-evidence:'); + }); +}); diff --git a/src/agents/worker.completionEvidence.test.ts b/src/agents/worker.completionEvidence.test.ts new file mode 100644 index 00000000..def5d0ed --- /dev/null +++ b/src/agents/worker.completionEvidence.test.ts @@ -0,0 +1,93 @@ +// A worker report that claims a check passed must show what ran; a document +// written in the same diff is not evidence (AGT-4900). These run through +// runWorker so the gate is checked against the git-derived file list and after +// the promote-to-success step, which must not flip it back. +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const spawnCli = vi.fn(async () => ({ stdout: 'raw' })); +const parseWorkerOutput = vi.fn(); +const getChangedFilesSinceSnapshot = vi.fn(); + +vi.mock('../adapters/index.js', () => ({ + getAdapter: () => ({ parseWorkerOutput }), + getDefaultAdapterName: () => 'gpt', + spawnCli: (...args: unknown[]) => spawnCli(...(args as [])), +})); +vi.mock('../support/gitTracker.js', () => ({ + isGitRepo: vi.fn(async () => true), + takeSnapshot: vi.fn(async () => 'snapshot-tree'), + getChangedFilesSinceSnapshot, +})); + +const { runWorker } = await import('./worker.js'); + +const run = () => runWorker({ + taskTitle: 'fix projection', taskDescription: 'AX-1', projectPath: '/repo', adapterName: 'gpt', +}); + +describe('runWorker completion-report evidence (AGT-4900)', () => { + beforeEach(() => { + vi.clearAllMocks(); + getChangedFilesSinceSnapshot.mockResolvedValue(['src/projection.py', 'docs/readiness.md']); + }); + + it('does not accept "tests pass" with no execution evidence as success', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'Codename: Atlas\nFixed the projection. All tests pass.', + }); + + const result = await run(); + + expect(result.success).toBe(false); + expect(result.unverifiedCompletionClaim).toBe(true); + expect(result.error).toMatch(/^worker-evidence: Report claims a check passed but shows no execution evidence/); + }); + + it('does not accept a document from the same diff as the evidence', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'No credentials are committed; the readiness check passes.\nEvidence: docs/readiness.md', + }); + + const result = await run(); + + expect(result.success).toBe(false); + expect(result.error).toContain('docs/readiness.md'); + }); + + it('is not undone by git promotion when the model reported no JSON success', async () => { + // Text-only reports start as success=false and are promoted by git changes; + // the evidence gate must run after that promotion. + parseWorkerOutput.mockReturnValue({ + success: false, summary: 'Done', filesChanged: [], commands: [], + output: 'All tests pass.', + }); + + const result = await run(); + + expect(result.success).toBe(false); + expect(result.unverifiedCompletionClaim).toBe(true); + }); + + it('accepts the same claim when the report shows the run', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'All tests pass.\nRan: uv run pytest -> exit 0, 2385 passed, 0 failed', + }); + + const result = await run(); + + expect(result.success).toBe(true); + expect(result.unverifiedCompletionClaim).toBeUndefined(); + }); + + it('leaves a report without any claim alone', async () => { + parseWorkerOutput.mockReturnValue({ + success: true, summary: 'Done', filesChanged: [], commands: [], + output: 'Codename: Atlas\nRenamed the projection helper.', + }); + + expect((await run()).success).toBe(true); + }); +}); diff --git a/src/agents/worker.ts b/src/agents/worker.ts index 48e5136f..52cdec60 100644 --- a/src/agents/worker.ts +++ b/src/agents/worker.ts @@ -30,6 +30,7 @@ import { getCoordinationStore } from '../coordination/coordinationStore.js'; import { filesOutsideWriteScope } from '../orchestration/writeScope.js'; import { acceptWorkerPaths, noteRejectedWorkerPaths, rejectedWorkerPaths } from '../support/rejectedWorkerPaths.js'; import { formatProjectGoalSection } from '../support/projectGoal.js'; +import { completionReportEvidenceIssues } from './completionReportEvidence.js'; // Types @@ -549,6 +550,20 @@ export async function runWorker(options: WorkerOptions): Promise { } } + // A report that claims a check passed must show what ran: command, exit + // code and, for test runners, counts. Documents written in this same diff + // are not evidence. Runs after git promotion so it cannot be flipped back, + // and against the git-derived filesChanged, not the model's list. (AGT-4900) + if (parsedResult.success) { + const evidenceIssues = completionReportEvidenceIssues(parsedResult); + if (evidenceIssues.length > 0) { + parsedResult.success = false; + parsedResult.unverifiedCompletionClaim = true; + parsedResult.error = `worker-evidence: ${evidenceIssues.join(' ')}`; + emitWorkerStatus(options, `[Worker] Completion claim without execution evidence: ${evidenceIssues[0].slice(0, 300)}`); + } + } + // The agent introduces itself with a `Codename:` line (AGT-4019); the name // is its display identity on the coordination board. const codenameMatch = (parsedResult.output || '').match(/^\s*Codename:\s*(.{1,60}?)\s*$/mi); diff --git a/src/agents/workerValidationEvidence.ts b/src/agents/workerValidationEvidence.ts index a78d04b2..ac15e822 100644 --- a/src/agents/workerValidationEvidence.ts +++ b/src/agents/workerValidationEvidence.ts @@ -23,6 +23,11 @@ const INSPECTION_ONLY_COMMAND_RE = /^\s*(?:rg|grep|sed|cat|ls|find|pwd|git\s+(?: const SCRIPT_SMOKE_COMMAND_RE = /^\s*(?:python3?|node|tsx|ts-node|uv\s+run|npx|bunx|bash|sh)\b.*\b(?:scripts?\/|tests?\/|\.py|\.js|\.ts|\.sh|--help|--dry-run|smoke|verify|validate|check|test|build|compile)\b/i; const TESTER_CODE_FILE_RE = /\.(ts|tsx|mts|cts|js|jsx|mjs|cjs|py|rs|go|java|rb|c|cpp|h|hpp)$/; +/** README/CHANGELOG/LICENSE/NOTICE, anything under docs/, and prose formats. */ +export function isDocOnlyFile(file: string): boolean { + return DOC_ONLY_FILE_RE.test(file); +} + export function isValidationRelevantFile(file: string): boolean { if (/(^|\/)docs?\//i.test(file)) return false; if (VALIDATION_RELEVANT_BASENAME_RE.test(file)) return true; diff --git a/src/locale/prompts/en.ts b/src/locale/prompts/en.ts index c93461f8..f59661b7 100644 --- a/src/locale/prompts/en.ts +++ b/src/locale/prompts/en.ts @@ -386,6 +386,17 @@ That name is how you appear on the team board and in conversation with your reviewer; keep the same one across iterations. Then write the summary itself as if you were telling your reviewer what you did and what they should look at. +Report what you ran, not whether a gate passed. For every check you ran, write +one line in this exact form (test runners give counts; lint/typecheck/build +give the exit code): + +Ran: npm test -> exit 0, 412 passed, 0 failed +Ran: npm run lint -> exit 0 + +A report that says tests or lint pass without such a line is not accepted as a +success. A document you wrote or edited in this change is not evidence. If you +could not run a check, say so instead of claiming it passed. + If (and only if) you want to flag low confidence or a blocker, end with this JSON: \`\`\`json { "success": false, "confidencePercent": 40, "haltReason": "why you're stuck" } diff --git a/src/locale/prompts/ko.ts b/src/locale/prompts/ko.ts index daf679b2..09871299 100644 --- a/src/locale/prompts/ko.ts +++ b/src/locale/prompts/ko.ts @@ -374,6 +374,16 @@ Codename: <스스로 지은 표시 이름 — 스타일·언어 자유> 바뀌어도 같은 이름을 유지하라. 요약 본문은 리뷰어에게 말하듯 — 무엇을 했고 어디를 봐 달라는지 — 써라. +게이트 통과 여부가 아니라 무엇을 실행했는지를 보고하라. 실행한 검사마다 정확히 이 +형식으로 한 줄씩 적어라(테스트 러너는 통과/실패 수, lint·typecheck·build는 종료코드): + +Ran: npm test -> exit 0, 412 passed, 0 failed +Ran: npm run lint -> exit 0 + +이런 줄 없이 테스트·린트가 통과했다고 쓴 보고는 성공으로 인정되지 않는다. 이번 +변경에서 네가 쓰거나 고친 문서는 근거가 아니다. 검사를 실행하지 못했다면 통과했다고 +쓰지 말고 실행하지 못했다고 적어라. + 낮은 확신이나 블로커를 알릴 때만(그럴 때만) 마지막에 이 JSON을 붙여라: \`\`\`json { "success": false, "confidencePercent": 40, "haltReason": "막힌 이유" }