From 55efc7efec8334fd7616182a1e7a565f5bc58f70 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 07:38:14 +0900 Subject: [PATCH] =?UTF-8?q?feat(gate):=20=EC=BB=A4=EB=B2=84=EB=A6=AC?= =?UTF-8?q?=EC=A7=80=20ratchet=20=E2=80=94=20=EB=B3=91=ED=95=A9=20?= =?UTF-8?q?=EA=B8=B0=EC=A4=80=20main=EB=B3=B4=EB=8B=A4=20=EC=BB=A4?= =?UTF-8?q?=EB=B2=84=EB=A6=AC=EC=A7=80=EA=B0=80=20=EB=82=AE=EC=9C=BC?= =?UTF-8?q?=EB=A9=B4=20=EB=B0=9C=ED=96=89=20=EB=B3=B4=EB=A5=98=20(AGT-4898?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/automation/coverageRatchetGate.test.ts | 170 ++++++++++++++++++ src/automation/coverageRatchetGate.ts | 197 +++++++++++++++++++++ src/automation/publishOnPark.test.ts | 37 ++++ src/automation/publishOnPark.ts | 43 +++-- 4 files changed, 437 insertions(+), 10 deletions(-) create mode 100644 src/automation/coverageRatchetGate.test.ts create mode 100644 src/automation/coverageRatchetGate.ts diff --git a/src/automation/coverageRatchetGate.test.ts b/src/automation/coverageRatchetGate.test.ts new file mode 100644 index 00000000..4d9f93c2 --- /dev/null +++ b/src/automation/coverageRatchetGate.test.ts @@ -0,0 +1,170 @@ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + coverageRatchetHold, + fileBaselineStore, + setCoverageProvider, + type CoverageBaselineStore, + type CoverageMeasurement, + type CoverageProvider, +} from './coverageRatchetGate.js'; + +const BASE_SHA = 'b'.repeat(40); +const worktree = mkdtempSync(join(tmpdir(), 'openswarm-cov-gate-')); + +/** git seam: a repo with an origin and a merge base, or none. */ +function fakeGit(mergeBase: string = BASE_SHA) { + return async (args: string[]) => { + if (args[0] === 'merge-base') return mergeBase ? `${mergeBase}\n` : ''; + if (args[0] === 'config') return 'git@github.com:o/r.git\n'; + return ''; + }; +} + +function memoryStore(initial: Record = {}): CoverageBaselineStore & { data: Record } { + const data = { ...initial }; + return { data, get: (k) => data[k], set: async (k, v) => { data[k] = v; } }; +} + +function provider(base: CoverageMeasurement, head: CoverageMeasurement) { + return { + measureBase: vi.fn(async () => base), + measureHead: vi.fn(async () => head), + } satisfies CoverageProvider; +} + +const resolveBase = async () => ({ ref: 'origin/main' }); +const measured = (percent: number): CoverageMeasurement => ({ kind: 'measured', percent }); + +afterEach(() => setCoverageProvider(null)); + +describe('coverageRatchetHold (AGT-4898)', () => { + it('holds a change whose coverage is lower than its main commit', async () => { + const result = await coverageRatchetHold(worktree, { + provider: provider(measured(82.5), measured(79.1)), + store: memoryStore(), + resolveBase, + git: fakeGit(), + }); + + expect(result.status).toBe('hold'); + if (result.status !== 'hold') return; + expect(result.baseline).toBe(82.5); + expect(result.head).toBe(79.1); + expect(result.reason).toContain('82.50%'); + expect(result.reason).toContain('79.10%'); + }); + + it('passes when coverage is unchanged or higher', async () => { + for (const head of [82.5, 90]) { + const result = await coverageRatchetHold(worktree, { + provider: provider(measured(82.5), measured(head)), + store: memoryStore(), + resolveBase, + git: fakeGit(), + }); + expect(result).toEqual({ status: 'pass', baseline: 82.5, head }); + } + }); + + it('ignores a drop within rounding tolerance', async () => { + const result = await coverageRatchetHold(worktree, { + provider: provider(measured(82.5), measured(82.495)), + store: memoryStore(), + resolveBase, + git: fakeGit(), + }); + expect(result.status).toBe('pass'); + }); + + it('stores the baseline per repository and merge base, and reuses it', async () => { + const store = memoryStore(); + const p = provider(measured(70), measured(71)); + await coverageRatchetHold(worktree, { provider: p, store, resolveBase, git: fakeGit() }); + await coverageRatchetHold(worktree, { provider: p, store, resolveBase, git: fakeGit() }); + + expect(store.data).toEqual({ [`git@github.com:o/r.git@${BASE_SHA}`]: 70 }); + expect(p.measureBase).toHaveBeenCalledTimes(1); + }); + + it('compares a branch with the main it was cut from, not a newer one', async () => { + const older = 'a'.repeat(40); + // Newer main measured 90; this branch started from an older main at 60. + const store = memoryStore({ [`git@github.com:o/r.git@${BASE_SHA}`]: 90, [`git@github.com:o/r.git@${older}`]: 60 }); + const result = await coverageRatchetHold(worktree, { + provider: provider(measured(0), measured(65)), + store, + resolveBase, + git: fakeGit(older), + }); + expect(result).toEqual({ status: 'pass', baseline: 60, head: 65 }); + }); + + it('skips and says so when the base is not measurable', async () => { + const store = memoryStore(); + const result = await coverageRatchetHold(worktree, { + provider: provider({ kind: 'unsupported', why: 'no coverage tool configured' }, measured(10)), + store, + resolveBase, + git: fakeGit(), + }); + expect(result.status).toBe('skipped'); + if (result.status === 'skipped') expect(result.reason).toContain('no coverage tool configured'); + expect(store.data).toEqual({}); + }); + + it('holds when main was measurable but the change is not (coverage config removed)', async () => { + const result = await coverageRatchetHold(worktree, { + provider: provider(measured(80), { kind: 'unsupported', why: 'coverage config missing' }), + store: memoryStore(), + resolveBase, + git: fakeGit(), + }); + expect(result.status).toBe('hold'); + }); + + it('holds when the measurement throws', async () => { + const failing: CoverageProvider = { + measureBase: async () => measured(80), + measureHead: async () => { throw new Error('test runner crashed'); }, + }; + const result = await coverageRatchetHold(worktree, { provider: failing, store: memoryStore(), resolveBase, git: fakeGit() }); + expect(result.status).toBe('hold'); + if (result.status === 'hold') expect(result.reason).toContain('test runner crashed'); + }); + + it('skips without a merge base', async () => { + const result = await coverageRatchetHold(worktree, { + provider: provider(measured(80), measured(10)), + store: memoryStore(), + resolveBase, + git: fakeGit(''), + }); + expect(result.status).toBe('skipped'); + }); + + it('skips and records it when no provider is configured', async () => { + const result = await coverageRatchetHold(worktree, { store: memoryStore(), resolveBase, git: fakeGit() }); + expect(result).toEqual({ status: 'skipped', reason: expect.stringContaining('no coverage provider') }); + }); + + it('uses the registered provider by default', async () => { + setCoverageProvider(provider(measured(80), measured(50))); + const result = await coverageRatchetHold(worktree, { store: memoryStore(), resolveBase, git: fakeGit() }); + expect(result.status).toBe('hold'); + }); +}); + +describe('fileBaselineStore', () => { + it('round-trips baselines through a JSON file and treats a missing file as empty', async () => { + const path = join(mkdtempSync(join(tmpdir(), 'openswarm-cov-store-')), 'nested', 'baselines.json'); + const store = fileBaselineStore(path); + expect(store.get('r@x')).toBeUndefined(); + await store.set('r@x', 81.25); + await store.set('r@y', 50); + expect(fileBaselineStore(path).get('r@x')).toBe(81.25); + expect(fileBaselineStore(path).get('r@y')).toBe(50); + }); +}); diff --git a/src/automation/coverageRatchetGate.ts b/src/automation/coverageRatchetGate.ts new file mode 100644 index 00000000..3533027f --- /dev/null +++ b/src/automation/coverageRatchetGate.ts @@ -0,0 +1,197 @@ +// ============================================ +// OpenSwarm — an unattended run may not publish lower coverage than main +// ============================================ +// +// testRemovalGate.ts blocks a change that lowers the active test count. A test +// can also be kept and weakened — its assertions loosened, its fixture emptied — +// and that passes the count unchanged. AGT-4190 defence 2: compare the line +// coverage of the change with the coverage of the main commit it branched from, +// and park for a person when it drops. A ratchet, not a target: the baseline is +// whatever that main commit measured. +// +// The baseline is keyed by repository and merge-base SHA, not "latest main": a +// branch cut from an older main is compared with the main it actually started +// from. Support is decided on the base side — a worker that deletes the coverage +// configuration does not switch the gate off; its head then fails to measure and +// the publication is held. +// +// Measurement is a provider seam. The number has to come from something the +// harness ran, never from the worker's own report (tester.ts parses coverage out +// of agent text — the exact evasion this gate exists for). No provider is wired +// in by default yet, so until one is, every repository is recorded as skipped. + +import { execFile } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { promisify } from 'node:util'; +import { atomicWriteFile } from '../support/atomicFile.js'; + +const exec = promisify(execFile); + +/** NEEDS_HUMAN code for a run whose change lowers coverage below its base. */ +export const COVERAGE_DROP_PARK_REASON = 'coverage_dropped'; + +/** + * Coverage is compared in percentage points; drops smaller than this are + * rounding noise between two runs of the same tool, not a weakened test. + */ +export const COVERAGE_TOLERANCE_PCT = 0.01; + +export type CoverageMeasurement = + | { kind: 'measured'; percent: number } + | { kind: 'unsupported'; why: string }; + +export interface CoverageProvider { + /** Measure the main commit `baseSha`. `unsupported` here skips the gate. */ + measureBase(worktreePath: string, baseSha: string): Promise; + /** Measure the worktree as it will be published. */ + measureHead(worktreePath: string): Promise; +} + +export interface CoverageBaselineStore { + get(key: string): number | undefined; + set(key: string, percent: number): Promise; +} + +export type CoverageRatchetResult = + | { status: 'pass'; baseline: number; head: number } + | { status: 'skipped'; reason: string } + | { status: 'hold'; reason: string; baseline?: number; head?: number }; + +let registeredProvider: CoverageProvider | null = null; + +/** + * Install the provider the publication path uses. Until something calls this, + * the gate records every repository as skipped. + */ +export function setCoverageProvider(provider: CoverageProvider | null): void { + registeredProvider = provider; +} + +function defaultBaselinePath(): string { + return process.env.OPENSWARM_COVERAGE_BASELINES + ?? join(homedir(), '.openswarm', 'coverage-baselines.json'); +} + +/** A JSON file of `{ "@": percent }`. Unreadable means empty. */ +export function fileBaselineStore(path: string = defaultBaselinePath()): CoverageBaselineStore { + const load = (): Record => { + try { + const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown; + return parsed && typeof parsed === 'object' ? parsed as Record : {}; + } catch { + return {}; + } + }; + return { + get(key) { + const value = load()[key]; + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; + }, + async set(key, percent) { + const all = load(); + all[key] = percent; + if (!existsSync(dirname(path))) mkdirSync(dirname(path), { recursive: true }); + await atomicWriteFile(path, `${JSON.stringify(all, null, 2)}\n`, 0o600); + }, + }; +} + +type Git = (args: string[]) => Promise; + +function gitIn(worktreePath: string): Git { + return async (args) => (await exec('git', ['-C', worktreePath, ...args])).stdout; +} + +/** The repository identity a baseline belongs to: its origin URL, else its top level. */ +async function repoKey(git: Git): Promise { + const remote = (await git(['config', '--get', 'remote.origin.url']).catch(() => '')).trim(); + if (remote) return remote; + return (await git(['rev-parse', '--show-toplevel'])).trim(); +} + +function pct(value: number): string { + return `${value.toFixed(2)}%`; +} + +/** The stored baseline for `key`, measuring and storing it on first sight. */ +async function baselineFor( + provider: CoverageProvider, + store: CoverageBaselineStore, + key: string, + worktreePath: string, + baseSha: string, +): Promise { + const stored = store.get(key); + if (stored !== undefined) return { kind: 'measured', percent: stored }; + const base = await provider.measureBase(worktreePath, baseSha); + if (base.kind === 'measured') await store.set(key, base.percent); + return base; +} + +/** + * Decide whether a reviewed, unattended publication must be held for a drop in + * coverage. Fails CLOSED once the base is known to be measurable: a head that + * cannot be measured, or a measurement that throws, holds the publication. + */ +export async function coverageRatchetHold( + worktreePath: string, + deps: { + provider?: CoverageProvider | null; + store?: CoverageBaselineStore; + resolveBase?: (path: string) => Promise<{ ref: string }>; + git?: Git; + } = {}, +): Promise { + const provider = deps.provider === undefined ? registeredProvider : deps.provider; + if (!provider) { + return { status: 'skipped', reason: 'no coverage provider is configured; the coverage ratchet did not run' }; + } + if (!existsSync(worktreePath)) { + return { status: 'skipped', reason: `worktree ${worktreePath} does not exist` }; + } + try { + const git = deps.git ?? gitIn(worktreePath); + const resolveBase = deps.resolveBase ?? (async (path: string) => { + const { resolveBaseRef } = await import('../support/worktreeManager.js'); + return resolveBaseRef(path); + }); + const { ref } = await resolveBase(worktreePath); + const baseSha = (await git(['merge-base', ref, 'HEAD']).catch(() => '')).trim(); + if (!baseSha) { + return { status: 'skipped', reason: `no merge base with ${ref}; nothing to compare coverage against` }; + } + + const store = deps.store ?? fileBaselineStore(); + const base = await baselineFor(provider, store, `${await repoKey(git)}@${baseSha}`, worktreePath, baseSha); + if (base.kind === 'unsupported') { + return { status: 'skipped', reason: `coverage is not measurable for this repository at ${baseSha.slice(0, 12)}: ${base.why}` }; + } + const baseline = base.percent; + + const head = await provider.measureHead(worktreePath); + if (head.kind === 'unsupported') { + return { + status: 'hold', + baseline, + reason: `Coverage was measurable on main (${pct(baseline)} at ${baseSha.slice(0, 12)}) but not on this change: ${head.why}. ` + + 'Holding the publication rather than publishing unmeasured.', + }; + } + if (head.percent + COVERAGE_TOLERANCE_PCT < baseline) { + return { + status: 'hold', + baseline, + head: head.percent, + reason: `This change lowers line coverage from ${pct(baseline)} (main at ${baseSha.slice(0, 12)}) to ${pct(head.percent)}. ` + + 'Weakening a test keeps the count and lowers coverage, so an unattended run does not publish it. ' + + 'Restore the coverage, or have a person review and publish the change.', + }; + } + return { status: 'pass', baseline, head: head.percent }; + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + return { status: 'hold', reason: `The coverage check could not run (${detail}); holding the publication rather than publishing unmeasured.` }; + } +} diff --git a/src/automation/publishOnPark.test.ts b/src/automation/publishOnPark.test.ts index 55ac4c1b..5eed7aa1 100644 --- a/src/automation/publishOnPark.test.ts +++ b/src/automation/publishOnPark.test.ts @@ -6,12 +6,15 @@ vi.mock('../support/worktreeManager.js', () => ({ commitAndCreatePRWithHead })); vi.mock('../core/eventHub.js', () => ({ broadcastEvent: vi.fn() })); // The worktrees here are fake paths; the gate itself is tested in testRemovalGate.test.ts. vi.mock('./testRemovalGate.js', () => ({ TEST_REMOVAL_PARK_REASON: 'active_tests_removed', testRemovalHold: async () => null })); +const coverageRatchetHold = vi.hoisted(() => vi.fn()); +vi.mock('./coverageRatchetGate.js', () => ({ COVERAGE_DROP_PARK_REASON: 'coverage_dropped', coverageRatchetHold })); import { PublicationScopeMismatchError } from '../support/publicationScopeFence.js'; import { PUBLICATION_SCOPE_PARK_REASON, WORKER_NO_CHANGES_PARK_REASON, publishApprovedWork, publishFinishedRun, publishParkedIfNeeded, publishParkedWork, publishStuckWork, shouldPublishParkedWork, summarizeFailure } from './publishOnPark.js'; beforeEach(() => { commitAndCreatePRWithHead.mockReset(); + coverageRatchetHold.mockReset().mockResolvedValue({ status: 'skipped', reason: 'no coverage provider is configured' }); }); describe('shouldPublishParkedWork (AGT-4076)', () => { @@ -514,3 +517,37 @@ describe('an unfinished run opens no pull request (AGT-4786)', () => { }); }); }); + +describe('coverage ratchet before publication (AGT-4898)', () => { + const info = { worktreePath: '/tmp/w', originalPath: '/tmp/r', branchName: 'swarm/AGT-4898', issueId: 'AGT-4898' }; + const task = { id: 'task-1', issueIdentifier: 'AGT-4898', title: 'Coverage ratchet' }; + + it('parks an approved run whose coverage dropped, before the lease fence and without publishing', async () => { + coverageRatchetHold.mockResolvedValue({ status: 'hold', reason: 'coverage 80.00% -> 70.00%', baseline: 80, head: 70 }); + const beforePublish = vi.fn(async () => true); + const durability = { beforePublish, onPublication: vi.fn(async () => true) } as unknown as ExecutionDurabilityHooks; + const result: { success: boolean; finalStatus: string; failureDetail?: string; operatorPark?: { code: string; reason: string } } = { + success: true, + finalStatus: 'approved', + }; + + await publishApprovedWork(info, task, result, durability); + + expect(result.success).toBe(false); + expect(result.operatorPark).toEqual({ code: 'coverage_dropped', reason: 'coverage 80.00% -> 70.00%' }); + expect(beforePublish).not.toHaveBeenCalled(); + expect(commitAndCreatePRWithHead).not.toHaveBeenCalled(); + }); + + it('publishes when the ratchet passes or is skipped', async () => { + for (const outcome of [{ status: 'pass', baseline: 80, head: 80 }, { status: 'skipped', reason: 'unsupported' }]) { + commitAndCreatePRWithHead.mockReset().mockResolvedValue({ prUrl: 'https://github.com/o/r/pull/1', headSha: 'h' }); + coverageRatchetHold.mockResolvedValue(outcome); + const result: { success: boolean; finalStatus: string; prUrl?: string } = { success: true, finalStatus: 'approved' }; + + await publishApprovedWork(info, task, result, undefined); + + expect(result.prUrl).toBe('https://github.com/o/r/pull/1'); + } + }); +}); diff --git a/src/automation/publishOnPark.ts b/src/automation/publishOnPark.ts index a2aeec40..ecd62ea9 100644 --- a/src/automation/publishOnPark.ts +++ b/src/automation/publishOnPark.ts @@ -34,6 +34,7 @@ import { pipelineExecutionBoundary, type StageBoundaryResults } from '../agents/ import type { ExecutionDurabilityHooks } from './durableRunCoordinator.js'; import { TEST_REMOVAL_PARK_REASON, testRemovalHold } from './testRemovalGate.js'; +import { COVERAGE_DROP_PARK_REASON, coverageRatchetHold } from './coverageRatchetGate.js'; /** Runs once after a reviewed publication succeeded and was durably recorded. */ export type ApprovedPublicationHook = (publication: { @@ -297,6 +298,37 @@ export async function publishStuckWork( } } +type HoldableResult = Pick & { success?: boolean; finalStatus?: string }; + +/** + * The pre-publication gates of an approved run: fewer active tests + * (testRemovalGate) and lower coverage than its main (coverageRatchetGate). + * Returns true when the run was parked for a person instead of published. + */ +async function holdBeforePublication(worktreePath: string, task: PublishableTask, result: HoldableResult): Promise { + const taskId = task.issueId || task.id; + const park = (code: string, reason: string): true => { + result.success = false; + result.finalStatus = 'failed'; + result.failureDetail = `publication: ${reason}`; + result.operatorPark = { code, reason }; + broadcastEvent({ type: 'log', data: { taskId, stage: 'pr', line: `Publication held: ${reason}` } }); + console.warn(`[Runner] Publication held for ${task.issueIdentifier}: ${reason}`); + return true; + }; + const removal = await testRemovalHold(worktreePath, task.description); + if (removal) return park(TEST_REMOVAL_PARK_REASON, removal.reason); + const coverage = await coverageRatchetHold(worktreePath); + if (coverage.status === 'hold') return park(COVERAGE_DROP_PARK_REASON, coverage.reason); + if (coverage.status === 'skipped') { + // Recorded, not silent: a ratchet that quietly stops measuring is the + // failure it exists to catch. + broadcastEvent({ type: 'log', data: { taskId, stage: 'pr', line: `Coverage ratchet skipped: ${coverage.reason}` } }); + console.log(`[Runner] Coverage ratchet skipped for ${task.issueIdentifier}: ${coverage.reason}`); + } + return false; +} + /** * Publish the branch of a run that passed review. * @@ -318,16 +350,7 @@ export async function publishApprovedWork( if (worktreeInfo && result.success && result.finalStatus === 'approved') { // Before the lease fence: `beforePublish` moves the run to PUBLISHING, and a // held run must park from where it is, not from a publication it never made. - const removal = await testRemovalHold(worktreeInfo.worktreePath, task.description); - if (removal) { - result.success = false; - result.finalStatus = 'failed'; - result.failureDetail = `publication: ${removal.reason}`; - result.operatorPark = { code: TEST_REMOVAL_PARK_REASON, reason: removal.reason }; - broadcastEvent({ type: 'log', data: { taskId: task.issueId || task.id, stage: 'pr', line: `Publication held: ${removal.reason}` } }); - console.warn(`[Runner] Publication held for ${task.issueIdentifier}: ${removal.reason}`); - return; - } + if (await holdBeforePublication(worktreeInfo.worktreePath, task, result)) return; const publishAllowed = await durability?.beforePublish() ?? true; if (!publishAllowed) { result.success = false;