Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 170 additions & 0 deletions src/automation/coverageRatchetGate.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it, vi } from 'vitest';
import {
coverageRatchetHold,
fileBaselineStore,
setCoverageProvider,
type CoverageBaselineStore,
type CoverageMeasurement,
type CoverageProvider,
} from './coverageRatchetGate.js';

const BASE_SHA = 'b'.repeat(40);
const worktree = mkdtempSync(join(tmpdir(), 'openswarm-cov-gate-'));

/** git seam: a repo with an origin and a merge base, or none. */
function fakeGit(mergeBase: string = BASE_SHA) {
return async (args: string[]) => {
if (args[0] === 'merge-base') return mergeBase ? `${mergeBase}\n` : '';
if (args[0] === 'config') return 'git@github.com:o/r.git\n';
return '';
};
}

function memoryStore(initial: Record<string, number> = {}): CoverageBaselineStore & { data: Record<string, number> } {
const data = { ...initial };
return { data, get: (k) => data[k], set: async (k, v) => { data[k] = v; } };
}

function provider(base: CoverageMeasurement, head: CoverageMeasurement) {
return {
measureBase: vi.fn(async () => base),
measureHead: vi.fn(async () => head),
} satisfies CoverageProvider;
}

const resolveBase = async () => ({ ref: 'origin/main' });
const measured = (percent: number): CoverageMeasurement => ({ kind: 'measured', percent });

afterEach(() => setCoverageProvider(null));

describe('coverageRatchetHold (AGT-4898)', () => {
it('holds a change whose coverage is lower than its main commit', async () => {
const result = await coverageRatchetHold(worktree, {
provider: provider(measured(82.5), measured(79.1)),
store: memoryStore(),
resolveBase,
git: fakeGit(),
});

expect(result.status).toBe('hold');
if (result.status !== 'hold') return;
expect(result.baseline).toBe(82.5);
expect(result.head).toBe(79.1);
expect(result.reason).toContain('82.50%');
expect(result.reason).toContain('79.10%');
});

it('passes when coverage is unchanged or higher', async () => {
for (const head of [82.5, 90]) {
const result = await coverageRatchetHold(worktree, {
provider: provider(measured(82.5), measured(head)),
store: memoryStore(),
resolveBase,
git: fakeGit(),
});
expect(result).toEqual({ status: 'pass', baseline: 82.5, head });
}
});

it('ignores a drop within rounding tolerance', async () => {
const result = await coverageRatchetHold(worktree, {
provider: provider(measured(82.5), measured(82.495)),
store: memoryStore(),
resolveBase,
git: fakeGit(),
});
expect(result.status).toBe('pass');
});

it('stores the baseline per repository and merge base, and reuses it', async () => {
const store = memoryStore();
const p = provider(measured(70), measured(71));
await coverageRatchetHold(worktree, { provider: p, store, resolveBase, git: fakeGit() });
await coverageRatchetHold(worktree, { provider: p, store, resolveBase, git: fakeGit() });

expect(store.data).toEqual({ [`git@github.com:o/r.git@${BASE_SHA}`]: 70 });
expect(p.measureBase).toHaveBeenCalledTimes(1);
});

it('compares a branch with the main it was cut from, not a newer one', async () => {
const older = 'a'.repeat(40);
// Newer main measured 90; this branch started from an older main at 60.
const store = memoryStore({ [`git@github.com:o/r.git@${BASE_SHA}`]: 90, [`git@github.com:o/r.git@${older}`]: 60 });
const result = await coverageRatchetHold(worktree, {
provider: provider(measured(0), measured(65)),
store,
resolveBase,
git: fakeGit(older),
});
expect(result).toEqual({ status: 'pass', baseline: 60, head: 65 });
});

it('skips and says so when the base is not measurable', async () => {
const store = memoryStore();
const result = await coverageRatchetHold(worktree, {
provider: provider({ kind: 'unsupported', why: 'no coverage tool configured' }, measured(10)),
store,
resolveBase,
git: fakeGit(),
});
expect(result.status).toBe('skipped');
if (result.status === 'skipped') expect(result.reason).toContain('no coverage tool configured');
expect(store.data).toEqual({});
});

it('holds when main was measurable but the change is not (coverage config removed)', async () => {
const result = await coverageRatchetHold(worktree, {
provider: provider(measured(80), { kind: 'unsupported', why: 'coverage config missing' }),
store: memoryStore(),
resolveBase,
git: fakeGit(),
});
expect(result.status).toBe('hold');
});

it('holds when the measurement throws', async () => {
const failing: CoverageProvider = {
measureBase: async () => measured(80),
measureHead: async () => { throw new Error('test runner crashed'); },
};
const result = await coverageRatchetHold(worktree, { provider: failing, store: memoryStore(), resolveBase, git: fakeGit() });
expect(result.status).toBe('hold');
if (result.status === 'hold') expect(result.reason).toContain('test runner crashed');
});

it('skips without a merge base', async () => {
const result = await coverageRatchetHold(worktree, {
provider: provider(measured(80), measured(10)),
store: memoryStore(),
resolveBase,
git: fakeGit(''),
});
expect(result.status).toBe('skipped');
});

it('skips and records it when no provider is configured', async () => {
const result = await coverageRatchetHold(worktree, { store: memoryStore(), resolveBase, git: fakeGit() });
expect(result).toEqual({ status: 'skipped', reason: expect.stringContaining('no coverage provider') });
});

it('uses the registered provider by default', async () => {
setCoverageProvider(provider(measured(80), measured(50)));
const result = await coverageRatchetHold(worktree, { store: memoryStore(), resolveBase, git: fakeGit() });
expect(result.status).toBe('hold');
});
});

describe('fileBaselineStore', () => {
it('round-trips baselines through a JSON file and treats a missing file as empty', async () => {
const path = join(mkdtempSync(join(tmpdir(), 'openswarm-cov-store-')), 'nested', 'baselines.json');
const store = fileBaselineStore(path);
expect(store.get('r@x')).toBeUndefined();
await store.set('r@x', 81.25);
await store.set('r@y', 50);
expect(fileBaselineStore(path).get('r@x')).toBe(81.25);
expect(fileBaselineStore(path).get('r@y')).toBe(50);
});
});
197 changes: 197 additions & 0 deletions src/automation/coverageRatchetGate.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
// ============================================
// OpenSwarm — an unattended run may not publish lower coverage than main
// ============================================
//
// testRemovalGate.ts blocks a change that lowers the active test count. A test
// can also be kept and weakened — its assertions loosened, its fixture emptied —
// and that passes the count unchanged. AGT-4190 defence 2: compare the line
// coverage of the change with the coverage of the main commit it branched from,
// and park for a person when it drops. A ratchet, not a target: the baseline is
// whatever that main commit measured.
//
// The baseline is keyed by repository and merge-base SHA, not "latest main": a
// branch cut from an older main is compared with the main it actually started
// from. Support is decided on the base side — a worker that deletes the coverage
// configuration does not switch the gate off; its head then fails to measure and
// the publication is held.
//
// Measurement is a provider seam. The number has to come from something the
// harness ran, never from the worker's own report (tester.ts parses coverage out
// of agent text — the exact evasion this gate exists for). No provider is wired
// in by default yet, so until one is, every repository is recorded as skipped.

import { execFile } from 'node:child_process';
import { existsSync, mkdirSync, readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { dirname, join } from 'node:path';
import { promisify } from 'node:util';
import { atomicWriteFile } from '../support/atomicFile.js';

const exec = promisify(execFile);

/** NEEDS_HUMAN code for a run whose change lowers coverage below its base. */
export const COVERAGE_DROP_PARK_REASON = 'coverage_dropped';

/**
* Coverage is compared in percentage points; drops smaller than this are
* rounding noise between two runs of the same tool, not a weakened test.
*/
export const COVERAGE_TOLERANCE_PCT = 0.01;

export type CoverageMeasurement =
| { kind: 'measured'; percent: number }
| { kind: 'unsupported'; why: string };

export interface CoverageProvider {
/** Measure the main commit `baseSha`. `unsupported` here skips the gate. */
measureBase(worktreePath: string, baseSha: string): Promise<CoverageMeasurement>;
/** Measure the worktree as it will be published. */
measureHead(worktreePath: string): Promise<CoverageMeasurement>;
}

export interface CoverageBaselineStore {
get(key: string): number | undefined;
set(key: string, percent: number): Promise<void>;
}

export type CoverageRatchetResult =
| { status: 'pass'; baseline: number; head: number }
| { status: 'skipped'; reason: string }
| { status: 'hold'; reason: string; baseline?: number; head?: number };

let registeredProvider: CoverageProvider | null = null;

/**
* Install the provider the publication path uses. Until something calls this,
* the gate records every repository as skipped.
*/
export function setCoverageProvider(provider: CoverageProvider | null): void {
registeredProvider = provider;
}

function defaultBaselinePath(): string {
return process.env.OPENSWARM_COVERAGE_BASELINES
?? join(homedir(), '.openswarm', 'coverage-baselines.json');
}

/** A JSON file of `{ "<repo>@<sha>": percent }`. Unreadable means empty. */
export function fileBaselineStore(path: string = defaultBaselinePath()): CoverageBaselineStore {
const load = (): Record<string, number> => {
try {
const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown;
return parsed && typeof parsed === 'object' ? parsed as Record<string, number> : {};
} catch {
return {};
}
};
return {
get(key) {
const value = load()[key];
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
},
async set(key, percent) {
const all = load();
all[key] = percent;
if (!existsSync(dirname(path))) mkdirSync(dirname(path), { recursive: true });
await atomicWriteFile(path, `${JSON.stringify(all, null, 2)}\n`, 0o600);
},
};
}

type Git = (args: string[]) => Promise<string>;

function gitIn(worktreePath: string): Git {
return async (args) => (await exec('git', ['-C', worktreePath, ...args])).stdout;
}

/** The repository identity a baseline belongs to: its origin URL, else its top level. */
async function repoKey(git: Git): Promise<string> {
const remote = (await git(['config', '--get', 'remote.origin.url']).catch(() => '')).trim();
if (remote) return remote;
return (await git(['rev-parse', '--show-toplevel'])).trim();
}

function pct(value: number): string {
return `${value.toFixed(2)}%`;
}

/** The stored baseline for `key`, measuring and storing it on first sight. */
async function baselineFor(
provider: CoverageProvider,
store: CoverageBaselineStore,
key: string,
worktreePath: string,
baseSha: string,
): Promise<CoverageMeasurement> {
const stored = store.get(key);
if (stored !== undefined) return { kind: 'measured', percent: stored };
const base = await provider.measureBase(worktreePath, baseSha);
if (base.kind === 'measured') await store.set(key, base.percent);
return base;
}

/**
* Decide whether a reviewed, unattended publication must be held for a drop in
* coverage. Fails CLOSED once the base is known to be measurable: a head that
* cannot be measured, or a measurement that throws, holds the publication.
*/
export async function coverageRatchetHold(
worktreePath: string,
deps: {
provider?: CoverageProvider | null;
store?: CoverageBaselineStore;
resolveBase?: (path: string) => Promise<{ ref: string }>;
git?: Git;
} = {},
): Promise<CoverageRatchetResult> {
const provider = deps.provider === undefined ? registeredProvider : deps.provider;
if (!provider) {
return { status: 'skipped', reason: 'no coverage provider is configured; the coverage ratchet did not run' };
}
if (!existsSync(worktreePath)) {
return { status: 'skipped', reason: `worktree ${worktreePath} does not exist` };
}
try {
const git = deps.git ?? gitIn(worktreePath);
const resolveBase = deps.resolveBase ?? (async (path: string) => {
const { resolveBaseRef } = await import('../support/worktreeManager.js');
return resolveBaseRef(path);
});
const { ref } = await resolveBase(worktreePath);
const baseSha = (await git(['merge-base', ref, 'HEAD']).catch(() => '')).trim();
if (!baseSha) {
return { status: 'skipped', reason: `no merge base with ${ref}; nothing to compare coverage against` };
}

const store = deps.store ?? fileBaselineStore();
const base = await baselineFor(provider, store, `${await repoKey(git)}@${baseSha}`, worktreePath, baseSha);
if (base.kind === 'unsupported') {
return { status: 'skipped', reason: `coverage is not measurable for this repository at ${baseSha.slice(0, 12)}: ${base.why}` };
}
const baseline = base.percent;

const head = await provider.measureHead(worktreePath);
if (head.kind === 'unsupported') {
return {
status: 'hold',
baseline,
reason: `Coverage was measurable on main (${pct(baseline)} at ${baseSha.slice(0, 12)}) but not on this change: ${head.why}. `
+ 'Holding the publication rather than publishing unmeasured.',
};
}
if (head.percent + COVERAGE_TOLERANCE_PCT < baseline) {
return {
status: 'hold',
baseline,
head: head.percent,
reason: `This change lowers line coverage from ${pct(baseline)} (main at ${baseSha.slice(0, 12)}) to ${pct(head.percent)}. `
+ 'Weakening a test keeps the count and lowers coverage, so an unattended run does not publish it. '
+ 'Restore the coverage, or have a person review and publish the change.',
};
}
return { status: 'pass', baseline, head: head.percent };
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
return { status: 'hold', reason: `The coverage check could not run (${detail}); holding the publication rather than publishing unmeasured.` };
}
}
Loading
Loading