diff --git a/src/automation/dailyReporter.lock.test.ts b/src/automation/dailyReporter.lock.test.ts new file mode 100644 index 00000000..2762a567 --- /dev/null +++ b/src/automation/dailyReporter.lock.test.ts @@ -0,0 +1,123 @@ +// Two daemon instances firing the same cron used to both read an empty +// watermark and publish every project twice: `reportInFlight` only guards one +// process. A report run now holds a cross-process file lock from the watermark +// read to the last write. (AGT-4902) + +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { LinearClient, Project } from '@linear/sdk'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const testState = vi.hoisted(() => ({ + home: `/tmp/openswarm-daily-reporter-lock-${process.pid}`, + postStatusUpdate: vi.fn(), +})); + +vi.mock('node:os', async (importOriginal) => ({ + ...(await importOriginal()), + homedir: () => testState.home, +})); + +vi.mock('../linear/index.js', () => ({ + postStatusUpdate: testState.postStatusUpdate, +})); + +const stateDir = join(testState.home, '.openswarm'); +const watermarkFile = join(stateDir, 'daily-reporter-watermark.json'); +const lockFile = `${watermarkFile}.lock`; +const today = new Date().toISOString().slice(0, 10); + +const PROJECTS = [ + { id: 'project-a', name: 'Project A', state: 'started' }, + { id: 'project-b', name: 'Project B', state: 'started' }, +] as Project[]; + +type ReporterModule = typeof import('./dailyReporter.js'); + +/** A fresh module instance stands in for a separate daemon process. */ +async function daemonInstance(): Promise { + vi.resetModules(); + const reporter = await import('./dailyReporter.js'); + const team = { + projects: vi.fn().mockResolvedValue({ + nodes: PROJECTS, + pageInfo: { hasNextPage: false, endCursor: null }, + }), + }; + reporter.setLinearClient({ team: vi.fn().mockResolvedValue(team) } as unknown as LinearClient); + reporter.setTeamId('team-1'); + return reporter; +} + +function publishCounts(): Record { + const counts: Record = {}; + for (const call of testState.postStatusUpdate.mock.calls) { + const id = call[0] as string; + counts[id] = (counts[id] ?? 0) + 1; + } + return counts; +} + +beforeEach(() => { + rmSync(testState.home, { recursive: true, force: true }); + testState.postStatusUpdate.mockReset(); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); +}); + +afterAll(() => { + rmSync(testState.home, { recursive: true, force: true }); +}); + +describe('daily reporter cross-process serialization', () => { + it('publishes each project once when two instances run at the same time', async () => { + // Slow publishes keep the first run inside its critical section while the + // second one starts, which is exactly the overlapping cron fire. + testState.postStatusUpdate.mockImplementation( + () => new Promise((resolve) => setTimeout(resolve, 30)), + ); + const first = await daemonInstance(); + const second = await daemonInstance(); + + await Promise.all([first.generateDailyReports(), second.generateDailyReports()]); + + expect(publishCounts()).toEqual({ 'project-a': 1, 'project-b': 1 }); + const record = JSON.parse(readFileSync(watermarkFile, 'utf8')) as { + date: string; + publishedProjectIds: string[]; + complete: boolean; + }; + expect(record.date).toBe(today); + expect(record.complete).toBe(true); + expect(record.publishedProjectIds.slice().sort()).toEqual(['project-a', 'project-b']); + expect(existsSync(lockFile)).toBe(false); + }); + + it('skips the run without publishing while a live instance holds the lock', async () => { + mkdirSync(stateDir, { recursive: true }); + // This process is alive, so the lock is not stale. + writeFileSync(lockFile, JSON.stringify({ pid: process.pid, token: 'other-instance' }), 'utf8'); + testState.postStatusUpdate.mockResolvedValue(undefined); + const reporter = await daemonInstance(); + + await reporter.generateDailyReports({ lockWaitMs: 50 }); + + expect(testState.postStatusUpdate).not.toHaveBeenCalled(); + expect(existsSync(watermarkFile)).toBe(false); + // The other holder's lock is left in place. + expect(JSON.parse(readFileSync(lockFile, 'utf8')).token).toBe('other-instance'); + }); + + it('reclaims a lock left by an instance that died and publishes', async () => { + mkdirSync(stateDir, { recursive: true }); + // Far above any pid_max, so `kill(pid, 0)` reports ESRCH: the owner is dead. + writeFileSync(lockFile, JSON.stringify({ pid: 2_000_000_000, token: 'dead-instance' }), 'utf8'); + testState.postStatusUpdate.mockResolvedValue(undefined); + const reporter = await daemonInstance(); + + await reporter.generateDailyReports({ lockWaitMs: 1_000 }); + + expect(publishCounts()).toEqual({ 'project-a': 1, 'project-b': 1 }); + expect(existsSync(lockFile)).toBe(false); + }); +}); diff --git a/src/automation/dailyReporter.ts b/src/automation/dailyReporter.ts index 17e5a966..7304450c 100644 --- a/src/automation/dailyReporter.ts +++ b/src/automation/dailyReporter.ts @@ -9,6 +9,7 @@ import { postStatusUpdate } from '../linear/index.js'; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { homedir } from 'node:os'; +import { withFileLock } from '../support/fileLock.js'; let cronJob: Cron | null = null; let linearClient: LinearClient | null = null; @@ -25,6 +26,17 @@ let projectPathMapping = new Map(); // published; only then does a later run of the same day skip outright. const WATERMARK_FILE = join(homedir(), '.openswarm', 'daily-reporter-watermark.json'); +// Cross-process lock around a whole report run. `reportInFlight` only guards +// one process; two daemon instances firing the same cron used to both read an +// empty watermark and publish every project twice. The run holds this lock +// from the watermark read to the last write, so the second instance waits, +// then reads the first one's record and skips what it published. A holder that +// died is reclaimed by the lock's dead-pid check. +const REPORT_LOCK_FILE = `${WATERMARK_FILE}.lock`; +/** How long a second instance waits for a running report before giving up. */ +const REPORT_LOCK_WAIT_MS = 15 * 60_000; +const LOCK_TIMEOUT_PREFIX = 'Timed out waiting for file lock'; + interface DailyWatermark { /** Day (UTC, YYYY-MM-DD) this record describes. */ date: string; @@ -145,12 +157,33 @@ export function stopDailyReporter(): void { * republishes only the projects that failed; the day is marked complete once * every project succeeded. */ -export async function generateDailyReports(): Promise { +export async function generateDailyReports( + options: { lockWaitMs?: number } = {}, +): Promise { if (!linearClient || !teamId) { console.warn('[DailyReporter] Linear client or team ID not configured'); return; } + try { + await withFileLock(REPORT_LOCK_FILE, () => generateDailyReportsLocked(), { + timeoutMs: options.lockWaitMs ?? REPORT_LOCK_WAIT_MS, + }); + } catch (error) { + if (error instanceof Error && error.message.startsWith(LOCK_TIMEOUT_PREFIX)) { + // Another instance is still publishing; it records what it published, + // so the next window retries anything it could not finish. + console.warn('[DailyReporter] Another instance holds the report lock, skipping this run'); + return; + } + console.error('[DailyReporter] Failed to generate reports:', error); + } +} + +/** One report run. Must only be called while holding REPORT_LOCK_FILE. */ +async function generateDailyReportsLocked(): Promise { + if (!linearClient || !teamId) return; + const today = new Date().toISOString().slice(0, 10); const watermark = readWatermark(); if (watermark?.date === today && watermark.complete) {