Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 123 additions & 0 deletions src/automation/dailyReporter.lock.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// Two daemon instances firing the same cron used to both read an empty
// watermark and publish every project twice: `reportInFlight` only guards one
// process. A report run now holds a cross-process file lock from the watermark
// read to the last write. (AGT-4902)

import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import type { LinearClient, Project } from '@linear/sdk';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';

const testState = vi.hoisted(() => ({
home: `/tmp/openswarm-daily-reporter-lock-${process.pid}`,
postStatusUpdate: vi.fn(),
}));

vi.mock('node:os', async (importOriginal) => ({
...(await importOriginal<typeof import('node:os')>()),
homedir: () => testState.home,
}));

vi.mock('../linear/index.js', () => ({
postStatusUpdate: testState.postStatusUpdate,
}));

const stateDir = join(testState.home, '.openswarm');
const watermarkFile = join(stateDir, 'daily-reporter-watermark.json');
const lockFile = `${watermarkFile}.lock`;
const today = new Date().toISOString().slice(0, 10);

const PROJECTS = [
{ id: 'project-a', name: 'Project A', state: 'started' },
{ id: 'project-b', name: 'Project B', state: 'started' },
] as Project[];

type ReporterModule = typeof import('./dailyReporter.js');

/** A fresh module instance stands in for a separate daemon process. */
async function daemonInstance(): Promise<ReporterModule> {
vi.resetModules();
const reporter = await import('./dailyReporter.js');
const team = {
projects: vi.fn().mockResolvedValue({
nodes: PROJECTS,
pageInfo: { hasNextPage: false, endCursor: null },
}),
};
reporter.setLinearClient({ team: vi.fn().mockResolvedValue(team) } as unknown as LinearClient);
reporter.setTeamId('team-1');
return reporter;
}

function publishCounts(): Record<string, number> {
const counts: Record<string, number> = {};
for (const call of testState.postStatusUpdate.mock.calls) {
const id = call[0] as string;
counts[id] = (counts[id] ?? 0) + 1;
}
return counts;
}

beforeEach(() => {
rmSync(testState.home, { recursive: true, force: true });
testState.postStatusUpdate.mockReset();
vi.spyOn(console, 'log').mockImplementation(() => {});
vi.spyOn(console, 'warn').mockImplementation(() => {});
});

afterAll(() => {
rmSync(testState.home, { recursive: true, force: true });
});

describe('daily reporter cross-process serialization', () => {
it('publishes each project once when two instances run at the same time', async () => {
// Slow publishes keep the first run inside its critical section while the
// second one starts, which is exactly the overlapping cron fire.
testState.postStatusUpdate.mockImplementation(
() => new Promise<void>((resolve) => setTimeout(resolve, 30)),
);
const first = await daemonInstance();
const second = await daemonInstance();

await Promise.all([first.generateDailyReports(), second.generateDailyReports()]);

expect(publishCounts()).toEqual({ 'project-a': 1, 'project-b': 1 });
const record = JSON.parse(readFileSync(watermarkFile, 'utf8')) as {
date: string;
publishedProjectIds: string[];
complete: boolean;
};
expect(record.date).toBe(today);
expect(record.complete).toBe(true);
expect(record.publishedProjectIds.slice().sort()).toEqual(['project-a', 'project-b']);
expect(existsSync(lockFile)).toBe(false);
});

it('skips the run without publishing while a live instance holds the lock', async () => {
mkdirSync(stateDir, { recursive: true });
// This process is alive, so the lock is not stale.
writeFileSync(lockFile, JSON.stringify({ pid: process.pid, token: 'other-instance' }), 'utf8');
testState.postStatusUpdate.mockResolvedValue(undefined);
const reporter = await daemonInstance();

await reporter.generateDailyReports({ lockWaitMs: 50 });

expect(testState.postStatusUpdate).not.toHaveBeenCalled();
expect(existsSync(watermarkFile)).toBe(false);
// The other holder's lock is left in place.
expect(JSON.parse(readFileSync(lockFile, 'utf8')).token).toBe('other-instance');
});

it('reclaims a lock left by an instance that died and publishes', async () => {
mkdirSync(stateDir, { recursive: true });
// Far above any pid_max, so `kill(pid, 0)` reports ESRCH: the owner is dead.
writeFileSync(lockFile, JSON.stringify({ pid: 2_000_000_000, token: 'dead-instance' }), 'utf8');
testState.postStatusUpdate.mockResolvedValue(undefined);
const reporter = await daemonInstance();

await reporter.generateDailyReports({ lockWaitMs: 1_000 });

expect(publishCounts()).toEqual({ 'project-a': 1, 'project-b': 1 });
expect(existsSync(lockFile)).toBe(false);
});
});
35 changes: 34 additions & 1 deletion src/automation/dailyReporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { postStatusUpdate } from '../linear/index.js';
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { homedir } from 'node:os';
import { withFileLock } from '../support/fileLock.js';

let cronJob: Cron | null = null;
let linearClient: LinearClient | null = null;
Expand All @@ -25,6 +26,17 @@ let projectPathMapping = new Map<string, string>();
// published; only then does a later run of the same day skip outright.
const WATERMARK_FILE = join(homedir(), '.openswarm', 'daily-reporter-watermark.json');

// Cross-process lock around a whole report run. `reportInFlight` only guards
// one process; two daemon instances firing the same cron used to both read an
// empty watermark and publish every project twice. The run holds this lock
// from the watermark read to the last write, so the second instance waits,
// then reads the first one's record and skips what it published. A holder that
// died is reclaimed by the lock's dead-pid check.
const REPORT_LOCK_FILE = `${WATERMARK_FILE}.lock`;
/** How long a second instance waits for a running report before giving up. */
const REPORT_LOCK_WAIT_MS = 15 * 60_000;
const LOCK_TIMEOUT_PREFIX = 'Timed out waiting for file lock';

interface DailyWatermark {
/** Day (UTC, YYYY-MM-DD) this record describes. */
date: string;
Expand Down Expand Up @@ -145,12 +157,33 @@ export function stopDailyReporter(): void {
* republishes only the projects that failed; the day is marked complete once
* every project succeeded.
*/
export async function generateDailyReports(): Promise<void> {
export async function generateDailyReports(
options: { lockWaitMs?: number } = {},
): Promise<void> {
if (!linearClient || !teamId) {
console.warn('[DailyReporter] Linear client or team ID not configured');
return;
}

try {
await withFileLock(REPORT_LOCK_FILE, () => generateDailyReportsLocked(), {
timeoutMs: options.lockWaitMs ?? REPORT_LOCK_WAIT_MS,
});
} catch (error) {
if (error instanceof Error && error.message.startsWith(LOCK_TIMEOUT_PREFIX)) {
// Another instance is still publishing; it records what it published,
// so the next window retries anything it could not finish.
console.warn('[DailyReporter] Another instance holds the report lock, skipping this run');
return;
}
console.error('[DailyReporter] Failed to generate reports:', error);
}
}

/** One report run. Must only be called while holding REPORT_LOCK_FILE. */
async function generateDailyReportsLocked(): Promise<void> {
if (!linearClient || !teamId) return;

const today = new Date().toISOString().slice(0, 10);
const watermark = readWatermark();
if (watermark?.date === today && watermark.complete) {
Expand Down
Loading