From a9f351f9a5fd11baa24bc5c8abe5b0c9909d97a3 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 07:48:45 +0900 Subject: [PATCH] fix(telemetry): reclaim a stale telemetry lock atomically and only if it is the judged one (AGT-4903) recoverTelemetryLock judged the lock owner and then unlinked the path, so a live lock taken between the judgement and the unlink was deleted. The lock is now renamed aside first; the moved file is deleted only when its token (or, for a malformed lock, its mtime) matches what was judged. Otherwise it is put back with link(), which never overwrites a newer lock. --- src/telemetry/telemetry.lockReclaim.test.ts | 147 ++++++++++++++++++++ src/telemetry/telemetry.ts | 62 ++++++++- 2 files changed, 203 insertions(+), 6 deletions(-) create mode 100644 src/telemetry/telemetry.lockReclaim.test.ts diff --git a/src/telemetry/telemetry.lockReclaim.test.ts b/src/telemetry/telemetry.lockReclaim.test.ts new file mode 100644 index 00000000..12417679 --- /dev/null +++ b/src/telemetry/telemetry.lockReclaim.test.ts @@ -0,0 +1,147 @@ +// recoverTelemetryLock used to read the lock owner, judge it, then unlink the +// PATH. If a live process took a fresh lock between the judgement and the +// unlink, that live lock was deleted (AGT-4903). Reclaim now moves the file +// aside, deletes it only when it is the lock that was judged, and otherwise +// puts it back. +// +// Liveness and namespace proof are stubbed so the race can be injected at the +// exact point of judgement: the stub for processAppearsAlive swaps the lock on +// disk before answering "dead". +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const liveness = vi.hoisted(() => ({ + processAppearsAlive: vi.fn<(pid: number) => boolean>(), +})); + +vi.mock('node:os', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, default: actual, homedir: () => process.env.OSW_TEST_HOME ?? actual.homedir() }; +}); + +vi.mock('../support/processLiveness.js', () => ({ + isProofCapableSpace: () => true, + sameProcessNamespace: () => true, + processNamespaceId: () => 'ns-test', + processAppearsAlive: liveness.processAppearsAlive, +})); + +const DEAD_PID = 0x7fffffff; + +let home: string; +let previousHome: string | undefined; + +function stateDir(): string { + return join(home, '.config', 'openswarm'); +} + +function stateFile(): string { + return join(stateDir(), 'telemetry.json'); +} + +function lockFile(): string { + return `${stateFile()}.lock`; +} + +function writeLock(owner: { pid: number; token: string }): void { + mkdirSync(stateDir(), { recursive: true }); + writeFileSync(lockFile(), JSON.stringify({ ...owner, ns: 'ns-test' }), { mode: 0o600 }); +} + +function leftoverQuarantines(): string[] { + return readdirSync(stateDir()).filter((entry) => entry.includes('.reclaim-')); +} + +async function loadTelemetry() { + vi.resetModules(); + const mod = await import('./telemetry.js'); + mod.initTelemetry({ version: '0.0.0-test', enabled: true }); + return mod; +} + +/** Run with the 2 s lock wait collapsed: first Date.now() sets the deadline. */ +function withExpiredLockWait(run: () => void): void { + const base = Date.now(); + let calls = 0; + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => { + calls += 1; + return calls === 1 ? base : base + 6_000; + }); + try { + run(); + } finally { + nowSpy.mockRestore(); + } +} + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'osw-telemetry-reclaim-')); + previousHome = process.env.OSW_TEST_HOME; + process.env.OSW_TEST_HOME = home; + delete process.env.OPENSWARM_TELEMETRY; + delete process.env.DO_NOT_TRACK; + delete process.env.CI; + delete process.env.GITHUB_ACTIONS; + liveness.processAppearsAlive.mockReset(); + vi.spyOn(process.stderr, 'write').mockImplementation(() => true); +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OSW_TEST_HOME; + else process.env.OSW_TEST_HOME = previousHome; + vi.restoreAllMocks(); +}); + +describe('telemetry stale lock reclaim — atomic and ownership-safe', () => { + it('keeps a live lock taken right after the dead owner was judged', async () => { + const { maybeShowNotice } = await loadTelemetry(); + writeLock({ pid: DEAD_PID, token: 'dead-owner' }); + + // While the dead owner is being judged, another process removes the stale + // file and takes a fresh lock at the same path. Every later liveness check + // sees that new, live owner. + liveness.processAppearsAlive.mockImplementation((pid: number) => { + if (pid === DEAD_PID) { + writeFileSync(lockFile(), JSON.stringify({ pid: process.pid, token: 'fresh-owner', ns: 'ns-test' })); + return false; + } + return true; + }); + + withExpiredLockWait(() => maybeShowNotice()); + + expect(existsSync(lockFile())).toBe(true); + expect(JSON.parse(readFileSync(lockFile(), 'utf8')).token).toBe('fresh-owner'); + expect(leftoverQuarantines()).toEqual([]); + }); + + it('still reclaims the lock of a dead owner and writes state under it', async () => { + const { maybeShowNotice } = await loadTelemetry(); + writeLock({ pid: DEAD_PID, token: 'dead-owner' }); + liveness.processAppearsAlive.mockImplementation((pid: number) => pid !== DEAD_PID); + + maybeShowNotice(); + + expect(liveness.processAppearsAlive).toHaveBeenCalledWith(DEAD_PID); + // Reclaimed, then this process's own lock was taken and released. + expect(existsSync(lockFile())).toBe(false); + expect(leftoverQuarantines()).toEqual([]); + expect(JSON.parse(readFileSync(stateFile(), 'utf8')).installId).toMatch(/^[A-Za-z0-9_-]{21}$/); + }); + + it('reclaims a malformed lock once it is older than the stale window', async () => { + const { maybeShowNotice } = await loadTelemetry(); + mkdirSync(stateDir(), { recursive: true }); + writeFileSync(lockFile(), 'not-json', { mode: 0o600 }); + const old = new Date(Date.now() - 60_000); + utimesSync(lockFile(), old, old); + + maybeShowNotice(); + + expect(existsSync(lockFile())).toBe(false); + expect(leftoverQuarantines()).toEqual([]); + expect(JSON.parse(readFileSync(stateFile(), 'utf8')).noticeShown).toBe(true); + }); +}); diff --git a/src/telemetry/telemetry.ts b/src/telemetry/telemetry.ts index 3c41365b..e45ae789 100644 --- a/src/telemetry/telemetry.ts +++ b/src/telemetry/telemetry.ts @@ -26,6 +26,8 @@ import { unlinkSync, fsyncSync, statSync, + renameSync, + linkSync, } from 'node:fs'; import { randomUUID } from 'node:crypto'; import { nanoid } from 'nanoid'; @@ -125,24 +127,72 @@ function readTelemetryLockOwner(lockPath: string): { pid: number; token: string; } } +/** + * Remove the exact lock that was judged stale, and nothing else. + * + * Unlinking the path after judging its owner is a TOCTOU: between the read and + * the unlink the stale owner's file can be replaced by a live process's fresh + * lock, and deleting the path would delete THAT lock (AGT-4903). Instead the + * file at the path is first moved aside with an atomic rename, then the moved + * file is compared with what was judged. Only a match is deleted. Anything else + * is a live lock taken after the judgement and is put back with link(), which + * fails rather than overwrite a lock another process acquired in the meantime. + * + * @param judgedToken token read from the judged lock, or undefined when its + * content was malformed (then the judged mtime identifies it). + */ +function reclaimJudgedTelemetryLock(judgedToken: string | undefined, judgedMtimeMs: number): void { + const quarantine = `${TELEMETRY_LOCK}.reclaim-${randomUUID()}`; + try { + renameSync(TELEMETRY_LOCK, quarantine); + } catch { + // ENOENT: the holder released (or another reclaim won) — nothing to do. + return; + } + let judgedLock = false; + try { + const moved = readTelemetryLockOwner(quarantine); + judgedLock = judgedToken !== undefined + ? moved?.token === judgedToken + : moved === null && statSync(quarantine).mtimeMs === judgedMtimeMs; + } catch { + judgedLock = false; + } + if (!judgedLock) { + try { + linkSync(quarantine, TELEMETRY_LOCK); + } catch { + // EEXIST: a newer lock already sits at the path. The displaced holder's + // release is token-checked, so it will not remove that newer lock. + } + } + try { + unlinkSync(quarantine); + } catch { /* best-effort */ } +} + /** * Reclaim a crashed telemetry lock without permanently disabling persistence. - * Namespace-proofed: never free a live owner in another pid space. + * Namespace-proofed: never free a live owner in another pid space. The removal + * itself is ownership-safe: see reclaimJudgedTelemetryLock. */ function recoverTelemetryLock(): void { if (!existsSync(TELEMETRY_LOCK)) return; - const owner = readTelemetryLockOwner(TELEMETRY_LOCK); try { - const age = Date.now() - statSync(TELEMETRY_LOCK).mtimeMs; + // Stat before reading the owner: a lock replaced in between then shows a + // fresh mtime and a different token, and is never mistaken for the judged one. + const judgedMtimeMs = statSync(TELEMETRY_LOCK).mtimeMs; + const owner = readTelemetryLockOwner(TELEMETRY_LOCK); + const age = Date.now() - judgedMtimeMs; if (!owner) { - if (age > LOCK_STALE_MS) unlinkSync(TELEMETRY_LOCK); + if (age > LOCK_STALE_MS) reclaimJudgedTelemetryLock(undefined, judgedMtimeMs); return; } if (isProofCapableSpace(owner.ns ?? undefined) && sameProcessNamespace(owner.ns ?? undefined)) { - if (!processAppearsAlive(owner.pid)) unlinkSync(TELEMETRY_LOCK); + if (!processAppearsAlive(owner.pid)) reclaimJudgedTelemetryLock(owner.token, judgedMtimeMs); return; } - if (age > LOCK_ABANDON_MS) unlinkSync(TELEMETRY_LOCK); + if (age > LOCK_ABANDON_MS) reclaimJudgedTelemetryLock(owner.token, judgedMtimeMs); } catch { // Leave the lock; writeState will retry or degrade gracefully. }