Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 147 additions & 0 deletions src/telemetry/telemetry.lockReclaim.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
// recoverTelemetryLock used to read the lock owner, judge it, then unlink the
// PATH. If a live process took a fresh lock between the judgement and the
// unlink, that live lock was deleted (AGT-4903). Reclaim now moves the file
// aside, deletes it only when it is the lock that was judged, and otherwise
// puts it back.
//
// Liveness and namespace proof are stubbed so the race can be injected at the
// exact point of judgement: the stub for processAppearsAlive swaps the lock on
// disk before answering "dead".
import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, utimesSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';

const liveness = vi.hoisted(() => ({
processAppearsAlive: vi.fn<(pid: number) => boolean>(),
}));

vi.mock('node:os', async (importOriginal) => {
const actual = await importOriginal<typeof import('node:os')>();
return { ...actual, default: actual, homedir: () => process.env.OSW_TEST_HOME ?? actual.homedir() };
});

vi.mock('../support/processLiveness.js', () => ({
isProofCapableSpace: () => true,
sameProcessNamespace: () => true,
processNamespaceId: () => 'ns-test',
processAppearsAlive: liveness.processAppearsAlive,
}));

const DEAD_PID = 0x7fffffff;

let home: string;
let previousHome: string | undefined;

function stateDir(): string {
return join(home, '.config', 'openswarm');
}

function stateFile(): string {
return join(stateDir(), 'telemetry.json');
}

function lockFile(): string {
return `${stateFile()}.lock`;
}

function writeLock(owner: { pid: number; token: string }): void {
mkdirSync(stateDir(), { recursive: true });
writeFileSync(lockFile(), JSON.stringify({ ...owner, ns: 'ns-test' }), { mode: 0o600 });
}

function leftoverQuarantines(): string[] {
return readdirSync(stateDir()).filter((entry) => entry.includes('.reclaim-'));
}

async function loadTelemetry() {
vi.resetModules();
const mod = await import('./telemetry.js');
mod.initTelemetry({ version: '0.0.0-test', enabled: true });
return mod;
}

/** Run with the 2 s lock wait collapsed: first Date.now() sets the deadline. */
function withExpiredLockWait(run: () => void): void {
const base = Date.now();
let calls = 0;
const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => {
calls += 1;
return calls === 1 ? base : base + 6_000;
});
try {
run();
} finally {
nowSpy.mockRestore();
}
}

beforeEach(() => {
home = mkdtempSync(join(tmpdir(), 'osw-telemetry-reclaim-'));
previousHome = process.env.OSW_TEST_HOME;
process.env.OSW_TEST_HOME = home;
delete process.env.OPENSWARM_TELEMETRY;
delete process.env.DO_NOT_TRACK;
delete process.env.CI;
delete process.env.GITHUB_ACTIONS;
liveness.processAppearsAlive.mockReset();
vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
});

afterEach(() => {
if (previousHome === undefined) delete process.env.OSW_TEST_HOME;
else process.env.OSW_TEST_HOME = previousHome;
vi.restoreAllMocks();
});

describe('telemetry stale lock reclaim — atomic and ownership-safe', () => {
it('keeps a live lock taken right after the dead owner was judged', async () => {
const { maybeShowNotice } = await loadTelemetry();
writeLock({ pid: DEAD_PID, token: 'dead-owner' });

// While the dead owner is being judged, another process removes the stale
// file and takes a fresh lock at the same path. Every later liveness check
// sees that new, live owner.
liveness.processAppearsAlive.mockImplementation((pid: number) => {
if (pid === DEAD_PID) {
writeFileSync(lockFile(), JSON.stringify({ pid: process.pid, token: 'fresh-owner', ns: 'ns-test' }));
return false;
}
return true;
});

withExpiredLockWait(() => maybeShowNotice());

expect(existsSync(lockFile())).toBe(true);
expect(JSON.parse(readFileSync(lockFile(), 'utf8')).token).toBe('fresh-owner');
expect(leftoverQuarantines()).toEqual([]);
});

it('still reclaims the lock of a dead owner and writes state under it', async () => {
const { maybeShowNotice } = await loadTelemetry();
writeLock({ pid: DEAD_PID, token: 'dead-owner' });
liveness.processAppearsAlive.mockImplementation((pid: number) => pid !== DEAD_PID);

maybeShowNotice();

expect(liveness.processAppearsAlive).toHaveBeenCalledWith(DEAD_PID);
// Reclaimed, then this process's own lock was taken and released.
expect(existsSync(lockFile())).toBe(false);
expect(leftoverQuarantines()).toEqual([]);
expect(JSON.parse(readFileSync(stateFile(), 'utf8')).installId).toMatch(/^[A-Za-z0-9_-]{21}$/);
});

it('reclaims a malformed lock once it is older than the stale window', async () => {
const { maybeShowNotice } = await loadTelemetry();
mkdirSync(stateDir(), { recursive: true });
writeFileSync(lockFile(), 'not-json', { mode: 0o600 });
const old = new Date(Date.now() - 60_000);
utimesSync(lockFile(), old, old);

maybeShowNotice();

expect(existsSync(lockFile())).toBe(false);
expect(leftoverQuarantines()).toEqual([]);
expect(JSON.parse(readFileSync(stateFile(), 'utf8')).noticeShown).toBe(true);
});
});
62 changes: 56 additions & 6 deletions src/telemetry/telemetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import {
unlinkSync,
fsyncSync,
statSync,
renameSync,
linkSync,
} from 'node:fs';
import { randomUUID } from 'node:crypto';
import { nanoid } from 'nanoid';
Expand Down Expand Up @@ -125,24 +127,72 @@ function readTelemetryLockOwner(lockPath: string): { pid: number; token: string;
}
}

/**
* Remove the exact lock that was judged stale, and nothing else.
*
* Unlinking the path after judging its owner is a TOCTOU: between the read and
* the unlink the stale owner's file can be replaced by a live process's fresh
* lock, and deleting the path would delete THAT lock (AGT-4903). Instead the
* file at the path is first moved aside with an atomic rename, then the moved
* file is compared with what was judged. Only a match is deleted. Anything else
* is a live lock taken after the judgement and is put back with link(), which
* fails rather than overwrite a lock another process acquired in the meantime.
*
* @param judgedToken token read from the judged lock, or undefined when its
* content was malformed (then the judged mtime identifies it).
*/
function reclaimJudgedTelemetryLock(judgedToken: string | undefined, judgedMtimeMs: number): void {
const quarantine = `${TELEMETRY_LOCK}.reclaim-${randomUUID()}`;
try {
renameSync(TELEMETRY_LOCK, quarantine);
} catch {
// ENOENT: the holder released (or another reclaim won) — nothing to do.
return;
}
let judgedLock = false;
try {
const moved = readTelemetryLockOwner(quarantine);
judgedLock = judgedToken !== undefined
? moved?.token === judgedToken
: moved === null && statSync(quarantine).mtimeMs === judgedMtimeMs;
} catch {
judgedLock = false;
}
if (!judgedLock) {
try {
linkSync(quarantine, TELEMETRY_LOCK);
} catch {
// EEXIST: a newer lock already sits at the path. The displaced holder's
// release is token-checked, so it will not remove that newer lock.
}
}
try {
unlinkSync(quarantine);
} catch { /* best-effort */ }
}

/**
* Reclaim a crashed telemetry lock without permanently disabling persistence.
* Namespace-proofed: never free a live owner in another pid space.
* Namespace-proofed: never free a live owner in another pid space. The removal
* itself is ownership-safe: see reclaimJudgedTelemetryLock.
*/
function recoverTelemetryLock(): void {
if (!existsSync(TELEMETRY_LOCK)) return;
const owner = readTelemetryLockOwner(TELEMETRY_LOCK);
try {
const age = Date.now() - statSync(TELEMETRY_LOCK).mtimeMs;
// Stat before reading the owner: a lock replaced in between then shows a
// fresh mtime and a different token, and is never mistaken for the judged one.
const judgedMtimeMs = statSync(TELEMETRY_LOCK).mtimeMs;
const owner = readTelemetryLockOwner(TELEMETRY_LOCK);
const age = Date.now() - judgedMtimeMs;
if (!owner) {
if (age > LOCK_STALE_MS) unlinkSync(TELEMETRY_LOCK);
if (age > LOCK_STALE_MS) reclaimJudgedTelemetryLock(undefined, judgedMtimeMs);
return;
}
if (isProofCapableSpace(owner.ns ?? undefined) && sameProcessNamespace(owner.ns ?? undefined)) {
if (!processAppearsAlive(owner.pid)) unlinkSync(TELEMETRY_LOCK);
if (!processAppearsAlive(owner.pid)) reclaimJudgedTelemetryLock(owner.token, judgedMtimeMs);
return;
}
if (age > LOCK_ABANDON_MS) unlinkSync(TELEMETRY_LOCK);
if (age > LOCK_ABANDON_MS) reclaimJudgedTelemetryLock(owner.token, judgedMtimeMs);
} catch {
// Leave the lock; writeState will retry or degrade gracefully.
}
Expand Down
Loading