Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/FULL-GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -794,7 +794,7 @@ Resolution: **store config overrides / merges with planning** via `readGraphProj
| Empty / abstain answers | Need structured links or JSONL; free prose offline is weak |
| Hook never runs | `enabled`+`auto_update`; default branch; not CI; Bash+git commit pattern; CLI on PATH |
| Stale graph | `gsd-graph sync` or `sync --full`; check `.last-sync-status.json` |
| `build_locked` (exit 3) | Wait for other writer; remove stale `.build.lock` if PID dead |
| `build_locked` (exit 3) | Another writer is running; wait for it. A `.build.lock` left by a process that no longer exists is taken over automatically |
| Huge noisy graph | Don’t point corpus at repo root / all of `src/` |
| Review pile-up | `review list`; accept/reject; consider ontology pack fit |
| MCP can’t build | Expected — enable `--allow-build` deliberately |
Expand Down
223 changes: 188 additions & 35 deletions src/io/lock.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// gsd-graph — exclusive .build.lock acquire / release / stale steal

import { createHash } from 'node:crypto';
import fs from 'node:fs';
import { GSD_GRAPH_REASON, GraphError } from '../errors';
import { ensureStoreRoot, storeFile } from './paths';
Expand Down Expand Up @@ -34,21 +35,47 @@ export interface AcquireBuildLockOptions {
cwd?: string;
/** Sleep helper for waitMs loop (injectable in tests). */
sleepMs?: (ms: number) => void;
/**
* One-line notice when a stale lock is taken over.
* Default: written to stderr (stdout stays machine output).
*/
onNotice?: (message: string) => void;
}

function isPidAlive(pid: number): boolean {
if (!Number.isInteger(pid) || pid <= 0) return false;
/**
* Probe a lock owner pid with signal 0.
*
* - `dead`: no such process (ESRCH), or the recorded pid is not a valid pid
* - `alive`: the process exists — including EPERM (it runs as another user)
* and any other error, which never counts as proof that the owner is gone
*/
export function lockOwnerState(pid: unknown): 'alive' | 'dead' {
if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) {
return 'dead';
}
try {
process.kill(pid, 0);
return true;
return 'alive';
} catch (err) {
return (err as NodeJS.ErrnoException).code === 'ESRCH' ? 'dead' : 'alive';
}
}

function readRaw(lockPath: string): string | null {
try {
return fs.readFileSync(lockPath, 'utf8');
} catch {
return false;
return null;
}
}

function readPayload(lockPath: string): BuildLockPayload | null {
const raw = readRaw(lockPath);
return raw === null ? null : parsePayload(raw);
}

function parsePayload(raw: string): BuildLockPayload | null {
try {
const raw = fs.readFileSync(lockPath, 'utf8');
const data = JSON.parse(raw) as Partial<BuildLockPayload>;
if (
typeof data.pid !== 'number' ||
Expand All @@ -67,17 +94,142 @@ function readPayload(lockPath: string): BuildLockPayload | null {
function isStale(
payload: BuildLockPayload,
now: number,
): { stale: boolean; reason?: string } {
): { stale: boolean; reason?: 'age' | 'dead_pid' } {
const started = Date.parse(payload.started_at);
if (!Number.isFinite(started) || now - started > STALE_MS) {
return { stale: true, reason: 'age' };
}
if (!isPidAlive(payload.pid)) {
if (lockOwnerState(payload.pid) === 'dead') {
return { stale: true, reason: 'dead_pid' };
}
return { stale: false };
}

function staleNotice(
lockPath: string,
payload: BuildLockPayload,
reason: 'age' | 'dead_pid',
): string {
const why =
reason === 'dead_pid'
? `pid ${payload.pid} is not running`
: `older than ${STALE_MS / 60_000} min`;
return (
`[gsd-graph] taking over stale build lock ${lockPath} ` +
`(${why}; started ${payload.started_at} by ${payload.owner})`
);
}

/**
* Grace for a lock or claim file that exists but has no readable content yet
* (created with an exclusive open, content written right after).
*/
const UNWRITTEN_GRACE_MS = 5_000;

function claimAbandoned(claimPath: string, holder: string): boolean {
if (holder.length === 0) {
// The pid is written right after the exclusive create; an empty claim is
// either being written now or was left by a crash in between.
try {
return Date.now() - fs.statSync(claimPath).mtimeMs > UNWRITTEN_GRACE_MS;
} catch {
return false;
}
}
return lockOwnerState(Number.parseInt(holder, 10)) === 'dead';
}

function unreadableLockAbandoned(lockPath: string): boolean {
try {
return Date.now() - fs.statSync(lockPath).mtimeMs > UNWRITTEN_GRACE_MS;
} catch {
return false;
}
}

function defaultNotice(message: string): void {
process.stderr.write(`${message}\n`);
}

function unlinkQuiet(p: string): void {
try {
fs.unlinkSync(p);
} catch {
/* already gone */
}
}

/**
* Take over one stale lock instance (`staleRaw` = its exact file content).
*
* Removing a lock and creating a new one are two steps, so two processes that
* both saw the same stale lock could otherwise both remove "it" — the second
* one removing the fresh lock the first had just created — and both run. To
* prevent that, a process must first win an exclusive claim file named after
* this stale instance (`openSync(…, 'wx')`); only the claim holder re-checks
* that the lock file still holds that instance and replaces it.
*
* A claim whose holder died (ESRCH) mid-takeover is skipped by claiming the
* next generation, so a crash here cannot wedge the lock again. Claims are
* removed once the stale instance is gone; a late process that then wins a
* fresh claim finds different content and backs off.
*
* Exported for tests only (not part of the package index).
*/
export function tryTakeOver(
lockPath: string,
staleRaw: string,
owner: BuildLockOwner,
now: number,
cwd: string,
): LockHandle | 'exists' {
const key = createHash('sha256')
.update(staleRaw, 'utf8')
.digest('hex')
.slice(0, 16);
const claims: string[] = [];
let claimed = false;
for (let gen = 0, attempts = 0; gen < 16 && attempts < 64; attempts++) {
const claimPath = `${lockPath}.takeover-${key}-${gen}`;
try {
const fd = fs.openSync(claimPath, 'wx');
try {
fs.writeFileSync(fd, `${process.pid}\n`, 'utf8');
} finally {
fs.closeSync(fd);
}
claims.push(claimPath);
claimed = true;
break;
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err;
}
const holder = readRaw(claimPath);
// Claim vanished: the takeover finished (or was abandoned); try again.
if (holder === null) continue;
if (claimAbandoned(claimPath, holder)) {
claims.push(claimPath);
gen += 1;
continue;
}
// Another live process is taking this lock over right now.
return 'exists';
}
if (!claimed) return 'exists';

try {
if (readRaw(lockPath) !== staleRaw) {
// Already replaced or released — not ours to remove.
return 'exists';
}
fs.unlinkSync(lockPath);
return tryAcquireOnce(lockPath, owner, now, cwd);
} finally {
// The stale instance is gone now (or never was ours): drop its claims.
for (const c of claims) unlinkQuiet(c);
}
}

function sleepSync(ms: number): void {
if (ms <= 0) return;
// Busy-wait free: Atomics.wait on a SharedArrayBuffer
Expand Down Expand Up @@ -142,7 +294,10 @@ function tryAcquireOnce(
* Acquire exclusive `.build.lock` under storeRoot (STORE-04, D-06).
*
* - openSync(wx) exclusive create
* - On EEXIST: steal if stale by age (15m) OR dead PID; else BUILD_LOCKED
* - On EEXIST: take over if stale by age (15m) OR dead owner PID (ESRCH;
* EPERM = alive under another user = held), with a one-line notice;
* else BUILD_LOCKED. Takeover is exclusive per stale lock instance
* (see tryTakeOver), so concurrent acquirers cannot both take it over.
* - Default fail-fast (waitMs=0); optional wait/retry loop
*
* Caller must call handle.release() (use try/finally).
Expand All @@ -158,6 +313,7 @@ export function acquireBuildLock(
const nowMs = opts?.nowMs ?? Date.now;
const cwd = opts?.cwd ?? process.cwd();
const sleep = opts?.sleepMs ?? sleepSync;
const notice = opts?.onNotice ?? defaultNotice;

const deadline = nowMs() + Math.max(0, waitMs);
let stoleOnce = false;
Expand All @@ -170,40 +326,37 @@ export function acquireBuildLock(
return result;
}

const payload = readPayload(lockPath);
if (payload) {
const { stale } = isStale(payload, now);
if (stale && !stoleOnce) {
// Steal: unlink and retry once
try {
fs.unlinkSync(lockPath);
} catch {
// race — fall through
}
const raw = readRaw(lockPath);
const payload = raw === null ? null : parsePayload(raw);
if (raw !== null && payload) {
const stale = isStale(payload, now);
if (stale.reason !== undefined && !stoleOnce) {
stoleOnce = true;
continue;
}
if (stale && stoleOnce) {
// Second steal attempt after failed race
try {
fs.unlinkSync(lockPath);
} catch {
/* ignore */
const taken = tryTakeOver(lockPath, raw, owner, now, cwd);
if (taken !== 'exists') {
notice(staleNotice(lockPath, payload, stale.reason));
return taken;
}
const retry = tryAcquireOnce(lockPath, owner, nowMs(), cwd);
if (retry !== 'exists') return retry;
// Another process took it over (or it was released): look again.
continue;
}
} else {
// Corrupt / unreadable lock — treat as stealable once
} else if (raw === null) {
// Released meanwhile → just retry.
if (!stoleOnce) {
try {
fs.unlinkSync(lockPath);
} catch {
/* ignore */
}
stoleOnce = true;
continue;
}
} else if (!stoleOnce && unreadableLockAbandoned(lockPath)) {
// Corrupt lock — stealable once, through the same exclusive takeover.
// A fresh unreadable lock is a new lock whose payload is still being
// written, so it counts as held.
stoleOnce = true;
const taken = tryTakeOver(lockPath, raw, owner, now, cwd);
if (taken !== 'exists') {
notice(`[gsd-graph] taking over unreadable build lock ${lockPath}`);
return taken;
}
continue;
}

if (nowMs() >= deadline) {
Expand Down
38 changes: 13 additions & 25 deletions src/pipeline/status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import fs from 'node:fs';
import path from 'node:path';
import { loadGraphV1Cached } from '../io/graph-cache';
import { STALE_MS } from '../io/lock';
import { lockOwnerState } from '../io/lock';
import { resolveStoreRoot, storeFile } from '../io/paths';
import { readJsonFile } from '../io/safe-json';
import { fingerprintFile } from '../sources/fingerprint';
Expand All @@ -21,7 +21,6 @@ import { loadReviewQueue } from './review';

interface LockPayloadLite {
pid?: number;
started_at?: string;
}

interface LastBuildStatus {
Expand All @@ -41,37 +40,26 @@ function readLastBuildStatus(storeRoot: string): LastBuildStatus | null {
}
}

function isPidAlive(pid: number): boolean {
if (!Number.isInteger(pid) || pid <= 0) return false;
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}

/**
* build_in_progress when .build.lock exists.
* Stale locks (age > STALE_MS or dead PID) still report true if the file exists
* (Phase 2: presence-based; steal is build's job).
* build_in_progress when .build.lock exists and its owner may still run.
*
* A lock whose recorded pid no longer exists (ESRCH) was left behind by a
* build that died without releasing it; the next writer takes it over
* (acquireBuildLock), so it is not reported as a build in progress. A lock
* older than STALE_MS whose pid is alive still counts: the build may simply
* be long-running. An unreadable lock still counts too (it may be mid-write).
*/
function lockInProgress(storeRoot: string): boolean {
const lockPath = storeFile(storeRoot, '.build.lock');
if (!fs.existsSync(lockPath)) return false;
// Presence is enough for status; optionally annotate via payload parse
try {
const raw = fs.readFileSync(lockPath, 'utf8');
const payload = JSON.parse(raw) as LockPayloadLite;
if (typeof payload.started_at === 'string') {
const started = Date.parse(payload.started_at);
if (Number.isFinite(started) && Date.now() - started > STALE_MS) {
// Still in progress from reader's POV if file remains (steal not run)
return true;
}
}
if (typeof payload.pid === 'number' && !isPidAlive(payload.pid)) {
return true;
if (
typeof payload.pid === 'number' &&
lockOwnerState(payload.pid) === 'dead'
) {
return false;
}
} catch {
// unreadable lock still means a build may be/was in progress
Expand Down
Loading