Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs-site/src/content/docs/getting-started/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,10 @@ It stops before publishing a new runtime or changing those settings. Retry start
local connectivity recovers; an installed background service retries through its service manager.
A timeout does not permanently turn the new process into a secondary instance.

IPv6 does not need to be enabled on loopback. Startup checks whether each loopback
address can be bound locally and skips an unavailable family. Timeouts on an available
family still stop startup until ownership can be verified.

## Next

- [How It Works](/getting-started/how-it-works/) — what happens to each request.
Expand Down
1 change: 1 addition & 0 deletions scripts/test-layout/layout.json
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,7 @@
"cli-start-auxiliary-bind.test.ts": "cli",
"cli-start-journal-order.test.ts": "cli",
"sibling-home-client-sync.test.ts": "cli",
"cross-home-loopback-family.test.ts": "cli",
"cli-status-hub-state.test.ts": "cli",
"cli-status-json.test.ts": "cli",
"cli-status-oauth-health.test.ts": "cli",
Expand Down
18 changes: 16 additions & 2 deletions src/cli/cross-home-owner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
} from "../server/proxy-liveness";
import { directLocalHttpFetch } from "../server/direct-local-http";
import type { RuntimePortState } from "../config/process-state";
import { loopbackFamilyAvailability } from "./loopback-family";

const MAX_HINT_BYTES = 256 * 1024;
// Far above any real Grok config; discovery must not stall startup when Grok sync is off.
Expand Down Expand Up @@ -128,13 +129,19 @@ function loopbackPort(raw: string | null): number | null {
}
}

type CrossHomeOwnerOptions = {
homeDir?: string;
io?: LivenessIo;
loopbackAvailabilityFn?: typeof loopbackFamilyAvailability;
};

/** Returns only a different process with an identity-checked /healthz response. */
export async function findCrossHomeOwner(options: { homeDir?: string; io?: LivenessIo } = {}): Promise<number | null> {
export async function findCrossHomeOwner(options: CrossHomeOwnerOptions = {}): Promise<number | null> {
const verdict = await findCrossHomeOwnerDetailed(options);
return verdict.kind === "owner" ? verdict.port : null;
}

export async function findCrossHomeOwnerDetailed(options: { homeDir?: string; io?: LivenessIo } = {}): Promise<CrossHomeOwnerVerdict> {
export async function findCrossHomeOwnerDetailed(options: CrossHomeOwnerOptions = {}): Promise<CrossHomeOwnerVerdict> {
const io = options.io ?? {};
const nowFn = io.nowFn ?? Date.now;
const timeoutMs = io.timeoutMs ?? START_OWNERSHIP_LIVENESS.timeoutMs ?? 1_500;
Expand Down Expand Up @@ -205,6 +212,8 @@ export async function findCrossHomeOwnerDetailed(options: { homeDir?: string; io
}

const queue = [...candidates];
const availability = new Map<string, ReturnType<typeof loopbackFamilyAvailability>>();
const checkAvailability = options.loopbackAvailabilityFn ?? loopbackFamilyAvailability;
const probed = new Set<number>();
for (const port of queue) {
if (probed.has(port)) continue;
Expand All @@ -225,6 +234,11 @@ export async function findCrossHomeOwnerDetailed(options: { homeDir?: string; io
noteIndeterminate(port, "ownership discovery ran out of its shared time budget");
break;
}
// Disabled loopback IPv6 can route ::1 off-host and time out. An ephemeral
// local bind proves address absence without treating an available family's
// timeout as empty. Unknown bind failures retain the existing probes.
if (!availability.has(hostname)) availability.set(hostname, checkAvailability(hostname));
if (availability.get(hostname) === "unavailable") continue;
const identity = await proxyIdentityAt(port, { hostname }, probeIo);
if (identity === null) {
// A null identity is not "free": distinguish a unanimous connect refusal
Expand Down
31 changes: 31 additions & 0 deletions src/cli/loopback-family.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/** A failed connect cannot distinguish an absent loopback address from a slow owner. */
export type LoopbackFamilyAvailability = "available" | "unavailable" | "unknown";

function bindLoopback(hostname: string): { stop(): void } {
return Bun.listen({ hostname, port: 0, socket: { data(socket) { socket.end(); } } });
}

/**
* Ask the kernel whether the concrete local address can be bound, independently of
* routing/connect timeouts. Only address/family absence permits skipping discovery;
* permissions, resource exhaustion and other failures leave owner evidence unknown.
* The ephemeral listener is closed synchronously and never accepts client traffic.
*/
export function loopbackFamilyAvailability(
hostname: string,
bindFn: (hostname: string) => { stop(): void } = bindLoopback,
): LoopbackFamilyAvailability {
const address = hostname === "[::1]" ? "::1" : hostname;
if (address !== "::1" && address !== "127.0.0.1") return "unknown";
let listener: { stop(): void };
try {
listener = bindFn(address);
} catch (error) {
const code = (error as { code?: unknown } | null)?.code;
return code === "EADDRNOTAVAIL" || code === "EAFNOSUPPORT"
|| code === "EPROTONOSUPPORT"
? "unavailable" : "unknown";
Comment thread
lidge-jun marked this conversation as resolved.
}
listener.stop();
return "available";
}
2 changes: 2 additions & 0 deletions structure/runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ their own files. `src/grok/inject.ts` holds the shared canonical config-write lo

## Lifecycle

Cross-home discovery in `src/cli/cross-home-owner.ts` checks concrete loopback address availability through `src/cli/loopback-family.ts` before probing each family. A synchronous ephemeral bind is immediately closed; only address/family-unavailable errors exclude that address. This prevents a disabled IPv6 loopback's off-host `::1` connect timeout from blocking startup. Other bind errors retain discovery, and a timeout on an available or unknown family still produces an indeterminate owner verdict. Live owners on either available family retain identity and attestation checks.

Startup catalog sync and native restore apply the [retired-native policy](catalog.md#shared-catalog). Foreground exit cleanup in `src/cli/index.ts` delegates restoration through the existing sibling/recycle teardown gates; approved external URLs follow [custom-provider consent](codex-provider-routing.md), while unapproved provider configuration and history remain unchanged. Guarded tests isolate the owner registry used by cross-home discovery beneath their selected home, including children without a registry override; production retains the shared locator. See [test state isolation](ops/test-sandbox-cleanup.md).
`src/server/background-lifecycle.ts` registers low-quota protection for each live server, flushes pending saves before releasing its owner, binds its own event ledger to management requests, and unregisters it on failed startup or stop; with no pending save, owner release remains synchronous. The protection writer loads config persistence only when a threshold schedules a save; see [configuration](config.md#codex-pool-low-quota-protection). Codex quota processing has shared and Reserve scopes; retired model evidence is suppressed as
described in [OpenAI quota ownership](providers/openai-tiers.md#public-provider-contract).
Expand Down
145 changes: 145 additions & 0 deletions tests/cli/cross-home-loopback-family.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
import { afterEach, expect, test } from "bun:test";
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { findCrossHomeOwnerDetailed, markCrossHomeSibling } from "../../src/cli/cross-home-owner";
import { loopbackFamilyAvailability, type LoopbackFamilyAvailability } from "../../src/cli/loopback-family";
import { resetSiblingStartForTests, siblingOfLivePort } from "../../src/codex/sibling-start";
import { LOCAL_ATTESTATION_CHALLENGE_HEADER, LOCAL_ATTESTATION_PROOF_HEADER,
createLocalAttestationProof } from "../../src/lib/local-management-attestation";
import { removeTreeWithRetry } from "../helpers/remove-tree";

const envKeys = ["OPENCODEX_HOME", "CODEX_HOME", "GROK_HOME", "OCX_OWNER_REGISTRY_DIR"] as const;
const originalEnv = Object.fromEntries(envKeys.map(key => [key, process.env[key]]));
const roots: string[] = [];
const port = 10102;
const ownerPid = process.pid + 1;
const secret = "A".repeat(43);

function fixture() {
const root = mkdtempSync(join(tmpdir(), "ocx-loopback-family-"));
roots.push(root);
const home = join(root, "home");
for (const key of envKeys) {
const dir = join(root, key.toLowerCase());
mkdirSync(dir, { recursive: true });
process.env[key] = dir;
}
mkdirSync(join(home, ".opencodex"), { recursive: true });
writeFileSync(join(process.env.CODEX_HOME!, "config.toml"),
`model_provider = "opencodex"\n[model_providers.opencodex]\nbase_url = "http://127.0.0.1:${port}/v1"\n`);
return { home };
}

afterEach(() => {
resetSiblingStartForTests();
for (const key of envKeys) {
if (originalEnv[key] === undefined) delete process.env[key];
else process.env[key] = originalEnv[key];
}
for (const root of roots.splice(0)) removeTreeWithRetry(root);
});

test.each(["EADDRNOTAVAIL", "EAFNOSUPPORT", "EPROTONOSUPPORT"])(
"%s from an IPv6 bind proves the family unavailable", code => {
expect(loopbackFamilyAvailability("[::1]", hostname => {
expect(hostname).toBe("::1");
throw Object.assign(new Error("unavailable"), { code });
})).toBe("unavailable");
},
);

test.each(["EACCES", "EPERM", "EMFILE", "EADDRINUSE", "ETIMEDOUT", "EINVAL", "ENETUNREACH"])(
"%s from a bind does not prove address absence", code => {
expect(loopbackFamilyAvailability("[::1]", () => {
throw Object.assign(new Error("ambiguous"), { code });
})).toBe("unknown");
},
);

test("successful availability check closes the ephemeral listener synchronously", () => {
let closed = 0;
expect(loopbackFamilyAvailability("127.0.0.1", () => ({ stop() { closed++; } }))).toBe("available");
expect(closed).toBe(1);
expect(loopbackFamilyAvailability("example.com", () => { throw new Error("must not bind"); })).toBe("unknown");
});

test("the real kernel bind can check IPv4 loopback availability", () => {
expect(loopbackFamilyAvailability("127.0.0.1")).toBe("available");
});

function probes(ipv6: LoopbackFamilyAvailability, liveHost?: string) {
const requests: string[] = [];
const checked: string[] = [];
return {
requests, checked,
loopbackAvailabilityFn: (hostname: string) => {
checked.push(hostname);
return hostname === "[::1]" ? ipv6 : "available" as const;
},
io: {
attempts: 1, timeoutMs: 50,
fetchFn: (async (input: string | URL | Request, init?: RequestInit) => {
const url = new URL(String(input));
requests.push(url.hostname);
if (url.hostname === liveHost) {
const headers = new Headers();
const challenge = new Headers(init?.headers).get(LOCAL_ATTESTATION_CHALLENGE_HEADER);
if (challenge) headers.set(LOCAL_ATTESTATION_PROOF_HEADER,
createLocalAttestationProof(secret, challenge, ownerPid, port));
return Response.json({ service: "opencodex", status: "ok", version: "0.0.0", uptime: 1, pid: ownerPid }, { headers });
}
// Reproduce the reported connect behavior: the disabled ::1 route times
// out while IPv4 refuses. Absence is proven by the bind check, not this.
throw Object.assign(new Error("connect failed"), {
code: url.hostname === "[::1]" ? "ETIMEDOUT" : "ECONNREFUSED",
});
}) as typeof fetch,
},
};
}

test("IPv4 refusal plus unavailable IPv6 permits startup without dialing the timeout route", async () => {
const fx = fixture();
const probe = probes("unavailable");
expect(await markCrossHomeSibling({ homeDir: fx.home, ...probe })).toBe(false);
expect(siblingOfLivePort()).toBeNull();
expect(probe.requests.length).toBeGreaterThan(0);
expect(probe.requests.every(host => host === "127.0.0.1")).toBe(true);
expect(probe.checked).toEqual(["127.0.0.1", "[::1]"]);
});

test.each(["available", "unknown"] as const)("IPv6 %s plus timeout still refuses startup", async availability => {
const fx = fixture();
const probe = probes(availability);
expect((await findCrossHomeOwnerDetailed({ homeDir: fx.home, ...probe })).kind).toBe("indeterminate");
await expect(markCrossHomeSibling({ homeDir: fx.home, ...probe })).rejects.toThrow("refusing startup");
expect(probe.requests).toContain("[::1]");
});

test("an unregistered live IPv4 listener still blocks startup when IPv6 is unavailable", async () => {
const fx = fixture();
await expect(markCrossHomeSibling({ homeDir: fx.home, ...probes("unavailable", "127.0.0.1") }))
.rejects.toThrow("refusing startup");
});

test("a network-unreachable bind does not hide an unregistered live IPv6 owner", async () => {
const fx = fixture();
const probe = probes("unknown", "[::1]");
const loopbackAvailabilityFn = (hostname: string) => loopbackFamilyAvailability(hostname, () => {
if (hostname === "[::1]") throw Object.assign(new Error("route failure"), { code: "ENETUNREACH" });
Comment thread
lidge-jun marked this conversation as resolved.
return { stop() {} };
});
await expect(markCrossHomeSibling({ homeDir: fx.home, ...probe, loopbackAvailabilityFn }))
.rejects.toThrow("refusing startup");
expect(probe.requests).toContain("[::1]");
});

test.each(["127.0.0.1", "[::1]"])("an attested live %s owner still makes the start a sibling", async host => {
const fx = fixture();
writeFileSync(join(fx.home, ".opencodex", "runtime-port.json"), JSON.stringify({
pid: ownerPid, port, attestationSecret: secret, hostname: host,
}));
expect(await markCrossHomeSibling({ homeDir: fx.home, ...probes("available", host) })).toBe(true);
expect(siblingOfLivePort()).toBe(port);
});
1 change: 1 addition & 0 deletions tests/fixtures/test-layout-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,7 @@
"cli-restore-back.test.ts": "cli", "cli-route-policy-not-found.test.ts": "cli",
"cli-start-auxiliary-bind.test.ts": "cli", "cli-start-journal-order.test.ts": "cli",
"sibling-home-client-sync.test.ts": "cli", "cli-status-hub-state.test.ts": "cli",
"cross-home-loopback-family.test.ts": "cli",
"cli-status-json.test.ts": "cli", "cli-status-oauth-health.test.ts": "cli",
"cli-stop-json.test.ts": "cli", "cli-storage-inspect.test.ts": "cli",
"cli-transport-honesty.test.ts": "cli", "cli-update-badge.test.ts": "cli",
Expand Down
Loading