diff --git a/docs-site/src/content/docs/getting-started/quickstart.md b/docs-site/src/content/docs/getting-started/quickstart.md index e8db9304b51..92e0ed79459 100644 --- a/docs-site/src/content/docs/getting-started/quickstart.md +++ b/docs-site/src/content/docs/getting-started/quickstart.md @@ -154,6 +154,10 @@ It stops before publishing a new runtime or changing those settings. Retry start local connectivity recovers; an installed background service retries through its service manager. A timeout does not permanently turn the new process into a secondary instance. +IPv6 does not need to be enabled on loopback. Startup checks whether each loopback +address can be bound locally and skips an unavailable family. Timeouts on an available +family still stop startup until ownership can be verified. + ## Next - [How It Works](/getting-started/how-it-works/) — what happens to each request. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index ef6d6d5c638..e10bd5345f9 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -518,6 +518,7 @@ "cli-start-auxiliary-bind.test.ts": "cli", "cli-start-journal-order.test.ts": "cli", "sibling-home-client-sync.test.ts": "cli", + "cross-home-loopback-family.test.ts": "cli", "cli-status-hub-state.test.ts": "cli", "cli-status-json.test.ts": "cli", "cli-status-oauth-health.test.ts": "cli", diff --git a/src/cli/cross-home-owner.ts b/src/cli/cross-home-owner.ts index 3d420e47598..a296d7698cb 100644 --- a/src/cli/cross-home-owner.ts +++ b/src/cli/cross-home-owner.ts @@ -26,6 +26,7 @@ import { } from "../server/proxy-liveness"; import { directLocalHttpFetch } from "../server/direct-local-http"; import type { RuntimePortState } from "../config/process-state"; +import { loopbackFamilyAvailability } from "./loopback-family"; const MAX_HINT_BYTES = 256 * 1024; // Far above any real Grok config; discovery must not stall startup when Grok sync is off. @@ -128,13 +129,19 @@ function loopbackPort(raw: string | null): number | null { } } +type CrossHomeOwnerOptions = { + homeDir?: string; + io?: LivenessIo; + loopbackAvailabilityFn?: typeof loopbackFamilyAvailability; +}; + /** Returns only a different process with an identity-checked /healthz response. */ -export async function findCrossHomeOwner(options: { homeDir?: string; io?: LivenessIo } = {}): Promise { +export async function findCrossHomeOwner(options: CrossHomeOwnerOptions = {}): Promise { const verdict = await findCrossHomeOwnerDetailed(options); return verdict.kind === "owner" ? verdict.port : null; } -export async function findCrossHomeOwnerDetailed(options: { homeDir?: string; io?: LivenessIo } = {}): Promise { +export async function findCrossHomeOwnerDetailed(options: CrossHomeOwnerOptions = {}): Promise { const io = options.io ?? {}; const nowFn = io.nowFn ?? Date.now; const timeoutMs = io.timeoutMs ?? START_OWNERSHIP_LIVENESS.timeoutMs ?? 1_500; @@ -205,6 +212,8 @@ export async function findCrossHomeOwnerDetailed(options: { homeDir?: string; io } const queue = [...candidates]; + const availability = new Map>(); + const checkAvailability = options.loopbackAvailabilityFn ?? loopbackFamilyAvailability; const probed = new Set(); for (const port of queue) { if (probed.has(port)) continue; @@ -225,6 +234,11 @@ export async function findCrossHomeOwnerDetailed(options: { homeDir?: string; io noteIndeterminate(port, "ownership discovery ran out of its shared time budget"); break; } + // Disabled loopback IPv6 can route ::1 off-host and time out. An ephemeral + // local bind proves address absence without treating an available family's + // timeout as empty. Unknown bind failures retain the existing probes. + if (!availability.has(hostname)) availability.set(hostname, checkAvailability(hostname)); + if (availability.get(hostname) === "unavailable") continue; const identity = await proxyIdentityAt(port, { hostname }, probeIo); if (identity === null) { // A null identity is not "free": distinguish a unanimous connect refusal diff --git a/src/cli/loopback-family.ts b/src/cli/loopback-family.ts new file mode 100644 index 00000000000..fdded605e57 --- /dev/null +++ b/src/cli/loopback-family.ts @@ -0,0 +1,31 @@ +/** A failed connect cannot distinguish an absent loopback address from a slow owner. */ +export type LoopbackFamilyAvailability = "available" | "unavailable" | "unknown"; + +function bindLoopback(hostname: string): { stop(): void } { + return Bun.listen({ hostname, port: 0, socket: { data(socket) { socket.end(); } } }); +} + +/** + * Ask the kernel whether the concrete local address can be bound, independently of + * routing/connect timeouts. Only address/family absence permits skipping discovery; + * permissions, resource exhaustion and other failures leave owner evidence unknown. + * The ephemeral listener is closed synchronously and never accepts client traffic. + */ +export function loopbackFamilyAvailability( + hostname: string, + bindFn: (hostname: string) => { stop(): void } = bindLoopback, +): LoopbackFamilyAvailability { + const address = hostname === "[::1]" ? "::1" : hostname; + if (address !== "::1" && address !== "127.0.0.1") return "unknown"; + let listener: { stop(): void }; + try { + listener = bindFn(address); + } catch (error) { + const code = (error as { code?: unknown } | null)?.code; + return code === "EADDRNOTAVAIL" || code === "EAFNOSUPPORT" + || code === "EPROTONOSUPPORT" + ? "unavailable" : "unknown"; + } + listener.stop(); + return "available"; +} diff --git a/structure/runtime.md b/structure/runtime.md index 669a2fb256b..e83b550e076 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -165,6 +165,8 @@ their own files. `src/grok/inject.ts` holds the shared canonical config-write lo ## Lifecycle +Cross-home discovery in `src/cli/cross-home-owner.ts` checks concrete loopback address availability through `src/cli/loopback-family.ts` before probing each family. A synchronous ephemeral bind is immediately closed; only address/family-unavailable errors exclude that address. This prevents a disabled IPv6 loopback's off-host `::1` connect timeout from blocking startup. Other bind errors retain discovery, and a timeout on an available or unknown family still produces an indeterminate owner verdict. Live owners on either available family retain identity and attestation checks. + Startup catalog sync and native restore apply the [retired-native policy](catalog.md#shared-catalog). Foreground exit cleanup in `src/cli/index.ts` delegates restoration through the existing sibling/recycle teardown gates; approved external URLs follow [custom-provider consent](codex-provider-routing.md), while unapproved provider configuration and history remain unchanged. Guarded tests isolate the owner registry used by cross-home discovery beneath their selected home, including children without a registry override; production retains the shared locator. See [test state isolation](ops/test-sandbox-cleanup.md). `src/server/background-lifecycle.ts` registers low-quota protection for each live server, flushes pending saves before releasing its owner, binds its own event ledger to management requests, and unregisters it on failed startup or stop; with no pending save, owner release remains synchronous. The protection writer loads config persistence only when a threshold schedules a save; see [configuration](config.md#codex-pool-low-quota-protection). Codex quota processing has shared and Reserve scopes; retired model evidence is suppressed as described in [OpenAI quota ownership](providers/openai-tiers.md#public-provider-contract). diff --git a/tests/cli/cross-home-loopback-family.test.ts b/tests/cli/cross-home-loopback-family.test.ts new file mode 100644 index 00000000000..83e57cea2fb --- /dev/null +++ b/tests/cli/cross-home-loopback-family.test.ts @@ -0,0 +1,145 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { findCrossHomeOwnerDetailed, markCrossHomeSibling } from "../../src/cli/cross-home-owner"; +import { loopbackFamilyAvailability, type LoopbackFamilyAvailability } from "../../src/cli/loopback-family"; +import { resetSiblingStartForTests, siblingOfLivePort } from "../../src/codex/sibling-start"; +import { LOCAL_ATTESTATION_CHALLENGE_HEADER, LOCAL_ATTESTATION_PROOF_HEADER, + createLocalAttestationProof } from "../../src/lib/local-management-attestation"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const envKeys = ["OPENCODEX_HOME", "CODEX_HOME", "GROK_HOME", "OCX_OWNER_REGISTRY_DIR"] as const; +const originalEnv = Object.fromEntries(envKeys.map(key => [key, process.env[key]])); +const roots: string[] = []; +const port = 10102; +const ownerPid = process.pid + 1; +const secret = "A".repeat(43); + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "ocx-loopback-family-")); + roots.push(root); + const home = join(root, "home"); + for (const key of envKeys) { + const dir = join(root, key.toLowerCase()); + mkdirSync(dir, { recursive: true }); + process.env[key] = dir; + } + mkdirSync(join(home, ".opencodex"), { recursive: true }); + writeFileSync(join(process.env.CODEX_HOME!, "config.toml"), + `model_provider = "opencodex"\n[model_providers.opencodex]\nbase_url = "http://127.0.0.1:${port}/v1"\n`); + return { home }; +} + +afterEach(() => { + resetSiblingStartForTests(); + for (const key of envKeys) { + if (originalEnv[key] === undefined) delete process.env[key]; + else process.env[key] = originalEnv[key]; + } + for (const root of roots.splice(0)) removeTreeWithRetry(root); +}); + +test.each(["EADDRNOTAVAIL", "EAFNOSUPPORT", "EPROTONOSUPPORT"])( + "%s from an IPv6 bind proves the family unavailable", code => { + expect(loopbackFamilyAvailability("[::1]", hostname => { + expect(hostname).toBe("::1"); + throw Object.assign(new Error("unavailable"), { code }); + })).toBe("unavailable"); + }, +); + +test.each(["EACCES", "EPERM", "EMFILE", "EADDRINUSE", "ETIMEDOUT", "EINVAL", "ENETUNREACH"])( + "%s from a bind does not prove address absence", code => { + expect(loopbackFamilyAvailability("[::1]", () => { + throw Object.assign(new Error("ambiguous"), { code }); + })).toBe("unknown"); + }, +); + +test("successful availability check closes the ephemeral listener synchronously", () => { + let closed = 0; + expect(loopbackFamilyAvailability("127.0.0.1", () => ({ stop() { closed++; } }))).toBe("available"); + expect(closed).toBe(1); + expect(loopbackFamilyAvailability("example.com", () => { throw new Error("must not bind"); })).toBe("unknown"); +}); + +test("the real kernel bind can check IPv4 loopback availability", () => { + expect(loopbackFamilyAvailability("127.0.0.1")).toBe("available"); +}); + +function probes(ipv6: LoopbackFamilyAvailability, liveHost?: string) { + const requests: string[] = []; + const checked: string[] = []; + return { + requests, checked, + loopbackAvailabilityFn: (hostname: string) => { + checked.push(hostname); + return hostname === "[::1]" ? ipv6 : "available" as const; + }, + io: { + attempts: 1, timeoutMs: 50, + fetchFn: (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(String(input)); + requests.push(url.hostname); + if (url.hostname === liveHost) { + const headers = new Headers(); + const challenge = new Headers(init?.headers).get(LOCAL_ATTESTATION_CHALLENGE_HEADER); + if (challenge) headers.set(LOCAL_ATTESTATION_PROOF_HEADER, + createLocalAttestationProof(secret, challenge, ownerPid, port)); + return Response.json({ service: "opencodex", status: "ok", version: "0.0.0", uptime: 1, pid: ownerPid }, { headers }); + } + // Reproduce the reported connect behavior: the disabled ::1 route times + // out while IPv4 refuses. Absence is proven by the bind check, not this. + throw Object.assign(new Error("connect failed"), { + code: url.hostname === "[::1]" ? "ETIMEDOUT" : "ECONNREFUSED", + }); + }) as typeof fetch, + }, + }; +} + +test("IPv4 refusal plus unavailable IPv6 permits startup without dialing the timeout route", async () => { + const fx = fixture(); + const probe = probes("unavailable"); + expect(await markCrossHomeSibling({ homeDir: fx.home, ...probe })).toBe(false); + expect(siblingOfLivePort()).toBeNull(); + expect(probe.requests.length).toBeGreaterThan(0); + expect(probe.requests.every(host => host === "127.0.0.1")).toBe(true); + expect(probe.checked).toEqual(["127.0.0.1", "[::1]"]); +}); + +test.each(["available", "unknown"] as const)("IPv6 %s plus timeout still refuses startup", async availability => { + const fx = fixture(); + const probe = probes(availability); + expect((await findCrossHomeOwnerDetailed({ homeDir: fx.home, ...probe })).kind).toBe("indeterminate"); + await expect(markCrossHomeSibling({ homeDir: fx.home, ...probe })).rejects.toThrow("refusing startup"); + expect(probe.requests).toContain("[::1]"); +}); + +test("an unregistered live IPv4 listener still blocks startup when IPv6 is unavailable", async () => { + const fx = fixture(); + await expect(markCrossHomeSibling({ homeDir: fx.home, ...probes("unavailable", "127.0.0.1") })) + .rejects.toThrow("refusing startup"); +}); + +test("a network-unreachable bind does not hide an unregistered live IPv6 owner", async () => { + const fx = fixture(); + const probe = probes("unknown", "[::1]"); + const loopbackAvailabilityFn = (hostname: string) => loopbackFamilyAvailability(hostname, () => { + if (hostname === "[::1]") throw Object.assign(new Error("route failure"), { code: "ENETUNREACH" }); + return { stop() {} }; + }); + await expect(markCrossHomeSibling({ homeDir: fx.home, ...probe, loopbackAvailabilityFn })) + .rejects.toThrow("refusing startup"); + expect(probe.requests).toContain("[::1]"); +}); + +test.each(["127.0.0.1", "[::1]"])("an attested live %s owner still makes the start a sibling", async host => { + const fx = fixture(); + writeFileSync(join(fx.home, ".opencodex", "runtime-port.json"), JSON.stringify({ + pid: ownerPid, port, attestationSecret: secret, hostname: host, + })); + expect(await markCrossHomeSibling({ homeDir: fx.home, ...probes("available", host) })).toBe(true); + expect(siblingOfLivePort()).toBe(port); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 9ae6ad80eb6..cf66ca95741 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -414,6 +414,7 @@ "cli-restore-back.test.ts": "cli", "cli-route-policy-not-found.test.ts": "cli", "cli-start-auxiliary-bind.test.ts": "cli", "cli-start-journal-order.test.ts": "cli", "sibling-home-client-sync.test.ts": "cli", "cli-status-hub-state.test.ts": "cli", + "cross-home-loopback-family.test.ts": "cli", "cli-status-json.test.ts": "cli", "cli-status-oauth-health.test.ts": "cli", "cli-stop-json.test.ts": "cli", "cli-storage-inspect.test.ts": "cli", "cli-transport-honesty.test.ts": "cli", "cli-update-badge.test.ts": "cli",