diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e0279..5e259ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Wirebot are documented in this file. ## [Unreleased] +### Fixed + +- An app-server that exits on its own — an OOM kill, a stray `kill -9`, or a crash — is now + relaunched automatically with a short back-off (about 50 seconds across five attempts) instead + of staying down until someone sends `/restart`. Turns wait during the relaunch and resume on the + new server; `/restart` remains the fallback once the attempts are exhausted. + ## [0.3.0] - 2026-09-05 ### Added diff --git a/README.md b/README.md index 5e9ea40..19b24ae 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,8 @@ Wirebot keeps the running Codex process synchronized using the [app-server mecha The runtime card in the Mini App shows the current outcome and offers **Apply changes** and **Restart Codex**. `/reload` and `/restart` provide the same private-chat controls. Restart is the fallback for startup-only state: Wirebot pauses new turns, lets active turns finish, restarts its child app-server with the same `CODEX_HOME`, reloads its resources, and lazily resumes persisted thread IDs. It does not restart the messaging bridges or discard authentication and conversation history. +If the app-server exits on its own — for example the kernel OOM-killer or a stray `kill -9` takes it out — Wirebot relaunches it automatically, retrying five times over roughly 50 seconds while new turns wait. Only when those attempts fail does the runtime stay degraded and ask for a manual `/restart`. + ## Source development Requirements: [Bun](https://bun.com) 1.4 or newer. diff --git a/src/codex/runtime-service.ts b/src/codex/runtime-service.ts index 80a0296..e621861 100644 --- a/src/codex/runtime-service.ts +++ b/src/codex/runtime-service.ts @@ -12,7 +12,7 @@ import type { RateLimitSnapshot } from "../generated/codex/v2/RateLimitSnapshot. import type { RateLimitWindow } from "../generated/codex/v2/RateLimitWindow.js"; import type { SkillMetadata } from "../generated/codex/v2/SkillMetadata.js"; import type { SkillsListResponse } from "../generated/codex/v2/SkillsListResponse.js"; -import { KeyedSerialQueue } from "../shared/async.js"; +import { delay, KeyedSerialQueue } from "../shared/async.js"; import { BridgeError, errorMessage } from "../shared/errors.js"; import type { Logger } from "../shared/logger.js"; import { type CodexConfigService, findBaseUserLayer } from "./config-service.js"; @@ -79,6 +79,13 @@ interface ReconcileOptions { readonly freshServer: boolean; } +/** + * Pauses before each automatic relaunch of an app-server that exited on its own, + * for example after an OOM kill or a stray `kill -9`; a manual `/restart` is the + * fallback once these are exhausted. + */ +const codexRecoveryDelaysMs: readonly number[] = [1_000, 2_000, 5_000, 10_000, 30_000]; + /** * Keeps Wirebot's long-lived app-server synchronized through Codex's native * config, MCP, and skill protocol surface. @@ -102,6 +109,7 @@ export class CodexRuntimeService { configPath: null, }; readonly #operations = new KeyedSerialQueue(); + #recovery: Promise | undefined; #unsubscribeNotification: (() => void) | undefined; #unsubscribeExit: (() => void) | undefined; #stopped = true; @@ -132,6 +140,7 @@ export class CodexRuntimeService { public async stop(): Promise { this.#stopped = true; + await this.#recovery; await this.serialize(async () => {}); this.#unsubscribeNotification?.(); this.#unsubscribeNotification = undefined; @@ -400,6 +409,68 @@ export class CodexRuntimeService { lastError: exit.error.message, restartRequired: true, }); + if (!exit.expected && !this.#stopped) this.startRecovery(exit); + } + + /** Relaunch an app-server that died on its own so nobody has to send `/restart`. */ + private startRecovery(exit: CodexAppServerExit): void { + if (this.#recovery !== undefined) return; + this.#logger.warn("Codex app-server exited unexpectedly; relaunching it automatically", { + code: exit.code, + signal: exit.signal, + attempts: codexRecoveryDelaysMs.length, + }); + this.#recovery = this.recover().finally(() => { + this.#recovery = undefined; + }); + } + + private async recover(): Promise { + for (const [index, delayMs] of codexRecoveryDelaysMs.entries()) { + await delay(delayMs); + if (this.#stopped) return; + const recovered = await this.serialize(() => this.tryRecover(index + 1)); + if (recovered) return; + } + this.#logger.error( + "Codex app-server could not be relaunched automatically; a manual restart is required", + undefined, + { lastError: this.#status.lastError }, + ); + } + + /** One relaunch attempt under the runtime lock; `true` ends the recovery loop. */ + private async tryRecover(attempt: number): Promise { + if (this.#stopped) return true; + // A manual restart or reload already brought the server back while we waited. + if (!this.#status.restartRequired) return true; + this.updateStatus({ state: "restarting", lastError: null }); + this.#codex.pause(); + try { + await this.#rpc.start(); + this.#serverModelProvider = undefined; + const status = await this.reconcile({ + hotReloadConfig: false, + reloadMcp: false, + freshServer: true, + }); + if (status.restartRequired) return false; + this.#logger.info("Codex app-server relaunched", { attempt, state: status.state }); + return true; + } catch (error) { + this.#logger.warn("Codex app-server relaunch attempt failed", { + attempt, + error: errorMessage(error), + }); + this.updateStatus({ + state: "degraded", + lastError: errorMessage(error), + restartRequired: true, + }); + return false; + } finally { + this.#codex.resume(); + } } private updateStatus(patch: Partial): void { diff --git a/src/core/bridge.ts b/src/core/bridge.ts index 88769bd..8eb750b 100644 --- a/src/core/bridge.ts +++ b/src/core/bridge.ts @@ -636,10 +636,12 @@ function runtimeStatusSummary(status: CodexRuntimeStatus): { } { const degraded = status.state === "degraded" || status.restartRequired; const detail = - status.lastError ?? - (status.restartRequired - ? "an app-server restart is required to apply startup-only changes" - : status.state); + status.state === "restarting" && status.restartRequired + ? `the app-server is being relaunched automatically${status.lastError === null ? "" : ` after: ${status.lastError}`}` + : (status.lastError ?? + (status.restartRequired + ? "an app-server restart is required to apply startup-only changes" + : status.state)); return { degraded, detail }; }