From fd79a3d8c8ae007526fb505bddead8170cb4b463 Mon Sep 17 00:00:00 2001 From: Oleg Zuev Date: Sat, 19 Sep 2026 23:19:02 +0700 Subject: [PATCH] Relaunch the Codex app-server automatically after an unexpected exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the app-server child died on its own (OOM kill, a stray `kill -9`, a crash), the runtime only flagged itself degraded and every turn failed with "Codex app-server is not running … Send /restart to recover it" until an operator noticed. Docker's restart policy does not help here because the wirebot process itself keeps running. CodexRuntimeService now reacts to an unexpected exit by relaunching the server under the runtime lock with a short back-off (1s, 2s, 5s, 10s, 30s). Turns are paused during each attempt and resumed afterwards so a failed attempt surfaces an error instead of hanging. A manual /restart or reload that lands first ends the loop, and stop() waits for an in-flight recovery. Once all attempts fail the previous behaviour returns: degraded status with the manual-restart hint. The status text shown while relaunching says so instead of asking for /restart. --- CHANGELOG.md | 7 ++++ README.md | 2 + src/codex/runtime-service.ts | 73 +++++++++++++++++++++++++++++++++++- src/core/bridge.ts | 10 +++-- 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e0279..5e259ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Wirebot are documented in this file. ## [Unreleased] +### Fixed + +- An app-server that exits on its own — an OOM kill, a stray `kill -9`, or a crash — is now + relaunched automatically with a short back-off (about 50 seconds across five attempts) instead + of staying down until someone sends `/restart`. Turns wait during the relaunch and resume on the + new server; `/restart` remains the fallback once the attempts are exhausted. + ## [0.3.0] - 2026-09-05 ### Added diff --git a/README.md b/README.md index 5e9ea40..19b24ae 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,8 @@ Wirebot keeps the running Codex process synchronized using the [app-server mecha The runtime card in the Mini App shows the current outcome and offers **Apply changes** and **Restart Codex**. `/reload` and `/restart` provide the same private-chat controls. Restart is the fallback for startup-only state: Wirebot pauses new turns, lets active turns finish, restarts its child app-server with the same `CODEX_HOME`, reloads its resources, and lazily resumes persisted thread IDs. It does not restart the messaging bridges or discard authentication and conversation history. +If the app-server exits on its own — for example the kernel OOM-killer or a stray `kill -9` takes it out — Wirebot relaunches it automatically, retrying five times over roughly 50 seconds while new turns wait. Only when those attempts fail does the runtime stay degraded and ask for a manual `/restart`. + ## Source development Requirements: [Bun](https://bun.com) 1.4 or newer. diff --git a/src/codex/runtime-service.ts b/src/codex/runtime-service.ts index 80a0296..e621861 100644 --- a/src/codex/runtime-service.ts +++ b/src/codex/runtime-service.ts @@ -12,7 +12,7 @@ import type { RateLimitSnapshot } from "../generated/codex/v2/RateLimitSnapshot. import type { RateLimitWindow } from "../generated/codex/v2/RateLimitWindow.js"; import type { SkillMetadata } from "../generated/codex/v2/SkillMetadata.js"; import type { SkillsListResponse } from "../generated/codex/v2/SkillsListResponse.js"; -import { KeyedSerialQueue } from "../shared/async.js"; +import { delay, KeyedSerialQueue } from "../shared/async.js"; import { BridgeError, errorMessage } from "../shared/errors.js"; import type { Logger } from "../shared/logger.js"; import { type CodexConfigService, findBaseUserLayer } from "./config-service.js"; @@ -79,6 +79,13 @@ interface ReconcileOptions { readonly freshServer: boolean; } +/** + * Pauses before each automatic relaunch of an app-server that exited on its own, + * for example after an OOM kill or a stray `kill -9`; a manual `/restart` is the + * fallback once these are exhausted. + */ +const codexRecoveryDelaysMs: readonly number[] = [1_000, 2_000, 5_000, 10_000, 30_000]; + /** * Keeps Wirebot's long-lived app-server synchronized through Codex's native * config, MCP, and skill protocol surface. @@ -102,6 +109,7 @@ export class CodexRuntimeService { configPath: null, }; readonly #operations = new KeyedSerialQueue(); + #recovery: Promise | undefined; #unsubscribeNotification: (() => void) | undefined; #unsubscribeExit: (() => void) | undefined; #stopped = true; @@ -132,6 +140,7 @@ export class CodexRuntimeService { public async stop(): Promise { this.#stopped = true; + await this.#recovery; await this.serialize(async () => {}); this.#unsubscribeNotification?.(); this.#unsubscribeNotification = undefined; @@ -400,6 +409,68 @@ export class CodexRuntimeService { lastError: exit.error.message, restartRequired: true, }); + if (!exit.expected && !this.#stopped) this.startRecovery(exit); + } + + /** Relaunch an app-server that died on its own so nobody has to send `/restart`. */ + private startRecovery(exit: CodexAppServerExit): void { + if (this.#recovery !== undefined) return; + this.#logger.warn("Codex app-server exited unexpectedly; relaunching it automatically", { + code: exit.code, + signal: exit.signal, + attempts: codexRecoveryDelaysMs.length, + }); + this.#recovery = this.recover().finally(() => { + this.#recovery = undefined; + }); + } + + private async recover(): Promise { + for (const [index, delayMs] of codexRecoveryDelaysMs.entries()) { + await delay(delayMs); + if (this.#stopped) return; + const recovered = await this.serialize(() => this.tryRecover(index + 1)); + if (recovered) return; + } + this.#logger.error( + "Codex app-server could not be relaunched automatically; a manual restart is required", + undefined, + { lastError: this.#status.lastError }, + ); + } + + /** One relaunch attempt under the runtime lock; `true` ends the recovery loop. */ + private async tryRecover(attempt: number): Promise { + if (this.#stopped) return true; + // A manual restart or reload already brought the server back while we waited. + if (!this.#status.restartRequired) return true; + this.updateStatus({ state: "restarting", lastError: null }); + this.#codex.pause(); + try { + await this.#rpc.start(); + this.#serverModelProvider = undefined; + const status = await this.reconcile({ + hotReloadConfig: false, + reloadMcp: false, + freshServer: true, + }); + if (status.restartRequired) return false; + this.#logger.info("Codex app-server relaunched", { attempt, state: status.state }); + return true; + } catch (error) { + this.#logger.warn("Codex app-server relaunch attempt failed", { + attempt, + error: errorMessage(error), + }); + this.updateStatus({ + state: "degraded", + lastError: errorMessage(error), + restartRequired: true, + }); + return false; + } finally { + this.#codex.resume(); + } } private updateStatus(patch: Partial): void { diff --git a/src/core/bridge.ts b/src/core/bridge.ts index 88769bd..8eb750b 100644 --- a/src/core/bridge.ts +++ b/src/core/bridge.ts @@ -636,10 +636,12 @@ function runtimeStatusSummary(status: CodexRuntimeStatus): { } { const degraded = status.state === "degraded" || status.restartRequired; const detail = - status.lastError ?? - (status.restartRequired - ? "an app-server restart is required to apply startup-only changes" - : status.state); + status.state === "restarting" && status.restartRequired + ? `the app-server is being relaunched automatically${status.lastError === null ? "" : ` after: ${status.lastError}`}` + : (status.lastError ?? + (status.restartRequired + ? "an app-server restart is required to apply startup-only changes" + : status.state)); return { degraded, detail }; }