diff --git a/README.md b/README.md index 94efe9c..cd0a7c0 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,9 @@ bunx @cortexkit/orw check --force "desktop_target": "/Applications/OpenCode.app", "install_cli": true, "install_desktop": true, - "notify_timeout": 120 + "notify_timeout": 120, + "retry_attempts": 3, + "retry_delay_ms": 10000 } ``` @@ -98,6 +100,13 @@ Relative paths in config are resolved from the directory containing `orw.config. - `prompt_path`: optional custom prompt template; if omitted, the packaged default prompt is used - `install_desktop`: defaults to `true` on macOS and `false` on Linux/Windows when generated by `init` +### Retry on provider errors + +When `opencode run` fails with a retryable provider error (e.g. "Provider returned error", rate limits, 5xx responses), ORW automatically retries using `opencode run --continue` to resume the last session. + +- `retry_attempts`: max number of attempts including the first one (default `3`) +- `retry_delay_ms`: delay between retries in milliseconds (default `10000`) + ## Commands ```bash diff --git a/config.example.json b/config.example.json index 2f23d02..28cb628 100644 --- a/config.example.json +++ b/config.example.json @@ -13,5 +13,7 @@ "desktop_target": "/Applications/OpenCode.app", "install_cli": true, "install_desktop": false, - "notify_timeout": 120 + "notify_timeout": 120, + "retry_attempts": 3, + "retry_delay_ms": 10000 } diff --git a/src/index.ts b/src/index.ts index 6f1ff70..f19f9e1 100755 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,13 @@ #!/usr/bin/env bun import fs from "node:fs/promises"; +import { readFileSync } from "node:fs"; import path from "node:path"; import os from "node:os"; import type { Buffer } from "node:buffer"; import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; +import { isRetryableError, readLogSlice, fileSize } from "./retry"; type RawCfg = { release_repo: string; @@ -24,13 +26,23 @@ type RawCfg = { install_desktop: boolean; notify_timeout: number; git_origin: string; + retry_attempts?: number; + retry_delay_ms?: number; + build_retry_attempts?: number; + build_retry_delay_ms?: number; + skip_permissions?: boolean; }; -type Cfg = Omit & { +type Cfg = Omit & { runtime_dir: string; prompt_path: string; config_file: string; config_dir: string; + retry_attempts: number; + retry_delay_ms: number; + build_retry_attempts: number; + build_retry_delay_ms: number; + skip_permissions: boolean; }; type Cli = { @@ -106,6 +118,7 @@ async function main() { if (cli.cmd === "preview") return preview(cfg); if (cli.cmd === "status") return status(cfg); if (cli.cmd === "check") return check(cfg, cli.force); + if (cli.cmd === "continue") return continueRun(cfg); } function parseCli(rawArgs: string[]): Cli { @@ -154,6 +167,7 @@ function parseCli(rawArgs: string[]): Cli { function needsConfig(cli: Cli) { if (cli.cmd === "check") return cli.positionals.length <= 1; + if (cli.cmd === "continue") return cli.positionals.length <= 1; if (cli.cmd === "preview") return cli.positionals.length === 1; if (cli.cmd === "status") return cli.positionals.length === 1; if (cli.cmd === "install-ready") return cli.positionals.length === 1; @@ -173,7 +187,7 @@ function printHelp() { } function helpText() { - return `OpenCode Release Watch\n\nUsage:\n orw [--config ] [command] [options]\n orw --help\n\nCommands:\n init Create orw.config.json in the current directory\n preview Print the integration prompt for the latest release\n check Build the latest release if needed; default command\n status Print the last successful build/install state\n install-ready Install the last verified artifacts\n install-when-closed Wait for OpenCode to quit, then install\n launchd install Install the macOS launchd scheduler\n launchd uninstall Remove the macOS launchd scheduler\n\nOptions:\n -c, --config Use a specific config file\n --force Rebuild even if the latest release was processed\n --wait-for-opencode With install-ready, wait until OpenCode quits\n -h, --help Show this help\n`; + return `OpenCode Release Watch\n\nUsage:\n orw [--config ] [command] [options]\n orw --help\n\nCommands:\n init Create orw.config.json in the current directory\n preview Print the integration prompt for the latest release\n check Build the latest release if needed; default command\n status Print the last successful build/install state\n install-ready Install the last verified artifacts\n install-when-closed Wait for OpenCode to quit, then install\n launchd install Install the macOS launchd scheduler\n launchd uninstall Remove the macOS launchd scheduler\n continue Resume the last interrupted opencode session\n\nOptions:\n -c, --config Use a specific config file\n --force Rebuild even if the latest release was processed\n --wait-for-opencode With install-ready, wait until OpenCode quits\n -h, --help Show this help\n\nConfig (orw.config.json):\n retry_attempts Number of retry attempts on provider error (default: 3)\n retry_delay_ms Delay between retries in milliseconds (default: 10000)\n`; } async function load(configPath?: string) { @@ -197,6 +211,11 @@ async function load(configPath?: string) { install_cli: raw.install_cli ?? true, install_desktop: raw.install_desktop ?? defaultInstallDesktop(), notify_timeout: raw.notify_timeout ?? 120, + retry_attempts: Math.max(1, raw.retry_attempts ?? 3), + retry_delay_ms: raw.retry_delay_ms ?? 10_000, + build_retry_attempts: Math.max(1, raw.build_retry_attempts ?? 3), + build_retry_delay_ms: raw.build_retry_delay_ms ?? 30_000, + skip_permissions: raw.skip_permissions ?? true, git_origin: raw.git_origin ?? `https://github.com/${releaseRepo}.git`, config_file: file, config_dir: configDir, @@ -258,6 +277,11 @@ function initConfig(): RawCfg & { runtime_dir: string } { install_cli: true, install_desktop: defaultInstallDesktop(), notify_timeout: 120, + retry_attempts: 3, + retry_delay_ms: 10_000, + build_retry_attempts: 3, + build_retry_delay_ms: 30_000, + skip_permissions: true, }; } @@ -315,32 +339,8 @@ async function check(cfg: Cfg, force: boolean) { await prep(cfg, sources, log); const env = releaseEnv(release); const prompt = await render(cfg, sources, release); - try { - await run( - [ - cfg.opencode_bin, - "run", - "--agent", - cfg.agent, - "--model", - cfg.model, - prompt, - ], - { - cwd: cfg.work_repo, - log, - env: { ...env, OPENCODE_DISABLE_PROJECT_CONFIG: "1" }, - }, - ); - } catch (err) { - await notify( - "OpenCode integration failed", - `${release.tag_name} failed. See ${log}.`, - ); - throw err; - } - - const next = await verifyBuild(cfg, release, log); + const next = await runOpenCodeWithBuildRetry(cfg, prompt, env, log, release); + await writeState(cfg, next); await writeState(cfg, next); await notify( "OpenCode build ready", @@ -383,6 +383,79 @@ async function check(cfg: Cfg, force: boolean) { } } +async function continueRun(cfg: Cfg) { + const release = await latest(cfg); + const prev = await readState(cfg); + if (!prev.tag) throw new Error("No previous build state found. Run `orw check` first."); + + const free = await hold(cfg, true); + try { + const log = prev.log ?? path.join( + logDir(cfg), + `${stamp()}-${release.tag_name.replaceAll("/", "-")}-continue.log`, + ); + await fs.mkdir(path.dirname(log), { recursive: true }); + await note(log, `\n--- Resuming: continue run for ${release.tag_name} ---\n`); + + const env = releaseEnv(release); + const sources = resolveSources(cfg); + const prompt = await render(cfg, sources, release); + + out(`Resuming last opencode session for ${release.tag_name}...`); + const next = await runOpenCodeWithBuildRetry(cfg, prompt, env, log, release); + await writeState(cfg, next); + out(`Continue completed for ${release.tag_name}`); + } finally { + await free(); + } +} + +async function runOpenCodeWithBuildRetry( + cfg: Cfg, + prompt: string, + env: Record, + log: string, + release: { tag_name: string; html_url: string }, +): Promise { + let lastErr: unknown; + for (let attempt = 1; attempt <= cfg.build_retry_attempts; attempt++) { + try { + await runOpenCodeWithRetry(cfg, prompt, env, log); + return await verifyBuild(cfg, release, log); + } catch (err) { + lastErr = err; + if (!await isBuildRetryableError(cfg, err)) { + throw err; + } + if (attempt >= cfg.build_retry_attempts) break; + out(`Build attempt ${attempt}/${cfg.build_retry_attempts} did not produce verified artifacts, retrying opencode (waiting ${cfg.build_retry_delay_ms / 1000}s)...`); + await sleep(cfg.build_retry_delay_ms); + } + } + throw lastErr; +} + +async function isBuildRetryableError(cfg: Cfg, err: unknown): Promise { + if (!(err instanceof Error)) return false; + const msg = err.message.toLowerCase(); + const code = "code" in err ? err.code : undefined; + + if (code === "ENOENT") { + const cli = cliPath(cfg); + return !(await exists(cli)); + } + + const buildErrorPatterns = [ + "expected a string starting with", + "build failed", + "compilation failed", + "typescript error", + "error ts", + "bun run build", + ]; + return buildErrorPatterns.some((p) => msg.includes(p)); +} + async function verifyBuild( cfg: Cfg, release: { tag_name: string; html_url: string }, @@ -408,14 +481,61 @@ async function verifyBuild( at: new Date().toISOString(), }; } catch (err) { - await notify( - "OpenCode integration failed", - `${release.tag_name} did not produce verified artifacts. See ${log}.`, - ); + if (await isBuildRetryableError(cfg, err)) { + await notify( + "OpenCode integration failed", + `${release.tag_name} did not produce verified artifacts. See ${log}.`, + ); + } throw err; } } +async function runOpenCodeWithRetry( + cfg: Cfg, + prompt: string, + env: Record, + log: string, +) { + const envWithFlag = { ...env, OPENCODE_DISABLE_PROJECT_CONFIG: "1" }; + let lastErr: unknown; + let logOffset = 0; + + for (let attempt = 1; attempt <= cfg.retry_attempts; attempt++) { + const isRetry = attempt > 1; + if (isRetry) { + out(`Retry attempt ${attempt}/${cfg.retry_attempts} after provider error (waiting ${cfg.retry_delay_ms / 1000}s)...`); + await note(log, `\n--- Retry attempt ${attempt}/${cfg.retry_attempts} ---\n`); + await sleep(cfg.retry_delay_ms); + } + + const beforeSize = await fileSize(log); + const args = [cfg.opencode_bin, "run"]; + if (cfg.skip_permissions) args.push("--dangerously-skip-permissions"); + if (isRetry) args.push("--continue"); + args.push("--agent", cfg.agent, "--model", cfg.model, prompt); + + try { + await run(args, { + cwd: cfg.work_repo, + log, + env: envWithFlag, + }); + return; + } catch (err) { + lastErr = err; + const afterSize = await fileSize(log); + const attemptLog = readLogSlice(log, beforeSize); + if (!isRetryableError(err, attemptLog)) { + throw err; + } + out(`Attempt ${attempt} failed with retryable provider error.`); + } + } + + throw lastErr; +} + async function latest(cfg: Cfg) { const url = `https://api.github.com/repos/${cfg.release_repo}/releases/latest`; const res = await fetch(url, { diff --git a/src/retry.test.ts b/src/retry.test.ts new file mode 100644 index 0000000..2865f49 --- /dev/null +++ b/src/retry.test.ts @@ -0,0 +1,75 @@ +import { describe, test, expect } from "bun:test"; +import { isRetryableError } from "./retry"; + +describe("isRetryableError", () => { + test("returns false for non-Error values", () => { + expect(isRetryableError("string error", "")).toBe(false); + expect(isRetryableError(42, "provider returned error")).toBe(false); + expect(isRetryableError(null, "provider returned error")).toBe(false); + }); + + test("returns false when error message lacks 'exited with'", () => { + const err = new Error("some other error"); + expect(isRetryableError(err, "provider returned error")).toBe(false); + }); + + test("detects 'provider returned error' in log slice", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: provider returned error")).toBe(true); + }); + + test("detects 'provider returned an error' in log slice", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: provider returned an error")).toBe(true); + }); + + test("detects rate limit patterns", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: rate limit exceeded")).toBe(true); + expect(isRetryableError(err, "Error: too many requests")).toBe(true); + }); + + test("detects network errors", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: fetch failed")).toBe(true); + expect(isRetryableError(err, "Error: socket hang up")).toBe(true); + expect(isRetryableError(err, "Error: ECONNRESET")).toBe(true); + expect(isRetryableError(err, "Error: ETIMEDOUT")).toBe(true); + }); + + test("detects HTTP status codes with context", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: HTTP 429")).toBe(true); + expect(isRetryableError(err, "Error: status 502")).toBe(true); + expect(isRetryableError(err, "Error: response status 503")).toBe(true); + }); + + test("rejects bare HTTP status codes without context", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: port 4292")).toBe(false); + expect(isRetryableError(err, "Error: line 5032")).toBe(false); + }); + + test("detects '500 internal server error'", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: 500 internal server error")).toBe(true); + }); + + test("returns false for non-retryable errors", () => { + const err = new Error("opencode exited with 1"); + expect(isRetryableError(err, "Error: build failed")).toBe(false); + expect(isRetryableError(err, "Error: syntax error")).toBe(false); + }); + + test("per-attempt log slicing: stale content from attempt 1 does not affect attempt 2", () => { + const err = new Error("opencode exited with 1"); + const attempt1Log = "Error: provider returned error"; + const attempt2Log = "Error: build failed"; + + // Attempt 1 log contains retryable error + expect(isRetryableError(err, attempt1Log)).toBe(true); + + // Attempt 2 log does NOT contain retryable error (only new content) + expect(isRetryableError(err, attempt2Log)).toBe(false); + }); +}); diff --git a/src/retry.ts b/src/retry.ts new file mode 100644 index 0000000..e70909c --- /dev/null +++ b/src/retry.ts @@ -0,0 +1,47 @@ +import { readFileSync } from "node:fs"; +import fs from "node:fs/promises"; + +export function isRetryableError(err: unknown, logSlice: string): boolean { + if (!(err instanceof Error)) return false; + const msg = err.message.toLowerCase(); + if (!msg.includes("exited with")) return false; + + const content = logSlice.toLowerCase(); + const textPatterns = [ + "provider returned error", + "provider returned an error", + "overloaded", + "rate limit", + "too many requests", + "connection error", + "fetch failed", + "socket hang up", + "econnreset", + "etimedout", + "context deadline exceeded", + "500 internal server error", + ]; + if (textPatterns.some((p) => content.includes(p))) return true; + + const httpStatusRegex = /(?:http|status|error|response)[\s:]+429\b/; + const httpServerErrorRegex = /(?:http|status|error|response)[\s:]+50[23]\b/; + return httpStatusRegex.test(content) || httpServerErrorRegex.test(content); +} + +export function readLogSlice(file: string, byteOffset: number): string { + try { + const buf = readFileSync(file); + if (byteOffset >= buf.length) return ""; + return buf.subarray(byteOffset).toString("utf8"); + } catch { + return ""; + } +} + +export async function fileSize(file: string): Promise { + try { + return (await fs.stat(file)).size; + } catch { + return 0; + } +}