From 82262c8a84a73f1db556809e1d5b133f429fb94b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 17:19:01 +0000 Subject: [PATCH 1/2] store, runner, server: process records for the independent Runner (P1R-C02) A gateway with an independent Runner records each execution in its operations database, in a new `processes` table, before it sends the execution to the Runner, and follows it to its end, so that a gateway that lost its memory can identify the same execution again. A record names the Runner by the identity its Hello states, the workspace, the approval it resumes, the DevGuard attempt, the participation, the tty and the SHA-256 digest of the argv; never the argv, its environment or a secret. It is `dispatching` until the Runner answers that it started the execution, `running` after that, and `terminal` once the Runner reports the end or answers that it did not start it. It keeps the deadline, the root process with its start identity and the furthest output cursor a gateway observed. The controller epoch stays empty until P1R-C03, and nothing records `unknown` until P1R-C05. An execution that resumes an approval is recorded in the transaction that marks the approval resuming. When the record cannot be written, the execution is refused with INTERNAL and nothing is sent. Exits that happened while no gateway was connected are recorded by the next gateway, and process_status and read_process record what the Runner reports. Updates only touch records of the Runner with the same full identity, so a Runner with an earlier Runner's pid changes nothing. A record never revives a handle and never alone proves that an execution started or how it ended. The database states its schema version in `user_version`, now 1. A database without one is brought up to it; a newer one is refused before anything is written. The patch ledger and approvals keep their columns, and an earlier CodeSpace still opens the database and leaves the new table alone. `--runner-lifecycle independent` now needs `--operations-db` and a Runner that states its identity. Such a gateway is the database's only writer: before it opens the database it takes an exclusive lock on `.gateway.lock`, which names its pid, waits up to 15 seconds for a gateway that is still ending, and then refuses to start. Only this mode takes the lock. The Runner reports, in the exec answer and the governed launch, when the process times out and its root process: the pid with, on Linux and macOS, a start identity read before the reaper runs. A PTY root is not named, since the pinned Codex PTY does not give its pid. The worker protocol is now 9. The docs describe the records, the single writer, the schema version and why an earlier CodeSpace must not share the database, in English and Korean. The macos-core validation stage runs the store tests and the runner's root report test. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 2 + crates/codex-runtime/tests/runtime_binary.rs | 2 +- crates/runner/src/api.rs | 16 + crates/runner/src/launch.rs | 33 +- crates/runner/src/lib.rs | 4 +- crates/runner/src/lifecycle.rs | 107 +++ crates/runner/src/process.rs | 75 +- crates/runner/src/uds.rs | 2 + crates/runner/src/wire.rs | 6 +- crates/runner/tests/uds_runner.rs | 2 +- crates/server/Cargo.toml | 2 + crates/server/src/config.rs | 22 +- crates/server/src/lib.rs | 1 + crates/server/src/main.rs | 57 +- crates/server/src/mcp.rs | 719 +++++++++++++- crates/server/src/records.rs | 183 ++++ crates/server/src/runtime.rs | 8 + crates/server/tests/independent_runner.rs | 253 ++++- crates/store/Cargo.toml | 1 + crates/store/src/approvals.rs | 51 +- crates/store/src/lib.rs | 322 ++++++- crates/store/src/processes.rs | 952 +++++++++++++++++++ crates/store/src/writer.rs | 246 +++++ docs/architecture.md | 4 +- docs/codex-reuse.md | 2 +- docs/devguard-integration.md | 2 +- docs/error-codes.md | 2 +- docs/ko/architecture.md | 4 +- docs/ko/codex-reuse.md | 2 +- docs/ko/devguard-integration.md | 2 +- docs/ko/error-codes.md | 2 +- docs/ko/operations.md | 8 +- docs/ko/runner-isolation.md | 17 +- docs/ko/security-model.md | 2 +- docs/operations.md | 8 +- docs/runner-isolation.md | 17 +- docs/security-model.md | 2 +- docs/translations.json | 30 +- scripts/validate-upstream.py | 5 + 39 files changed, 3039 insertions(+), 136 deletions(-) create mode 100644 crates/server/src/records.rs create mode 100644 crates/store/src/processes.rs create mode 100644 crates/store/src/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 6600bd5..144ccfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -851,6 +851,7 @@ dependencies = [ "http", "reqwest 0.12.28", "rmcp", + "rusqlite", "schemars 1.2.2", "serde", "serde_json", @@ -870,6 +871,7 @@ version = "0.6.0" dependencies = [ "codespace-domain", "hex", + "libc", "rusqlite", "serde", "serde_json", diff --git a/crates/codex-runtime/tests/runtime_binary.rs b/crates/codex-runtime/tests/runtime_binary.rs index c26c649..ee977b6 100644 --- a/crates/codex-runtime/tests/runtime_binary.rs +++ b/crates/codex-runtime/tests/runtime_binary.rs @@ -144,7 +144,7 @@ async fn hello_on_live_socket() { match parsed.result { Some(RunnerOpResult::Hello { protocol, .. }) => { assert_eq!(protocol, WIRE_PROTOCOL); - assert_eq!(protocol, 8); + assert_eq!(protocol, 9); } other => panic!("unexpected hello {other:?}"), } diff --git a/crates/runner/src/api.rs b/crates/runner/src/api.rs index 45444d6..e64d25b 100644 --- a/crates/runner/src/api.rs +++ b/crates/runner/src/api.rs @@ -96,6 +96,22 @@ pub fn default_exec_timeout_ms() -> u64 { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RunnerExecResult { pub process_id: ProcessId, + /// The root process the Runner created (P1R-C02). A PTY root is not named. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root: Option, + /// When the Runner times the process out, in milliseconds since the Unix epoch + /// (P1R-C02). The Runner's own timer decides; this only reports it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deadline_at_ms: Option, +} + +/// A process's root as the Runner created it (P1R-C02): its PID and, where the host reports +/// it, the start identity that tells it apart from a later process with the same PID. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunnerProcessRoot { + pub pid: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/runner/src/launch.rs b/crates/runner/src/launch.rs index 1b8ac7c..cbfce6a 100644 --- a/crates/runner/src/launch.rs +++ b/crates/runner/src/launch.rs @@ -53,7 +53,7 @@ use crate::admission::{ }; use crate::process::spawn_env; use crate::registration::error_code; -use crate::{RunnerCwd, RunnerExecRequest}; +use crate::{RunnerCwd, RunnerExecRequest, RunnerExecResult, RunnerProcessRoot}; /// How long a launch waits for the helper's transcript by default. pub const DEFAULT_READY_BOUND: Duration = Duration::from_secs(10); @@ -245,6 +245,13 @@ pub struct ManagedLaunch { /// The scope's root as DevGuard bound it: the helper's PID, which the executable kept. #[serde(default, skip_serializing_if = "Option::is_none")] pub scope_root_pid: Option, + /// The helper's process as the Runner created it, which the executable replaces + /// (P1R-C02). A PTY root is not named. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root: Option, + /// When the Runner times the process out, in milliseconds since the Unix epoch (P1R-C02). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deadline_at_ms: Option, } /// How a governed execution's launch ended for its request. @@ -389,11 +396,11 @@ impl PreparedExecution { .runner .spawn_managed(&self.ws, request.clone(), slot, invocation, hooks) .await; - if spawned.is_err() { + let Ok(spawned) = spawned else { // No helper exists, and none will be created for this grant. let attempt = self.abandon(Abandon::SpawnFailed).await; return self.not_launched(PreparationOutcome::HelperNotCreated, attempt); - } + }; phases.push(LaunchPhase::HelperCreated); let (sender, ended) = tokio::sync::oneshot::channel(); tokio::task::spawn_blocking(move || { @@ -420,7 +427,9 @@ impl PreparedExecution { } } }); - return self.launched(phases, LaunchDispatch::Unknown).await; + return self + .launched(phases, LaunchDispatch::Unknown, &spawned) + .await; } }; match end { @@ -432,7 +441,8 @@ impl PreparedExecution { LaunchPhase::PayloadExecAttempted, LaunchPhase::PayloadRunning, ]); - self.launched(phases, LaunchDispatch::Confirmed).await + self.launched(phases, LaunchDispatch::Confirmed, &spawned) + .await } TranscriptEnd::Uncertain => { // Whether the executable started is not known. Once the process has ended this @@ -442,7 +452,8 @@ impl PreparedExecution { tokio::spawn(async move { settle.after_exit().await; }); - self.launched(phases, LaunchDispatch::Unknown).await + self.launched(phases, LaunchDispatch::Unknown, &spawned) + .await } TranscriptEnd::ExecFailed { errno } => { // The helper claimed the grant before READY, so its scope settles it; the @@ -489,11 +500,13 @@ impl PreparedExecution { LaunchReport::NotLaunched(self.report(outcome, attempt)) } - /// The launched report, with what DevGuard's record shows of the run. + /// The launched report, with what DevGuard's record shows of the run and what the spawn + /// reported of its process. async fn launched( &mut self, phases: Vec, dispatch: LaunchDispatch, + spawned: &RunnerExecResult, ) -> LaunchReport { let mut account = self.account; let mut scope_root_pid = None; @@ -523,6 +536,8 @@ impl PreparedExecution { phases, account: Some(account), scope_root_pid, + root: spawned.root.clone(), + deadline_at_ms: spawned.deadline_at_ms, }) } @@ -865,6 +880,10 @@ report() {{ eval "printf '%s\n' \"\$1\" >&$report"; }} ); assert_eq!(launch.process_id.0, process); assert_eq!(launch.scope_root_pid, Some(4242)); + // The Runner names the helper's process, which the command replaced, and when its + // timeout ends (P1R-C02); the pinned Codex PTY does not name its child. + assert_eq!(launch.root.is_some(), !tty, "{launch:?}"); + assert!(launch.deadline_at_ms.is_some(), "{launch:?}"); assert_eq!( launch.account.unwrap().applied, Applied::Launched { diff --git a/crates/runner/src/lib.rs b/crates/runner/src/lib.rs index c7aca8b..1eebcbb 100644 --- a/crates/runner/src/lib.rs +++ b/crates/runner/src/lib.rs @@ -124,8 +124,8 @@ mod wire; pub use api::{ default_exec_timeout_ms, runner_local_exec_env, RunnerApplyPatchRequest, RunnerApplyPatchResult, RunnerCwd, RunnerError, RunnerExecEnv, RunnerExecPolicy, - RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, - RunnerResizeResult, RunnerWriteStdin, DEFAULT_TIMEOUT_MS, MAX_OUTPUT_BYTES, + RunnerExecRequest, RunnerExecResult, RunnerProcessRoot, RunnerProcessStatus, RunnerReadProcess, + RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, DEFAULT_TIMEOUT_MS, MAX_OUTPUT_BYTES, }; pub use codespace_domain::{DEFAULT_FIND_LIMIT, DEFAULT_READ_LIMIT}; pub use descriptors::exclude_unrelated; diff --git a/crates/runner/src/lifecycle.rs b/crates/runner/src/lifecycle.rs index 1125621..891430b 100644 --- a/crates/runner/src/lifecycle.rs +++ b/crates/runner/src/lifecycle.rs @@ -103,6 +103,54 @@ fn boot_id() -> Option { None } +/// The start identity of process `pid` (P1R-C02): with the PID it names that one process, and +/// tells it apart from a later process that reuses the PID. Read while the process cannot have +/// been reaped yet, as a child that only its own reaper waits for, it is that process's. `None` +/// where the host does not report it. +#[cfg(target_os = "linux")] +pub(crate) fn process_start_identity(pid: u32) -> Option { + // `stat`'s 22nd field: the start time in clock ticks since boot. The command name before it + // is in parentheses and may itself hold spaces and parentheses. + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + let fields = &stat[stat.rfind(')')? + 1..]; + let ticks: u64 = fields.split_whitespace().nth(19)?.parse().ok()?; + Some(format!("linux-start-ticks:{ticks}")) +} + +/// The start identity of process `pid` (P1R-C02): with the PID it names that one process, and +/// tells it apart from a later process that reuses the PID. Read while the process cannot have +/// been reaped yet, as a child that only its own reaper waits for, it is that process's. `None` +/// where the host does not report it. +#[cfg(target_os = "macos")] +pub(crate) fn process_start_identity(pid: u32) -> Option { + // SAFETY: an all-zero `proc_bsdinfo` is a valid value of this plain C structure. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::() as libc::c_int; + // SAFETY: proc_pidinfo writes at most `size` bytes into `info`. A non-zero argument asks + // for an exited process that is not reaped yet too. + let written = unsafe { + libc::proc_pidinfo( + libc::c_int::try_from(pid).ok()?, + libc::PROC_PIDTBSDINFO, + 1, + (&mut info as *mut libc::proc_bsdinfo).cast(), + size, + ) + }; + if written != size || info.pbi_pid != pid { + return None; + } + Some(format!( + "macos-start:{}.{:06}", + info.pbi_start_tvsec, info.pbi_start_tvusec + )) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) fn process_start_identity(_pid: u32) -> Option { + None +} + /// 128 random bits in hex. A host without `/dev/urandom` gets the time and the process ID /// instead: the nonce tells Runner processes apart, it guards nothing. fn nonce() -> String { @@ -136,6 +184,65 @@ mod tests { assert_eq!(RunnerIdentity::this_process(), identity); } + /// Whether the child `pid` has exited and is not reaped yet. + fn exited_unreaped(pid: u32) -> bool { + // SAFETY: an all-zero `siginfo_t` is valid, and zero `si_pid` is how `waitid` with + // `WNOHANG` reports that no child has exited. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a valid `siginfo_t`; `WNOWAIT` leaves the child waitable. + let result = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // SAFETY: `waitid` filled `info`, or left the zero it was given. + result == 0 && unsafe { info.si_pid() } != 0 + } + + /// A process's start identity is the same for as long as the process holds its PID, until + /// it is reaped: a root whose identity the Runner reads before its reaper runs is never + /// confused with a later process that reuses the PID (P1R-C02). + #[test] + fn a_start_identity_names_one_process_until_it_is_reaped() { + let own = process_start_identity(std::process::id()); + if cfg!(not(any(target_os = "linux", target_os = "macos"))) { + assert_eq!(own, None); + return; + } + let own = own.expect("this process's start identity"); + let mut child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let pid = child.id(); + let started = process_start_identity(pid).expect("a running child's start identity"); + assert_ne!(started, own); + assert_eq!(process_start_identity(pid).as_ref(), Some(&started)); + child.kill().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !exited_unreaped(pid) { + assert!( + std::time::Instant::now() < deadline, + "the child did not exit" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert_eq!( + process_start_identity(pid).as_ref(), + Some(&started), + "an exited child that is not reaped yet" + ); + child.wait().unwrap(); + assert_ne!( + process_start_identity(pid).as_ref(), + Some(&started), + "a reaped child" + ); + } + #[test] fn lifecycles_cross_the_wire_by_name() { assert_eq!( diff --git a/crates/runner/src/process.rs b/crates/runner/src/process.rs index 9fa2fb0..6f0fc38 100644 --- a/crates/runner/src/process.rs +++ b/crates/runner/src/process.rs @@ -30,9 +30,10 @@ use tokio::process::{Child, ChildStdin, Command}; use tokio::sync::mpsc; use tokio::task::JoinHandle; +use crate::lifecycle::process_start_identity; use crate::{ - runner_local_exec_env, RunnerCwd, RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, - RunnerReadProcess, RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, + runner_local_exec_env, RunnerCwd, RunnerExecRequest, RunnerExecResult, RunnerProcessRoot, + RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, }; pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); @@ -679,6 +680,12 @@ impl InProcessRunner { format!("failed to spawn process: {err}"), ) })?; + // Before its reaper exists, so the PID is still this child's (P1R-C02). + let root = spawned.id().map(|pid| RunnerProcessRoot { + pid, + started: process_start_identity(pid), + }); + let deadline_at_ms = deadline_after(timeout); let stdin = spawned.stdin.take(); let stdout = spawned.stdout.take(); let stderr = spawned.stderr.take(); @@ -750,6 +757,8 @@ impl InProcessRunner { Ok(RunnerExecResult { process_id: req.process_id, + root, + deadline_at_ms, }) } @@ -856,6 +865,8 @@ impl InProcessRunner { format!("failed to spawn process: {err}"), ) })?; + // The pinned Codex PTY does not name its child, so a PTY root has no PID here. + let deadline_at_ms = deadline_after(timeout); let stdout = session .take_stdout() .ok_or_else(|| ErrorBody::new(ErrorCode::InvalidPatch, "PTY stdout missing"))?; @@ -949,6 +960,8 @@ impl InProcessRunner { Ok(RunnerExecResult { process_id: req.process_id, + root: None, + deadline_at_ms, }) } @@ -1198,6 +1211,15 @@ async fn reap_child( } } +/// When a timeout that starts now ends, in milliseconds since the Unix epoch (P1R-C02). +fn deadline_after(timeout: Duration) -> Option { + std::time::SystemTime::now() + .checked_add(timeout)? + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|deadline| u64::try_from(deadline.as_millis()).ok()) +} + /// Whether `child` has exited, seen without reaping it. Until its reaper waits, an exited child /// still holds its PID, so no other process can be reached through that PID. fn exited_unreaped(child: &Child) -> bool { @@ -3024,4 +3046,53 @@ mod tests { assert_eq!(read.retained_from, status.output_retained_from); assert!(read.cursor >= read.retained_from); } + + /// A spawn reports its root process and when its timeout ends (P1R-C02): a pipe root by its + /// PID and start identity, read before its reaper runs, so a command that exits at once is + /// named too; a PTY root, which the pinned Codex PTY does not name, by its deadline only. + #[tokio::test] + async fn a_spawn_reports_its_root_and_its_deadline() { + fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64 + } + let dir = tempdir().unwrap(); + let ws = workspace(dir.path()); + let runner = InProcessRunner::new(Arc::new(|_| {})); + for (process_id, script, tty) in [ + ("proc-root-pipe", "sleep 30", false), + ("proc-root-exits", "exit 0", false), + ("proc-root-pty", "sleep 30", true), + ] { + let mut req = RunnerExecRequest::for_host( + vec!["/bin/sh".into(), "-c".into(), script.into()], + ProcessId(process_id.into()), + Profile::WorkspaceWrite, + ); + req.tty = tty; + req.timeout_ms = 60_000; + let before = now_ms(); + let spawned = runner.spawn_host(&ws, req).await.unwrap(); + let after = now_ms(); + let deadline = spawned.deadline_at_ms.expect("a deadline"); + assert!( + (before + 60_000..=after + 60_000).contains(&deadline), + "{process_id}: {before} {deadline} {after}" + ); + if tty { + assert_eq!(spawned.root, None); + continue; + } + let root = spawned.root.expect("a pipe root"); + if cfg!(any(target_os = "linux", target_os = "macos")) { + assert!(root.started.is_some(), "{process_id}: {root:?}"); + } + if script == "sleep 30" { + assert_eq!(process_start_identity(root.pid), root.started); + } + } + runner.kill_host_workspace("demo").unwrap(); + } } diff --git a/crates/runner/src/uds.rs b/crates/runner/src/uds.rs index 816900b..89ccc93 100644 --- a/crates/runner/src/uds.rs +++ b/crates/runner/src/uds.rs @@ -404,6 +404,8 @@ impl UdsRunner { phases: Vec::new(), account: None, scope_root_pid: None, + root: None, + deadline_at_ms: None, })) } Err(err) => Err(err), diff --git a/crates/runner/src/wire.rs b/crates/runner/src/wire.rs index 6939396..7a510b7 100644 --- a/crates/runner/src/wire.rs +++ b/crates/runner/src/wire.rs @@ -27,7 +27,7 @@ use crate::{ RunnerResizeResult, RunnerWriteStdin, ShellRelease, }; -pub const WIRE_PROTOCOL: u32 = 8; +pub const WIRE_PROTOCOL: u32 = 9; const MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; const MAX_REPLAY: usize = 32; /// How long a stopping independent Runner waits for its terminated processes to be reaped and @@ -1049,8 +1049,8 @@ mod tests { } #[test] - fn wire_protocol_is_v8() { - assert_eq!(WIRE_PROTOCOL, 8); + fn wire_protocol_is_v9() { + assert_eq!(WIRE_PROTOCOL, 9); } #[tokio::test] diff --git a/crates/runner/tests/uds_runner.rs b/crates/runner/tests/uds_runner.rs index 910a043..9d79120 100644 --- a/crates/runner/tests/uds_runner.rs +++ b/crates/runner/tests/uds_runner.rs @@ -121,7 +121,7 @@ async fn uds_process_status_echo_and_protocol_6_hello() { .expect("serve runner"); }); let runner = UdsRunner::from_stream(client, Arc::new(|_| {})); - runner.handshake().await.expect("hello protocol 8"); + runner.handshake().await.expect("hello protocol 9"); let dir = tempdir().unwrap(); let ws = workspace(dir.path()); let process_id = ProcessId("proc-uds-status".into()); diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index c55d2c3..194ca41 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -55,3 +55,5 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus tokio = { version = "1", features = ["macros", "rt-multi-thread", "process"] } tempfile = "3" codespace-store = { path = "../store" } +# Reads and damages operations databases in the process record tests (P1R-C02). +rusqlite = { version = "0.32", features = ["bundled"] } diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index dda4fa3..cbde08c 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -78,6 +78,8 @@ pub struct Cli { pub config: Option, /// SQLite file for operations. Unset uses an in-memory database (no replay across restarts). + /// Required with `--runner-lifecycle independent`, whose gateway records its executions there + /// and holds the lock file beside it (P1R-C02). #[arg(long, env = "CODESPACE_OPERATIONS_DB")] pub operations_db: Option, @@ -153,6 +155,10 @@ impl Cli { if self.runner_socket.is_none() { return Err("--runner-lifecycle independent needs --runner-socket".into()); } + // Its executions outlive the gateway, so their records must too (P1R-C02). + if self.operations_db.is_none() { + return Err("--runner-lifecycle independent needs --operations-db".into()); + } Ok(GatewayLifecycle::Independent { exit_intent }) } @@ -226,9 +232,19 @@ mod tests { if !args.contains(&"--runtime-bin") { cli.runtime_bin = None; } + if !args.contains(&"--operations-db") { + cli.operations_db = None; + } cli.lifecycle() }; - let socket = ["--runner", "uds", "--runner-socket", "/run/cs/runner.sock"]; + let socket = [ + "--runner", + "uds", + "--runner-socket", + "/run/cs/runner.sock", + "--operations-db", + "/var/lib/cs/operations.sqlite", + ]; let independent = ["--runner-lifecycle", "independent"]; assert_eq!(lifecycle(&[]), Ok(GatewayLifecycle::Managed)); assert_eq!(lifecycle(&socket), Ok(GatewayLifecycle::Managed)); @@ -253,6 +269,10 @@ mod tests { [&["--runner", "uds"][..], &independent, &detach].concat(), "needs --runner-socket", ), + ( + [&socket[..4], &independent, &detach].concat(), + "needs --operations-db", + ), ( [ &socket[..], diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index d6038dd..4df41ce 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -10,6 +10,7 @@ pub mod logging; pub mod mcp; pub mod patch_helper; pub mod protocol; +pub mod records; pub mod runtime; pub mod stdio; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 1c663af..598eeb2 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -1,5 +1,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::time::Duration; use anyhow::Result; use clap::Parser; @@ -10,10 +11,16 @@ use codespace_server::config::{ }; use codespace_server::http::serve_http_handler; use codespace_server::logging; +use codespace_server::records::ProcessRecords; use codespace_server::runtime::RuntimeProcess; use codespace_server::stdio; use codespace_server::CodeSpace; -use codespace_store::Store; +use codespace_store::{Store, WriterLock}; + +/// How long a gateway with an independent Runner waits for the gateway before it to let go of +/// the operations database (P1R-C02): one that is ending detaches at once, or stops its Runner, +/// which waits up to 5 seconds for its processes. +const OPERATIONS_DB_LOCK_WAIT: Duration = Duration::from_secs(15); fn main() -> Result<()> { // First, while this is the only thread: no later fork can then copy libnotify's @@ -26,6 +33,8 @@ fn main() -> Result<()> { logging::init(); let cli = Cli::parse(); let lifecycle = cli.lifecycle().map_err(anyhow::Error::msg)?; + // Released as the gateway exits, once its runtime is shut down. + let _writer = lock_operations_db(&cli, lifecycle)?; let result = runtime.block_on(serve(cli, lifecycle)); if let GatewayLifecycle::Independent { .. } = lifecycle { // A signal can end the gateway while a read of its stdin waits on one of the runtime's @@ -36,11 +45,37 @@ fn main() -> Result<()> { result } +/// A gateway with an independent Runner is its operations database's only writer (P1R-C02): it +/// takes the lock beside the database before it opens it, waiting a bounded time for a gateway +/// that is still ending, and refuses to start after that. +fn lock_operations_db(cli: &Cli, lifecycle: GatewayLifecycle) -> Result> { + let (GatewayLifecycle::Independent { .. }, Some(database)) = (lifecycle, &cli.operations_db) + else { + return Ok(None); + }; + let lock = WriterLock::acquire(database, OPERATIONS_DB_LOCK_WAIT, |holder| { + tracing::info!( + holder = ?holder, + wait = ?OPERATIONS_DB_LOCK_WAIT, + "another gateway holds the operations database; waiting for it to end" + ) + }) + .map_err(anyhow::Error::msg)?; + tracing::info!( + lock = %lock.path().display(), + "this gateway is the only writer of the operations database" + ); + Ok(Some(lock)) +} + async fn serve(cli: Cli, lifecycle: GatewayLifecycle) -> Result<()> { let registry = load_registry(&cli)?; let store = load_store(&cli)?; let started = start_runner(&cli, lifecycle, store.clone()).await?; - let handler = CodeSpace::from_cli(&cli, registry, store, started.runner.clone()); + let mut handler = CodeSpace::from_cli(&cli, registry, store, started.runner.clone()); + if let Some(records) = &started.records { + handler = handler.with_records(records.clone()); + } #[cfg(feature = "devguard")] handler.register_owner_now(); let (result, ending) = match lifecycle { @@ -103,6 +138,8 @@ struct StartedRunner { lifecycle: GatewayLifecycle, /// Set before the gateway ends its connection to an independent Runner itself. leaving: Arc, + /// The records of the executions sent to an independent Runner (P1R-C02). + records: Option>, } impl StartedRunner { @@ -115,6 +152,7 @@ impl StartedRunner { process, lifecycle, leaving, + records: _, } = self; if let (GatewayLifecycle::Independent { exit_intent }, RuntimeBackend::Uds(runner)) = (lifecycle, &runner) @@ -175,11 +213,14 @@ async fn start_runner( process: None, lifecycle, leaving, + records: None, }), RunnerMode::Uds => { if let (GatewayLifecycle::Independent { .. }, Some(socket)) = (lifecycle, &cli.runner_socket) { + // Each exit releases its lease and is recorded (P1R-C02). + let (on_release, exits) = ProcessRecords::release_hook(store.clone()); // The gateway only connects; the Runner registers itself, from its own // credential file, and nothing is handed to it (P1R-C01). let runner = @@ -189,11 +230,18 @@ async fn start_runner( if cli.devguard.worker().is_some() { codespace_server::runtime::require_registration(&runner, socket)?; } + let identity = runner + .hello() + .identity + .ok_or_else(|| anyhow::anyhow!("the independent Runner has no identity"))?; + let runner = RuntimeBackend::Uds(runner); + let records = ProcessRecords::start(store, &identity, runner.clone(), exits); return Ok(StartedRunner { - runner: RuntimeBackend::Uds(runner), + runner, process: None, lifecycle, leaving, + records: Some(records), }); } if let Some(bin) = &cli.runtime_bin { @@ -213,6 +261,7 @@ async fn start_runner( process: Some(process), lifecycle, leaving, + records: None, }); } let (process, runner) = @@ -223,6 +272,7 @@ async fn start_runner( process: Some(process), lifecycle, leaving, + records: None, }); } let socket = cli.runner_socket.as_ref().ok_or_else(|| { @@ -236,6 +286,7 @@ async fn start_runner( process: None, lifecycle, leaving, + records: None, }) } } diff --git a/crates/server/src/mcp.rs b/crates/server/src/mcp.rs index 66adc08..f8591bb 100644 --- a/crates/server/src/mcp.rs +++ b/crates/server/src/mcp.rs @@ -8,10 +8,11 @@ use codespace_domain::{ ErrorBody, ErrorCode, ExecCommandParams, ExecCommandResult, ExecDispatchStatus, FindParams, FindResult, NetworkPolicyState, OperationResumeParams, OperationResumeResult, OperationStatusParams, OperationStatusResult, PatchStatus, ProcessId, ProcessResizeParams, - ProcessResizeResult, ProcessStatusParams, ProcessStatusResult, ReadParams, ReadProcessParams, - ReadProcessResult, ReadResult, SteerClaimNextResult, SteerCompleteParams, SteerStatusResult, - TerminateProcessParams, WorkFinishResult, WorkId, WorkIdParams, WorkOpenParams, WorkOpenResult, - WorkspaceExecutionInfo, WorkspaceInfo, WorkspaceInfoParams, WriteStdinParams, + ProcessResizeResult, ProcessState, ProcessStatusParams, ProcessStatusResult, ReadParams, + ReadProcessParams, ReadProcessResult, ReadResult, SteerClaimNextResult, SteerCompleteParams, + SteerStatusResult, TerminateProcessParams, WorkFinishResult, WorkId, WorkIdParams, + WorkOpenParams, WorkOpenResult, WorkspaceExecutionInfo, WorkspaceInfo, WorkspaceInfoParams, + WriteStdinParams, }; use codespace_policy::{ allow, Action, ClientClaims, EnvironmentKind, NetworkAxis, Participation, PermissionProfile, @@ -52,6 +53,8 @@ pub struct CodeSpace { pub(crate) registration: Option>, /// Whether the runner is an independent Runner, and what a shutdown asks of it (P1R-C01). pub(crate) lifecycle: crate::config::GatewayLifecycle, + /// The records of the executions sent to an independent Runner (P1R-C02). + pub(crate) records: Option>, } fn err_json(err: ErrorBody) -> String { @@ -240,6 +243,7 @@ impl CodeSpace { #[cfg(feature = "devguard")] registration: None, lifecycle: crate::config::GatewayLifecycle::Managed, + records: None, } } @@ -250,6 +254,13 @@ impl CodeSpace { self } + /// The handler of a gateway that records the executions it sends to an independent Runner + /// (P1R-C02). + pub fn with_records(mut self, records: Arc) -> Self { + self.records = Some(records); + self + } + /// New work is refused when the runner is an independent Runner that an earlier gateway /// controlled (P1R-C01), before any approval, lease or claim: what that gateway left is not /// recovered yet. @@ -445,6 +456,9 @@ impl CodeSpace { }) .await .map_err(runner_err_json)?; + if let Some(records) = &self.records { + records.observed_output(&result.process_id, result.cursor); + } Ok(Json(ReadProcessResult { process_id: result.process_id, cursor: result.cursor, @@ -469,6 +483,14 @@ impl CodeSpace { .process_status(¶ms.process_id) .await .map_err(runner_err_json)?; + if let Some(records) = &self.records { + match result.state { + ProcessState::Exited => records.ended(&result), + ProcessState::Running => { + records.observed_output(&result.process_id, result.output_total) + } + } + } Ok(Json(ProcessStatusResult { process_id: result.process_id, state: result.state, @@ -975,8 +997,23 @@ impl CodeSpace { .store .acquire_shell_busy(¶ms.workspace_id.0, &process_id.0) .await?; - if let Some(id) = resume { - self.store.mark_resuming(id)?; + // A governed execution's DevGuard attempt, minted before its record names it. + let attempt_id = participation.attempt_id(); + match &self.records { + // Recorded before it is sent to an independent Runner, in the transaction that marks + // the approval it resumes (P1R-C02). + Some(records) => records.dispatching( + &process_id, + ¶ms, + participation.recorded(), + attempt_id.as_deref(), + resume, + )?, + None => { + if let Some(id) = resume { + self.store.mark_resuming(id)?; + } + } } let mut req = RunnerExecRequest::for_host(params.command, process_id.clone(), ws.profile); req.policy.network = ws.network; @@ -985,6 +1022,7 @@ impl CodeSpace { Participating::Off => {} #[cfg(feature = "devguard")] Participating::Managed => { + let attempt_id = attempt_id.unwrap_or_else(new_attempt_id); if !self .registration .as_ref() @@ -992,15 +1030,17 @@ impl CodeSpace { { // The workspace lease is held while the owner prepares; nothing is started, // so it is released with the answer whatever the attempt's own state. - let refused = self.prepare_governed(ws, req).await; + let refused = self.prepare_governed(ws, req, attempt_id).await; + self.record_not_started(&process_id); reservation.abort(); return Err(self.refused_governed(resume, refused)); } // The owner may create a process from here on (CSRG-U4). reservation.arm_dispatch(); - return match self.launch_governed(ws, req).await { + return match self.launch_governed(ws, req, attempt_id).await { Ok(launch) => { reservation.confirm(); + self.record_launch(&launch); Ok(ExecCommandResult { process_id: launch.process_id, dispatch_status: match launch.dispatch { @@ -1018,6 +1058,7 @@ impl CodeSpace { Err(refused) => { // The command did not start; a helper that existed was reaped. reservation.abort(); + self.record_not_started(&process_id); Err(self.refused_governed(resume, refused)) } }; @@ -1027,12 +1068,20 @@ impl CodeSpace { match self.runner.exec(ws, req).await { Ok(result) => { reservation.confirm(); + if let Some(records) = &self.records { + records.running( + &result.process_id, + result.root.as_ref(), + result.deadline_at_ms, + ); + } Ok(ExecCommandResult { process_id: result.process_id, dispatch_status: ExecDispatchStatus::Confirmed, coordination: self.hint(¶ms.workspace_id.0, params.work_id.as_ref()), }) } + // The record stays dispatching: the execution may or may not have started. Err(RunnerError::TransportAmbiguous { .. }) => { reservation.confirm(); Ok(ExecCommandResult { @@ -1043,10 +1092,36 @@ impl CodeSpace { } Err(err) => { reservation.abort(); + self.record_not_started(&process_id); Err(err.into_error_body()) } } } + + /// Record that the Runner did not start the execution, when this gateway records them. + fn record_not_started(&self, process_id: &ProcessId) { + if let Some(records) = &self.records { + records.not_started(process_id); + } + } + + /// Record a governed launch: running once its helper exists, and still dispatching when its + /// answer was lost (P1R-C02). + #[cfg(feature = "devguard")] + fn record_launch(&self, launch: &codespace_runner::launch::ManagedLaunch) { + if let Some(records) = &self.records { + if launch + .phases + .contains(&codespace_runner::launch::LaunchPhase::HelperCreated) + { + records.running( + &launch.process_id, + launch.root.as_ref(), + launch.deadline_at_ms, + ); + } + } + } } /// A governed execution that did not start, and whether the resource authority proves that it @@ -1084,6 +1159,32 @@ enum Participating { Managed, } +impl Participating { + /// A governed execution's DevGuard attempt ID. + fn attempt_id(self) -> Option { + match self { + Self::Off => None, + #[cfg(feature = "devguard")] + Self::Managed => Some(new_attempt_id()), + } + } + + /// How a process record names it (P1R-C02). + fn recorded(self) -> codespace_store::RecordedParticipation { + match self { + Self::Off => codespace_store::RecordedParticipation::Off, + #[cfg(feature = "devguard")] + Self::Managed => codespace_store::RecordedParticipation::Required, + } + } +} + +/// A new DevGuard attempt ID, minted by the gateway. +#[cfg(feature = "devguard")] +fn new_attempt_id() -> String { + format!("cs-attempt-{}", Uuid::new_v4().simple()) +} + impl CodeSpace { /// How a new execution in `ws` takes part, or why this gateway refuses it. A workspace that /// requires resource participation is prepared through the execution owner's admission @@ -1113,14 +1214,18 @@ impl CodeSpace { /// Have the execution owner prepare `req` without launching it (CSRG-U3): an owner that /// cannot launch cancels each preparation, so it ends in a refusal and nothing is started. #[cfg(feature = "devguard")] - async fn prepare_governed(&self, ws: &Workspace, req: RunnerExecRequest) -> NotStarted { + async fn prepare_governed( + &self, + ws: &Workspace, + req: RunnerExecRequest, + attempt_id: String, + ) -> NotStarted { let Some(registration) = &self.registration else { return NotStarted::unproven(ErrorBody::new( ErrorCode::Internal, "no execution owner registration", )); }; - let attempt_id = format!("cs-attempt-{}", Uuid::new_v4().simple()); match registration .prepare(&self.runner, ws, req, attempt_id) .await @@ -1137,6 +1242,7 @@ impl CodeSpace { &self, ws: &Workspace, req: RunnerExecRequest, + attempt_id: String, ) -> Result { let Some(registration) = &self.registration else { return Err(NotStarted::unproven(ErrorBody::new( @@ -1144,7 +1250,6 @@ impl CodeSpace { "no execution owner registration", ))); }; - let attempt_id = format!("cs-attempt-{}", Uuid::new_v4().simple()); match registration .launch(&self.runner, ws, req, attempt_id) .await @@ -4290,11 +4395,15 @@ mod independent_tests { use super::tests::{exec_params, participation_registry}; use super::*; use crate::config::{GatewayLifecycle, RunnerExitIntent}; + use crate::records::ProcessRecords; use codespace_domain::{ ApprovalDecision, ProcessDisconnectAction, ProcessState, ProcessStatusParams, - TerminateProcessParams, WorkspaceId, + ProcessTermination, TerminateProcessParams, WorkspaceId, + }; + use codespace_runner::{host_worker, IndependentRunner, RunnerIdentity, StopHandle, UdsRunner}; + use codespace_store::{ + ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, RunnerRef, }; - use codespace_runner::{host_worker, IndependentRunner, StopHandle, UdsRunner}; use std::collections::BTreeMap; use std::time::{Duration, Instant}; use tokio::net::UnixStream; @@ -4335,26 +4444,37 @@ mod independent_tests { Self { streams, stop } } - /// A gateway with its own handle on the operations database, connected to the Runner. + /// A gateway with its own handle on the operations database, connected to the Runner, + /// that records its executions there as `codespace-mcp` does (P1R-C02). async fn gateway(&self, root: &std::path::Path, database: &std::path::Path) -> CodeSpace { - let store = Arc::new(Store::open(database).unwrap()); let (client, server) = UnixStream::pair().unwrap(); self.streams.send(server).unwrap(); - let releases = store.clone(); - let runner = UdsRunner::from_stream( - client, - Arc::new(move |process_id: &str| releases.release_process(process_id)), - ); - runner.handshake().await.unwrap(); - CodeSpace::with_store_and_runner( - participation_registry(root, "off"), - store, - RuntimeBackend::Uds(runner), - ) - .with_lifecycle(DETACHING) + recording_gateway(root, database, client).await } } + /// A gateway connected to an independent Runner through `stream`, recording its executions + /// in `database`. + async fn recording_gateway( + root: &std::path::Path, + database: &std::path::Path, + stream: UnixStream, + ) -> CodeSpace { + let store = Arc::new(Store::open(database).unwrap()); + let (release, exits) = ProcessRecords::release_hook(store.clone()); + let runner = UdsRunner::from_stream(stream, release); + runner.handshake().await.unwrap(); + let identity = runner + .hello() + .identity + .expect("an independent Runner's identity"); + let runner = RuntimeBackend::Uds(runner); + let records = ProcessRecords::start(store.clone(), &identity, runner.clone(), exits); + CodeSpace::with_store_and_runner(participation_registry(root, "off"), store, runner) + .with_lifecycle(DETACHING) + .with_records(records) + } + impl Drop for Runner { fn drop(&mut self) { self.stop.request(); @@ -4375,6 +4495,118 @@ mod independent_tests { } } + fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64 + } + + /// How a record names the Runner with `identity`. + fn runner_ref(identity: RunnerIdentity) -> RunnerRef { + RunnerRef { + boot_id: identity.boot_id, + pid: identity.pid, + started_at_ms: identity.started_at_ms, + nonce: identity.nonce, + } + } + + fn terminal(record: &ProcessRecord) -> bool { + record.phase == ProcessPhase::Terminal + } + + /// The record of `process_id` once `done` holds of it, read as a gateway that has only the + /// operations database reads it. + async fn record_once( + database: &std::path::Path, + process_id: &ProcessId, + done: impl Fn(&ProcessRecord) -> bool, + ) -> ProcessRecord { + let store = Store::open(database).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match store.process_record(&process_id.0).unwrap() { + Some(record) if done(&record) => return record, + record => assert!(Instant::now() < deadline, "{record:?}"), + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + /// The whole output of a process that ended. + async fn output_of(gateway: &CodeSpace, process_id: &ProcessId) -> ReadProcessResult { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let read = gateway + .read_process(Parameters(ReadProcessParams { + process_id: process_id.clone(), + cursor: 0, + })) + .await + .unwrap() + .0; + if read.eof { + return read; + } + assert!(Instant::now() < deadline, "{read:?}"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + /// Wait until the workspace is free again: the gateway heard that its process exited. + async fn released(gateway: &CodeSpace, workspace_id: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + while gateway.store.try_acquire_write(workspace_id).is_err() { + assert!(Instant::now() < deadline, "{workspace_id} stays busy"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + /// Another Runner than the one in this process. + fn fake_identity() -> RunnerIdentity { + RunnerIdentity { + boot_id: Some("fake-boot".into()), + pid: 4_194_000, + started_at_ms: 1_000, + nonce: "fake-runner".into(), + } + } + + /// A Runner that answers `Hello` as the independent Runner `identity`, hands the first + /// execution it is sent to `on_exec`, and then is lost without answering it. + async fn fake_runner( + mut stream: UnixStream, + identity: RunnerIdentity, + on_exec: impl FnOnce(RunnerExecRequest), + ) { + use codespace_runner::{ + read_frame, write_frame, RunnerLifecycle, RunnerOp, RunnerOpResult, WireEnvelope, + WIRE_PROTOCOL, + }; + while let Ok(Some(frame)) = read_frame(&mut stream).await { + let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); + match request.op { + Some(RunnerOp::Hello) => { + // From JSON, so that it suits a Runner built with DevGuard or without. + let hello: RunnerOpResult = serde_json::from_value(serde_json::json!({ + "type": "hello", + "data": { + "protocol": WIRE_PROTOCOL, + "lifecycle": RunnerLifecycle::Independent, + "identity": identity, + }, + })) + .unwrap(); + let reply = WireEnvelope::response(request.request_id.unwrap(), Ok(hello)); + write_frame(&mut stream, &reply).await.unwrap(); + } + Some(RunnerOp::Exec { request, .. }) => return on_exec(request), + other => panic!("a request the fake Runner does not answer: {other:?}"), + } + } + } + /// After a detach the next gateway reads and controls the processes it inherits, but new /// executions, patches and approval resumes are refused before anything is claimed, so a /// granted approval stays granted. @@ -4550,6 +4782,439 @@ mod independent_tests { assert_eq!(started.dispatch_status, ExecDispatchStatus::Confirmed); } + /// A gateway records each execution it sends to its independent Runner, naming the Runner, + /// and follows it to its end (P1R-C02): the root process the Runner created with its start + /// identity, the deadline the Runner set, how the execution ended and how far its output + /// went. The record keeps the argv's digest, not the argv. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_gateway_records_an_execution_from_its_dispatch_to_its_end() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + let before = now_ms(); + let started = gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/sh", "-c", "echo $$; exit 3"], + false, + ))) + .await + .unwrap() + .0; + let after = now_ms(); + record_once(&database, &started.process_id, terminal).await; + let output = output_of(&gateway, &started.process_id).await; + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!(record.runner, runner_ref(RunnerIdentity::this_process())); + assert_eq!(record.controller_epoch, None); + assert_eq!(record.workspace_id, "demo"); + assert_eq!(record.approval_id, None); + assert_eq!(record.attempt_id, None); + assert_eq!(record.participation, RecordedParticipation::Off); + assert!(!record.tty); + assert!( + record.argv_digest.starts_with("sha256:") && !record.argv_digest.contains("exit"), + "{record:?}" + ); + assert_eq!( + record.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Exited)) + ); + assert_eq!((record.exit_code, record.signal), (Some(3), None)); + let root_process = record.root.clone().expect("the root process"); + assert_eq!(root_process.pid.to_string(), output.chunk.trim()); + if cfg!(any(target_os = "linux", target_os = "macos")) { + assert!(root_process.started.is_some(), "{record:?}"); + } + let timeout = codespace_runner::default_exec_timeout_ms(); + let deadline = record.deadline_at_ms.expect("the deadline"); + assert!( + (before + timeout..=after + timeout).contains(&deadline), + "{before} {deadline} {after}" + ); + assert_eq!(record.output_cursor, Some(output.cursor)); + assert!(before <= record.created_at_ms && record.created_at_ms <= record.updated_at_ms); + } + + /// An execution that ends while no gateway is connected is recorded by the next gateway, to + /// which the Runner gives the exit it kept (P1R-C02): that gateway finds the execution by + /// the record the gateway before it wrote. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_end_while_no_gateway_is_connected_is_recorded_by_the_next_gateway() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let first = runner.gateway(&root, &database).await; + let started = first + .exec_command(Parameters(exec_params( + "demo", + &[ + "/bin/sh", + "-c", + "while [ ! -e go ]; do sleep 0.02; done; touch gone; exit 7", + ], + false, + ))) + .await + .unwrap() + .0; + detach(&first).await; + // The Runner keeps an exit for its next connection once this one has ended. + tokio::time::sleep(Duration::from_millis(200)).await; + std::fs::write(root.join("go"), "").unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !root.join("gone").exists() { + assert!(Instant::now() < deadline, "the process does not end"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + tokio::time::sleep(Duration::from_millis(200)).await; + let left = Store::open(&database) + .unwrap() + .process_record(&started.process_id.0) + .unwrap() + .expect("the record the first gateway wrote"); + assert_eq!(left.phase, ProcessPhase::Running, "{left:?}"); + + let _next = runner.gateway(&root, &database).await; + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!( + record.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Exited)) + ); + assert_eq!(record.exit_code, Some(7)); + assert_eq!((&record.runner, &record.root), (&left.runner, &left.root)); + } + + /// An end that reached a gateway that went away before recording it is recorded once the + /// next gateway asks the Runner about the execution (P1R-C02): a record never says more + /// than the Runner reported, and follows what the Runner reports. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_end_a_gateway_missed_is_recorded_when_the_next_gateway_asks_the_runner() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + // A gateway that records its executions but loses each exit it hears of, as one that + // crashes before recording it does. + let first = { + let (client, server) = UnixStream::pair().unwrap(); + runner.streams.send(server).unwrap(); + let store = Arc::new(Store::open(&database).unwrap()); + let (release, lost) = ProcessRecords::release_hook(store.clone()); + drop(lost); + let uds = UdsRunner::from_stream(client, release); + uds.handshake().await.unwrap(); + let identity = uds.hello().identity.unwrap(); + let backend = RuntimeBackend::Uds(uds); + let (_, none) = ProcessRecords::release_hook(store.clone()); + let records = ProcessRecords::start(store.clone(), &identity, backend.clone(), none); + CodeSpace::with_store_and_runner(participation_registry(&root, "off"), store, backend) + .with_lifecycle(DETACHING) + .with_records(records) + }; + let started = first + .exec_command(Parameters(exec_params( + "demo", + &["/bin/sh", "-c", "exit 9"], + false, + ))) + .await + .unwrap() + .0; + released(&first, "demo").await; + detach(&first).await; + let next = runner.gateway(&root, &database).await; + tokio::time::sleep(Duration::from_millis(200)).await; + let missed = Store::open(&database) + .unwrap() + .process_record(&started.process_id.0) + .unwrap() + .unwrap(); + assert_eq!(missed.phase, ProcessPhase::Running, "{missed:?}"); + let status = next + .process_status(Parameters(ProcessStatusParams { + process_id: started.process_id.clone(), + })) + .await + .unwrap() + .0; + assert_eq!(status.state, ProcessState::Exited); + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!( + record.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Exited)) + ); + assert_eq!(record.exit_code, Some(9)); + } + + /// An execution that ends at once is recorded as it ended, whether its end or the Runner's + /// answer that it started reaches the gateway first (P1R-C02). + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn an_execution_that_ends_at_once_is_recorded_as_it_ended() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + for code in 0..24 { + let started = gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/sh", "-c", &format!("exit {code}")], + false, + ))) + .await + .unwrap() + .0; + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!( + record.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Exited)), + "{record:?}" + ); + assert_eq!(record.exit_code, Some(code), "{record:?}"); + assert!( + record.root.is_some() && record.deadline_at_ms.is_some(), + "{record:?}" + ); + // The workspace is free for the next execution once its exit was heard. + released(&gateway, "demo").await; + } + } + + /// An execution that resumes an approval is recorded with it (P1R-C02), in the transaction + /// that marks the approval resuming. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_resumed_execution_is_recorded_with_its_approval() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + let held = refused( + gateway + .exec_command(Parameters(exec_params( + "gov-confirm", + &["/bin/sh", "-c", "exit 5"], + false, + ))) + .await, + ); + assert_eq!(held.code, ErrorCode::ApprovalRequired, "{held:?}"); + let approval_id = ApprovalId(held.approval_id.unwrap()); + gateway + .approval_resolve(Parameters(ApprovalResolveParams { + approval_id: approval_id.clone(), + decision: ApprovalDecision::Grant, + })) + .await + .unwrap(); + let resumed = gateway + .operation_resume(Parameters(OperationResumeParams { + approval_id: approval_id.clone(), + })) + .await + .unwrap() + .0; + let started = resumed.exec_command.expect("the execution"); + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!(record.approval_id.as_ref(), Some(&approval_id)); + assert_eq!(record.workspace_id, "gov-confirm"); + assert_eq!(record.exit_code, Some(5)); + assert_eq!( + gateway.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Consumed + ); + } + + /// An execution whose record cannot be written is not sent (P1R-C02): nothing starts, the + /// workspace is free again, and an approval it would resume is answered with that failure + /// instead of being left resuming. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_execution_that_cannot_be_recorded_is_not_sent() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + let touch = format!("{}/started", root.display()); + let held = refused( + gateway + .exec_command(Parameters(exec_params( + "gov-confirm", + &["/usr/bin/touch", &touch], + false, + ))) + .await, + ); + let approval_id = ApprovalId(held.approval_id.unwrap()); + gateway + .approval_resolve(Parameters(ApprovalResolveParams { + approval_id: approval_id.clone(), + decision: ApprovalDecision::Grant, + })) + .await + .unwrap(); + rusqlite::Connection::open(&database) + .unwrap() + .execute_batch("DROP TABLE processes") + .unwrap(); + + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/usr/bin/touch", &touch], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::Internal, "{refusal:?}"); + assert!( + refusal.message.contains("nothing was started"), + "{refusal:?}" + ); + drop( + gateway + .store + .try_acquire_write("demo") + .expect("the workspace is free"), + ); + let refusal = refused( + gateway + .operation_resume(Parameters(OperationResumeParams { + approval_id: approval_id.clone(), + })) + .await, + ); + assert_eq!(refusal.code, ErrorCode::Internal, "{refusal:?}"); + assert_eq!( + gateway.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Consumed + ); + drop( + gateway + .store + .try_acquire_write("gov-confirm") + .expect("the workspace is free"), + ); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!(!root.join("started").exists()); + } + + /// An execution whose answer is lost with the Runner may or may not have started: its + /// record stays dispatching, under the Runner it was sent to (P1R-C02). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_execution_whose_answer_is_lost_stays_dispatching() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("operations.db"); + let (client, server) = UnixStream::pair().unwrap(); + tokio::spawn(fake_runner(server, fake_identity(), |_| {})); + let gateway = recording_gateway(dir.path(), &database, client).await; + let sent = gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "lost"], + false, + ))) + .await + .unwrap() + .0; + assert_eq!(sent.dispatch_status, ExecDispatchStatus::Unknown); + let record = gateway + .store + .process_record(&sent.process_id.0) + .unwrap() + .expect("the record"); + assert_eq!(record.phase, ProcessPhase::Dispatching); + assert_eq!(record.runner, runner_ref(fake_identity())); + assert_eq!((record.root, record.outcome), (None, None)); + } + + /// A gateway records an execution before it sends it (P1R-C02): the record is there, + /// dispatching, when the Runner receives the execution, and a gateway that crashes at that + /// moment leaves it for the next gateway, naming the Runner it was sent to. + #[test] + fn an_execution_is_recorded_before_the_runner_receives_it() { + use std::os::unix::process::ExitStatusExt; + const DIRECTORY: &str = "CODESPACE_TEST_CRASH_AT_DISPATCH"; + // SIGABRT, on Linux and macOS. + const ABORTED: i32 = 6; + if let Some(dir) = std::env::var_os(DIRECTORY) { + // The crashing gateway, in a copy of this test binary. + let dir = std::path::PathBuf::from(dir); + let database = dir.join("operations.db"); + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() + .block_on(async { + let (client, server) = UnixStream::pair().unwrap(); + let (seen, received) = (database.clone(), dir.join("received")); + tokio::spawn(fake_runner(server, fake_identity(), move |request| { + let record = Store::open(&seen) + .unwrap() + .process_record(&request.process_id.0) + .unwrap(); + let phase = record.map(|record| record.phase); + std::fs::write(&received, format!("{} {phase:?}", request.process_id.0)) + .unwrap(); + std::process::abort(); + })); + let gateway = recording_gateway(&dir, &database, client).await; + let sent = gateway + .exec_command(Parameters(exec_params("demo", &["/bin/echo", "x"], false))) + .await + .map(|sent| sent.0); + panic!("the gateway did not crash: {sent:?}"); + }); + return; + } + let dir = tempfile::tempdir().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "mcp::independent_tests::an_execution_is_recorded_before_the_runner_receives_it", + "--test-threads=1", + "--nocapture", + ]) + .env(DIRECTORY, dir.path()) + .output() + .unwrap(); + assert_eq!( + output.status.signal(), + Some(ABORTED), + "the copy did not crash as the Runner received the execution:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let received = std::fs::read_to_string(dir.path().join("received")).unwrap(); + let (process_id, phase) = received.split_once(' ').unwrap(); + assert_eq!(phase, "Some(Dispatching)"); + let record = Store::open(&dir.path().join("operations.db")) + .unwrap() + .process_record(process_id) + .unwrap() + .expect("the record the crashed gateway left"); + assert_eq!(record.phase, ProcessPhase::Dispatching); + assert_eq!(record.runner, runner_ref(fake_identity())); + assert_eq!(record.workspace_id, "demo"); + assert_eq!( + (record.root, record.deadline_at_ms, record.outcome), + (None, None, None) + ); + } + /// Each independent mode states only what its Runner guarantees, and the rest of the /// instructions stay as they are. #[test] diff --git a/crates/server/src/records.rs b/crates/server/src/records.rs new file mode 100644 index 0000000..c7e8194 --- /dev/null +++ b/crates/server/src/records.rs @@ -0,0 +1,183 @@ +//! Process records of a gateway connected to an independent Runner (P1R-C02). Each execution is +//! recorded in the operations database before it is sent to the Runner, and followed to its +//! end, so that a gateway that lost its memory can identify the same execution again. A record +//! never revives a handle, and never by itself proves that an execution started or how it +//! ended: the Runner is the authority on live processes. + +use std::sync::Arc; + +use codespace_domain::{ + ApprovalId, ErrorBody, ErrorCode, ExecCommandParams, ProcessId, ProcessState, +}; +use codespace_runner::{ + Runner, RunnerIdentity, RunnerProcessRoot, RunnerProcessStatus, RuntimeBackend, ShellRelease, +}; +use codespace_store::{ + NewProcessRecord, ProcessEnd, RecordedParticipation, RootProcess, RunnerRef, Store, +}; +use tokio::sync::mpsc; + +/// The exits that a gateway's release hook hands to its process records. +pub struct Exits(mpsc::UnboundedReceiver); + +/// What a gateway records of the executions it sends to one independent Runner. +pub struct ProcessRecords { + store: Arc, + runner: RunnerRef, +} + +impl ProcessRecords { + /// The release hook of a gateway connected to an independent Runner: it releases the lease + /// of each process that exits, as every gateway's hook does, and hands the exit on to the + /// records. An exit that comes before the records exist waits for them. + pub fn release_hook(store: Arc) -> (ShellRelease, Exits) { + let (exits, exited) = mpsc::unbounded_channel(); + let release: ShellRelease = Arc::new(move |process_id: &str| { + store.release_process(process_id); + let _ = exits.send(ProcessId(process_id.to_string())); + }); + (release, Exits(exited)) + } + + /// The records of the executions sent to the Runner with `identity`, which from now on + /// follow its exit events: each one is recorded as the Runner reports it. + pub fn start( + store: Arc, + identity: &RunnerIdentity, + runner: RuntimeBackend, + exits: Exits, + ) -> Arc { + let records = Arc::new(Self { + store, + runner: RunnerRef { + boot_id: identity.boot_id.clone(), + pid: identity.pid, + started_at_ms: identity.started_at_ms, + nonce: identity.nonce.clone(), + }, + }); + let following = records.clone(); + let Exits(mut exited) = exits; + tokio::spawn(async move { + while let Some(process_id) = exited.recv().await { + match runner.process_status(&process_id).await { + Ok(status) if status.state == ProcessState::Exited => following.ended(&status), + Ok(_) => {} + // The Runner reported the exit; how it ended is not known. + Err(_) => following.end( + &process_id, + &ProcessEnd::Ended { + termination: None, + exit_code: None, + signal: None, + output_total: None, + }, + ), + } + } + }); + records + } + + /// Record an execution before it is sent to the Runner, and mark the approval it resumes + /// `resuming` with it. When the record cannot be written, nothing is sent. + pub(crate) fn dispatching( + &self, + process_id: &ProcessId, + params: &ExecCommandParams, + participation: RecordedParticipation, + attempt_id: Option<&str>, + resume: Option<&ApprovalId>, + ) -> Result<(), ErrorBody> { + self.store + .record_dispatch(&NewProcessRecord { + process_id: &process_id.0, + runner: &self.runner, + workspace_id: ¶ms.workspace_id.0, + approval_id: resume, + attempt_id, + participation, + tty: params.tty, + argv: ¶ms.command, + }) + .map_err(|err| match err.code { + ErrorCode::Internal => ErrorBody::new( + ErrorCode::Internal, + format!( + "the execution could not be recorded before it was sent, so nothing was \ + started: {}", + err.message + ), + ), + _ => err, + }) + } + + /// The Runner answered that it started the execution. + pub(crate) fn running( + &self, + process_id: &ProcessId, + root: Option<&RunnerProcessRoot>, + deadline_at_ms: Option, + ) { + let root = root.map(|root| RootProcess { + pid: root.pid, + started: root.started.clone(), + }); + self.write( + process_id, + "running", + self.store + .record_running(&self.runner, &process_id.0, root.as_ref(), deadline_at_ms), + ); + } + + /// The Runner answered that it did not start the execution. + pub(crate) fn not_started(&self, process_id: &ProcessId) { + self.end(process_id, &ProcessEnd::NotStarted); + } + + /// The Runner reported the execution's status after it exited. + pub(crate) fn ended(&self, status: &RunnerProcessStatus) { + self.end( + &status.process_id, + &ProcessEnd::Ended { + termination: status.termination, + exit_code: status.exit_code, + signal: status.signal.as_ref().map(|signal| signal.number), + output_total: Some(status.output_total), + }, + ); + } + + /// A gateway observed the execution's output up to `cursor`. + pub(crate) fn observed_output(&self, process_id: &ProcessId, cursor: u64) { + self.write( + process_id, + "output", + self.store + .record_output(&self.runner, &process_id.0, cursor), + ); + } + + fn end(&self, process_id: &ProcessId, end: &ProcessEnd) { + self.write( + process_id, + "end", + self.store.record_end(&self.runner, &process_id.0, end), + ); + } + + /// A record that cannot be updated keeps what it said: the execution goes on, and the + /// Runner remains the authority on it. + fn write(&self, process_id: &ProcessId, what: &str, written: Result) { + if let Err(err) = written { + tracing::warn!( + process_id = %process_id.0, + record = what, + error = %err.message, + "the process record was not updated" + ); + } + } +} diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index d2f783e..bd61b83 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -177,6 +177,14 @@ impl RuntimeProcess { hello.lifecycle.as_str() )); } + // Its executions are recorded under its identity (P1R-C02). + if hello.identity.is_none() { + runner.close().await; + return Err(anyhow!( + "--runner-lifecycle independent: the Runner at {} did not state its identity", + socket.display() + )); + } tracing::info!( runner_pid = hello.identity.as_ref().map(|identity| identity.pid), previous_controller = hello.previous_controller, diff --git a/crates/server/tests/independent_runner.rs b/crates/server/tests/independent_runner.rs index 81d82c7..4672e68 100644 --- a/crates/server/tests/independent_runner.rs +++ b/crates/server/tests/independent_runner.rs @@ -1,18 +1,24 @@ //! The gateway with an independent Runner (P1R-C01), each started as an operator starts it: the //! Runner outlives a gateway, the next gateway inherits its processes but starts nothing, and a -//! gateway that ends leaves or stops the Runner as its exit intent says. These tests need the -//! worker binary (`CODESPACE_RUNTIME_BIN`) and return early without it, as the gateway's other -//! worker tests do. +//! gateway that ends leaves or stops the Runner as its exit intent says. Each gateway records +//! its executions in the operations database, which it is the only writer of (P1R-C02). The +//! tests with a Runner need the worker binary (`CODESPACE_RUNTIME_BIN`) and return early +//! without it, as the gateway's other worker tests do. use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Stdio}; use std::time::Duration; -use codespace_domain::{TOOL_EXEC_COMMAND, TOOL_PROCESS_STATUS, TOOL_READ_PROCESS}; +use codespace_domain::{ + ProcessTermination, TOOL_EXEC_COMMAND, TOOL_PROCESS_STATUS, TOOL_READ_PROCESS, +}; +use codespace_store::{ProcessOutcome, ProcessPhase, Store, WriterLock}; use rmcp::model::{CallToolRequestParams, CallToolResult}; use rmcp::service::{RunningService, ServiceError}; use rmcp::{object, RoleClient, ServiceExt}; +use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::{Child, Command}; +use tokio::sync::mpsc; use tokio::time::{sleep, timeout, Instant}; /// The process timeout that every gateway here gives its executions. @@ -95,6 +101,11 @@ impl Setup { runner } + /// The operations database that every gateway here records into. + fn database(&self) -> PathBuf { + self.root.path().join("ops.sqlite") + } + /// The gateway binary with these settings alone: the environment of the test run may name a /// worker for the gateway to start, which an independent Runner's gateway refuses. fn gateway_command(&self, args: &[&str]) -> Command { @@ -109,10 +120,7 @@ impl Setup { } command .env("CODESPACE_CONFIG", &self.cfg) - .env( - "CODESPACE_OPERATIONS_DB", - self.root.path().join("ops.sqlite"), - ) + .env("CODESPACE_OPERATIONS_DB", self.database()) .env( "CODESPACE_PROCESS_TIMEOUT_SECS", TIMEOUT.as_secs().to_string(), @@ -122,21 +130,30 @@ impl Setup { command } + /// The settings of a gateway connected to the independent Runner, whose normal shutdown + /// asks `exit_intent` of it. + fn independent(&self, exit_intent: &str) -> Vec { + [ + "--runner", + "uds", + "--runner-socket", + &self.socket.display().to_string(), + "--runner-lifecycle", + "independent", + "--runner-exit-intent", + exit_intent, + ] + .map(String::from) + .to_vec() + } + /// A gateway connected to the independent Runner, whose normal shutdown asks `exit_intent` /// of it. async fn gateway(&self, exit_intent: &str) -> Gateway { - let socket = self.socket.display().to_string(); + let args = self.independent(exit_intent); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); let mut process = self - .gateway_command(&[ - "--runner", - "uds", - "--runner-socket", - &socket, - "--runner-lifecycle", - "independent", - "--runner-exit-intent", - exit_intent, - ]) + .gateway_command(&args) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() @@ -149,18 +166,49 @@ impl Setup { /// The error of a gateway that does not start with `args`. async fn refused(&self, args: &[&str]) -> String { - let output = timeout( - Duration::from_secs(30), - self.gateway_command(args).stdin(Stdio::null()).output(), - ) - .await - .expect("the gateway ends") - .expect("run the gateway"); - assert!(!output.status.success(), "{args:?}: {output:?}"); - String::from_utf8_lossy(&output.stderr).into_owned() + refused(self.gateway_command(args)).await } } +/// The error of a gateway `command` that does not start. +async fn refused(mut command: Command) -> String { + let output = timeout( + Duration::from_secs(30), + command.stdin(Stdio::null()).output(), + ) + .await + .expect("the gateway ends") + .expect("run the gateway"); + assert!(!output.status.success(), "{output:?}"); + String::from_utf8_lossy(&output.stderr).into_owned() +} + +/// The lines a process writes to `stderr`, read for as long as it writes them. +fn log_lines(stderr: tokio::process::ChildStderr) -> mpsc::UnboundedReceiver { + let (lines, logged) = mpsc::unbounded_channel(); + tokio::spawn(async move { + let mut read = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = read.next_line().await { + let _ = lines.send(line); + } + }); + logged +} + +/// The first line logged that contains `text`. +async fn logged(lines: &mut mpsc::UnboundedReceiver, text: &str) -> String { + timeout(Duration::from_secs(15), async { + loop { + let line = lines.recv().await.expect("the gateway logs it"); + if line.contains(text) { + return line; + } + } + }) + .await + .unwrap_or_else(|_| panic!("the gateway does not log {text:?}")) +} + struct Gateway { client: RunningService, process: Child, @@ -198,6 +246,22 @@ impl Gateway { payload(&status) } + /// The process's status once it has exited. + async fn ended(&self, process_id: &str) -> serde_json::Value { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let status = self.status(process_id).await; + if status["state"] == "exited" { + return status; + } + assert!( + Instant::now() < deadline, + "the process does not end: {status}" + ); + sleep(Duration::from_millis(40)).await; + } + } + /// The process's output once it contains `expected`. async fn output(&self, process_id: &str, expected: &str) -> String { let deadline = Instant::now() + Duration::from_secs(15); @@ -444,3 +508,136 @@ async fn a_gateway_refuses_independent_settings_or_a_runner_that_do_not_fit() { ); exited(&mut managed).await; } + +/// A gateway with an independent Runner needs an operations database, and is its only writer +/// (P1R-C02): before it connects to the Runner, it waits a bounded time for the gateway that +/// holds the database's lock, and then refuses to start. A gateway in another mode takes no +/// lock. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_gateway_with_an_independent_runner_is_its_databases_only_writer() { + let setup = Setup::new(); + let args = setup.independent("detach"); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + let mut without = setup.gateway_command(&args); + without.env_remove("CODESPACE_OPERATIONS_DB"); + let refusal = refused(without).await; + assert!(refusal.contains("needs --operations-db"), "{refusal}"); + + let held = WriterLock::acquire(&setup.database(), Duration::ZERO, |_| { + unreachable!("nothing holds the lock yet") + }) + .unwrap(); + let started = Instant::now(); + let mut waiting = setup.gateway_command(&args); + waiting.env("RUST_LOG", "info"); + let refusal = refused(waiting).await; + let waited = started.elapsed(); + assert!(refusal.contains("waiting for it to end"), "{refusal}"); + assert!( + refusal.contains("has another writer") + && refusal.contains(&format!("process {}", std::process::id())), + "{refusal}" + ); + assert!(waited >= Duration::from_secs(15), "{waited:?}"); + + // A gateway in another mode records nothing and takes no lock. + let mut managed = setup + .gateway_command(&[]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn a managed gateway"); + let stdout = managed.stdout.take().unwrap(); + let stdin = managed.stdin.take().unwrap(); + let client = ().serve((stdout, stdin)).await.expect("initialize"); + let ended = Gateway { + client, + process: managed, + } + .shut_down() + .await; + assert!(ended.success(), "{ended:?}"); + drop(held); +} + +/// A gateway started while the gateway before it still ends, as when a client restarts its +/// stdio gateway, waits for it to let go of the operations database and then serves (P1R-C02). +/// The records the gateway before it wrote name each execution, the Runner it was sent to and +/// its root process, and say how it ended or that it still runs. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_restarted_gateway_waits_for_the_one_before_it_and_finds_its_records() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let first = setup.gateway("detach").await; + let ended = first.started(&["/bin/sh", "-c", "echo $$; exit 4"]).await; + first.ended(&ended).await; + let root_pid = first.output(&ended, "\n").await; + let running = first + .started(&["/bin/sh", "-c", "echo started; exec sleep 30"]) + .await; + first.output(&running, "started").await; + + let args = setup.independent("stop"); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + let mut process = setup + .gateway_command(&args) + .env("RUST_LOG", "info") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn the next gateway"); + let mut lines = log_lines(process.stderr.take().unwrap()); + let first_pid = first.process.id().unwrap(); + let line = logged(&mut lines, "waiting for it to end").await; + assert!(line.contains(&format!("Some({first_pid})")), "{line}"); + let stdout = process.stdout.take().unwrap(); + let stdin = process.stdin.take().unwrap(); + let serving = tokio::spawn(async move { ().serve((stdout, stdin)).await }); + sleep(Duration::from_millis(500)).await; + assert!( + !serving.is_finished(), + "the next gateway serves while the first one holds the database" + ); + let shut = first.shut_down().await; + assert!(shut.success(), "{shut:?}"); + logged(&mut lines, "the only writer of the operations database").await; + let client = timeout(Duration::from_secs(15), serving) + .await + .expect("the next gateway serves") + .unwrap() + .expect("initialize"); + let next = Gateway { client, process }; + assert_eq!(next.status(&running).await["state"], "running"); + + let store = Store::open(&setup.database()).unwrap(); + let runner_pid = runner.id().unwrap(); + let record = store.process_record(&ended).unwrap().expect("its record"); + assert_eq!(record.phase, ProcessPhase::Terminal, "{record:?}"); + assert_eq!( + record.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Exited)) + ); + assert_eq!(record.exit_code, Some(4)); + assert_eq!(record.runner.pid, runner_pid); + assert_eq!( + record.root.map(|root| root.pid.to_string()), + Some(root_pid.trim().to_string()) + ); + let record = store.process_record(&running).unwrap().expect("its record"); + assert_eq!(record.phase, ProcessPhase::Running, "{record:?}"); + assert_eq!(record.runner.pid, runner_pid); + let version: i64 = rusqlite::Connection::open(setup.database()) + .unwrap() + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .unwrap(); + assert_eq!(version, codespace_store::SCHEMA_VERSION); + + let shut = next.shut_down().await; + assert!(shut.success(), "{shut:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); +} diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index f645487..af54af5 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -14,6 +14,7 @@ serde = { workspace = true } serde_json = { workspace = true } sha2 = "0.10" hex = "0.4" +libc = "0.2" uuid = { version = "1", features = ["v4"] } tokio = { version = "1", features = ["sync"] } diff --git a/crates/store/src/approvals.rs b/crates/store/src/approvals.rs index 13a3445..6b65dd2 100644 --- a/crates/store/src/approvals.rs +++ b/crates/store/src/approvals.rs @@ -337,27 +337,7 @@ impl Store { pub fn mark_resuming(&self, approval_id: &ApprovalId) -> Result<(), ErrorBody> { let conn = self.conn.lock().expect("sqlite mutex"); - let row = load(&conn, &approval_id.0)?.ok_or_else(|| not_found(approval_id))?; - match row.state.as_str() { - "resuming" => Ok(()), - "queued" => { - let updated = conn - .execute( - "UPDATE approvals - SET state = 'resuming' - WHERE approval_id = ?1 AND state = 'queued'", - params![approval_id.0], - ) - .map_err(db_err)?; - if updated != 1 { - return Err(conflict("approval is not queued for dispatch")); - } - Ok(()) - } - other => Err(conflict(format!( - "approval is in an unknown state `{other}`" - ))), - } + mark_resuming_on(&conn, approval_id) } pub fn fail_next_finish(&self) { @@ -422,6 +402,35 @@ impl Store { } } +/// Mark a queued approval as being resumed, on `conn`: the store's connection, or a transaction +/// that also records the execution it resumes (P1R-C02). +pub(crate) fn mark_resuming_on( + conn: &rusqlite::Connection, + approval_id: &ApprovalId, +) -> Result<(), ErrorBody> { + let row = load(conn, &approval_id.0)?.ok_or_else(|| not_found(approval_id))?; + match row.state.as_str() { + "resuming" => Ok(()), + "queued" => { + let updated = conn + .execute( + "UPDATE approvals + SET state = 'resuming' + WHERE approval_id = ?1 AND state = 'queued'", + params![approval_id.0], + ) + .map_err(db_err)?; + if updated != 1 { + return Err(conflict("approval is not queued for dispatch")); + } + Ok(()) + } + other => Err(conflict(format!( + "approval is in an unknown state `{other}`" + ))), + } +} + fn load(conn: &rusqlite::Connection, approval_id: &str) -> Result, ErrorBody> { conn.query_row( "SELECT approval_id, workspace_id, tool, fingerprint, params_json, state, resolved_at, result_json diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 6478725..a0764f1 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -1,13 +1,18 @@ -//! Single-instance SQLite operations, confirmation holds, and in-memory -//! resource occupancy. Occupancy is not a SQLite schema. Request-owned +//! Single-instance SQLite operations, confirmation holds, process records and +//! in-memory resource occupancy. Occupancy is not a SQLite schema. Request-owned //! conflicts wait on a per-resource FIFO that starts at `acquire()`. A live //! process rejects overlapping mutation immediately with `WORKSPACE_BUSY` //! and fails trailing waiters. Queue saturation is `RESOURCE_QUEUE_FULL`. //! HTTP/JSON-RPC request ids are never stored as [`OperationId`] values. +//! +//! The database carries a schema version (P1R-C02): a database that a newer +//! CodeSpace wrote is never opened, so nothing here changes what it recorded. mod approvals; mod coord; +mod processes; mod resource; +mod writer; use std::collections::HashSet; use std::path::Path; @@ -23,8 +28,17 @@ use sha2::{Digest, Sha256}; pub use approvals::{ApprovalRecord, ResumeClaim}; pub use coord::CreateIntent; +pub use processes::{ + NewProcessRecord, ProcessEnd, ProcessOutcome, ProcessPhase, ProcessRecord, + RecordedParticipation, RootProcess, RunnerRef, +}; use resource::AcquireOutcome; pub use resource::{LockMode, Resource, ResourceGuard}; +pub use writer::WriterLock; + +/// The schema version this CodeSpace writes in `PRAGMA user_version` (P1R-C02). A database +/// without one (0) predates it and is brought up to it; a newer one is refused. +pub const SCHEMA_VERSION: i64 = 1; #[derive(Debug, Clone)] pub struct StoredOperation { @@ -153,6 +167,14 @@ impl Store { } fn from_connection(conn: Connection) -> Result { + // Before anything is written: a database a newer CodeSpace wrote is left as it is. + let version = schema_version(&conn)?; + if version > SCHEMA_VERSION { + return Err(format!( + "the operations database has schema version {version}, newer than this \ + CodeSpace's {SCHEMA_VERSION}: open it with the CodeSpace that wrote it" + )); + } conn.execute_batch( "CREATE TABLE IF NOT EXISTS operations ( operation_id TEXT PRIMARY KEY, @@ -214,8 +236,15 @@ impl Store { );", ) .map_err(|e| e.to_string())?; + conn.execute_batch(processes::SCHEMA) + .map_err(|e| e.to_string())?; migrate_operations(&conn)?; migrate_approvals(&conn)?; + // Last, so a database whose upgrade was cut short is upgraded again. + if version < SCHEMA_VERSION { + conn.execute_batch(&format!("PRAGMA user_version = {SCHEMA_VERSION}")) + .map_err(|e| e.to_string())?; + } Ok(Self { conn: Mutex::new(conn), locks: Mutex::new(resource::ResourceSerializer::default()), @@ -669,6 +698,12 @@ fn migrate_approvals(conn: &Connection) -> Result<(), String> { Ok(()) } +/// The database's schema version: 0 for one that predates schema versions (P1R-C02). +fn schema_version(conn: &Connection) -> Result { + conn.query_row("PRAGMA user_version", [], |row| row.get(0)) + .map_err(|err| format!("cannot read the operations database: {err}")) +} + fn table_columns(conn: &Connection, table: &str) -> Result, String> { let mut stmt = conn .prepare(&format!("PRAGMA table_info({table})")) @@ -1049,4 +1084,287 @@ mod tests { assert_eq!(err.code, ErrorCode::Internal); assert_eq!(err.message, "resource waiter closed"); } + + /// The statements with which CodeSpace before process records (main `feb5f52`) opens its + /// database, which an earlier binary still runs on one that this binary wrote. + const PRE_PROCESS_RECORDS_OPEN: &str = " + CREATE TABLE IF NOT EXISTS operations ( + operation_id TEXT PRIMARY KEY, + operation_key TEXT, + workspace_id TEXT NOT NULL, + fingerprint TEXT NOT NULL, + status TEXT NOT NULL, + result_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + finished_at INTEGER, + events_json TEXT + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_operations_key + ON operations(operation_key) + WHERE operation_key IS NOT NULL; + CREATE TABLE IF NOT EXISTS works ( + work_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + title TEXT, + state TEXT NOT NULL, + queue_revision INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + closed_at INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_works_workspace ON works(workspace_id); + CREATE TABLE IF NOT EXISTS intents ( + intent_id TEXT PRIMARY KEY, + work_id TEXT, + workspace_id TEXT NOT NULL, + position INTEGER NOT NULL, + revision INTEGER NOT NULL DEFAULT 1, + kind TEXT NOT NULL, + delivery TEXT NOT NULL, + body TEXT NOT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + claimed_at INTEGER, + completed_at INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_intents_work_state + ON intents(work_id, state, position); + CREATE INDEX IF NOT EXISTS idx_intents_workspace_next + ON intents(workspace_id, state) + WHERE work_id IS NULL; + CREATE TABLE IF NOT EXISTS approvals ( + approval_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + tool TEXT NOT NULL, + fingerprint TEXT NOT NULL, + params_json TEXT NOT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + resolved_at INTEGER, + result_json TEXT + ); + DROP INDEX IF EXISTS idx_approvals_active_fp; + CREATE UNIQUE INDEX IF NOT EXISTS idx_approvals_active_fp + ON approvals(workspace_id, fingerprint) + WHERE state IN ('pending','granted','queued','resuming') AND length(fingerprint) > 0;"; + + fn user_version(path: &Path) -> i64 { + rusqlite::Connection::open(path) + .unwrap() + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .unwrap() + } + + fn open_refused(path: &Path) -> String { + match Store::open(path) { + Ok(_) => panic!("{} was opened", path.display()), + Err(err) => err, + } + } + + fn runner() -> RunnerRef { + RunnerRef { + boot_id: Some("boot-a".into()), + pid: 4242, + started_at_ms: 1_760_000_000_000, + nonce: "nonce-a".into(), + } + } + + fn dispatch(store: &Store, process_id: &str) { + store + .record_dispatch(&NewProcessRecord { + process_id, + runner: &runner(), + workspace_id: "demo", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["true".to_string()], + }) + .unwrap(); + } + + /// A database that CodeSpace before process records left, with no schema version and no + /// `processes` table, opens with its patches and approvals as they were, and is brought up + /// to this schema version. + #[test] + fn a_database_from_before_schema_versions_is_brought_up_to_this_one() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ops.sqlite"); + let (operation_id, approval_id) = { + let store = Store::open(&path).unwrap(); + let first = params("before", "k-before"); + let Begin::Fresh(id) = store + .begin( + first.operation_key.as_ref(), + "demo", + &Store::fingerprint(&first), + ) + .unwrap() + else { + panic!("fresh"); + }; + store + .finish( + &id, + &ApplyPatchResult::new(PatchStatus::Applied, id.clone()), + ) + .unwrap(); + let approval = store + .create_approval( + "demo", + codespace_domain::ApprovalTargetTool::ApplyPatch, + &Store::fingerprint(&first), + serde_json::to_value(&first).unwrap(), + ) + .unwrap(); + (id, approval.approval_id) + }; + { + let conn = rusqlite::Connection::open(&path).unwrap(); + conn.execute_batch("DROP TABLE processes; PRAGMA user_version = 0;") + .unwrap(); + conn.execute_batch(PRE_PROCESS_RECORDS_OPEN).unwrap(); + } + assert_eq!(user_version(&path), 0); + + let store = Store::open(&path).unwrap(); + assert_eq!(user_version(&path), SCHEMA_VERSION); + assert_eq!( + store.status(&operation_id).unwrap().status, + PatchStatus::Applied + ); + assert_eq!( + store.inspect_approval(&approval_id).unwrap().0, + codespace_domain::ApprovalState::Pending + ); + assert!(store.process_record("proc-1").unwrap().is_none()); + dispatch(&store, "proc-1"); + assert!(store.process_record("proc-1").unwrap().is_some()); + } + + /// A database that a newer CodeSpace wrote is refused before anything is written to it. + #[test] + fn a_newer_database_is_refused_and_left_as_it_is() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ops.sqlite"); + dispatch(&Store::open(&path).unwrap(), "proc-1"); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch(&format!("PRAGMA user_version = {}", SCHEMA_VERSION + 1)) + .unwrap(); + let before = std::fs::read(&path).unwrap(); + let err = open_refused(&path); + assert!( + err.contains(&format!( + "schema version {}, newer than this CodeSpace's {SCHEMA_VERSION}", + SCHEMA_VERSION + 1 + )), + "{err}" + ); + assert_eq!(std::fs::read(&path).unwrap(), before); + assert_eq!( + std::fs::read_dir(dir.path()).unwrap().count(), + 1, + "nothing but the database" + ); + } + + /// An earlier CodeSpace does not know process records: it opens a database this one wrote + /// with its own statements and records patches, and leaves the records and the schema + /// version as they are. + #[test] + fn an_earlier_codespace_leaves_the_records_it_does_not_know() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ops.sqlite"); + let recorded = { + let store = Store::open(&path).unwrap(); + dispatch(&store, "proc-1"); + store + .record_running( + &runner(), + "proc-1", + Some(&RootProcess { + pid: 777, + started: Some("linux:100".into()), + }), + Some(1_760_000_030_000), + ) + .unwrap(); + store.process_record("proc-1").unwrap().unwrap() + }; + { + let conn = rusqlite::Connection::open(&path).unwrap(); + conn.execute_batch(PRE_PROCESS_RECORDS_OPEN).unwrap(); + conn.execute( + "INSERT INTO operations + (operation_id, operation_key, workspace_id, fingerprint, status, result_json, + created_at, finished_at, events_json) + VALUES ('op-earlier', NULL, 'demo', 'fp', 'unknown', '{}', 1, NULL, NULL)", + [], + ) + .unwrap(); + } + assert_eq!(user_version(&path), SCHEMA_VERSION); + let store = Store::open(&path).unwrap(); + assert_eq!(store.process_record("proc-1").unwrap(), Some(recorded)); + assert_eq!( + store + .status(&OperationId("op-earlier".into())) + .unwrap() + .status, + PatchStatus::Unknown + ); + } + + /// A file that is not a database, such as one cut short by a crash, is refused and kept: it + /// is never replaced with an empty database. + #[test] + fn a_damaged_database_is_refused_and_kept() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ops.sqlite"); + let damaged = b"this is not an operations database, and it was cut sh".repeat(40); + std::fs::write(&path, &damaged).unwrap(); + let err = open_refused(&path); + assert!(err.contains("cannot read the operations database"), "{err}"); + assert_eq!(std::fs::read(&path).unwrap(), damaged); + } + + /// The patch operations ledger and the approvals table keep the columns they had before + /// process records. + #[test] + fn the_patch_ledger_and_the_approvals_keep_their_columns() { + let store = Store::memory().unwrap(); + let conn = store.conn.lock().unwrap(); + assert_eq!( + table_columns(&conn, "operations").unwrap(), + [ + "operation_id", + "operation_key", + "workspace_id", + "fingerprint", + "status", + "result_json", + "created_at", + "finished_at", + "events_json" + ] + ); + assert_eq!( + table_columns(&conn, "approvals").unwrap(), + [ + "approval_id", + "workspace_id", + "tool", + "fingerprint", + "params_json", + "state", + "created_at", + "resolved_at", + "result_json" + ] + ); + } } diff --git a/crates/store/src/processes.rs b/crates/store/src/processes.rs new file mode 100644 index 0000000..d350298 --- /dev/null +++ b/crates/store/src/processes.rs @@ -0,0 +1,952 @@ +//! Process records (P1R-C02). A gateway connected to an independent Runner records each +//! execution before it sends it to the Runner, and follows it to its end, so that a gateway +//! that lost its memory can identify the same execution again. A record never revives a +//! handle, and never by itself proves that an execution started or how it ended: the Runner is +//! the authority on live processes. +//! +//! A record keeps the argv's digest, never the argv, its environment or a secret. + +use codespace_domain::{ApprovalId, ErrorBody, ErrorCode, ProcessTermination}; +use rusqlite::{params, Connection, OptionalExtension}; + +use crate::{approvals, hash_canonical, Store}; + +/// The `processes` table. What a record learns after its execution was sent is NULL until then. +pub(crate) const SCHEMA: &str = "CREATE TABLE IF NOT EXISTS processes ( + process_id TEXT PRIMARY KEY, + runner_boot_id TEXT, + runner_pid INTEGER NOT NULL, + runner_started_at_ms INTEGER NOT NULL, + runner_nonce TEXT NOT NULL, + controller_epoch INTEGER, + workspace_id TEXT NOT NULL, + approval_id TEXT, + attempt_id TEXT, + participation TEXT NOT NULL, + tty INTEGER NOT NULL, + argv_digest TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ('dispatching', 'running', 'terminal', 'unknown')), + deadline_at_ms INTEGER, + root_pid INTEGER, + root_started TEXT, + termination TEXT, + exit_code INTEGER, + signal INTEGER, + output_cursor INTEGER, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_processes_workspace_phase ON processes(workspace_id, phase);"; + +/// The Runner an execution was sent to, by the identity its `Hello` states (P1R-C01). A Runner +/// with an earlier Runner's PID, after a restart or a reboot, is another Runner. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RunnerRef { + pub boot_id: Option, + pub pid: u32, + pub started_at_ms: u64, + pub nonce: String, +} + +/// Where an execution's record stands. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessPhase { + /// Recorded before it was sent to the Runner: it may or may not have started. + Dispatching, + /// The Runner answered that it started it. + Running, + /// The Runner reported that it ended, or answered that it did not start it. + Terminal, + /// What became of it cannot be learned any more (P1R-C05). Nothing records it yet. + Unknown, +} + +impl ProcessPhase { + fn as_str(self) -> &'static str { + match self { + Self::Dispatching => "dispatching", + Self::Running => "running", + Self::Terminal => "terminal", + Self::Unknown => "unknown", + } + } + + fn parse(raw: &str) -> Option { + Some(match raw { + "dispatching" => Self::Dispatching, + "running" => Self::Running, + "terminal" => Self::Terminal, + "unknown" => Self::Unknown, + _ => return None, + }) + } +} + +/// How an execution takes part in the resource authority. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedParticipation { + Off, + Required, +} + +impl RecordedParticipation { + fn as_str(self) -> &'static str { + match self { + Self::Off => "off", + Self::Required => "required", + } + } + + fn parse(raw: &str) -> Option { + Some(match raw { + "off" => Self::Off, + "required" => Self::Required, + _ => return None, + }) + } +} + +/// An execution about to be sent to the Runner. +#[derive(Debug, Clone)] +pub struct NewProcessRecord<'a> { + pub process_id: &'a str, + pub runner: &'a RunnerRef, + pub workspace_id: &'a str, + /// The approval this execution resumes: it is marked `resuming` with the record, in one + /// transaction. + pub approval_id: Option<&'a ApprovalId>, + /// A governed execution's DevGuard attempt. + pub attempt_id: Option<&'a str>, + pub participation: RecordedParticipation, + pub tty: bool, + /// Recorded as its digest only. + pub argv: &'a [String], +} + +/// The root process the Runner created: its PID and, where the Runner can read it, the start +/// identity that tells it apart from a later process with the same PID. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RootProcess { + pub pid: u32, + pub started: Option, +} + +impl RootProcess { + /// Whether `observed` is this root: the same PID with the same start identity. The same + /// PID alone never is, since the system reuses PIDs. + pub fn is(&self, observed: &RootProcess) -> bool { + self.pid == observed.pid && self.started.is_some() && self.started == observed.started + } +} + +/// How an execution ended, as far as the gateway learned it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProcessEnd { + /// The Runner answered that it did not start the execution. + NotStarted, + /// The Runner reported that the execution ended. `termination` is `None` when the gateway + /// could not learn how. + Ended { + termination: Option, + exit_code: Option, + signal: Option, + output_total: Option, + }, +} + +/// How a terminal execution's record says it ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessOutcome { + /// The Runner answered that it did not start the execution. + NotStarted, + /// The Runner's termination of the execution. + Ended(ProcessTermination), +} + +impl ProcessOutcome { + fn as_str(self) -> &'static str { + match self { + Self::NotStarted => "not_started", + Self::Ended(ProcessTermination::Exited) => "exited", + Self::Ended(ProcessTermination::Signaled) => "signaled", + Self::Ended(ProcessTermination::Timeout) => "timeout", + Self::Ended(ProcessTermination::Terminated) => "terminated", + Self::Ended(ProcessTermination::Unknown) => "unknown", + } + } + + fn parse(raw: &str) -> Option { + Some(match raw { + "not_started" => Self::NotStarted, + "exited" => Self::Ended(ProcessTermination::Exited), + "signaled" => Self::Ended(ProcessTermination::Signaled), + "timeout" => Self::Ended(ProcessTermination::Timeout), + "terminated" => Self::Ended(ProcessTermination::Terminated), + "unknown" => Self::Ended(ProcessTermination::Unknown), + _ => return None, + }) + } +} + +/// An execution's record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProcessRecord { + pub process_id: String, + pub runner: RunnerRef, + /// The controller epoch the execution was sent under (P1R-C03). Nothing issues one yet. + pub controller_epoch: Option, + pub workspace_id: String, + pub approval_id: Option, + pub attempt_id: Option, + pub participation: RecordedParticipation, + pub tty: bool, + /// `sha256:` and the hex SHA-256 of the argv as a JSON array. + pub argv_digest: String, + pub phase: ProcessPhase, + /// When the Runner times the execution out, in milliseconds since the Unix epoch, as it + /// reported. The Runner's own timer decides. + pub deadline_at_ms: Option, + pub root: Option, + pub outcome: Option, + pub exit_code: Option, + pub signal: Option, + /// The furthest output offset a gateway observed. + pub output_cursor: Option, + pub created_at_ms: u64, + pub updated_at_ms: u64, +} + +impl ProcessRecord { + /// The Runner answered that it started the execution. A record that already ended stays + /// ended, since the end can be reported before the answer, but learns the root and the + /// deadline. + fn started(&mut self, root: Option<&RootProcess>, deadline_at_ms: Option) { + if self.phase == ProcessPhase::Dispatching { + self.phase = ProcessPhase::Running; + } + if self.root.is_none() { + self.root = root.cloned(); + } + if self.deadline_at_ms.is_none() { + self.deadline_at_ms = deadline_at_ms; + } + } + + /// The execution ended, or never started. Only a record still dispatching can be told that + /// its execution never started. How it ended is recorded once. + fn ended(&mut self, end: &ProcessEnd) { + match end { + ProcessEnd::NotStarted => { + if self.phase == ProcessPhase::Dispatching { + self.phase = ProcessPhase::Terminal; + self.outcome = Some(ProcessOutcome::NotStarted); + } + } + // The Runner answered that it did not start it: nothing reported later changes that. + ProcessEnd::Ended { .. } if self.outcome == Some(ProcessOutcome::NotStarted) => {} + ProcessEnd::Ended { + termination, + exit_code, + signal, + output_total, + } => { + self.phase = ProcessPhase::Terminal; + if let (None, Some(termination)) = (self.outcome, termination) { + self.outcome = Some(ProcessOutcome::Ended(*termination)); + self.exit_code = *exit_code; + self.signal = *signal; + } + if let Some(total) = output_total { + self.observed_output(*total); + } + } + } + } + + fn observed_output(&mut self, cursor: u64) { + if self.output_cursor.is_none_or(|seen| seen < cursor) { + self.output_cursor = Some(cursor); + } + } +} + +impl Store { + /// Record an execution before it is sent to the Runner. An approval it resumes is marked + /// `resuming` in the same transaction: both are recorded, or neither. + pub fn record_dispatch(&self, record: &NewProcessRecord<'_>) -> Result<(), ErrorBody> { + let argv_digest = hash_canonical(&serde_json::json!(record.argv)); + let now = to_sql(now_ms()); + let mut conn = self.conn.lock().expect("sqlite mutex"); + let tx = conn.transaction().map_err(record_err)?; + if let Some(approval_id) = record.approval_id { + approvals::mark_resuming_on(&tx, approval_id)?; + } + tx.execute( + "INSERT INTO processes ( + process_id, runner_boot_id, runner_pid, runner_started_at_ms, runner_nonce, + controller_epoch, workspace_id, approval_id, attempt_id, participation, tty, + argv_digest, phase, created_at_ms, updated_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?13)", + params![ + record.process_id, + record.runner.boot_id, + record.runner.pid, + to_sql(record.runner.started_at_ms), + record.runner.nonce, + record.workspace_id, + record.approval_id.map(|id| id.0.as_str()), + record.attempt_id, + record.participation.as_str(), + record.tty, + argv_digest, + ProcessPhase::Dispatching.as_str(), + now, + ], + ) + .map_err(record_err)?; + #[cfg(test)] + if CRASH_BEFORE_COMMIT.load(std::sync::atomic::Ordering::SeqCst) { + std::process::abort(); + } + tx.commit().map_err(record_err) + } + + /// Record that the Runner answered that it started the execution, with the root process + /// and the deadline it reported. Returns whether the record changed: `false` also when + /// `runner` sent no execution with this ID. + pub fn record_running( + &self, + runner: &RunnerRef, + process_id: &str, + root: Option<&RootProcess>, + deadline_at_ms: Option, + ) -> Result { + self.change_record(runner, process_id, |record| { + record.started(root, deadline_at_ms) + }) + } + + /// Record how the execution ended, as the Runner reported it. Returns whether the record + /// changed: `false` also when `runner` sent no execution with this ID. + pub fn record_end( + &self, + runner: &RunnerRef, + process_id: &str, + end: &ProcessEnd, + ) -> Result { + self.change_record(runner, process_id, |record| record.ended(end)) + } + + /// Record the furthest output offset a gateway observed. Returns whether the record changed. + pub fn record_output( + &self, + runner: &RunnerRef, + process_id: &str, + cursor: u64, + ) -> Result { + self.change_record(runner, process_id, |record| record.observed_output(cursor)) + } + + /// The record of the execution with this ID, whichever Runner it was sent to. + pub fn process_record(&self, process_id: &str) -> Result, ErrorBody> { + let conn = self.conn.lock().expect("sqlite mutex"); + load(&conn, process_id) + } + + /// Change the record of an execution that `runner` was sent, and write it back when it + /// changed. Records of executions sent to another Runner are never changed. + fn change_record( + &self, + runner: &RunnerRef, + process_id: &str, + change: impl FnOnce(&mut ProcessRecord), + ) -> Result { + let conn = self.conn.lock().expect("sqlite mutex"); + let Some(before) = load(&conn, process_id)? else { + return Ok(false); + }; + if before.runner != *runner { + return Ok(false); + } + let mut record = before.clone(); + change(&mut record); + if record == before { + return Ok(false); + } + let updated = conn + .execute( + "UPDATE processes + SET phase = ?6, deadline_at_ms = ?7, root_pid = ?8, root_started = ?9, + termination = ?10, exit_code = ?11, signal = ?12, output_cursor = ?13, + updated_at_ms = ?14 + WHERE process_id = ?1 AND runner_boot_id IS ?2 AND runner_pid = ?3 + AND runner_started_at_ms = ?4 AND runner_nonce = ?5", + params![ + process_id, + runner.boot_id, + runner.pid, + to_sql(runner.started_at_ms), + runner.nonce, + record.phase.as_str(), + record.deadline_at_ms.map(to_sql), + record.root.as_ref().map(|root| root.pid), + record + .root + .as_ref() + .and_then(|root| root.started.as_deref()), + record.outcome.map(ProcessOutcome::as_str), + record.exit_code, + record.signal, + record.output_cursor.map(to_sql), + to_sql(now_ms()), + ], + ) + .map_err(record_err)?; + Ok(updated == 1) + } +} + +/// A crash between a record's statements and its commit, for the torn-write test. +#[cfg(test)] +pub(crate) static CRASH_BEFORE_COMMIT: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +const RECORD_SELECT: &str = "SELECT process_id, runner_boot_id, runner_pid, runner_started_at_ms, + runner_nonce, controller_epoch, workspace_id, approval_id, attempt_id, participation, tty, + argv_digest, phase, deadline_at_ms, root_pid, root_started, termination, exit_code, signal, + output_cursor, created_at_ms, updated_at_ms FROM processes"; + +fn load(conn: &Connection, process_id: &str) -> Result, ErrorBody> { + conn.query_row( + &format!("{RECORD_SELECT} WHERE process_id = ?1"), + params![process_id], + map_record, + ) + .optional() + .map_err(record_err)? + .transpose() +} + +fn map_record(row: &rusqlite::Row<'_>) -> rusqlite::Result> { + let process_id: String = row.get(0)?; + let participation: String = row.get(9)?; + let phase: String = row.get(12)?; + let root_pid: Option = row.get(14)?; + let root_started: Option = row.get(15)?; + let termination: Option = row.get(16)?; + let unreadable = |what: &str, value: &str| { + Err(ErrorBody::new( + ErrorCode::Internal, + format!("the record of process `{process_id}` has an unknown {what} `{value}`"), + )) + }; + let Some(participation) = RecordedParticipation::parse(&participation) else { + return Ok(unreadable("participation", &participation)); + }; + let Some(phase) = ProcessPhase::parse(&phase) else { + return Ok(unreadable("phase", &phase)); + }; + let outcome = match termination { + None => None, + Some(raw) => match ProcessOutcome::parse(&raw) { + Some(outcome) => Some(outcome), + None => return Ok(unreadable("termination", &raw)), + }, + }; + Ok(Ok(ProcessRecord { + runner: RunnerRef { + boot_id: row.get(1)?, + pid: row.get(2)?, + started_at_ms: from_sql(row.get(3)?), + nonce: row.get(4)?, + }, + controller_epoch: row.get::<_, Option>(5)?.map(from_sql), + workspace_id: row.get(6)?, + approval_id: row.get::<_, Option>(7)?.map(ApprovalId), + attempt_id: row.get(8)?, + participation, + tty: row.get(10)?, + argv_digest: row.get(11)?, + phase, + deadline_at_ms: row.get::<_, Option>(13)?.map(from_sql), + root: root_pid.map(|pid| RootProcess { + pid, + started: root_started, + }), + outcome, + exit_code: row.get(17)?, + signal: row.get(18)?, + output_cursor: row.get::<_, Option>(19)?.map(from_sql), + created_at_ms: from_sql(row.get(20)?), + updated_at_ms: from_sql(row.get(21)?), + process_id, + })) +} + +fn record_err(err: rusqlite::Error) -> ErrorBody { + ErrorBody::new(ErrorCode::Internal, format!("process record: {err}")) +} + +fn to_sql(value: u64) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} + +fn from_sql(value: i64) -> u64 { + u64::try_from(value).unwrap_or(0) +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use codespace_domain::{ApprovalDecision, ApprovalState, ApprovalTargetTool}; + use std::os::unix::process::ExitStatusExt; + use std::path::Path; + use std::process::Command; + use std::sync::atomic::Ordering; + + use crate::ResumeClaim; + + fn runner() -> RunnerRef { + RunnerRef { + boot_id: Some("boot-a".into()), + pid: 4242, + started_at_ms: 1_760_000_000_000, + nonce: "nonce-a".into(), + } + } + + fn argv() -> Vec { + vec!["cargo".into(), "test".into()] + } + + fn dispatch( + store: &Store, + process_id: &str, + runner: &RunnerRef, + approval_id: Option<&ApprovalId>, + ) -> Result<(), ErrorBody> { + store.record_dispatch(&NewProcessRecord { + process_id, + runner, + workspace_id: "demo", + approval_id, + attempt_id: Some("cs-attempt-1"), + participation: RecordedParticipation::Required, + tty: true, + argv: &argv(), + }) + } + + fn record(store: &Store, process_id: &str) -> ProcessRecord { + store.process_record(process_id).unwrap().expect("a record") + } + + fn root() -> RootProcess { + RootProcess { + pid: 777, + started: Some("linux:100".into()), + } + } + + fn exited(code: i32, output_total: u64) -> ProcessEnd { + ProcessEnd::Ended { + termination: Some(ProcessTermination::Exited), + exit_code: Some(code), + signal: None, + output_total: Some(output_total), + } + } + + /// A granted exec approval that `operation_resume` claimed: `queued`, about to dispatch. + fn queued_approval(store: &Store, command: &str) -> ApprovalId { + let created = store + .create_approval( + "demo", + ApprovalTargetTool::ExecCommand, + &format!("fp-{command}"), + serde_json::json!({ "workspace_id": "demo", "command": [command] }), + ) + .unwrap(); + store + .resolve_approval(&created.approval_id, ApprovalDecision::Grant) + .unwrap(); + let ResumeClaim::Execute(_, _guard) = store.claim_resume(&created.approval_id).unwrap() + else { + panic!("expected execute"); + }; + created.approval_id + } + + fn approval_state(store: &Store, approval_id: &ApprovalId) -> ApprovalState { + store.inspect_approval(approval_id).unwrap().0 + } + + /// A gateway that lost its memory finds the execution by its `process_id`, from the record + /// written before it was sent to the one that says how it ended. + #[test] + fn a_new_gateway_finds_each_execution_from_dispatch_to_its_end() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let runner = runner(); + dispatch(&Store::open(&database).unwrap(), "proc-1", &runner, None).unwrap(); + + let store = Store::open(&database).unwrap(); + let sent = record(&store, "proc-1"); + assert_eq!(sent.phase, ProcessPhase::Dispatching); + assert_eq!(sent.runner, runner); + assert_eq!(sent.controller_epoch, None); + assert_eq!(sent.workspace_id, "demo"); + assert_eq!(sent.approval_id, None); + assert_eq!(sent.attempt_id.as_deref(), Some("cs-attempt-1")); + assert_eq!(sent.participation, RecordedParticipation::Required); + assert!(sent.tty); + assert_eq!(sent.argv_digest, hash_canonical(&serde_json::json!(argv()))); + assert_eq!( + ( + sent.root, + sent.deadline_at_ms, + sent.outcome, + sent.output_cursor + ), + (None, None, None, None) + ); + assert_eq!(sent.created_at_ms, sent.updated_at_ms); + assert!(store + .record_running(&runner, "proc-1", Some(&root()), Some(1_760_000_030_000)) + .unwrap()); + drop(store); + + let store = Store::open(&database).unwrap(); + let running = record(&store, "proc-1"); + assert_eq!(running.phase, ProcessPhase::Running); + assert_eq!(running.root, Some(root())); + assert_eq!(running.deadline_at_ms, Some(1_760_000_030_000)); + assert!(store.record_output(&runner, "proc-1", 10).unwrap()); + // A cursor never goes back. + assert!(!store.record_output(&runner, "proc-1", 4).unwrap()); + assert!(store.record_end(&runner, "proc-1", &exited(0, 12)).unwrap()); + drop(store); + + let store = Store::open(&database).unwrap(); + let ended = record(&store, "proc-1"); + assert_eq!(ended.phase, ProcessPhase::Terminal); + assert_eq!( + ended.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Exited)) + ); + assert_eq!((ended.exit_code, ended.signal), (Some(0), None)); + assert_eq!(ended.output_cursor, Some(12)); + assert_eq!(ended.root, Some(root())); + // How it ended is recorded once, and an ended record stays ended. + let signaled = ProcessEnd::Ended { + termination: Some(ProcessTermination::Signaled), + exit_code: None, + signal: Some(9), + output_total: Some(12), + }; + assert!(!store.record_end(&runner, "proc-1", &signaled).unwrap()); + assert!(!store + .record_running(&runner, "proc-1", Some(&root()), Some(1)) + .unwrap()); + assert_eq!(record(&store, "proc-1"), ended); + } + + /// The Runner's exit event can reach the gateway before its answer to the execution: the + /// answer then leaves the record ended, and fills in only the root and the deadline. + #[test] + fn an_end_reported_before_the_start_answer_stays_the_end() { + let store = Store::memory().unwrap(); + let runner = runner(); + dispatch(&store, "proc-race", &runner, None).unwrap(); + assert!(store + .record_end(&runner, "proc-race", &exited(3, 5)) + .unwrap()); + assert!(store + .record_running(&runner, "proc-race", Some(&root()), Some(1_760_000_030_000)) + .unwrap()); + let raced = record(&store, "proc-race"); + assert_eq!(raced.phase, ProcessPhase::Terminal); + assert_eq!( + raced.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Exited)) + ); + assert_eq!(raced.exit_code, Some(3)); + assert_eq!(raced.root, Some(root())); + assert_eq!(raced.deadline_at_ms, Some(1_760_000_030_000)); + } + + /// An end whose details the gateway could not learn is completed by the first report + /// that has them, and only by that one. + #[test] + fn an_end_without_details_is_completed_once() { + let store = Store::memory().unwrap(); + let runner = runner(); + dispatch(&store, "proc-1", &runner, None).unwrap(); + store + .record_running(&runner, "proc-1", Some(&root()), None) + .unwrap(); + let unobserved = ProcessEnd::Ended { + termination: None, + exit_code: None, + signal: None, + output_total: None, + }; + assert!(store.record_end(&runner, "proc-1", &unobserved).unwrap()); + let ended = record(&store, "proc-1"); + assert_eq!((ended.phase, ended.outcome), (ProcessPhase::Terminal, None)); + let timeout = ProcessEnd::Ended { + termination: Some(ProcessTermination::Timeout), + exit_code: None, + signal: None, + output_total: Some(5), + }; + assert!(store.record_end(&runner, "proc-1", &timeout).unwrap()); + assert!(!store.record_end(&runner, "proc-1", &exited(0, 5)).unwrap()); + let ended = record(&store, "proc-1"); + assert_eq!( + ended.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Timeout)) + ); + assert_eq!((ended.exit_code, ended.output_cursor), (None, Some(5))); + } + + /// Only an execution that is still dispatching can be one the Runner did not start. + #[test] + fn an_execution_the_runner_did_not_start_ends_as_not_started() { + let store = Store::memory().unwrap(); + let runner = runner(); + dispatch(&store, "proc-refused", &runner, None).unwrap(); + assert!(store + .record_end(&runner, "proc-refused", &ProcessEnd::NotStarted) + .unwrap()); + let refused = record(&store, "proc-refused"); + assert_eq!(refused.phase, ProcessPhase::Terminal); + assert_eq!(refused.outcome, Some(ProcessOutcome::NotStarted)); + assert!(!store + .record_end(&runner, "proc-refused", &exited(0, 0)) + .unwrap()); + + dispatch(&store, "proc-started", &runner, None).unwrap(); + store + .record_running(&runner, "proc-started", None, None) + .unwrap(); + assert!(!store + .record_end(&runner, "proc-started", &ProcessEnd::NotStarted) + .unwrap()); + assert_eq!(record(&store, "proc-started").phase, ProcessPhase::Running); + } + + /// A record names its Runner by the whole identity of its `Hello`. A later Runner with the + /// same PID, as after a reboot, never changes it, nor does a gateway connected to it. + #[test] + fn a_runner_with_the_same_pid_is_another_runner() { + let store = Store::memory().unwrap(); + let runner = runner(); + dispatch(&store, "proc-1", &runner, None).unwrap(); + let before = record(&store, "proc-1"); + for other in [ + RunnerRef { + nonce: "nonce-b".into(), + ..runner.clone() + }, + RunnerRef { + started_at_ms: runner.started_at_ms + 1, + ..runner.clone() + }, + RunnerRef { + boot_id: Some("boot-b".into()), + ..runner.clone() + }, + RunnerRef { + boot_id: None, + ..runner.clone() + }, + RunnerRef { + pid: runner.pid + 1, + ..runner.clone() + }, + ] { + assert!(!store + .record_running(&other, "proc-1", Some(&root()), Some(1)) + .unwrap()); + assert!(!store.record_end(&other, "proc-1", &exited(0, 1)).unwrap()); + assert!(!store.record_output(&other, "proc-1", 1).unwrap()); + } + assert_eq!(record(&store, "proc-1"), before); + } + + /// A root process is the one recorded only with the same PID and the same start identity, + /// so a later process that reuses the PID is told apart. A PID without its start identity + /// proves nothing. + #[test] + fn a_root_is_the_recorded_process_only_with_its_start_identity() { + let recorded = root(); + assert!(recorded.is(&root())); + let reused = RootProcess { + pid: 777, + started: Some("linux:250".into()), + }; + assert!(!recorded.is(&reused)); + let other = RootProcess { + pid: 778, + started: Some("linux:100".into()), + }; + assert!(!recorded.is(&other)); + let unnamed = RootProcess { + pid: 777, + started: None, + }; + assert!(!recorded.is(&unnamed)); + assert!(!unnamed.is(&unnamed.clone())); + } + + /// An execution that resumes an approval is recorded in the transaction that marks the + /// approval `resuming`: both are written, or neither. + #[test] + fn a_resumed_execution_is_recorded_with_its_approval_or_not_at_all() { + let store = Store::memory().unwrap(); + let runner = runner(); + let approval = queued_approval(&store, "one"); + dispatch(&store, "proc-resume", &runner, Some(&approval)).unwrap(); + assert_eq!(approval_state(&store, &approval), ApprovalState::Resuming); + assert_eq!( + record(&store, "proc-resume").approval_id, + Some(approval.clone()) + ); + + // The record cannot be written: the approval stays queued. + let second = queued_approval(&store, "two"); + let err = dispatch(&store, "proc-resume", &runner, Some(&second)).unwrap_err(); + assert_eq!(err.code, ErrorCode::Internal); + assert_eq!(approval_state(&store, &second), ApprovalState::Queued); + assert_eq!(record(&store, "proc-resume").approval_id, Some(approval)); + + // The approval cannot be resumed: nothing is recorded. + let pending = store + .create_approval( + "demo", + ApprovalTargetTool::ExecCommand, + "fp-pending", + serde_json::json!({ "workspace_id": "demo", "command": ["three"] }), + ) + .unwrap() + .approval_id; + let err = dispatch(&store, "proc-pending", &runner, Some(&pending)).unwrap_err(); + assert_eq!(err.code, ErrorCode::ApprovalConflict); + assert!(store.process_record("proc-pending").unwrap().is_none()); + assert_eq!(approval_state(&store, &pending), ApprovalState::Pending); + } + + /// A gateway that crashes while it writes an execution's record, after the statements and + /// before the commit, leaves neither the record nor the approval's resume: the next gateway + /// finds the approval queued, and the database intact. + #[test] + fn a_crash_while_writing_leaves_neither_the_record_nor_the_resume() { + const DATABASE: &str = "CODESPACE_STORE_TORN_DATABASE"; + const APPROVAL: &str = "CODESPACE_STORE_TORN_APPROVAL"; + if let (Some(database), Ok(approval)) = + (std::env::var_os(DATABASE), std::env::var(APPROVAL)) + { + // The crashing gateway, in a copy of this test binary. + let store = Store::open(Path::new(&database)).unwrap(); + CRASH_BEFORE_COMMIT.store(true, Ordering::SeqCst); + let written = dispatch(&store, "proc-torn", &runner(), Some(&ApprovalId(approval))); + panic!("the write returned instead of crashing: {written:?}"); + } + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let approval = queued_approval(&Store::open(&database).unwrap(), "torn"); + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "processes::tests::a_crash_while_writing_leaves_neither_the_record_nor_the_resume", + "--test-threads=1", + "--nocapture", + ]) + .env(DATABASE, &database) + .env(APPROVAL, &approval.0) + .output() + .unwrap(); + assert_eq!( + output.status.signal(), + Some(libc::SIGABRT), + "the copy did not crash at the commit:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let store = Store::open(&database).unwrap(); + assert!(store.process_record("proc-torn").unwrap().is_none()); + assert_eq!(approval_state(&store, &approval), ApprovalState::Queued); + let check: String = rusqlite::Connection::open(&database) + .unwrap() + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(check, "ok"); + // The next attempt records both. + dispatch(&store, "proc-torn", &runner(), Some(&approval)).unwrap(); + assert_eq!(approval_state(&store, &approval), ApprovalState::Resuming); + } + + /// A record keeps the argv's digest: neither the argv nor a secret in it reaches the + /// database file. + #[test] + fn a_record_keeps_no_argv_and_no_secret() { + const SECRET: &str = "cs-test-secret-6d1f0c9a"; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let argv = vec![ + "curl".to_string(), + "--header".to_string(), + format!("Authorization: Bearer {SECRET}"), + ]; + let runner = runner(); + { + let store = Store::open(&database).unwrap(); + store + .record_dispatch(&NewProcessRecord { + process_id: "proc-secret", + runner: &runner, + workspace_id: "demo", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &argv, + }) + .unwrap(); + store + .record_running(&runner, "proc-secret", Some(&root()), Some(1)) + .unwrap(); + store + .record_end(&runner, "proc-secret", &exited(0, 1)) + .unwrap(); + let kept = record(&store, "proc-secret"); + assert_eq!(kept.argv_digest, hash_canonical(&serde_json::json!(argv))); + assert!( + kept.argv_digest.starts_with("sha256:"), + "{}", + kept.argv_digest + ); + } + let mut bytes = Vec::new(); + for entry in std::fs::read_dir(dir.path()).unwrap() { + bytes.extend(std::fs::read(entry.unwrap().path()).unwrap()); + } + for needle in [SECRET, "Authorization", "--header"] { + assert!( + !bytes + .windows(needle.len()) + .any(|window| window == needle.as_bytes()), + "{needle} reached the database" + ); + } + } +} diff --git a/crates/store/src/writer.rs b/crates/store/src/writer.rs new file mode 100644 index 0000000..3b41754 --- /dev/null +++ b/crates/store/src/writer.rs @@ -0,0 +1,246 @@ +//! The single writer of an operations database (P1R-C02). A gateway connected to an independent +//! Runner holds an exclusive lock on a file beside its database for as long as it runs, so that +//! two gateways never record into the same database. The lock is advisory: a gateway in another +//! mode, or a CodeSpace from before process records, does not take it. + +use std::ffi::OsStr; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +/// How often a waiting gateway tries the lock again. +const RETRY: Duration = Duration::from_millis(100); + +/// An exclusive lock on `.gateway.lock`, held until it is dropped or its process +/// ends, however it ends. +#[derive(Debug)] +pub struct WriterLock { + _file: File, + path: PathBuf, +} + +impl WriterLock { + /// The lock file beside `database`. + pub fn path_for(database: &Path) -> PathBuf { + let mut name = database + .file_name() + .map(OsStr::to_os_string) + .unwrap_or_default(); + name.push(".gateway.lock"); + database.with_file_name(name) + } + + /// Take the lock beside `database`. The gateway that holds it may still be ending, as when + /// a client starts a new stdio gateway while the old one detaches, so this waits up to + /// `wait` for it, and calls `waiting` with the holder's process ID once it has to wait. + /// Then it refuses. + pub fn acquire( + database: &Path, + wait: Duration, + waiting: impl FnOnce(Option), + ) -> Result { + let path = Self::path_for(database); + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent) + .map_err(|err| format!("cannot create {}: {err}", parent.display()))?; + } + let mut file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .open(&path) + .map_err(|err| format!("cannot open the lock file {}: {err}", path.display()))?; + let deadline = Instant::now() + wait; + let mut waiting = Some(waiting); + loop { + if try_lock(&file).map_err(|err| format!("cannot lock {}: {err}", path.display()))? { + break; + } + let holder = holder(&mut file); + if let Some(waiting) = waiting.take() { + waiting(holder); + } + let now = Instant::now(); + if now >= deadline { + let holder = holder + .map(|pid| format!("process {pid}")) + .unwrap_or_else(|| "another process".into()); + return Err(format!( + "the operations database {} has another writer: {holder} holds {} and \ + did not let go within {wait:?}. One gateway records into a database at a \ + time; end that gateway first", + database.display(), + path.display(), + )); + } + std::thread::sleep(RETRY.min(deadline - now)); + } + // Who holds it, for a gateway that has to wait. + file.set_len(0) + .and_then(|()| file.seek(SeekFrom::Start(0))) + .and_then(|_| file.write_all(format!("{}\n", std::process::id()).as_bytes())) + .map_err(|err| format!("cannot write {}: {err}", path.display()))?; + Ok(Self { _file: file, path }) + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +/// Take the lock if no other open file holds it. +fn try_lock(file: &File) -> std::io::Result { + loop { + // SAFETY: flock locks the open file description behind a descriptor this file owns. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { + return Ok(true); + } + let err = std::io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EWOULDBLOCK) => return Ok(false), + Some(libc::EINTR) => continue, + _ => return Err(err), + } + } +} + +/// The process ID the holder wrote, when it has. +fn holder(file: &mut File) -> Option { + let mut text = String::new(); + file.seek(SeekFrom::Start(0)).ok()?; + file.read_to_string(&mut text).ok()?; + text.trim().parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + use std::process::{Command, Stdio}; + + fn not_waiting(holder: Option) { + panic!("waited for {holder:?}"); + } + + #[test] + fn the_lock_is_a_private_file_beside_the_database_naming_its_holder() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("state").join("ops.sqlite"); + let lock = WriterLock::acquire(&database, Duration::ZERO, not_waiting).unwrap(); + assert_eq!( + lock.path(), + dir.path().join("state").join("ops.sqlite.gateway.lock") + ); + let mode = std::fs::metadata(lock.path()).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + assert_eq!( + std::fs::read_to_string(lock.path()).unwrap(), + format!("{}\n", std::process::id()) + ); + // Only the lock: the database is not created here. + assert!(!database.exists()); + } + + #[test] + fn a_second_writer_waits_up_to_its_bound_and_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let _first = WriterLock::acquire(&database, Duration::ZERO, not_waiting).unwrap(); + let started = Instant::now(); + let mut waited_for = None; + let err = WriterLock::acquire(&database, Duration::from_millis(400), |holder| { + waited_for = Some(holder) + }) + .unwrap_err(); + assert!(started.elapsed() >= Duration::from_millis(400), "{err}"); + assert_eq!(waited_for, Some(Some(std::process::id()))); + assert!(err.contains("has another writer"), "{err}"); + assert!( + err.contains(&format!("process {}", std::process::id())), + "{err}" + ); + assert!(err.contains("ops.sqlite.gateway.lock"), "{err}"); + } + + #[test] + fn a_writer_that_lets_go_within_the_bound_is_waited_for() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let first = WriterLock::acquire(&database, Duration::ZERO, not_waiting).unwrap(); + let ending = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(300)); + drop(first); + }); + let started = Instant::now(); + let mut waited = false; + let second = + WriterLock::acquire(&database, Duration::from_secs(10), |_| waited = true).unwrap(); + ending.join().unwrap(); + assert!(waited); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(250), "{elapsed:?}"); + assert!(elapsed < Duration::from_secs(10), "{elapsed:?}"); + assert_eq!( + std::fs::read_to_string(second.path()).unwrap(), + format!("{}\n", std::process::id()) + ); + } + + /// A gateway that is killed while it holds the lock lets go of it with its process: the + /// next one never has to remove a stale lock file. + #[test] + fn a_writer_killed_holding_the_lock_lets_go_of_it() { + const HOLDER: &str = "CODESPACE_STORE_LOCK_HOLDER"; + if let Some(database) = std::env::var_os(HOLDER) { + // The holder, in a copy of this test binary: it holds the lock until it is killed. + let _lock = + WriterLock::acquire(Path::new(&database), Duration::from_secs(30), |_| {}).unwrap(); + loop { + std::thread::sleep(Duration::from_secs(60)); + } + } + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let mut holder = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "writer::tests::a_writer_killed_holding_the_lock_lets_go_of_it", + "--test-threads=1", + "--nocapture", + ]) + .env(HOLDER, &database) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let named = format!("process {}", holder.id()); + let deadline = Instant::now() + Duration::from_secs(30); + loop { + // A lock this test takes meanwhile is let go at once, before the next try. + let held = matches!( + WriterLock::acquire(&database, Duration::ZERO, |_| {}), + Err(err) if err.contains(&named) + ); + if held { + break; + } + assert!(Instant::now() < deadline, "the holder never took the lock"); + std::thread::sleep(Duration::from_millis(50)); + } + holder.kill().unwrap(); + holder.wait().unwrap(); + let lock = WriterLock::acquire(&database, Duration::from_secs(5), |_| {}).unwrap(); + assert_eq!( + std::fs::read_to_string(lock.path()).unwrap(), + format!("{}\n", std::process::id()) + ); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index e905fbd..3f79183 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,11 +35,11 @@ The UDS worker and default runner both execute on the same host. A Linux command | `server` | MCP transport, HTTP authentication/inbox, request validation and orchestration | | `domain` | CodeSpace tool parameters, results, IDs, error and execution types | | `policy` | Registered roots, environments, profiles, and network policy | -| `store` | SQLite patch operations, confirmation holds, logical works, user instructions; in-memory occupancy | +| `store` | SQLite patch operations, confirmation holds, logical works, user instructions, records of an independent Runner's executions; in-memory occupancy | | `runner` | Execution DTOs, file scope, patch transaction, process supervision, UDS client/server protocol | | Isolated adapters | Codex patch, PTY, filesystem, worker hardening/socket, Linux sandbox mechanisms | -Patch operations, confirmation holds, and works/intents survive restart only with a configured SQLite file. Process handles and occupancy leases are memory-only. `operation_status` does not track exec requests. The transport request ID, patch operation ID, process ID, work ID, instruction ID, and approval ID serve different purposes. +Patch operations, confirmation holds, and works/intents survive restart only with a configured SQLite file. Process handles and occupancy leases are memory-only; the record that a gateway with an [independent Runner](runner-isolation.md#process-records) keeps of each execution identifies it again but never revives its handle. `operation_status` does not track exec requests. The transport request ID, patch operation ID, process ID, work ID, instruction ID, and approval ID serve different purposes. diff --git a/docs/codex-reuse.md b/docs/codex-reuse.md index 63a1605..05e9982 100644 --- a/docs/codex-reuse.md +++ b/docs/codex-reuse.md @@ -37,7 +37,7 @@ Agent → CodeSpace MCP/policy/store → Runner contract Core crates have no direct Codex dependencies. Adapter crates are separate Cargo workspaces to accommodate the pinned upstream workspace dependencies. The filesystem and PTY adapters are library dependencies of the Runner; patch and Linux sandbox operations use helper processes. An isolated Cargo workspace alone does not create a process or security boundary. -The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 8 and sandbox-helper protocol version 1 are separate contracts. +The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 9 and sandbox-helper protocol version 1 are separate contracts. diff --git a/docs/devguard-integration.md b/docs/devguard-integration.md index 1f5d958..a12fdb7 100644 --- a/docs/devguard-integration.md +++ b/docs/devguard-integration.md @@ -338,7 +338,7 @@ Default participation remains `off`; `required` is an explicit operator choice i ## Initial Gateway recovery scope -**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. A gateway does not reconnect yet, and a gateway that connects after another one started work refuses new executions, patches and approval resumes with `RECOVERY_PENDING` until the Runner is stopped and a new one is started. Reconnection, process records, reconciliation and Runner loss come in the later units, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. +**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. A gateway does not reconnect yet, and a gateway that connects after another one started work refuses new executions, patches and approval resumes with `RECOVERY_PENDING` until the Runner is stopped and a new one is started. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. Reconnection, reconciliation and Runner loss come in the later units, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. P1-RECOVERY introduces an operator-selected independent Runner mode and explicit capability. Existing modes keep their current shutdown/disconnect contracts. InProcess shares the Gateway PID and is excluded from live Gateway restart recovery. diff --git a/docs/error-codes.md b/docs/error-codes.md index 305a888..5cf4a39 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -56,7 +56,7 @@ Errors use uppercase identifiers and a message. An `operation_id` may be present | `APPROVAL_AMBIGUOUS` | Resume was interrupted and the terminal result is not on disk. Includes `approval_id`; patch cases may also include `operation_id` | | `RECOVERY_PENDING` | An earlier gateway started work on this gateway's [independent Runner](runner-isolation.md#independent-runner). The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts until the Runner is stopped and a new one is started. The approval stays as it was. Not occupancy; retrying does not help | | `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and does not reconnect. The request was not sent, and the Runner's processes keep running; restart the gateway to reach them | -| `INTERNAL` | Scheduler or store invariant failed. Not occupancy; do not retry as `WORKSPACE_BUSY` | +| `INTERNAL` | Scheduler or store invariant failed, or a gateway with an [independent Runner](runner-isolation.md#process-records) could not record an execution, which it then did not start. Not occupancy; do not retry as `WORKSPACE_BUSY` | ## Dispatch and completion diff --git a/docs/ko/architecture.md b/docs/ko/architecture.md index ef8a41a..5fd7f18 100644 --- a/docs/ko/architecture.md +++ b/docs/ko/architecture.md @@ -41,11 +41,11 @@ UDS worker와 기본 러너는 모두 같은 호스트에서 실행합니다. | `server` | MCP 전송, HTTP 인증·inbox, 요청 검증과 실행 연결 | | `domain` | CodeSpace 도구 인자·결과, ID, 오류·실행 타입 | | `policy` | 등록 경로, 환경, 권한 프로필, 네트워크 정책 | -| `store` | SQLite의 패치 작업·확인 홀드·논리적 작업·사용자 지시, 메모리의 점유 상태 | +| `store` | SQLite의 패치 작업·확인 홀드·논리적 작업·사용자 지시·독립 Runner의 실행 기록, 메모리의 점유 상태 | | `runner` | 실행 데이터 타입, 파일 범위, 패치 처리, 프로세스 관리, UDS 통신 | | 분리된 어댑터 | Codex 패치·PTY·파일 시스템·worker 보호와 소켓·Linux 샌드박스 구현 연결 | -패치 작업, 확인 홀드, 작업·지시 큐는 SQLite 파일을 설정한 경우에만 재시작 후 유지됩니다. 프로세스 핸들과 점유 상태는 메모리에만 있습니다. `operation_status`는 명령 실행을 조회하지 않습니다. 전송 요청 ID, 패치 작업 ID, 프로세스 ID, 작업 ID, 지시 ID, 승인 ID는 서로 다른 대상을 가리킵니다. +패치 작업, 확인 홀드, 작업·지시 큐는 SQLite 파일을 설정한 경우에만 재시작 후 유지됩니다. 프로세스 핸들과 점유 상태는 메모리에만 있습니다. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이가 남기는 실행 기록은 그 실행을 다시 알아보게 할 뿐 핸들을 되살리지 않습니다. `operation_status`는 명령 실행을 조회하지 않습니다. 전송 요청 ID, 패치 작업 ID, 프로세스 ID, 작업 ID, 지시 ID, 승인 ID는 서로 다른 대상을 가리킵니다. diff --git a/docs/ko/codex-reuse.md b/docs/ko/codex-reuse.md index 89c4480..72f89e5 100644 --- a/docs/ko/codex-reuse.md +++ b/docs/ko/codex-reuse.md @@ -42,7 +42,7 @@ CodeSpace는 특정 버전에 고정한 Codex 소스의 실행 라이브러리 핵심 crate에는 직접적인 Codex 의존성이 없습니다. 어댑터는 고정된 업스트림의 workspace 의존성을 수용하기 위해 별도의 Cargo workspace로 구성합니다. 파일 시스템·PTY 어댑터는 Runner의 라이브러리 의존성이며, 패치와 Linux 샌드박스는 도우미 프로세스를 사용합니다. Cargo workspace를 분리하는 것만으로 프로세스나 보안 경계가 생기지는 않습니다. -Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 8과 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. +Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 9와 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. diff --git a/docs/ko/devguard-integration.md b/docs/ko/devguard-integration.md index 363daea..bdecd99 100644 --- a/docs/ko/devguard-integration.md +++ b/docs/ko/devguard-integration.md @@ -338,7 +338,7 @@ contract/client pin, daemon/helper artifact와 CodeSpace Runner wire는 각각 ## 최초 Gateway 복구 범위 -**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. gateway는 아직 다시 연결하지 않으며, 다른 gateway가 작업을 시작한 뒤에 연결한 gateway는 Runner를 중지하고 새로 시작할 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. 재연결, process 기록, 대조, Runner 손실 처리는 이후 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. +**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. gateway는 아직 다시 연결하지 않으며, 다른 gateway가 작업을 시작한 뒤에 연결한 gateway는 Runner를 중지하고 새로 시작할 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. 두 번째 단위 P1R-C02는 [실행 기록](runner-isolation.md#process-records)을 더한다. gateway는 작업 데이터베이스에 쓰는 유일한 gateway로서 각 실행을 Runner에 보내기 전에 기록하고 끝날 때까지 따라간다. 재연결, 대조, Runner 손실 처리는 이후 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. P1-RECOVERY는 운영자가 선택하는 독립 Runner 모드와 명시적인 capability를 도입한다. 기존 모드의 종료·연결 소실 계약은 유지한다. InProcess는 Gateway와 PID를 공유하므로 Gateway 재시작 중 live 복구 대상에서 제외한다. diff --git a/docs/ko/error-codes.md b/docs/ko/error-codes.md index 0e45668..b225875 100644 --- a/docs/ko/error-codes.md +++ b/docs/ko/error-codes.md @@ -59,7 +59,7 @@ | `APPROVAL_AMBIGUOUS` | 재개가 중단되어 단말 결과가 디스크에 없음. `approval_id` 포함. 패치는 `operation_id`도 있을 수 있음 | | `RECOVERY_PENDING` | 이 게이트웨이의 [독립 Runner](runner-isolation.md#independent-runner)에서 이전 게이트웨이가 작업을 시작함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, Runner를 중지하고 새로 시작하기 전에는 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 점유가 아니므로 재시도해도 소용없음 | | `RUNNER_DISCONNECTED` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 다시 연결하지 않음. 요청은 보내지 않았고 Runner의 프로세스는 계속 실행됨. 그 프로세스에 닿으려면 게이트웨이를 재시작할 것 | -| `INTERNAL` | 스케줄러·스토어 불변조건 실패. 점유가 아니므로 `WORKSPACE_BUSY`로 재시도하지 말 것 | +| `INTERNAL` | 스케줄러·스토어 불변조건 실패, 또는 [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이가 실행을 기록하지 못해 시작하지 않음. 점유가 아니므로 `WORKSPACE_BUSY`로 재시도하지 말 것 | ## 실행 요청과 완료의 구분 diff --git a/docs/ko/operations.md b/docs/ko/operations.md index 1df8474..c22b858 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -115,7 +115,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 사는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. +worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 살고 게이트웨이가 그 프로세스를 작업 데이터베이스에 기록하는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. @@ -127,7 +127,7 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 | 설정 또는 제한 | 동작 | | --- | --- | | `RUST_LOG` | stderr 로그 수준. 기본값 `info` | -| `CODESPACE_OPERATIONS_DB` 미설정 | 패치 작업과 지시 큐를 메모리에 보관하며 재시작 시 사라짐 | +| `CODESPACE_OPERATIONS_DB` 미설정 | 패치 작업과 지시 큐를 메모리에 보관하며 재시작 시 사라짐. 독립 Runner에 연결하는 게이트웨이는 이 설정 없이 시작하지 않음 | | `CODESPACE_PROCESS_TIMEOUT_SECS` | 양의 정수. 기본 30초. 러너 환경에 설정 | | `CODESPACE_MAX_PROCESSES` | 러너 전체의 실행 중 프로세스 기본 상한 8개. 넘는 요청은 프로세스를 만들기 전에 거부. 작업 공간별 점유 규칙도 적용 | | 프로세스 출력 | 마지막 256 KiB 보관. stdout/stderr를 합침. `read_process`는 `output_lost`와 `retained_from`을 보고하고, 종료는 `process_status`로 조회. 실행 중인 PTY 크기는 `process_resize`로 변경 | @@ -135,6 +135,10 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 로그와 데이터베이스는 토큰·게이트웨이 설정과 같이 관리 대상 작업 공간 밖에 두세요. stderr 로그의 보관·순환은 운영자가 관리합니다. Bearer 토큰을 로그나 커밋에 넣지 마세요. 데이터베이스를 삭제하면 패치 중복 실행 방지 기록과 확인 홀드 행도 사라집니다. + + +**스키마 버전과 이전 버전.** 작업 데이터베이스에는 스키마 버전이 적혀 있습니다. CodeSpace는 자기보다 새 스키마 버전의 데이터베이스를 아무것도 쓰기 전에 거부하고 그대로 둡니다. 그런 데이터베이스는 그것을 쓴 CodeSpace 버전으로 여세요. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 이 데이터베이스에 기록하며, 실행 중에는 그 옆의 lock 파일 `.gateway.lock`을 잡고 있습니다. 실행 기록이 생기기 전의 CodeSpace 버전도 이런 데이터베이스를 열 수 있고 기록은 건드리지 않지만, 스키마 버전을 확인하지도 lock을 걸지도 않습니다. 그러니 독립 Runner의 게이트웨이가 쓰는 데이터베이스에서는 이전 버전을 실행하지 마세요. 독립 모드에서 되돌아갈 때는 그 모드를 쓰지 않고 데이터베이스는 그대로 둡니다. 이전 사본을 복원하면 그 뒤에 쓴 패치 중복 실행 방지 기록, 확인 홀드, 실행 기록이 사라집니다. + 패치 응답을 받지 못했다면 `operation_id` 또는 `operation_key` 중 하나만 지정해 `operation_status`를 조회합니다. 재시작 후 미완료 기록은 `unknown`이 되므로 파일을 확인한 뒤 다음 행동을 결정하세요. 프로세스는 `process_id`로 관리하며 `operation_status`로 복구할 수 없습니다. 프로세스가 죽을 때 `queued`이던 확인 홀드는 다시 acquire합니다. `resuming`이던 홀드는 패치 원장에서 복구하거나, 저장된 단말 결과를 재현하거나, `APPROVAL_AMBIGUOUS`를 반환할 수 있습니다. exec는 다시 spawn하지 않습니다. [재시도와 복구 규칙](agent-integration.md)을 참고하세요. diff --git a/docs/ko/runner-isolation.md b/docs/ko/runner-isolation.md index c293036..bbb4d74 100644 --- a/docs/ko/runner-isolation.md +++ b/docs/ko/runner-isolation.md @@ -13,7 +13,7 @@ `in-process`는 `codespace-mcp` 안에서 프로세스를 관리합니다. `uds`(Unix domain socket, Unix 도메인 소켓)는 같은 호스트의 `codespace-codex-runtime` 안에서 같은 관리 코드를 실행합니다. worker는 시작 시 Codex의 프로세스 보호 설정을 적용하고 전용 Unix 소켓을 엽니다. worker 자체를 보호하는 것과 실행할 명령에 샌드박스를 적용하는 것은 별개입니다. -게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 8 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. +게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 9 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. 관리 프로세스의 수명은 MCP 연결이 아니라 러너 인스턴스가 소유합니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지합니다. 게이트웨이와 worker의 UDS 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 관리 중인 worker와 자식 프로세스가 종료되고 핸들이 사라집니다. 종료할 때 게이트웨이는 worker와의 연결을 닫으므로 worker는 종료하면서 자기 프로세스를 끝내고, 5초 뒤에도 실행 중인 worker는 강제로 종료됩니다. 재시작은 `process_id`를 복구하지 않습니다. [독립 Runner](#independent-runner)는 예외로, 그 프로세스는 게이트웨이보다 오래 삽니다. @@ -23,19 +23,20 @@ ## 독립 Runner -> **아직 지원하는 모드가 아닙니다.** 독립 Runner는 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 독립 Runner에 다시 연결하지 않으며, 다른 게이트웨이 뒤에 연결한 게이트웨이는 새 작업을 시작할 수 없습니다. 재연결과 상태 대조는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. +> **아직 지원하는 모드가 아닙니다.** 독립 Runner와 그 실행의 기록은 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 독립 Runner에 다시 연결하지 않으며, 다른 게이트웨이 뒤에 연결한 게이트웨이는 새 작업을 시작할 수 없습니다. 재연결과 상태 대조는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. 독립 Runner는 자신에게 연결하는 게이트웨이보다 오래 삽니다. 게이트웨이가 종료되거나 재시작되거나 강제로 종료되어도 프로세스와 그 출력, timeout은 계속됩니다. 운영자나 하네스가 독립 Runner를 시작하고, 게이트웨이는 연결만 합니다. ```bash codespace-codex-runtime "$RUNNER_DIR/runner.sock" --lifecycle independent codespace-mcp --runner uds --runner-socket "$RUNNER_DIR/runner.sock" \ - --runner-lifecycle independent --runner-exit-intent detach + --runner-lifecycle independent --runner-exit-intent detach \ + --operations-db "$STATE_DIR/operations.sqlite" ``` Runner는 소켓 디렉터리를 권한 0700으로 직접 만듭니다. 이미 있는 디렉터리는 같은 사용자의 디렉터리이고 권한이 0700이며 심볼릭 링크가 아닐 때만 받아들입니다. 이미 있는 디렉터리를 바꾸지 않으며 `/`, `/tmp`, `/var/tmp`, `$HOME`은 거부합니다. 게이트웨이 연결은 한 번에 하나씩 받고, 앞 연결의 요청이 모두 끝난 뒤에 다음 연결을 받습니다. 중지할 때는 소켓을 지우고, 디렉터리도 직접 만든 것이면 지웁니다. -게이트웨이의 `--runner-lifecycle`이 Runner의 수명 모드와 다르거나, `--runner-lifecycle independent`에 `--runner uds`, `--runner-socket`, `--runner-exit-intent` 중 하나가 없거나, `--runtime-bin`이나 `--runner-dir`가 함께 있으면 게이트웨이는 시작하지 않습니다. `--runner-exit-intent`는 `--runner-lifecycle independent`와 함께일 때만 받습니다. +게이트웨이의 `--runner-lifecycle`이 Runner의 수명 모드와 다르거나, Runner가 자기 식별 정보를 알리지 않거나, `--runner-lifecycle independent`에 `--runner uds`, `--runner-socket`, `--runner-exit-intent`, `--operations-db` 중 하나가 없거나, `--runtime-bin`이나 `--runner-dir`가 함께 있으면 게이트웨이는 시작하지 않습니다. `--runner-exit-intent`는 `--runner-lifecycle independent`와 함께일 때만 받습니다. | 사건 | 관리 worker(기본값) | 독립 Runner | | --- | --- | --- | @@ -52,6 +53,14 @@ Runner는 소켓 디렉터리를 권한 0700으로 직접 만듭니다. 이미 **이전 게이트웨이가 있었을 때.** 새로 시작한 Runner에 처음 연결한 게이트웨이는 관리 worker와 같이 동작합니다. 한 게이트웨이가 그 Runner에서 작업(실행, 패치, DevGuard를 쓰면 준비나 launch)을 시작한 뒤에는, 다음 게이트웨이들이 Runner의 프로세스를 넘겨받습니다. 그 게이트웨이는 `process_id`로 프로세스를 조회하고, 출력을 읽고, stdin에 쓰고, 크기를 바꾸고, 종료할 수 있지만, 새 실행, 패치, 승인 재개는 `RECOVERY_PENDING`으로 거부하고 승인은 그대로 둡니다. 넘겨받은 프로세스가 모두 끝난 뒤에도 마찬가지이므로, 작업을 다시 시작하려면 Runner를 중지하고 새로 시작합니다. 게이트웨이는 Runner의 식별 정보(프로세스 ID, 부팅과 시작 시각), 이전에 다른 게이트웨이가 제어했는지, 넘겨받은 프로세스 수를 로그에 남깁니다. + + +**실행 기록.** 게이트웨이는 각 실행을 Runner에 보내기 전에 작업 데이터베이스에 기록하고, 실행이 끝날 때까지 따라갑니다. 그래서 메모리를 잃은 게이트웨이도 같은 실행을 다시 알아볼 수 있습니다. 기록에는 `process_id`, 실행을 보낸 Runner의 식별 정보(프로세스 ID, 부팅, 시작 시각, 임의의 nonce), workspace, 실행이 재개하는 승인, DevGuard attempt, DevGuard 참여 여부, tty 여부, argv의 SHA-256 digest가 들어가며, argv 자체와 그 환경, secret은 들어가지 않습니다. 기록의 단계는 Runner가 실행을 시작했다고 답할 때까지 `dispatching`이고, 그 뒤로는 `running`입니다. Runner가 실행이 끝났다고 알리면 Runner가 보고한 종료 방식, 종료 코드, 신호와 함께 `terminal`이 되고, Runner가 실행을 시작하지 않았다고 답할 때도 `terminal`이 됩니다. 기록에는 Runner가 정한 deadline, Runner가 만든 루트 프로세스, 게이트웨이가 출력을 어디까지 읽었는지도 남습니다. 루트 프로세스는 프로세스 ID와, 나중에 같은 ID를 받은 프로세스와 구별해 주는 시작 시각으로 남기며, `tty: true` 실행의 루트는 남기지 않습니다. + +승인을 재개하는 실행은 그 승인을 resuming으로 바꾸는 transaction 안에서 기록합니다. 기록을 쓸 수 없으면 실행을 `INTERNAL`로 거부하고 아무것도 보내지 않습니다. 연결된 게이트웨이가 없는 동안 끝난 실행은 Runner가 보관했다가 다음 게이트웨이에 알리고, 그 게이트웨이가 기록합니다. `process_status`와 `read_process`도 Runner가 보고한 내용을 기록합니다. 기록이 핸들을 되살리는 일은 없으며, 기록만으로 실행이 시작되었는지나 어떻게 끝났는지가 증명되지도 않습니다. 기준은 Runner이며, 응답을 잃은 실행의 기록은 `dispatching`으로 남습니다. 기록으로 작업을 복구하는 일은 다음 단계에서 더합니다. + +**writer는 하나.** 독립 Runner에 연결한 게이트웨이는 자기 작업 데이터베이스에 쓰는 유일한 게이트웨이입니다. 데이터베이스를 열기 전에 그 옆의 파일 `.gateway.lock`에 배타 lock을 걸고, 어떤 식으로 종료하든 종료할 때까지 lock을 유지합니다. 이 파일에는 lock을 가진 게이트웨이의 프로세스 ID가 적힙니다. 앞 게이트웨이가 아직 detach하는 동안 새 게이트웨이를 시작했을 때처럼 lock이 이미 잡혀 있으면, 새 게이트웨이는 기다린다는 로그를 남기고 최대 15초 동안 기다린 뒤 시작을 거부합니다. 이 lock은 이 모드에서만 겁니다. 데이터베이스의 스키마 버전과 이전 CodeSpace 버전은 [운영](operations.md#operations-database)을 참고하세요. + **DevGuard.** `devguard` feature로 빌드한 독립 Runner는 worker의 DevGuard 설정으로 스스로 등록하며, 소비자 secret은 `--devguard-credential-file`로 지정한 비공개 파일에서 직접 읽습니다. `--devguard register`로 시작한 게이트웨이는 Runner에 secret을 넘기지 않습니다. 자원 참여를 요구하는 workspace는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 실행하며, 그렇지 않으면 `RESOURCE_POLICY_UNSUPPORTED`로 거부합니다. [DevGuard 결합](devguard-integration.md#resource-gateway-recovery)을 참고하세요. diff --git a/docs/ko/security-model.md b/docs/ko/security-model.md index dfdfbb9..1d6f2b7 100644 --- a/docs/ko/security-model.md +++ b/docs/ko/security-model.md @@ -44,7 +44,7 @@ Linux 도우미가 없으면 restricted 정책의 작업 공간에서도 비격 버전 검사는 예상하지 못한 파일 버전에 패치를 적용하는 것을 막습니다. 작업 키는 패치 중복 요청을 구분하며 인증 토큰이 아닙니다. 작업 공간 점유는 파일을 변경할 수 있는 패치·명령 실행을 직렬화합니다. 요청 소유 충돌은 `acquire()`에서 시작하는 인메모리 FIFO에서 기다리고, 라이브 프로세스는 뒤 대기자와 새 요청에 `WORKSPACE_BUSY`를 반환합니다. 점유 상태는 재시작 후에도 남지 않습니다. 영속적인 프로세스 복구는 없습니다. -`CODESPACE_OPERATIONS_DB`의 `approvals` 확인 홀드는 행이 `pending`·`granted`·`queued`·`resuming`인 동안 V4A 패치나 exec argv를 보관합니다. 패치 원장은 패치 본문이 아니라 해시를 보관합니다. `denied` 또는 `consumed` 뒤에는 홀드 본문을 digest 메타(도구, 작업 공간, fingerprint)로 바꿉니다. 이 데이터베이스는 토큰·게이트웨이 설정과 같이 작업 공간 루트 밖에 두세요. 홀드는 워크플로 일시정지이며 같은 MCP 호출자가 grant할 수 있습니다. 격리 경계가 아닙니다. +`CODESPACE_OPERATIONS_DB`의 `approvals` 확인 홀드는 행이 `pending`·`granted`·`queued`·`resuming`인 동안 V4A 패치나 exec argv를 보관합니다. 패치 원장은 패치 본문이 아니라 해시를 보관합니다. `denied` 또는 `consumed` 뒤에는 홀드 본문을 digest 메타(도구, 작업 공간, fingerprint)로 바꿉니다. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 여기에 기록하는데(`processes`), argv는 SHA-256 digest로만 남기고 argv 자체와 그 환경, secret은 남기지 않습니다. 이 데이터베이스는 토큰·게이트웨이 설정과 같이 작업 공간 루트 밖에 두세요. 홀드는 워크플로 일시정지이며 같은 MCP 호출자가 grant할 수 있습니다. 격리 경계가 아닙니다. 패치 스냅샷 복원은 가능한 범위에서 수행하며 모든 실패의 롤백을 보장하지 않습니다. `unknown`, 부분 실패, 적용 후 검증 오류가 발생하면 해당 파일을 확인하세요. [패치 동작](behavior-differences.md)과 [연동 복구 규칙](agent-integration.md)을 참고하세요. diff --git a/docs/operations.md b/docs/operations.md index 0eb8ec7..9d46fff 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -109,7 +109,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, is not a supported mode yet. +The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway and whose gateway records them in the operations database, is not a supported mode yet. @@ -119,7 +119,7 @@ The worker runs on the same host and is not a container. The gateway creates a p | Setting or limit | Behavior | | --- | --- | | `RUST_LOG` | stderr tracing level; default `info` | -| `CODESPACE_OPERATIONS_DB` unset | In-memory patch operations and instruction queue; lost on restart | +| `CODESPACE_OPERATIONS_DB` unset | In-memory patch operations and instruction queue; lost on restart. A gateway with an independent Runner does not start without it | | `CODESPACE_PROCESS_TIMEOUT_SECS` | Positive integer; default 30 seconds; set in the runner environment | | `CODESPACE_MAX_PROCESSES` | Default 8 live processes across the runner; a request over it is refused before any process is created; workspace occupancy still applies | | Process output | Last 256 KiB retained; stdout/stderr combined; `read_process` reports `output_lost` and `retained_from`; `process_status` reports termination; `process_resize` resizes a running PTY | @@ -127,6 +127,10 @@ The worker runs on the same host and is not a container. The gateway creates a p Store logs and the database outside the managed workspace, with tokens and gateway configuration. Rotate stderr capture yourself. Do not log Bearer tokens or commit real credentials. Deleting the database also deletes patch idempotency records and confirmation-hold rows. + + +**Schema version and earlier versions.** The operations database states its schema version. CodeSpace refuses a database with a newer schema version than its own before writing anything, and leaves it as it is: open it with the CodeSpace version that wrote it. A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions in the database, and holds the lock file `.gateway.lock` beside it while it runs. A CodeSpace version from before these records still opens such a database and leaves the records alone, but it neither checks the schema version nor takes the lock, so do not run one on a database that a gateway with an independent Runner uses. To go back from the independent mode, stop using it and keep the database: restoring an earlier copy would drop the patch idempotency records, confirmation holds and process records written since. + After losing a patch response, query `operation_status` with exactly one of `operation_id` or `operation_key`. An unfinished record becomes `unknown` after restart; inspect files before deciding what to do. Processes use `process_id` and cannot be recovered through `operation_status`. A confirmation hold that was `queued` when the process died can be resumed and will reacquire. A hold that was `resuming` may recover a patch from that ledger, replay a stored terminal result, or return `APPROVAL_AMBIGUOUS`; exec is not respawned. See [retry and recovery rules](agent-integration.md). diff --git a/docs/runner-isolation.md b/docs/runner-isolation.md index 0ba8417..6b6a7aa 100644 --- a/docs/runner-isolation.md +++ b/docs/runner-isolation.md @@ -10,7 +10,7 @@ There are three separate questions: which process runs the work, whether command `in-process` runs the supervisor inside `codespace-mcp`. `uds` (Unix domain socket) runs that supervisor inside `codespace-codex-runtime` on the same host. The worker starts with Codex process hardening and binds a private Unix socket. Hardening the worker does not sandbox its commands. -The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 8, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. +The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 9, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. Managed process lifetime is owned by the runner instance, not the MCP connection. Streamable HTTP or MCP client disconnect keeps processes running. Gateway/worker UDS disconnect or gateway shutdown (including stdio EOF) ends the owned worker and its children; process handles are lost. At shutdown the gateway closes the worker's connection, so the worker ends its processes as it exits; a worker still running 5 seconds later is killed. Restart does not restore `process_id`. An [independent Runner](#independent-runner) is the exception: its processes outlive the gateway. @@ -20,19 +20,20 @@ Managed process lifetime is owned by the runner instance, not the MCP connection ## Independent Runner -> **Not a supported mode yet.** The independent Runner is the first step of Gateway recovery. A gateway does not reconnect to it, and a gateway that connects after another one cannot start new work. Later steps add reconnection and reconciliation, and the mode is supported only once it is qualified. +> **Not a supported mode yet.** The independent Runner and the records of its executions are the first steps of Gateway recovery. A gateway does not reconnect to it, and a gateway that connects after another one cannot start new work. Later steps add reconnection and reconciliation, and the mode is supported only once it is qualified. An independent Runner outlives the gateways that connect to it: its processes, their output and their timeouts carry on when a gateway exits, restarts or is killed. An operator or a harness starts it, and a gateway only connects to it: ```bash codespace-codex-runtime "$RUNNER_DIR/runner.sock" --lifecycle independent codespace-mcp --runner uds --runner-socket "$RUNNER_DIR/runner.sock" \ - --runner-lifecycle independent --runner-exit-intent detach + --runner-lifecycle independent --runner-exit-intent detach \ + --operations-db "$STATE_DIR/operations.sqlite" ``` The Runner creates its socket's directory with mode 0700, or accepts an existing one only when it is a directory of the same user with mode 0700 and not a symbolic link. It never changes an existing directory, and it refuses `/`, `/tmp`, `/var/tmp` and `$HOME`. It serves one gateway connection at a time, and accepts the next only after every request of the previous one has ended. When it stops, it removes its socket, and the directory if it created it. -A gateway does not start when its `--runner-lifecycle` differs from the Runner's lifecycle, or when `--runner-lifecycle independent` comes without `--runner uds`, `--runner-socket` or `--runner-exit-intent`, or with `--runtime-bin` or `--runner-dir`. `--runner-exit-intent` is accepted only with `--runner-lifecycle independent`. +A gateway does not start when its `--runner-lifecycle` differs from the Runner's lifecycle, when the Runner does not state its identity, or when `--runner-lifecycle independent` comes without `--runner uds`, `--runner-socket`, `--runner-exit-intent` or `--operations-db`, or with `--runtime-bin` or `--runner-dir`. `--runner-exit-intent` is accepted only with `--runner-lifecycle independent`. | Event | Managed worker (default) | Independent Runner | | --- | --- | --- | @@ -49,6 +50,14 @@ Ending a process signals a tty process's process group, but a pipe process alone **After an earlier gateway.** The first gateway that connects to a newly started Runner works as it does with a managed worker. Once a gateway has started work there (an execution, a patch, or with DevGuard a preparation or a launch), every later gateway inherits the Runner's processes: it can query, read, write to, resize and terminate them with their `process_id`, but it refuses new executions, patches and approval resumes with `RECOVERY_PENDING`, and leaves the approvals as they are. This holds after the inherited processes have ended too; to start work again, stop the Runner and start a new one. The gateway logs the Runner's identity (its process ID, boot and start time), whether a gateway controlled it before and how many processes it inherited. + + +**Process records.** The gateway records each execution in its operations database before it sends the execution to the Runner, and follows it to its end, so that a gateway that lost its memory can identify the same execution again. A record holds the `process_id`, the identity of the Runner it was sent to (its process ID, boot, start time and a random nonce), the workspace, the approval it resumes, its DevGuard attempt, whether it takes part in DevGuard, whether it has a tty, and the SHA-256 digest of its argv, never the argv, its environment or a secret. Its phase is `dispatching` until the Runner answers that it started the execution, and `running` after that. It becomes `terminal` once the Runner reports that the execution ended, with the termination, exit code and signal the Runner reported, or once the Runner answers that it did not start it. The record also keeps the deadline the Runner set, the root process the Runner created, by its process ID and the start time that tells it apart from a later process with the same ID (a `tty: true` root is not named), and how far a gateway read the output. + +An execution that resumes an approval is recorded in the transaction that marks the approval resuming. When the record cannot be written, the execution is refused with `INTERNAL` and nothing is sent. The next gateway records the exits that happened while no gateway was connected, which the Runner keeps for it, and `process_status` and `read_process` record what the Runner reports. A record never revives a handle, and never by itself proves that an execution started or how it ended: the Runner is the authority, and the record of an execution whose answer was lost stays `dispatching`. Recovering work from the records comes in later steps. + +**One writer.** A gateway with an independent Runner is its operations database's only writer. Before it opens the database it takes an exclusive lock on a file beside it, `.gateway.lock`, which names the process ID of the gateway that holds it, and it keeps the lock until it exits, however it exits. A gateway that finds the lock held, such as one started while the gateway before it still detaches, logs that it waits, waits up to 15 seconds, and then refuses to start. Only this mode takes the lock. See [Operations](operations.md#operations-database) for the database's schema version and earlier CodeSpace versions. + **DevGuard.** With the `devguard` feature, the independent Runner registers itself with the worker's DevGuard settings and reads the consumer secret from the private file given with `--devguard-credential-file`; its gateway, started with `--devguard register`, hands it no secret. Workspaces that require resource participation run only when the Runner states that it registers and launches through a launch helper, and are refused with `RESOURCE_POLICY_UNSUPPORTED` otherwise. See [DevGuard integration](devguard-integration.md#resource-gateway-recovery). diff --git a/docs/security-model.md b/docs/security-model.md index b220d4e..de4d951 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -34,7 +34,7 @@ The operator-registered root is a trust anchor. Keep tokens, the operations data Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery. -Confirmation holds in `CODESPACE_OPERATIONS_DB` (`approvals`) store the V4A patch or exec argv while the row is `pending`, `granted`, `queued`, or `resuming`. The patch operations ledger stores hashes, not patch text. After `denied` or `consumed`, the hold body is replaced with digest metadata (tool, workspace, fingerprint). Keep this database outside the workspace root, with tokens and gateway configuration. The hold is a workflow pause: the same MCP caller can grant it. It is not an isolation boundary. +Confirmation holds in `CODESPACE_OPERATIONS_DB` (`approvals`) store the V4A patch or exec argv while the row is `pending`, `granted`, `queued`, or `resuming`. The patch operations ledger stores hashes, not patch text. After `denied` or `consumed`, the hold body is replaced with digest metadata (tool, workspace, fingerprint). A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions there (`processes`), each with the SHA-256 digest of its argv, never the argv, its environment or a secret. Keep this database outside the workspace root, with tokens and gateway configuration. The hold is a workflow pause: the same MCP caller can grant it. It is not an isolation boundary. Patch snapshot restoration is best effort and is not an all-failure rollback guarantee. On `unknown`, partial failure, or post-apply verification error, inspect affected files. See [patch behavior](behavior-differences.md) and [integration recovery rules](agent-integration.md). diff --git a/docs/translations.json b/docs/translations.json index 716660e..c0e8e7f 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -78,8 +78,8 @@ "최소-도입-조건", "최초-gateway-복구-범위" ], - "source_sha256": "2baf08aad540a8065010299f5f48d2614492938e2df400133dfaa08cb10bfb2c", - "translation_sha256": "9d0ee4193edbc99e725622d78b6178850c7658a7354a9f40184c36c127ec7f09" + "source_sha256": "e5cfae184723094b5cbc3b271c3d704cf755fbdd0a2866be544017accbab5647", + "translation_sha256": "b566cc59248d0913b80cf337a4809202de48e3d251a628c0873dac4ee69c2649" }, { "id": "cs-rg-qualification", @@ -278,6 +278,7 @@ "mvp-흐름-재현", "mvp-흐름-재현", "operations", + "operations-database", "optional-unix-socket-worker", "recovery-after-disconnect-or-restart", "register-a-workspace", @@ -309,8 +310,8 @@ "작업-공간-등록", "첫-연결-확인" ], - "source_sha256": "99d45a3addf7a209b087b3663af59dddf73b3a0043c0f0e9194e73fd24732c07", - "translation_sha256": "972fd7b98c226bb7607b17e6cbd872ae584cd189c6017ae49a8c9b6a3f8bd385" + "source_sha256": "6771c8a61ac3106aec333a32dbb72d7dd6a9f28f1d79fe43fff4a36d066592d7", + "translation_sha256": "179443bfeaf66b429399e7f199977544833958add18416c1266cd06f85f7ad0b" }, { "id": "chatgpt-connector", @@ -399,8 +400,8 @@ "현재-실행-구조", "확장-시-유지할-경계" ], - "source_sha256": "ff67a5d9dcfa1943b2a54eb27c41e3ca3d2e24f532c12f4b06091f0f5e31821c", - "translation_sha256": "6ea5c1e57e1dafea7e2d2806fca18574584b9308b411e810e9ae704a7c6ef751" + "source_sha256": "1e7fb7ea6df71ceb47c00c43948c1d82bd99461c4dc5798974cd9508ad4ab690", + "translation_sha256": "d46c5e472191b3071cd0e27c08d37195f637eb15c28644653f0292057458daea" }, { "id": "execution-substrate", @@ -575,8 +576,8 @@ "프로세스-정직성", "프로필-mvp" ], - "source_sha256": "76c0e770f66e5fdffce7c6ec21be14ea4204e801fc3f373c9f63687be9d3c9b2", - "translation_sha256": "be0b5e577dcf358df4e453f201e915adbeda9465b2d73a3ddad9c92c22cb3ba0" + "source_sha256": "c500143209c56fc7949beb8fd3d46ccb5dffa59d34ef987d4095388d8965f69b", + "translation_sha256": "262d567a19dad4f1f1ab617ac24f215bb45e6578b41b71dd48b11729445dbd86" }, { "id": "runner-isolation", @@ -598,6 +599,7 @@ "macos-docker-없음", "macos-no-docker", "network-and-filesystem-scope", + "process-records", "runner-isolation", "what-the-compose-fixture-does", "이후-프로세스-분리", @@ -608,8 +610,8 @@ "컨테이너-실험-구성과-검증", "호스트와-worker-실행" ], - "source_sha256": "513bdd237464fa077354b3279b336dae6d9276403b13ca3bf3465dea9898d08c", - "translation_sha256": "c49af49c145ae3e13c07ee7eb58da5179685720aa039fd66c6dc05680a9061f1" + "source_sha256": "4afc4358cc0ae2c5437e27c8530ab69de89d8b9b5a92447896cb8aff18359b94", + "translation_sha256": "f1eb75de13e8037baddc37b256d9e89e77a44489296996e4ef84e161d9f12c36" }, { "id": "error-codes", @@ -637,8 +639,8 @@ "전송-실패", "전송-실패-작업-없음" ], - "source_sha256": "d37b93d073a059ff993b2309efdedc98d51a80f3f32b5802f66b62d0e9e1338a", - "translation_sha256": "9e100facbb0cc372e235733e6c093eb1a4a3e4d9903343bf6b098df8e745e294" + "source_sha256": "6ef6e724e3cf3b46d259fcf7fb60fa2d7912d7e2b7ca438eb7a230ac822d022f", + "translation_sha256": "9ccea1fedaf5129cf948c5d4671d54369753b923e9ca016a6dcce11e1d27661b" }, { "id": "codex-reuse", @@ -719,8 +721,8 @@ "핀-6b9826e의-후보", "핵심-대-어댑터" ], - "source_sha256": "cfacce5889ed00ed9a7338f98b80c20f05947ee0af5e913fb29e4b64869a403b", - "translation_sha256": "bef8eb0b952cf5c47e6df069076dca6dfbae97dcb07f78609f52f977172a79cb" + "source_sha256": "d2d099d4d51afb70810a3512a2463eeaf2d94efc272697dd278b82fac7c0d9c3", + "translation_sha256": "0db076e7693509c777c47d3aa9819f81463f763be63705ad0eb426c5f62121e1" }, { "id": "upstream-lock", diff --git a/scripts/validate-upstream.py b/scripts/validate-upstream.py index 752e1f8..0dd9c92 100644 --- a/scripts/validate-upstream.py +++ b/scripts/validate-upstream.py @@ -76,6 +76,11 @@ def stages(): # macOS path the Linux legs cannot exercise. + [cargo('test', 'root', '-p', 'codespace-runner', '--lib', '--', 'descriptors', 'fork_handlers', 'missing_executable_is_process_spawn_failed'), cargo('test', 'root', '-p', 'codespace-server', '--lib', '--', 'descriptors')] + # The process records of the gateway with an independent Runner (P1R-C02): the operations + # database's schema version, its records and its single writer's lock, and the root + # process that a spawn reports with its macOS start identity. + + [cargo('test', 'root', '-p', 'codespace-store', '--lib'), + cargo('test', 'root', '-p', 'codespace-runner', '--lib', '--', 'a_spawn_reports_its_root')] # The DevGuard adapter against an authority with native host evidence and DevGuard's # launch helper, and the DevGuard tests of the runner (registration, its wire, the # preparation of governed executions, CSRG-U3, and their launch, CSRG-U4), the worker From a4f362c8026276793eebd0add5f57c7a122a10f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 17:37:25 +0000 Subject: [PATCH 2/2] server: compare a root with the shell's pid only unsandboxed (P1R-C02) Inside the Linux sandbox the shell's `$$` is its PID in the sandbox's own PID namespace, and the root the Runner reports is the sandbox helper, so the two record tests compared a host PID with a namespace PID wherever bubblewrap works, as in CI's integration stage. They now compare them only without the sandbox, and the restarted-gateway test checks, sandboxed or not, that the running execution's recorded root is a live child of the Runner. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/server/src/mcp.rs | 6 +++++- crates/server/tests/independent_runner.rs | 22 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/server/src/mcp.rs b/crates/server/src/mcp.rs index f8591bb..569a4dc 100644 --- a/crates/server/src/mcp.rs +++ b/crates/server/src/mcp.rs @@ -4825,7 +4825,11 @@ mod independent_tests { ); assert_eq!((record.exit_code, record.signal), (Some(3), None)); let root_process = record.root.clone().expect("the root process"); - assert_eq!(root_process.pid.to_string(), output.chunk.trim()); + // Unsandboxed, the root is the shell itself. The Linux sandbox's root is its helper, and + // the shell's `$$` is its PID in the sandbox's own PID namespace. + if !codespace_runner::linux_sandbox_available() { + assert_eq!(root_process.pid.to_string(), output.chunk.trim()); + } if cfg!(any(target_os = "linux", target_os = "macos")) { assert!(root_process.started.is_some(), "{record:?}"); } diff --git a/crates/server/tests/independent_runner.rs b/crates/server/tests/independent_runner.rs index 4672e68..2556640 100644 --- a/crates/server/tests/independent_runner.rs +++ b/crates/server/tests/independent_runner.rs @@ -340,6 +340,15 @@ async fn still_beating(beat: &Path) -> bool { before != read() } +/// The parent of the live process `pid`, as `ps` reports it on Linux and macOS. +fn parent_of(pid: u32) -> Option { + let output = std::process::Command::new("ps") + .args(["-o", "ppid=", "-p", &pid.to_string()]) + .output() + .ok()?; + String::from_utf8_lossy(&output.stdout).trim().parse().ok() +} + /// A gateway killed while its process runs leaves the process to the Runner: the next gateway /// reads it and sees it end at its original timeout, not at one restarted when that gateway /// connected, but starts nothing itself, and its normal shutdown with the intent `stop` then @@ -623,13 +632,18 @@ async fn a_restarted_gateway_waits_for_the_one_before_it_and_finds_its_records() ); assert_eq!(record.exit_code, Some(4)); assert_eq!(record.runner.pid, runner_pid); - assert_eq!( - record.root.map(|root| root.pid.to_string()), - Some(root_pid.trim().to_string()) - ); + let root = record.root.expect("its root process"); + // Unsandboxed, the root is the shell itself. The Linux sandbox's root is its helper, and + // the shell's `$$` is its PID in the sandbox's own PID namespace. + if !codespace_runner::linux_sandbox_available() { + assert_eq!(root.pid.to_string(), root_pid.trim()); + } let record = store.process_record(&running).unwrap().expect("its record"); assert_eq!(record.phase, ProcessPhase::Running, "{record:?}"); assert_eq!(record.runner.pid, runner_pid); + // The running execution's root is a live process that the Runner created, sandboxed or not. + let root = record.root.expect("its root process"); + assert_eq!(parent_of(root.pid), Some(runner_pid), "{root:?}"); let version: i64 = rusqlite::Connection::open(setup.database()) .unwrap() .query_row("PRAGMA user_version", [], |row| row.get(0))