From 4e54309671a2fc55658a16fd3c8a9511e8e790af Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 20:17:09 +0000 Subject: [PATCH] runner, server, store: Claim and controller epoch for the independent Runner (P1R-C03) An independent Runner now serves every connection it accepts, and one connection at a time controls it after a Claim that names the gateway's operations database. The Runner grants a Claim only when the peer credentials of the connection show its own user, and only for the operations database of its first Claim, whose random identity the database keeps in a new `store_identity` table: a guard against a gateway with the wrong database, not a credential. A Claim is refused while another connection controls the Runner and is open, a stopped gateway's included. Once that connection ended, a Claim waits up to 10 seconds for its requests, preparations and launches included, and is refused past that bound. Each granted Claim raises the controller epoch, which never decreases, and answers with every process the Runner holds and the number of the last exit event that list reflects. Exit events are numbered and go to the controller only; one whose write failed goes to the next controller. Only the controller's requests are answered beyond Hello, Claim and Detach. Exec, apply_patch, write_stdin, resize, terminate, terminate workspace, Stop, and with DevGuard Prepare and Launch carry the epoch of the Claim they were sent under, and the Runner refuses one from an earlier epoch with the new STALE_CONTROLLER. The PR1 barrier holds: a controller after one that started work, its own earlier connection included, starts nothing and gets RECOVERY_PENDING. A gateway claims its Runner at start, asking again for up to 30 seconds while another connection controls it. When its connection is lost, or it closed it after a call deadline, it connects again, checks that the same Runner answers, and claims it again, pausing from 100 ms up to 5 s between tries. Meanwhile it refuses requests that need the Runner with the new RUNNER_RECONNECTING; once nothing accepts connections at the socket, another Runner answers there, or the Runner refuses the Claim for a reason that waiting cannot end, it refuses them with RUNNER_DISCONNECTED. Each exit is handed on once. Executions are recorded with the epoch they are sent under. A request refused with STALE_CONTROLLER, RECOVERY_PENDING, RUNNER_RECONNECTING or RUNNER_DISCONNECTED never ran: the approval it resumes is given back with return_resume or stays queued, a patch's operation is forgotten so its operation_key begins afresh, and an execution is recorded as not started. Whether new work can be sent is checked before claim_resume, and again once the workspace is held. The worker binds Tokio's listener, which codex-uds wraps on Unix, to read each connection's peer credentials, and logs each Claim with its peer's pid. The worker protocol is now 10, and Hello no longer states previous_controller and inherited_processes, which the Claim's answer replaces. The docs, English and Korean, describe the Claim, the epoch, the reconnect, the new codes, and the exposure that remains: while no gateway controls the Runner, any process of the same user that can reach its socket can claim it, a command run without the Linux sandbox included. The macOS validation stage, which runs these tests, now says that it covers the Claim's peer credentials. The mode stays unsupported. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/bin/codespace-codex-runtime.rs | 70 +- crates/codex-runtime/tests/runtime_binary.rs | 95 +- crates/domain/src/error.rs | 20 +- crates/runner/src/lib.rs | 39 +- crates/runner/src/process.rs | 48 +- crates/runner/src/uds.rs | 769 ++++- crates/runner/src/wire.rs | 2540 ++++++++++++++--- crates/runner/tests/uds_runner.rs | 2 +- crates/server/src/devguard.rs | 2 - crates/server/src/main.rs | 25 +- crates/server/src/mcp.rs | 677 ++++- crates/server/src/records.rs | 7 +- crates/server/src/runtime.rs | 150 +- crates/server/tests/independent_runner.rs | 206 +- crates/store/src/approvals.rs | 24 +- crates/store/src/lib.rs | 97 + crates/store/src/processes.rs | 12 +- docs/agent-integration.md | 2 +- docs/architecture.md | 2 +- docs/codex-reuse.md | 4 +- docs/devguard-integration.md | 2 +- docs/error-codes.md | 6 +- docs/ko/agent-integration.md | 2 +- docs/ko/architecture.md | 2 +- docs/ko/codex-reuse.md | 4 +- docs/ko/devguard-integration.md | 2 +- docs/ko/error-codes.md | 6 +- docs/ko/operations.md | 4 +- docs/ko/runner-isolation.md | 30 +- docs/ko/security-model.md | 4 +- docs/operations.md | 4 +- docs/runner-isolation.md | 30 +- docs/security-model.md | 4 +- docs/translations.json | 34 +- scripts/validate-upstream.py | 7 +- 35 files changed, 4156 insertions(+), 776 deletions(-) diff --git a/crates/codex-runtime/src/bin/codespace-codex-runtime.rs b/crates/codex-runtime/src/bin/codespace-codex-runtime.rs index 66016c8..85cd208 100644 --- a/crates/codex-runtime/src/bin/codespace-codex-runtime.rs +++ b/crates/codex-runtime/src/bin/codespace-codex-runtime.rs @@ -15,11 +15,12 @@ //! //! With `--lifecycle independent` (P1R-C01) an operator or a harness starts the worker, not a //! gateway: it is an independent Runner. It prepares its socket's directory itself, created -//! with mode 0700 or accepted only as a private directory of its user, and serves one gateway -//! connection at a time. Its processes, their output and their timeouts outlive each connection -//! until a gateway's `Stop`, or the Runner's own SIGTERM or SIGINT, ends them and the Runner. -//! With DevGuard it reads the consumer secret from `--devguard-credential-file`, as no gateway -//! hands it one. +//! with mode 0700 or accepted only as a private directory of its user, and serves every +//! connection it accepts, of which one at a time, after its `Claim`, controls it (P1R-C03). +//! It logs each Claim with the process ID of its peer. Its processes, their output and their +//! timeouts outlive each connection until its controller's `Stop`, or the Runner's own SIGTERM +//! or SIGINT, ends them and the Runner. With DevGuard it reads the consumer secret from +//! `--devguard-credential-file`, as no gateway hands it one. use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -30,7 +31,7 @@ use codespace_runner::serve_runner_connection; use codespace_runner::serve_runner_connection_with_registration; use codespace_runner::{ allocate_private_runner_dir, host_worker, prepare_runner_socket_dir, reclaim_leftover_socket, - runner_socket_path, IndependentRunner, RunnerLifecycle, Served, STOP_DRAIN_BOUND, + runner_socket_path, ClaimNote, IndependentRunner, RunnerLifecycle, Served, STOP_DRAIN_BOUND, }; fn main() { @@ -138,7 +139,8 @@ async fn serve() { refuse(2, "--lifecycle independent needs the socket's path first") }); let (runner, events) = host_worker(); - let independent = IndependentRunner::new(runner, events); + let independent = IndependentRunner::new(runner, events) + .with_claim_observer(std::sync::Arc::new(log_claim)); #[cfg(feature = "devguard")] let independent = independent.with_registration(registration, launcher); std::process::exit(serve_independent(&socket, independent).await); @@ -146,9 +148,39 @@ async fn serve() { } } -/// An independent Runner's life (P1R-C01): its socket, the connections it serves one at a -/// time, and its stop, after which its socket, and the directory it created for it, are -/// removed. The exit code. +/// What an independent Runner logs of each Claim (P1R-C03): any process of its user that can +/// reach its socket may claim it while no connection controls it. +fn log_claim(note: ClaimNote) { + let peer = |pid: Option| pid.map_or_else(|| "unknown".into(), |pid| pid.to_string()); + match note { + ClaimNote::Claimed { + epoch, + previous_controller, + peer_pid, + } => eprintln!( + "codespace-codex-runtime: process {} claimed the Runner under epoch {epoch}{}", + peer(peer_pid), + if previous_controller { + "; an earlier controller started work, so it starts nothing" + } else { + "" + } + ), + ClaimNote::Refused { + refusal, + message, + peer_pid, + } => eprintln!( + "codespace-codex-runtime: refused the Claim of process {} ({}): {message}", + peer(peer_pid), + refusal.as_str() + ), + } +} + +/// An independent Runner's life (P1R-C01): its socket, the connections it serves, and its +/// stop, after which its socket, and the directory it created for it, are removed. The exit +/// code. async fn serve_independent(socket: &Path, independent: IndependentRunner) -> i32 { let dir = match socket.parent() { Some(dir) if !dir.as_os_str().is_empty() => dir, @@ -163,7 +195,9 @@ async fn serve_independent(socket: &Path, independent: IndependentRunner) -> i32 rendezvous.remove_dir(); refuse(1, err); } - let listener = match codex_uds::UnixListener::bind(socket).await { + // Tokio's listener, as codex-uds binds on Unix: a Claim needs its peer's credentials from + // the accepted stream (P1R-C03). + let listener = match tokio::net::UnixListener::bind(socket) { Ok(listener) => listener, Err(err) => { rendezvous.remove_dir(); @@ -196,20 +230,20 @@ async fn serve_independent(socket: &Path, independent: IndependentRunner) -> i32 std::process::id(), socket.display() ); - let listener = std::sync::Arc::new(tokio::sync::Mutex::new(listener)); + let listener = std::sync::Arc::new(listener); let terminated = independent .run( move || { let listener = listener.clone(); - async move { listener.lock().await.accept().await } + async move { listener.accept().await.map(|(stream, _)| stream) } }, |ending| match ending { - Ok(Served::Detached) => { - eprintln!("codespace-codex-runtime: a gateway detached; its processes go on") - } + Ok(Served::Detached) => eprintln!( + "codespace-codex-runtime: the controlling gateway detached; its processes go on" + ), Ok(Served::Lost) => eprintln!( - "codespace-codex-runtime: a gateway's connection ended without a detach or a \ - stop; its processes go on" + "codespace-codex-runtime: the controlling gateway's connection ended without a \ + detach or a stop; its processes go on" ), Ok(Served::Stopped { .. }) => {} Err(err) => eprintln!("codespace-codex-runtime: accept: {err}"), diff --git a/crates/codex-runtime/tests/runtime_binary.rs b/crates/codex-runtime/tests/runtime_binary.rs index ee977b6..bc98943 100644 --- a/crates/codex-runtime/tests/runtime_binary.rs +++ b/crates/codex-runtime/tests/runtime_binary.rs @@ -9,10 +9,12 @@ use std::time::Duration; use codespace_domain::{ErrorCode, ProcessId, ProcessState, Profile, WorkspaceId}; use codespace_policy::Workspace; use codespace_runner::{ - read_frame, write_frame, Runner, RunnerError, RunnerExecRequest, RunnerLifecycle, RunnerOp, - RunnerOpResult, UdsRunner, WireEnvelope, WIRE_PROTOCOL, + read_frame, write_frame, Runner, RunnerClaim, RunnerError, RunnerExecRequest, RunnerLifecycle, + RunnerOp, RunnerOpResult, UdsRunner, WireEnvelope, WIRE_PROTOCOL, }; +use tokio::io::AsyncBufReadExt; use tokio::net::UnixStream; +use tokio::sync::mpsc; const BIN: &str = env!("CARGO_BIN_EXE_codespace-codex-runtime"); @@ -144,7 +146,7 @@ async fn hello_on_live_socket() { match parsed.result { Some(RunnerOpResult::Hello { protocol, .. }) => { assert_eq!(protocol, WIRE_PROTOCOL); - assert_eq!(protocol, 9); + assert_eq!(protocol, 10); } other => panic!("unexpected hello {other:?}"), } @@ -212,21 +214,47 @@ async fn worker_socket_drop_terminates_owned_child() { ); } -/// An independent Runner (P1R-C01) at `socket`, whose directory it prepares itself. -fn spawn_independent(socket: &Path) -> tokio::process::Child { - tokio::process::Command::new(BIN) +/// An independent Runner (P1R-C01) at `socket`, whose directory it prepares itself, and the +/// lines it logs. +fn spawn_independent(socket: &Path) -> (tokio::process::Child, mpsc::UnboundedReceiver) { + let mut runner = tokio::process::Command::new(BIN) .arg(socket) .args(["--lifecycle", "independent"]) + .stderr(std::process::Stdio::piped()) .kill_on_drop(true) .spawn() - .expect("spawn an independent Runner") + .expect("spawn an independent Runner"); + let (lines, logged) = mpsc::unbounded_channel(); + let mut read = tokio::io::BufReader::new(runner.stderr.take().unwrap()).lines(); + tokio::spawn(async move { + while let Ok(Some(line)) = read.next_line().await { + let _ = lines.send(line); + } + }); + (runner, logged) +} + +/// The first line logged that contains `text`. +async fn logged(lines: &mut mpsc::UnboundedReceiver, text: &str) -> String { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let line = lines.recv().await.expect("the Runner logs it"); + if line.contains(text) { + return line; + } + } + }) + .await + .unwrap_or_else(|_| panic!("the Runner does not log {text:?}")) } -/// A gateway's connection to the Runner at `socket`, after its handshake. -async fn gateway(socket: &Path) -> UdsRunner { +/// A gateway's connection to the Runner at `socket`, after its handshake and its Claim +/// (P1R-C03). +async fn gateway(socket: &Path) -> (UdsRunner, RunnerClaim) { let runner = UdsRunner::from_stream(wait_connect(socket).await, Arc::new(|_| {})); runner.handshake().await.expect("hello"); - runner + let claim = runner.claim("store-test").await.expect("claim"); + (runner, claim) } fn mode(path: &Path) -> u32 { @@ -285,15 +313,16 @@ fn demo_workspace(root: &Path) -> Workspace { /// An independent Runner keeps its pipe and tty processes after its gateway detaches, the next /// gateway inherits them but starts nothing, and a stop ends them and the Runner, which removes -/// its socket and the directory it created (P1R-C01). +/// its socket and the directory it created (P1R-C01). The Runner logs each Claim with the +/// process that made it (P1R-C03). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn an_independent_runner_outlives_its_gateways_until_one_stops_it() { let base = tempfile::tempdir().unwrap(); std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); let dir = base.path().join("runner"); let socket = runner_socket(&dir); - let mut runner = spawn_independent(&socket); - let first = gateway(&socket).await; + let (mut runner, mut lines) = spawn_independent(&socket); + let (first, claim) = gateway(&socket).await; assert_eq!(mode(&dir), 0o700); assert_eq!(mode(base.path()), 0o755); let hello = first.hello(); @@ -302,7 +331,15 @@ async fn an_independent_runner_outlives_its_gateways_until_one_stops_it() { hello.identity.as_ref().map(|identity| identity.pid), runner.id() ); - assert!(!hello.previous_controller); + assert_eq!((claim.epoch, claim.previous_controller), (1, false)); + let line = logged(&mut lines, "claimed the Runner").await; + assert!( + line.contains(&format!( + "process {} claimed the Runner under epoch 1", + std::process::id() + )), + "{line}" + ); // A second Runner neither takes nor removes the live socket. let second = Command::new(BIN) .arg(&socket) @@ -329,11 +366,27 @@ async fn an_independent_runner_outlives_its_gateways_until_one_stops_it() { // No gateway is connected, and both keep running. assert!(still_beating(&pipe_beat).await && still_beating(&tty_beat).await); - let next = gateway(&socket).await; - let inherited = next.hello(); - assert_eq!(inherited.identity, hello.identity); - assert!(inherited.previous_controller); - assert_eq!(inherited.inherited_processes, 2); + let (next, inherited) = gateway(&socket).await; + assert_eq!(next.hello().identity, hello.identity); + assert_eq!((inherited.epoch, inherited.previous_controller), (2, true)); + let mut processes: Vec<_> = inherited + .processes + .iter() + .map(|entry| (entry.status.process_id.0.as_str(), entry.status.state)) + .collect(); + processes.sort_by_key(|(process_id, _)| *process_id); + assert_eq!( + processes, + [ + ("proc-pipe", ProcessState::Running), + ("proc-tty", ProcessState::Running) + ] + ); + let line = logged(&mut lines, "under epoch 2").await; + assert!( + line.contains("an earlier controller started work"), + "{line}" + ); assert_eq!( next.process_status(&ProcessId("proc-pipe".into())) .await @@ -367,8 +420,8 @@ async fn sigterm_stops_an_independent_runner_and_its_processes() { let base = tempfile::tempdir().unwrap(); let dir = base.path().join("runner"); let socket = runner_socket(&dir); - let mut runner = spawn_independent(&socket); - let connected = gateway(&socket).await; + let (mut runner, _lines) = spawn_independent(&socket); + let (connected, _) = gateway(&socket).await; let ws_dir = tempfile::tempdir().unwrap(); let ws = demo_workspace(ws_dir.path()); let beat = ws_dir.path().join("beat"); diff --git a/crates/domain/src/error.rs b/crates/domain/src/error.rs index 1074bb0..149dc95 100644 --- a/crates/domain/src/error.rs +++ b/crates/domain/src/error.rs @@ -64,10 +64,20 @@ pub enum ErrorCode { /// approval being resumed is left as it was. Stopping the Runner and starting a new one ends /// it. Not occupancy; do not retry as busy. RecoveryPending, - /// This gateway lost its connection to its independent Runner and does not reconnect - /// (P1R-C01). The request never reached the Runner, whose processes keep running; - /// restarting the gateway reaches them again. + /// This gateway lost its connection to its independent Runner and cannot claim it again + /// (P1R-C01, P1R-C03): the Runner stopped, another Runner answers at its socket, or it + /// refused this gateway. The request never reached a Runner. Restart the gateway once the + /// Runner can be reached; a Runner that did not stop keeps its processes running. RunnerDisconnected, + /// This gateway lost its connection to its independent Runner and is claiming it again + /// (P1R-C03). The request never reached the Runner, whose processes keep running; retry + /// once the gateway has reconnected. Not occupancy. + RunnerReconnecting, + /// The independent Runner refused the request because it does not come from its current + /// controller (P1R-C03): another connection claimed the Runner after the request was + /// prepared, or the request's connection never claimed it. Nothing was started or changed. + /// Not occupancy; do not retry as busy. + StaleController, /// Scheduler or store invariant failed. Not occupancy; do not retry as busy. Internal, } @@ -187,6 +197,10 @@ mod tests { assert_eq!(json, "\"RECOVERY_PENDING\""); let json = serde_json::to_string(&ErrorCode::RunnerDisconnected).unwrap(); assert_eq!(json, "\"RUNNER_DISCONNECTED\""); + let json = serde_json::to_string(&ErrorCode::RunnerReconnecting).unwrap(); + assert_eq!(json, "\"RUNNER_RECONNECTING\""); + let json = serde_json::to_string(&ErrorCode::StaleController).unwrap(); + assert_eq!(json, "\"STALE_CONTROLLER\""); let json = serde_json::to_string(&ErrorCode::Internal).unwrap(); assert_eq!(json, "\"INTERNAL\""); } diff --git a/crates/runner/src/lib.rs b/crates/runner/src/lib.rs index 1eebcbb..f17982c 100644 --- a/crates/runner/src/lib.rs +++ b/crates/runner/src/lib.rs @@ -150,14 +150,18 @@ pub use socket::{ allocate_private_runner_dir, is_forbidden_runner_dir, prepare_runner_socket_dir, reclaim_leftover_socket, runner_socket_path, RUNNER_SOCKET_NAME, }; -pub use uds::{runner_disconnected, DisconnectHook, RunnerHello, UdsRunner, RUNNER_CALL_DEADLINE}; +pub use uds::{ + runner_disconnected, runner_reconnecting, ClaimError, DisconnectHook, LinkEvent, LinkObserver, + RunnerHello, UdsRunner, CLAIM_START_BOUND, RECLAIM_BACKOFF_MAX, RUNNER_CALL_DEADLINE, +}; pub use watch::{FsWatchEvent, FsWatchKind, WatchClosed, WatchSubscription}; #[cfg(feature = "devguard")] pub use wire::serve_runner_connection_with_registration; pub use wire::{ - host_worker, read_frame, recovery_pending, serve_runner_connection, write_frame, - IndependentRunner, RunnerEvent, RunnerOp, RunnerOpResult, Served, StopHandle, WireEnvelope, - WireKind, STOP_DRAIN_BOUND, WIRE_PROTOCOL, + host_worker, read_frame, recovery_pending, serve_runner_connection, write_frame, ClaimNote, + ClaimObserver, ClaimRefusal, IndependentRunner, RunnerClaim, RunnerEvent, RunnerEvents, + RunnerInventoryEntry, RunnerOp, RunnerOpResult, Served, StopHandle, WireEnvelope, WireKind, + CLAIM_WAIT_BOUND, STOP_DRAIN_BOUND, WIRE_PROTOCOL, }; /// Execution-plane API. Control-plane fields (`work_id`, coordination, @@ -322,13 +326,28 @@ impl RuntimeBackend { Self::InProcess(InProcessRunner::new(on_release)) } - /// The backend is an independent Runner that an earlier gateway controlled, so this gateway - /// starts no new execution, patch or approval resume (P1R-C01): see - /// [`UdsRunner::refuses_new_work`]. - pub fn refuses_new_work(&self) -> bool { + /// The controller epoch under which this gateway sends a new execution, patch or approval + /// resume, none for a runner it does not claim, or why it starts none (P1R-C01, P1R-C03): + /// see [`UdsRunner::dispatch_epoch`]. + pub fn dispatch_epoch(&self) -> Result, ErrorBody> { match self { - Self::InProcess(_) => false, - Self::Uds(runner) => runner.refuses_new_work(), + Self::InProcess(_) => Ok(None), + Self::Uds(runner) => runner.dispatch_epoch(), + } + } + + /// Why this gateway starts no new execution, patch or approval resume, if it does not + /// (P1R-C01, P1R-C03): see [`UdsRunner::dispatch_epoch`]. + pub fn new_work_refusal(&self) -> Option { + self.dispatch_epoch().err() + } + + /// This backend, sending its requests under the controller epoch `epoch` when there is one + /// (P1R-C03): see [`UdsRunner::at_epoch`]. + pub fn at_epoch(&self, epoch: Option) -> Self { + match (self, epoch) { + (Self::Uds(runner), Some(epoch)) => Self::Uds(runner.at_epoch(epoch)), + _ => self.clone(), } } } diff --git a/crates/runner/src/process.rs b/crates/runner/src/process.rs index 6f0fc38..c98116a 100644 --- a/crates/runner/src/process.rs +++ b/crates/runner/src/process.rs @@ -1043,18 +1043,26 @@ impl InProcessRunner { let slot = map .get(&process_id.0) .ok_or_else(|| missing(&process_id.0))?; - let snap = slot.lifecycle.lock().expect("lifecycle").snapshot; - let buf = slot.output.lock().expect("output"); - Ok(RunnerProcessStatus { - process_id: process_id.clone(), - state: snap.state, - exit_code: snap.exit_code, - termination: snap.termination, - signal: snap.signal.map(process_signal), - output_total: buf.total, - output_retained_from: buf.dropped, - eof: buf.eof, - }) + Ok(slot_status(process_id, slot)) + } + + /// Every process the runner holds, with its workspace and status, ended ones it still + /// keeps included, in `process_id` order: what a gateway that claims an independent Runner + /// inherits (P1R-C03). + pub fn inventory(&self) -> Vec<(String, RunnerProcessStatus)> { + let mut map = self.inner.lock().expect("runner"); + self.evict_completed(&mut map); + let mut inventory: Vec<_> = map + .iter() + .map(|(process_id, slot)| { + ( + slot.workspace_id.clone(), + slot_status(&ProcessId(process_id.clone()), slot), + ) + }) + .collect(); + inventory.sort_by(|a, b| a.1.process_id.0.cmp(&b.1.process_id.0)); + inventory } pub fn host_resize( @@ -1320,6 +1328,22 @@ async fn pump_reader( } } +/// A process's status, from its slot. +fn slot_status(process_id: &ProcessId, slot: &Slot) -> RunnerProcessStatus { + let snap = slot.lifecycle.lock().expect("lifecycle").snapshot; + let buf = slot.output.lock().expect("output"); + RunnerProcessStatus { + process_id: process_id.clone(), + state: snap.state, + exit_code: snap.exit_code, + termination: snap.termination, + signal: snap.signal.map(process_signal), + output_total: buf.total, + output_retained_from: buf.dropped, + eof: buf.eof, + } +} + fn live_count(map: &HashMap) -> usize { map.values() .filter(|slot| slot.output.lock().map(|buf| !buf.eof).unwrap_or(false)) diff --git a/crates/runner/src/uds.rs b/crates/runner/src/uds.rs index 89ccc93..2e302f9 100644 --- a/crates/runner/src/uds.rs +++ b/crates/runner/src/uds.rs @@ -4,16 +4,21 @@ //! not claim Linux container isolation. P0 is 1:1: one gateway connection //! per worker. Replay is connection-local. //! -//! An independent Runner (P1R-C01) outlives the gateway's connection. The client never -//! reconnects: once its connection ends, every later request is refused with -//! `RUNNER_DISCONNECTED` and the gateway has to be restarted. +//! An independent Runner (P1R-C01) outlives the gateway's connection. The gateway takes control +//! of it with a `Claim` (P1R-C03), and each request that changes a process or the workspace +//! carries the epoch of that Claim. A client that claimed through +//! [`UdsRunner::claim_independent`] connects and claims the same Runner again when its +//! connection is lost, pausing longer after each failed try up to [`RECLAIM_BACKOFF_MAX`], and +//! refuses requests with `RUNNER_RECONNECTING` meanwhile; once it cannot claim that Runner +//! again, it refuses them with `RUNNER_DISCONNECTED`. Any other client refuses every request +//! with `RUNNER_DISCONNECTED` once its connection to an independent Runner ended. use std::collections::HashMap; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, OnceLock, PoisonError}; use std::task::{Context, Poll}; use std::time::Duration; @@ -25,8 +30,8 @@ use tokio::sync::{oneshot, Mutex}; use tokio::time::timeout; use crate::wire::{ - read_frame, write_frame, RunnerEvent, RunnerOp, RunnerOpResult, WireEnvelope, WireKind, - WIRE_PROTOCOL, + read_frame, recovery_pending, write_frame, ClaimRefusal, RunnerClaim, RunnerEvent, RunnerOp, + RunnerOpResult, WireEnvelope, WireKind, WIRE_PROTOCOL, }; use crate::{ Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, RunnerError, RunnerExecRequest, @@ -38,6 +43,14 @@ use crate::{ /// so `apply_patch` is not cut short; short enough to fail a hung /// mismatch instead of waiting forever. pub const RUNNER_CALL_DEADLINE: Duration = Duration::from_secs(120); +/// How long a gateway that starts with an independent Runner keeps claiming it while another +/// connection controls it, or that connection's requests run (P1R-C03). +pub const CLAIM_START_BOUND: Duration = Duration::from_secs(30); +/// The pause after a client's first failed try to claim its independent Runner, doubled after +/// each further one up to [`RECLAIM_BACKOFF_MAX`] (P1R-C03). +const RECLAIM_BACKOFF_START: Duration = Duration::from_millis(100); +/// The longest pause between two tries to claim an independent Runner again (P1R-C03). +pub const RECLAIM_BACKOFF_MAX: Duration = Duration::from_secs(5); pub type DisconnectHook = Arc; @@ -46,28 +59,116 @@ pub type DisconnectHook = Arc; pub struct RunnerHello { pub lifecycle: RunnerLifecycle, pub identity: Option, - /// An earlier gateway controlled this independent Runner. - pub previous_controller: bool, - /// Processes still running when this connection began. - pub inherited_processes: u32, } -/// The refusal of a request to an independent Runner after the gateway's connection to it -/// ended (P1R-C01): the request never left, and the gateway does not reconnect. -pub fn runner_disconnected() -> ErrorBody { +/// The refusal of a request to an independent Runner that this gateway cannot reach again +/// (P1R-C01, P1R-C03): the request was not sent. +pub fn runner_disconnected(reason: &str) -> ErrorBody { ErrorBody::new( ErrorCode::RunnerDisconnected, - "this gateway lost its connection to the independent Runner and does not reconnect; \ - the Runner's processes keep running. Restart the gateway to reach them", + format!( + "this gateway cannot reach its independent Runner again ({reason}), so the request \ + was not sent. Restart the gateway once the Runner can be reached; a Runner that did \ + not stop keeps its processes running" + ), ) } -struct Shared { +/// The refusal of a request while this gateway claims its independent Runner again +/// (P1R-C03): the request was not sent. +pub fn runner_reconnecting() -> ErrorBody { + ErrorBody::new( + ErrorCode::RunnerReconnecting, + "this gateway lost its connection to the independent Runner and is claiming it again, \ + so the request was not sent; the Runner's processes keep running. Retry once the \ + gateway has reconnected", + ) +} + +/// What becomes of a client's connection to the independent Runner it claimed (P1R-C03). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LinkEvent { + /// The connection was lost; the client claims the Runner again. + Lost, + /// A try to claim the Runner again failed, and why; the next comes after `retry`. + Retrying { reason: String, retry: Duration }, + /// The Runner was claimed again. + Reclaimed(RunnerClaim), + /// The Runner cannot be claimed again, and why: every later request is refused with + /// `RUNNER_DISCONNECTED`. + Gone(String), +} + +pub type LinkObserver = Arc; + +/// Why a client does not control an independent Runner (P1R-C03). +#[derive(Debug, Clone)] +pub enum ClaimError { + /// The Runner refused the Claim. + Refused { + refusal: ClaimRefusal, + message: String, + }, + /// The worker is not the independent Runner this client claims. + NotIndependent(String), + /// The Claim could not be sent, or its answer was lost. + Transport(RunnerError), +} + +impl std::fmt::Display for ClaimError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Refused { refusal, message } => { + write!( + f, + "the Runner refused the Claim ({}): {message}", + refusal.as_str() + ) + } + Self::NotIndependent(message) => f.write_str(message), + Self::Transport(err) => write!(f, "{err}"), + } + } +} + +/// One connection to the worker. +struct Link { writer: Mutex, pending: Mutex>>>, alive: AtomicBool, + /// The Runner's answer to this connection's Claim (P1R-C03). + claim: OnceLock, +} + +/// Which connection a client's requests go over. +enum State { + Linked(Arc), + /// The connection was lost and the client claims the Runner again (P1R-C03). + Reclaiming, + /// The Runner cannot be claimed again, and why (P1R-C03). + Gone(String), +} + +/// Where and for what a client claims its independent Runner again (P1R-C03). +struct Reclaim { + socket: PathBuf, + store_id: String, + observer: LinkObserver, +} + +/// How one try to claim an independent Runner ended. +enum Attempt { + /// Trying again may succeed. + Retry(String), + /// Trying again cannot succeed. + Gone(String), +} + +struct Client { + next_id: AtomicU64, + on_process_exit: ShellRelease, on_disconnect: DisconnectHook, - /// What the worker's `Hello` stated (P1R-C01). + /// What the worker's first `Hello` stated (P1R-C01). hello: OnceLock, /// The worker's `Hello` stated that it answers `Registration` (CSRG-U2). #[cfg(feature = "devguard")] @@ -78,20 +179,30 @@ struct Shared { /// The worker's `Hello` stated that it answers `Launch` (CSRG-U4). #[cfg(feature = "devguard")] launch: AtomicBool, + state: std::sync::Mutex, + /// Set once the client claimed an independent Runner that it claims again when its + /// connection is lost (P1R-C03). + reclaim: OnceLock, + /// The gateway leaves the Runner: no connection is made again. + leaving: AtomicBool, + /// The number of the last exit event delivered (P1R-C03). + last_event: AtomicU64, } pub struct UdsRunner { - shared: Arc, - next_id: Arc, + client: Arc, call_deadline: Duration, + /// The controller epoch this handle's requests carry instead of the current Claim's + /// (P1R-C03). + epoch: Option, } impl Clone for UdsRunner { fn clone(&self) -> Self { Self { - shared: self.shared.clone(), - next_id: self.next_id.clone(), + client: self.client.clone(), call_deadline: self.call_deadline, + epoch: self.epoch, } } } @@ -126,28 +237,13 @@ impl UdsRunner { on_process_exit: ShellRelease, on_disconnect: DisconnectHook, ) -> Self { - let (read, write) = stream.into_split(); - let shared = Arc::new(Shared { - writer: Mutex::new(write), - pending: Mutex::new(HashMap::new()), - alive: AtomicBool::new(true), - on_disconnect, - hello: OnceLock::new(), - #[cfg(feature = "devguard")] - registration: AtomicBool::new(false), - #[cfg(feature = "devguard")] - admission: AtomicBool::new(false), - #[cfg(feature = "devguard")] - launch: AtomicBool::new(false), - }); - let reader_shared = shared.clone(); - tokio::spawn(async move { - read_loop(read, reader_shared, on_process_exit).await; - }); + let client = Arc::new(Client::new(on_process_exit, on_disconnect)); + let link = client.open_link(stream); + *client.state() = State::Linked(link); Self { - shared, - next_id: Arc::new(AtomicU64::new(1)), + client, call_deadline: RUNNER_CALL_DEADLINE, + epoch: None, } } @@ -164,16 +260,12 @@ impl UdsRunner { if let RunnerOpResult::Hello { lifecycle, identity, - previous_controller, - inherited_processes, .. } = &hello { - let _ = self.shared.hello.set(RunnerHello { + let _ = self.client.hello.set(RunnerHello { lifecycle: *lifecycle, identity: identity.clone(), - previous_controller: *previous_controller, - inherited_processes: *inherited_processes, }); } #[cfg(feature = "devguard")] @@ -184,11 +276,11 @@ impl UdsRunner { .. } = hello { - self.shared + self.client .registration .store(registration, Ordering::SeqCst); - self.shared.admission.store(admission, Ordering::SeqCst); - self.shared.launch.store(launch, Ordering::SeqCst); + self.client.admission.store(admission, Ordering::SeqCst); + self.client.launch.store(launch, Ordering::SeqCst); } #[cfg(not(feature = "devguard"))] let _ = hello; @@ -217,28 +309,174 @@ impl UdsRunner { /// What the worker's `Hello` stated about its lifecycle (P1R-C01); before the handshake, a /// managed worker's defaults. pub fn hello(&self) -> RunnerHello { - self.shared.hello.get().cloned().unwrap_or_default() + self.client.hello.get().cloned().unwrap_or_default() } - fn independent(&self) -> bool { - self.shared - .hello - .get() - .is_some_and(|hello| hello.lifecycle == RunnerLifecycle::Independent) + /// Take control of the independent Runner over this connection, for the operations + /// database `store_id` (P1R-C03). The connection's later requests carry the Claim's epoch. + pub async fn claim(&self, store_id: &str) -> Result { + let link = self.client.current().map_err(ClaimError::Transport)?; + self.client + .claim_on(&link, store_id, self.call_deadline) + .await } - /// The worker is an independent Runner that an earlier gateway controlled (P1R-C01). This - /// gateway then queries, reads, writes to, resizes and terminates the processes it inherits, - /// but starts no new execution, patch or approval resume: the Runner refuses them as well. - pub fn refuses_new_work(&self) -> bool { - self.shared.hello.get().is_some_and(|hello| { - hello.lifecycle == RunnerLifecycle::Independent && hello.previous_controller - }) + /// Connect to the independent Runner at `socket` and claim it for the operations database + /// `store_id` (P1R-C03). While another connection controls the Runner, or that connection's + /// requests run, and when the Claim's answer was lost, the Claim is asked again over a new + /// connection with a growing pause for up to [`CLAIM_START_BOUND`]; any other refusal, + /// nothing accepting connections at `socket`, and a worker that is not an independent + /// Runner stating its identity, end the try. Once claimed, the client claims the same + /// Runner again whenever its connection is lost, and tells `observer` how that goes. + pub async fn claim_independent( + socket: &Path, + store_id: &str, + on_process_exit: ShellRelease, + observer: LinkObserver, + ) -> Result<(Self, RunnerClaim), ClaimError> { + let client = Arc::new(Client::new(on_process_exit, Arc::new(|| {}))); + let runner = Self { + client: client.clone(), + call_deadline: RUNNER_CALL_DEADLINE, + epoch: None, + }; + let deadline = tokio::time::Instant::now() + CLAIM_START_BOUND; + let mut backoff = RECLAIM_BACKOFF_START; + loop { + let stream = UnixStream::connect(socket).await.map_err(|err| { + ClaimError::Transport(RunnerError::before_dispatch(format!( + "connect {}: {err}", + socket.display() + ))) + })?; + let link = client.open_link(stream); + *client.state() = State::Linked(link.clone()); + let claimed = match runner.first_or_same_hello(socket).await { + Ok(()) => runner.claim(store_id).await, + Err(err) => Err(err), + }; + match claimed { + Ok(claim) => { + let _ = client.reclaim.set(Reclaim { + socket: socket.to_path_buf(), + store_id: store_id.to_string(), + observer, + }); + // A connection lost before the client was set to claim again. + if !link.alive.load(Ordering::SeqCst) { + client.start_reclaim(&link); + } + return Ok((runner, claim)); + } + Err(err) + if tokio::time::Instant::now() + backoff < deadline + && match &err { + ClaimError::Refused { refusal, .. } => refusal.is_transient(), + // The connection was lost after the Runner's `Hello`, perhaps with + // the Claim's answer: a Runner that granted it lets go once that + // connection ended. + ClaimError::Transport(_) => client.hello.get().is_some(), + ClaimError::NotIndependent(_) => false, + } => + { + observer(LinkEvent::Retrying { + reason: match err { + ClaimError::Refused { message, .. } => message, + err => err.to_string(), + }, + retry: backoff, + }); + client.close_link(&link).await; + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(RECLAIM_BACKOFF_MAX); + } + Err(err) => { + client.leaving.store(true, Ordering::SeqCst); + client.close_link(&link).await; + return Err(err); + } + } + } + } + + /// The handshake of a client's first connection, which must find an independent Runner + /// that states its identity, or the `Hello` of a later one, which must find the same Runner. + async fn first_or_same_hello(&self, socket: &Path) -> Result<(), ClaimError> { + if self.client.hello.get().is_none() { + self.handshake().await.map_err(ClaimError::Transport)?; + let hello = self.hello(); + if hello.lifecycle != RunnerLifecycle::Independent { + return Err(ClaimError::NotIndependent(format!( + "the worker at {} is {}, not an independent Runner", + socket.display(), + hello.lifecycle.as_str() + ))); + } + if hello.identity.is_none() { + return Err(ClaimError::NotIndependent(format!( + "the Runner at {} did not state its identity", + socket.display() + ))); + } + return Ok(()); + } + let link = self.client.current().map_err(ClaimError::Transport)?; + match self.client.same_runner(&link).await { + Ok(()) => Ok(()), + Err(Attempt::Retry(message)) => { + Err(ClaimError::Transport(RunnerError::before_dispatch(message))) + } + Err(Attempt::Gone(message)) => Err(ClaimError::NotIndependent(message)), + } + } + + /// The Claim of this client's current connection, if it claimed the Runner (P1R-C03). + pub fn claimed(&self) -> Option { + match &*self.client.state() { + State::Linked(link) if link.alive.load(Ordering::SeqCst) => link.claim.get().cloned(), + _ => None, + } + } + + /// This client, sending every request under the controller epoch `epoch` instead of the + /// current Claim's (P1R-C03): a request prepared under one Claim is refused by the Runner + /// once another Claim replaced it. + pub fn at_epoch(&self, epoch: u64) -> Self { + Self { + epoch: Some(epoch), + ..self.clone() + } + } + + /// The controller epoch under which this client sends a new execution, patch or approval + /// resume, none for a worker that is not claimed, or why it starts none (P1R-C01, + /// P1R-C03): the connection to its independent Runner is lost or being made again, or an + /// earlier controller started work on the Runner. + pub fn dispatch_epoch(&self) -> Result, ErrorBody> { + if !self.client.independent() { + return Ok(None); + } + let link = self + .client + .current() + .map_err(RunnerError::into_error_body)?; + match link.claim.get() { + Some(claim) if claim.previous_controller => Err(recovery_pending()), + Some(claim) => Ok(Some(self.epoch.unwrap_or(claim.epoch))), + None => Ok(self.epoch), + } + } + + /// Why this client starts no new execution, patch or approval resume, if it does not: see + /// [`Self::dispatch_epoch`]. + pub fn new_work_refusal(&self) -> Option { + self.dispatch_epoch().err() } /// Leave the independent Runner, whose processes, output and timeouts carry on, and close - /// the connection (P1R-C01). + /// the connection (P1R-C01). No connection is made again. pub async fn detach(&self) -> Result<(), RunnerError> { + self.client.leaving.store(true, Ordering::SeqCst); let detached = match self.call(RunnerOp::Detach).await { Ok(RunnerOpResult::Detached) => Ok(()), Ok(other) => Err(unexpected(other)), @@ -249,8 +487,9 @@ impl UdsRunner { } /// Stop the independent Runner: it ends every process and exits (P1R-C01). How many - /// processes were still running. + /// processes were still running. No connection is made again. pub async fn stop_runner(&self) -> Result { + self.client.leaving.store(true, Ordering::SeqCst); let stopped = match self.call(RunnerOp::Stop).await { Ok(RunnerOpResult::Stopped { terminated }) => Ok(terminated), Ok(other) => Err(unexpected(other)), @@ -263,7 +502,7 @@ impl UdsRunner { /// Whether the worker's `Hello` stated that it answers `Registration` (CSRG-U2). #[cfg(feature = "devguard")] pub fn states_registration(&self) -> bool { - self.shared.registration.load(Ordering::SeqCst) + self.client.registration.load(Ordering::SeqCst) } /// The worker's report of its own DevGuard registration (CSRG-U2), or why it could not be @@ -281,7 +520,14 @@ impl UdsRunner { } match self.call(RunnerOp::Registration).await { Ok(RunnerOpResult::Registration(info)) => Ok(info), - Err(RunnerError::Execution(body)) if body.code == ErrorCode::RunnerDisconnected => { + Err(RunnerError::Execution(body)) + if matches!( + body.code, + ErrorCode::RunnerDisconnected + | ErrorCode::RunnerReconnecting + | ErrorCode::StaleController + ) => + { Err(ResourceRegistrationState::OwnerUnreachable) } Err(RunnerError::Execution(_)) => Err(ResourceRegistrationState::UnsupportedMode), @@ -292,7 +538,7 @@ impl UdsRunner { /// Whether the worker's `Hello` stated that it answers `Prepare` (CSRG-U3). #[cfg(feature = "devguard")] pub fn states_admission(&self) -> bool { - self.shared.admission.load(Ordering::SeqCst) + self.client.admission.load(Ordering::SeqCst) } /// Have the worker prepare a governed execution under its own registration (CSRG-U3). The @@ -350,7 +596,7 @@ impl UdsRunner { /// Whether the worker's `Hello` stated that it answers `Launch` (CSRG-U4). #[cfg(feature = "devguard")] pub fn states_launch(&self) -> bool { - self.shared.launch.load(Ordering::SeqCst) + self.client.launch.load(Ordering::SeqCst) } /// Have the worker prepare a governed execution under its own registration and launch it @@ -412,60 +658,336 @@ impl UdsRunner { } } + /// Close the connection. No connection is made again. pub async fn close(&self) { - self.shared.alive.store(false, Ordering::SeqCst); - { - let mut writer = self.shared.writer.lock().await; - let _ = writer.shutdown().await; + self.client.leaving.store(true, Ordering::SeqCst); + let link = match &*self.client.state() { + State::Linked(link) => Some(link.clone()), + _ => None, + }; + if let Some(link) = link { + self.client.close_link(&link).await; } - fail_pending(&self.shared, RunnerError::ambiguous("runner socket closed")).await; } async fn call(&self, op: RunnerOp) -> Result { - if !self.shared.alive.load(Ordering::SeqCst) { - // An independent Runner's connection is not made again (P1R-C01). - if self.independent() { - return Err(RunnerError::execution(runner_disconnected())); + let link = self.client.current()?; + let epoch = self + .epoch + .or_else(|| link.claim.get().map(|claim| claim.epoch)); + self.client + .call_on(&link, op, epoch, self.call_deadline) + .await + } +} + +impl Client { + fn new(on_process_exit: ShellRelease, on_disconnect: DisconnectHook) -> Self { + Self { + next_id: AtomicU64::new(1), + on_process_exit, + on_disconnect, + hello: OnceLock::new(), + #[cfg(feature = "devguard")] + registration: AtomicBool::new(false), + #[cfg(feature = "devguard")] + admission: AtomicBool::new(false), + #[cfg(feature = "devguard")] + launch: AtomicBool::new(false), + state: std::sync::Mutex::new(State::Gone("not connected".into())), + reclaim: OnceLock::new(), + leaving: AtomicBool::new(false), + last_event: AtomicU64::new(0), + } + } + + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn independent(&self) -> bool { + self.hello + .get() + .is_some_and(|hello| hello.lifecycle == RunnerLifecycle::Independent) + } + + /// A new connection over `stream`, read from now on. It carries no request until the + /// client sends one over it. + fn open_link(self: &Arc, stream: UnixStream) -> Arc { + let (read, write) = stream.into_split(); + let link = Arc::new(Link { + writer: Mutex::new(write), + pending: Mutex::new(HashMap::new()), + alive: AtomicBool::new(true), + claim: OnceLock::new(), + }); + tokio::spawn(read_loop(read, self.clone(), link.clone())); + link + } + + /// The connection a request goes over, or why none can take it. + fn current(&self) -> Result, RunnerError> { + match &*self.state() { + State::Linked(link) if link.alive.load(Ordering::SeqCst) => Ok(link.clone()), + State::Linked(_) if !self.independent() => { + Err(RunnerError::before_dispatch("runner socket closed")) } - return Err(RunnerError::before_dispatch("runner socket closed")); + // A lost connection that is about to be made again. + State::Linked(_) + if self.reclaim.get().is_some() && !self.leaving.load(Ordering::SeqCst) => + { + Err(RunnerError::execution(runner_reconnecting())) + } + State::Linked(_) if self.leaving.load(Ordering::SeqCst) => Err(RunnerError::execution( + runner_disconnected("this gateway left the Runner"), + )), + State::Linked(_) => Err(RunnerError::execution(runner_disconnected( + "the connection to the Runner was lost", + ))), + State::Reclaiming => Err(RunnerError::execution(runner_reconnecting())), + State::Gone(reason) => Err(RunnerError::execution(runner_disconnected(reason))), } + } + + async fn call_on( + self: &Arc, + link: &Arc, + op: RunnerOp, + epoch: Option, + deadline: Duration, + ) -> Result { let request_id = format!("rrpc-{}", self.next_id.fetch_add(1, Ordering::SeqCst)); let (tx, rx) = oneshot::channel(); - { - self.shared - .pending - .lock() - .await - .insert(request_id.clone(), tx); - } - let envelope = WireEnvelope::request(request_id.clone(), op); + link.pending.lock().await.insert(request_id.clone(), tx); + let envelope = WireEnvelope::request(request_id.clone(), op).with_epoch(epoch); let write_result = { - let mut writer = self.shared.writer.lock().await; + let mut writer = link.writer.lock().await; write_envelope(&mut *writer, &envelope).await }; if let Err(err) = write_result { - self.shared.pending.lock().await.remove(&request_id); + link.pending.lock().await.remove(&request_id); if matches!(err, RunnerError::TransportAmbiguous { .. }) { - self.close().await; + self.close_link(link).await; } return Err(err); } - match timeout(self.call_deadline, rx).await { + match timeout(deadline, rx).await { Ok(Ok(Ok(response))) => decode_response(response), Ok(Ok(Err(err))) => Err(err), Ok(Err(_)) => Err(RunnerError::ambiguous("runner socket closed")), Err(_) => { - self.close().await; + self.close_link(link).await; Err(RunnerError::ambiguous("runner call deadline exceeded")) } } } + + /// Claim the Runner over `link` for the operations database `store_id` (P1R-C03). + async fn claim_on( + self: &Arc, + link: &Arc, + store_id: &str, + deadline: Duration, + ) -> Result { + let claim = RunnerOp::Claim { + store_id: store_id.to_string(), + }; + match self.call_on(link, claim, None, deadline).await { + Ok(RunnerOpResult::Claimed(claim)) => { + let _ = link.claim.set(claim.clone()); + Ok(claim) + } + Ok(RunnerOpResult::ClaimRefused { refusal, message }) => { + Err(ClaimError::Refused { refusal, message }) + } + Ok(other) => Err(ClaimError::Transport(unexpected(other))), + Err(err) => Err(ClaimError::Transport(err)), + } + } + + /// End `link`: no request goes over it again, and those waiting on it end ambiguous. A + /// client that claims its Runner again does so when `link` is its connection. + async fn close_link(self: &Arc, link: &Arc) { + link.alive.store(false, Ordering::SeqCst); + { + let mut writer = link.writer.lock().await; + let _ = writer.shutdown().await; + } + fail_pending(link, RunnerError::ambiguous("runner socket closed")).await; + self.start_reclaim(link); + } + + /// Claim the Runner again once `link`, the client's connection, is lost (P1R-C03): only for + /// a client set to, and not once the gateway leaves. + fn start_reclaim(self: &Arc, link: &Arc) { + let Some(reclaim) = self.reclaim.get() else { + return; + }; + if self.leaving.load(Ordering::SeqCst) { + return; + } + { + let mut state = self.state(); + match &*state { + State::Linked(current) if Arc::ptr_eq(current, link) => *state = State::Reclaiming, + _ => return, + } + } + (reclaim.observer)(LinkEvent::Lost); + tokio::spawn(self.clone().reclaim_loop()); + } + + /// Claim the Runner again until it is claimed, it cannot be, or the gateway leaves. + async fn reclaim_loop(self: Arc) { + let Some(reclaim) = self.reclaim.get() else { + return; + }; + let mut backoff = RECLAIM_BACKOFF_START; + loop { + if self.leaving.load(Ordering::SeqCst) { + *self.state() = State::Gone("this gateway left the Runner".into()); + return; + } + match self + .link_and_claim(&reclaim.socket, &reclaim.store_id) + .await + { + Ok((link, claim)) => { + { + let mut state = self.state(); + if !self.leaving.load(Ordering::SeqCst) { + *state = State::Linked(link); + drop(state); + (reclaim.observer)(LinkEvent::Reclaimed(claim)); + return; + } + *state = State::Gone("this gateway left the Runner".into()); + } + // The gateway left while the Runner was claimed again: it leaves the Runner + // as a detach does. + let _ = self + .call_on( + &link, + RunnerOp::Detach, + Some(claim.epoch), + RUNNER_CALL_DEADLINE, + ) + .await; + self.close_link(&link).await; + return; + } + Err(Attempt::Gone(reason)) => { + *self.state() = State::Gone(reason.clone()); + (reclaim.observer)(LinkEvent::Gone(reason)); + return; + } + Err(Attempt::Retry(reason)) => { + (reclaim.observer)(LinkEvent::Retrying { + reason, + retry: backoff, + }); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(RECLAIM_BACKOFF_MAX); + } + } + } + } + + /// One try to claim the Runner at `socket` again over a new connection, which is not the + /// client's until the caller makes it so. Nothing answering at the socket, another Runner + /// there, or a refusal that waiting cannot end, means the Runner cannot be claimed again. + async fn link_and_claim( + self: &Arc, + socket: &Path, + store_id: &str, + ) -> Result<(Arc, RunnerClaim), Attempt> { + let stream = match UnixStream::connect(socket).await { + Ok(stream) => stream, + Err(err) + if matches!( + err.kind(), + io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused + ) => + { + return Err(Attempt::Gone(format!( + "nothing accepts connections at {}: {err}", + socket.display() + ))) + } + Err(err) => { + return Err(Attempt::Retry(format!( + "connect {}: {err}", + socket.display() + ))) + } + }; + let link = self.open_link(stream); + let claimed = match self.same_runner(&link).await { + Ok(()) => match self.claim_on(&link, store_id, RUNNER_CALL_DEADLINE).await { + Ok(claim) => Ok(claim), + Err(ClaimError::Refused { refusal, message }) if refusal.is_transient() => { + Err(Attempt::Retry(message)) + } + Err(ClaimError::Refused { message, .. }) => Err(Attempt::Gone(message)), + Err(err) => Err(Attempt::Retry(err.to_string())), + }, + Err(attempt) => Err(attempt), + }; + match claimed { + Ok(claim) => Ok((link, claim)), + Err(attempt) => { + self.close_link(&link).await; + Err(attempt) + } + } + } + + /// The Runner answering over `link` is the one the client's first `Hello` found. + async fn same_runner(self: &Arc, link: &Arc) -> Result<(), Attempt> { + let expected = self.hello.get().and_then(|hello| hello.identity.clone()); + match self + .call_on(link, RunnerOp::Hello, None, RUNNER_CALL_DEADLINE) + .await + { + Ok(RunnerOpResult::Hello { + protocol, identity, .. + }) if protocol == WIRE_PROTOCOL => { + if identity.is_some() && identity == expected { + Ok(()) + } else { + Err(Attempt::Gone(format!( + "another Runner answers at its socket (pid {})", + identity.map_or_else(|| "unknown".into(), |found| found.pid.to_string()) + ))) + } + } + Ok(other) => Err(Attempt::Gone(format!( + "the worker at its socket answered Hello with {other:?}" + ))), + Err(RunnerError::Execution(body)) => Err(Attempt::Gone(format!( + "the worker at its socket refused Hello: {}", + body.message + ))), + Err(err) => Err(Attempt::Retry(format!("Hello: {err}"))), + } + } + + /// Deliver an exit event once: the Runner sends an event again when its write to a lost + /// connection failed (P1R-C03). + fn exited(&self, seq: Option, process_id: &str) { + if let Some(seq) = seq { + if self.last_event.fetch_max(seq, Ordering::SeqCst) >= seq { + return; + } + } + (self.on_process_exit)(process_id); + } } async fn read_loop( mut read: tokio::net::unix::OwnedReadHalf, - shared: Arc, - on_process_exit: ShellRelease, + client: Arc, + link: Arc, ) { loop { match read_frame(&mut read).await { @@ -473,12 +995,12 @@ async fn read_loop( Ok(envelope) => match envelope.kind { WireKind::Event => { if let Some(RunnerEvent::ProcessExited { process_id }) = envelope.event { - on_process_exit(&process_id.0); + client.exited(envelope.seq, &process_id.0); } } WireKind::Response => { if let Some(id) = envelope.request_id.clone() { - if let Some(tx) = shared.pending.lock().await.remove(&id) { + if let Some(tx) = link.pending.lock().await.remove(&id) { let _ = tx.send(Ok(envelope)); } } @@ -487,7 +1009,7 @@ async fn read_loop( }, Err(err) => { fail_pending( - &shared, + &link, RunnerError::ambiguous(format!("invalid runner rpc: {err}")), ) .await; @@ -495,17 +1017,18 @@ async fn read_loop( } }, Ok(None) | Err(_) => { - fail_pending(&shared, RunnerError::ambiguous("runner socket closed")).await; + fail_pending(&link, RunnerError::ambiguous("runner socket closed")).await; break; } } } - shared.alive.store(false, Ordering::SeqCst); - (shared.on_disconnect)(); + link.alive.store(false, Ordering::SeqCst); + (client.on_disconnect)(); + client.start_reclaim(&link); } -async fn fail_pending(shared: &Shared, err: RunnerError) { - let mut pending = shared.pending.lock().await; +async fn fail_pending(link: &Link, err: RunnerError) { + let mut pending = link.pending.lock().await; for (_, tx) in pending.drain() { let _ = tx.send(Err(err.clone())); } @@ -846,4 +1369,46 @@ mod tests { "{err:?}" ); } + + /// An exit event is delivered once, however often the Runner sends it, and an event the + /// client already passed is not delivered again; an event without a number is always + /// delivered (P1R-C03). + #[tokio::test] + async fn an_exit_event_is_delivered_once() { + let (client, server) = UnixStream::pair().unwrap(); + let delivered = Arc::new(std::sync::Mutex::new(Vec::new())); + let hook = delivered.clone(); + let runner = UdsRunner::from_stream( + client, + Arc::new(move |process_id: &str| hook.lock().unwrap().push(process_id.to_string())), + ); + let (mut read, mut write) = server.into_split(); + let exited = |process_id: &str| RunnerEvent::ProcessExited { + process_id: ProcessId(process_id.into()), + }; + for (seq, process_id) in [(1, "proc-1"), (1, "proc-1"), (3, "proc-3"), (2, "proc-2")] { + write_frame(&mut write, &WireEnvelope::event(seq, exited(process_id))) + .await + .unwrap(); + } + let mut unnumbered = WireEnvelope::event(0, exited("proc-4")); + unnumbered.seq = None; + write_frame(&mut write, &unnumbered).await.unwrap(); + // The client reads its frames in order: once a request is answered, each event sent + // before the answer was handled. + let answering = tokio::spawn(async move { + let frame = read_frame(&mut read).await.unwrap().unwrap(); + let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); + write_frame( + &mut write, + &WireEnvelope::response(request.request_id.unwrap(), Ok(RunnerOpResult::Terminate)), + ) + .await + .unwrap(); + (read, write) + }); + runner.terminate(&ProcessId("proc-5".into())).await.unwrap(); + let _ = answering.await.unwrap(); + assert_eq!(*delivered.lock().unwrap(), ["proc-1", "proc-3", "proc-4"]); + } } diff --git a/crates/runner/src/wire.rs b/crates/runner/src/wire.rs index 7a510b7..9b0cbff 100644 --- a/crates/runner/src/wire.rs +++ b/crates/runner/src/wire.rs @@ -5,18 +5,22 @@ //! id, `operation_id`, `operation_key`, or `process_id`. //! //! A `managed` worker serves the one connection of the gateway that started it. An -//! `independent` Runner (P1R-C01) serves one connection at a time through [`IndependentRunner`]: -//! its processes, their output, their timeouts and their exit events outlive each connection. +//! `independent` Runner (P1R-C01) serves its connections through [`IndependentRunner`]: its +//! processes, their output, their timeouts and their exit events outlive each connection. A +//! connection controls the Runner only after its `Claim` (P1R-C03), and every request that +//! changes a process or the workspace carries the epoch of that Claim, which the Runner checks. use std::collections::VecDeque; -use std::sync::atomic::{AtomicU32, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, PoisonError}; use std::time::Duration; use codespace_domain::{ErrorBody, ErrorCode, FindResult, ProcessId, ReadResult}; use codespace_policy::Workspace; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::unix::OwnedWriteHalf; +use tokio::net::UnixStream; use tokio::sync::{mpsc, oneshot, watch, Mutex}; use tokio::task::JoinSet; @@ -27,12 +31,20 @@ use crate::{ RunnerResizeResult, RunnerWriteStdin, ShellRelease, }; -pub const WIRE_PROTOCOL: u32 = 9; +pub const WIRE_PROTOCOL: u32 = 10; const MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; const MAX_REPLAY: usize = 32; /// How long a stopping independent Runner waits for its terminated processes to be reaped and /// their output to end before it exits anyway (P1R-C01). pub const STOP_DRAIN_BOUND: Duration = Duration::from_secs(5); +/// How long an independent Runner's `Claim` waits for the requests of the connection that +/// controlled it before, once that connection has ended, before it is refused (P1R-C03). +pub const CLAIM_WAIT_BOUND: Duration = Duration::from_secs(10); +/// How often a waiting `Claim` looks again, besides each change of control. +const CLAIM_POLL: Duration = Duration::from_millis(20); +/// How many of a connection's requests an independent Runner reads ahead of the one it +/// answers, so that it sees the connection end while a request runs. +const READ_AHEAD: usize = 8; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -48,6 +60,10 @@ pub struct WireEnvelope { pub kind: WireKind, #[serde(default, skip_serializing_if = "Option::is_none")] pub request_id: Option, + /// The controller epoch a request is sent under (P1R-C03): an independent Runner refuses a + /// request that changes a process or the workspace unless it carries the current one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epoch: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub op: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -56,6 +72,10 @@ pub struct WireEnvelope { pub result: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, + /// An event's number: the worker numbers its exit events from 1 in the order they + /// happened (P1R-C03). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub seq: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub event: Option, } @@ -66,48 +86,52 @@ impl WireEnvelope { protocol: WIRE_PROTOCOL, kind: WireKind::Request, request_id: Some(request_id), + epoch: None, op: Some(op), ok: None, result: None, error: None, + seq: None, event: None, } } + /// This request, sent under the controller epoch `epoch` (P1R-C03). + pub fn with_epoch(mut self, epoch: Option) -> Self { + self.epoch = epoch; + self + } + pub fn response(request_id: String, result: Result) -> Self { - match result { - Ok(value) => Self { - protocol: WIRE_PROTOCOL, - kind: WireKind::Response, - request_id: Some(request_id), - op: None, - ok: Some(true), - result: Some(value), - error: None, - event: None, - }, - Err(error) => Self { - protocol: WIRE_PROTOCOL, - kind: WireKind::Response, - request_id: Some(request_id), - op: None, - ok: Some(false), - result: None, - error: Some(error), - event: None, - }, + let (ok, result, error) = match result { + Ok(value) => (true, Some(value), None), + Err(error) => (false, None, Some(error)), + }; + Self { + protocol: WIRE_PROTOCOL, + kind: WireKind::Response, + request_id: Some(request_id), + epoch: None, + op: None, + ok: Some(ok), + result, + error, + seq: None, + event: None, } } - pub fn event(event: RunnerEvent) -> Self { + pub fn event(seq: u64, event: RunnerEvent) -> Self { Self { protocol: WIRE_PROTOCOL, kind: WireKind::Event, request_id: None, + epoch: None, op: None, ok: None, result: None, error: None, + seq: Some(seq), event: Some(event), } } @@ -177,13 +201,19 @@ pub enum RunnerOp { request_id: String, }, Hello, + /// Take control of an independent Runner for the operations database `store_id` + /// (P1R-C03). Answered `Claimed` with a new controller epoch, or `ClaimRefused`. A managed + /// worker refuses it. + Claim { + store_id: String, + }, /// The gateway leaves an independent Runner and its processes keep running (P1R-C01). The - /// Runner answers `Detached`, reads nothing more from this connection and serves the next - /// one once this connection's requests have ended. A managed worker refuses it. + /// Runner answers `Detached`, reads nothing more from this connection and lets another one + /// claim it once this connection's requests have ended. A managed worker refuses it. Detach, - /// Stop an independent Runner (P1R-C01): it reads nothing more, lets this connection's - /// requests end, terminates every process, waits for them, answers `Stopped` and exits. A - /// managed worker refuses it. + /// Stop an independent Runner (P1R-C01): it reads nothing more, lets the requests in + /// progress end, terminates every process, waits for them, answers `Stopped` and exits. + /// Only its controller stops it (P1R-C03). A managed worker refuses it. Stop, /// The execution owner's DevGuard registration (CSRG-U2). Sent only to a worker whose /// `Hello` states `registration`; a worker without it would close the connection. @@ -235,14 +265,6 @@ pub enum RunnerOpResult { /// Which Runner process answered (P1R-C01). #[serde(default, skip_serializing_if = "Option::is_none")] identity: Option, - /// An earlier connection of this independent Runner started work, so this connection - /// may observe and control the processes it inherits but starts nothing (P1R-C01). - #[serde(default)] - previous_controller: bool, - /// Processes still running when this connection began, which it inherits from - /// earlier connections (P1R-C01). - #[serde(default)] - inherited_processes: u32, /// The worker answers `Registration` (CSRG-U2). Left out when false, so a worker built /// without DevGuard says `Hello` as before. #[cfg(feature = "devguard")] @@ -258,6 +280,14 @@ pub enum RunnerOpResult { #[serde(default, skip_serializing_if = "is_false")] launch: bool, }, + /// An independent Runner's answer to a `Claim` it accepted (P1R-C03). + Claimed(RunnerClaim), + /// An independent Runner's answer to a `Claim` it refused (P1R-C03): the connection may + /// ask again, and nothing else changed. + ClaimRefused { + refusal: ClaimRefusal, + message: String, + }, /// An independent Runner's answer to `Detach` (P1R-C01). Detached, /// An independent Runner's answer to `Stop`, sent once its processes ended or @@ -274,13 +304,72 @@ pub enum RunnerOpResult { Launched(crate::launch::LaunchReport), } +/// What a connection that claimed an independent Runner controls (P1R-C03). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunnerClaim { + /// The controller epoch of this Claim, one more than the one before. Every request of this + /// connection that changes a process or the workspace carries it. + pub epoch: u64, + /// An earlier controller started work, so this connection may observe and control the + /// processes it inherits but starts nothing (P1R-C01). + pub previous_controller: bool, + /// Every process the Runner holds when the Claim is answered, those that ended included. + pub processes: Vec, + /// The number of the last exit event before the inventory was taken: an event numbered up + /// to it is already reflected in the inventory. Events this connection receives may start + /// below it, with those no controller received. + pub events_through: u64, +} + +/// A process an independent Runner holds (P1R-C03). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunnerInventoryEntry { + pub workspace_id: String, + pub status: RunnerProcessStatus, +} + +/// Why an independent Runner refused a `Claim` (P1R-C03). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ClaimRefusal { + /// The connection's peer runs as another user than the Runner, or its credentials could + /// not be read. + PeerUid, + /// The Runner was first claimed for another operations database. + StoreMismatch, + /// Another connection controls the Runner and is still open, or claims it now. + ControllerActive, + /// The connection that controlled the Runner ended, but its requests did not end within + /// [`CLAIM_WAIT_BOUND`]. + RequestsRunning, + /// The Runner is stopping. + Stopping, +} + +impl ClaimRefusal { + pub fn as_str(self) -> &'static str { + match self { + Self::PeerUid => "peer_uid", + Self::StoreMismatch => "store_mismatch", + Self::ControllerActive => "controller_active", + Self::RequestsRunning => "requests_running", + Self::Stopping => "stopping", + } + } + + /// Asking again later may succeed: the controller may end, or its requests may. + pub fn is_transient(self) -> bool { + matches!(self, Self::ControllerActive | Self::RequestsRunning) + } +} + #[cfg(feature = "devguard")] fn is_false(value: &bool) -> bool { !*value } -/// The refusal of new work on a connection to an independent Runner that an earlier connection -/// controlled (P1R-C01): recovering what that controller left is not available yet. +/// The refusal of new work on a connection to an independent Runner that an earlier controller +/// started work on (P1R-C01): recovering what that controller left is not available yet. pub fn recovery_pending() -> ErrorBody { ErrorBody::new( ErrorCode::RecoveryPending, @@ -291,6 +380,12 @@ pub fn recovery_pending() -> ErrorBody { ) } +/// The refusal of a request that does not come from an independent Runner's current +/// controller (P1R-C03). +fn stale_controller(message: impl Into) -> ErrorBody { + ErrorBody::new(ErrorCode::StaleController, message) +} + /// The worker's registration, when it was started with one, and its launch helper. #[cfg(feature = "devguard")] #[derive(Clone, Default)] @@ -313,17 +408,14 @@ impl Registrar { fn hello( registrar: &Registrar, lifecycle: RunnerLifecycle, - previous_controller: bool, - inherited_processes: u32, + identity: &RunnerIdentity, ) -> RunnerOpResult { #[cfg(not(feature = "devguard"))] let () = *registrar; RunnerOpResult::Hello { protocol: WIRE_PROTOCOL, lifecycle, - identity: Some(RunnerIdentity::this_process()), - previous_controller, - inherited_processes, + identity: Some(identity.clone()), #[cfg(feature = "devguard")] registration: true, #[cfg(feature = "devguard")] @@ -334,21 +426,40 @@ fn hello( } } -/// Host worker: `InProcessRunner` plus a `ProcessExited` event stream. -pub fn host_worker() -> (InProcessRunner, mpsc::UnboundedReceiver) { - let (tx, rx) = mpsc::unbounded_channel(); +/// The exit events of a host worker's processes, numbered from 1 in the order the processes +/// ended (P1R-C03). +pub struct RunnerEvents { + receiver: mpsc::UnboundedReceiver<(u64, RunnerEvent)>, + /// The number of the last event, shared with the worker's release hook. + numbered: Arc>, +} + +/// Host worker: `InProcessRunner` plus its numbered `ProcessExited` events. +pub fn host_worker() -> (InProcessRunner, RunnerEvents) { + let (tx, receiver) = mpsc::unbounded_channel(); + let numbered = Arc::new(std::sync::Mutex::new(0u64)); + let numbering = numbered.clone(); let on_release: ShellRelease = Arc::new(move |process_id: &str| { - let _ = tx.send(RunnerEvent::ProcessExited { - process_id: ProcessId(process_id.to_string()), - }); + // Numbered and queued under one lock, so the queue's order is the numbers' order. + let mut last = numbering.lock().unwrap_or_else(PoisonError::into_inner); + *last += 1; + let _ = tx.send(( + *last, + RunnerEvent::ProcessExited { + process_id: ProcessId(process_id.to_string()), + }, + )); }); - (InProcessRunner::new(on_release), rx) + ( + InProcessRunner::new(on_release), + RunnerEvents { receiver, numbered }, + ) } pub async fn serve_runner_connection( stream: S, runner: InProcessRunner, - events: mpsc::UnboundedReceiver, + events: RunnerEvents, ) -> Result<(), std::io::Error> where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, @@ -368,7 +479,7 @@ where pub async fn serve_runner_connection_with_registration( stream: S, runner: InProcessRunner, - events: mpsc::UnboundedReceiver, + events: RunnerEvents, registration: Option>, launcher: Option, ) -> Result<(), std::io::Error> @@ -391,7 +502,7 @@ where async fn serve( stream: S, runner: InProcessRunner, - mut events: mpsc::UnboundedReceiver, + events: RunnerEvents, registrar: Registrar, ) -> Result<(), std::io::Error> where @@ -400,10 +511,11 @@ where let (mut read, write) = tokio::io::split(stream); let write = Arc::new(Mutex::new(write)); let event_write = write.clone(); + let mut events = events.receiver; tokio::spawn(async move { - while let Some(event) = events.recv().await { + while let Some((seq, event)) = events.recv().await { let mut writer = event_write.lock().await; - if write_frame(&mut *writer, &WireEnvelope::event(event)) + if write_frame(&mut *writer, &WireEnvelope::event(seq, event)) .await .is_err() { @@ -412,14 +524,18 @@ where } }); - let hello = hello(®istrar, RunnerLifecycle::Managed, false, 0); + let hello = hello( + ®istrar, + RunnerLifecycle::Managed, + &RunnerIdentity::this_process(), + ); let mut cache: VecDeque<(String, WireEnvelope)> = VecDeque::new(); loop { let payload = match read_frame(&mut read).await? { Some(payload) => payload, None => return Ok(()), }; - let (request_id, op) = match parse_request(&payload) { + let (request_id, _, op) = match parse_request(&payload) { Ok(request) => request, Err(response) => { let mut writer = write.lock().await; @@ -445,14 +561,16 @@ where op => op, }; let response = match op { - Some(RunnerOp::Detach | RunnerOp::Stop) => WireEnvelope::response( - request_id, - Err(ErrorBody::new( - ErrorCode::InvalidCommand, - "this worker belongs to the gateway that started it; only an independent \ - Runner detaches or stops on request", - )), - ), + Some(RunnerOp::Claim { .. } | RunnerOp::Detach | RunnerOp::Stop) => { + WireEnvelope::response( + request_id, + Err(ErrorBody::new( + ErrorCode::InvalidCommand, + "this worker belongs to the gateway that started it; only an \ + independent Runner is claimed, detaches or stops on request", + )), + ) + } op => answer(&mut cache, &runner, request_id, op, &hello).await, }; let mut writer = write.lock().await; @@ -460,8 +578,11 @@ where } } -/// A request's id and operation, or the response that refuses it and ends the connection. -fn parse_request(payload: &[u8]) -> Result<(String, Option), Box> { +/// A request's id, epoch and operation, or the response that refuses it and ends the +/// connection. +fn parse_request( + payload: &[u8], +) -> Result<(String, Option, Option), Box> { let request: WireEnvelope = serde_json::from_slice(payload).map_err(|err| { Box::new(WireEnvelope::response( String::new(), @@ -480,7 +601,11 @@ fn parse_request(payload: &[u8]) -> Result<(String, Option), Box bool { match op { RunnerOp::Exec { .. } | RunnerOp::ApplyPatch { .. } => true, @@ -607,7 +732,24 @@ fn starts_work(op: &RunnerOp) -> bool { } } -/// How an independent Runner's connection ended (P1R-C01). +/// A request that changes a process or the workspace, or stops the Runner: an independent +/// Runner refuses it unless it carries the current controller epoch (P1R-C03). +fn fenced(op: &RunnerOp) -> bool { + match op { + RunnerOp::Exec { .. } + | RunnerOp::ApplyPatch { .. } + | RunnerOp::WriteStdin { .. } + | RunnerOp::Resize { .. } + | RunnerOp::Terminate { .. } + | RunnerOp::TerminateWorkspace { .. } + | RunnerOp::Stop => true, + #[cfg(feature = "devguard")] + RunnerOp::Prepare { .. } | RunnerOp::Launch { .. } => true, + _ => false, + } +} + +/// How an independent Runner's controlling connection ended (P1R-C01). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Served { /// The gateway detached: the processes keep running. @@ -655,15 +797,15 @@ impl StopHandle { } } -/// The exit events of an independent Runner's processes, kept between its connections. +/// The exit events of an independent Runner's processes, kept between its controllers. struct EventHub { - events: mpsc::UnboundedReceiver, - /// An event whose write failed, which the next connection gets first. - undelivered: Option, + events: mpsc::UnboundedReceiver<(u64, RunnerEvent)>, + /// An event whose write failed, which the next controller gets first. + undelivered: Option<(u64, RunnerEvent)>, } -/// Forward exit events to one connection until `stop`, or until a write fails. Events left in -/// the hub go to the next connection. +/// Forward exit events to the controller's connection until `stop`, or until a write fails. +/// Events left in the hub go to the next controller. async fn forward_events( hub: Arc>, write: Arc>, @@ -673,7 +815,7 @@ async fn forward_events( { let mut hub = hub.lock().await; loop { - let event = match hub.undelivered.take() { + let (seq, event) = match hub.undelivered.take() { Some(event) => event, None => tokio::select! { biased; @@ -685,33 +827,56 @@ async fn forward_events( }, }; let mut writer = write.lock().await; - if write_frame(&mut *writer, &WireEnvelope::event(event.clone())) + if write_frame(&mut *writer, &WireEnvelope::event(seq, event.clone())) .await .is_err() { - hub.undelivered = Some(event); + hub.undelivered = Some((seq, event)); return; } } } +/// What an independent Runner tells its operator about each `Claim` (P1R-C03). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ClaimNote { + Claimed { + epoch: u64, + previous_controller: bool, + peer_pid: Option, + }, + Refused { + refusal: ClaimRefusal, + message: String, + peer_pid: Option, + }, +} + +pub type ClaimObserver = Arc; + /// An independent Runner (P1R-C01), started by an operator or a harness rather than a gateway. -/// It serves one connection at a time, and its processes, their output, their timeouts and -/// their exit events outlive each connection. A connection becomes the Runner's controller with -/// the first request that starts work; every later connection states `previous_controller` in -/// its `Hello` and may only query, read, write stdin, resize and terminate, until the Runner -/// stops. Recovering what an earlier controller left is later work (P1R-C03, P1R-C04). +/// It serves any number of connections at once, and its processes, their output, their +/// timeouts and their exit events outlive each connection. One connection at a time controls +/// it, after its `Claim` (P1R-C03): only that connection's requests are answered beyond `Hello` +/// and `Claim`, only it receives the exit events, and each of its requests that changes a +/// process or the workspace must carry the epoch of its Claim. A Claim is refused while +/// another connection controls the Runner and is open; once that connection ends, the Claim +/// waits up to [`CLAIM_WAIT_BOUND`] for its requests. A controller after one that started work +/// may observe and control the processes it inherits but starts nothing (P1R-C01); recovering +/// what an earlier controller left is later work (P1R-C04). pub struct IndependentRunner { runner: InProcessRunner, - events: Arc>, + events: RunnerEvents, registrar: Registrar, - /// A connection has started work. - controlled: bool, stop: StopHandle, + identity: RunnerIdentity, + owner_uid: u32, + claim_wait: Duration, + observer: Option, } impl IndependentRunner { - pub fn new(runner: InProcessRunner, events: mpsc::UnboundedReceiver) -> Self { + pub fn new(runner: InProcessRunner, events: RunnerEvents) -> Self { let stop = StopHandle { requested: Arc::new(watch::Sender::new(false)), terminated: Arc::new(AtomicU32::new(0)), @@ -719,16 +884,17 @@ impl IndependentRunner { }; Self { runner, - events: Arc::new(Mutex::new(EventHub { - events, - undelivered: None, - })), + events, #[cfg(feature = "devguard")] registrar: Registrar::default(), #[cfg(not(feature = "devguard"))] registrar: (), - controlled: false, stop, + identity: RunnerIdentity::this_process(), + // SAFETY: geteuid has no preconditions and cannot fail. + owner_uid: unsafe { libc::geteuid() }, + claim_wait: CLAIM_WAIT_BOUND, + observer: None, } } @@ -747,164 +913,445 @@ impl IndependentRunner { self } + /// Tell `observer` about each Claim, accepted or refused (P1R-C03). + pub fn with_claim_observer(mut self, observer: ClaimObserver) -> Self { + self.observer = Some(observer); + self + } + + /// The user a Claim's peer must run as, instead of this process's effective user. For + /// tests: a peer of another user cannot be made without privileges. + #[doc(hidden)] + pub fn with_owner_uid(mut self, uid: u32) -> Self { + self.owner_uid = uid; + self + } + + /// How long a Claim waits for the requests of the controller before it, instead of + /// [`CLAIM_WAIT_BOUND`]. For tests. + #[doc(hidden)] + pub fn with_claim_wait(mut self, wait: Duration) -> Self { + self.claim_wait = wait; + self + } + + /// The identity the Runner states, instead of this process's. For tests: two Runners in one + /// process. + #[doc(hidden)] + pub fn with_identity(mut self, identity: RunnerIdentity) -> Self { + self.identity = identity; + self + } + pub fn stop_handle(&self) -> StopHandle { self.stop.clone() } - /// Serve each connection that `accept` yields, one at a time, until the Runner stops: through - /// `Stop`, or a stop request, which ends a connection being served or the wait for the next - /// one. `served` hears how each connection ended, or why one could not be accepted; a failed - /// accept is tried again after a pause. How many processes the stop terminated. - pub async fn run( - mut self, + /// Serve each connection that `accept` yields, all at once, until the Runner stops: + /// through its controller's `Stop`, or a stop request, which ends every connection and the + /// wait for the next one. `served` hears how each controlling connection ended, the stop, + /// and why a connection could not be accepted; a failed accept is tried again after a + /// pause. How many processes the stop terminated. + pub async fn run( + self, mut accept: A, mut served: impl FnMut(std::io::Result), ) -> u32 where A: FnMut() -> F, - F: std::future::Future>, - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + F: std::future::Future>, { - let stop = self.stop.clone(); + let shared = Arc::new(Shared::new(self)); + let stop = shared.stop.clone(); + let mut connections: JoinSet = JoinSet::new(); + let next_connection = AtomicU64::new(1); loop { - let accepted = tokio::select! { + tokio::select! { biased; - () = stop.requested() => { - let terminated = self.stop().await; - served(Ok(Served::Stopped { terminated })); - return terminated; - } - accepted = accept() => accepted, - }; - match accepted { - Ok(stream) => { - let ending = self.serve(stream).await; - served(Ok(ending)); - if let Served::Stopped { terminated } = ending { - return terminated; + () = stop.requested() => break, + Some(ended) = connections.join_next(), if !connections.is_empty() => { + if let Ok(Ended { served: Some(ending), .. }) = ended { + served(Ok(ending)); } } - Err(err) => { - served(Err(err)); - tokio::time::sleep(Duration::from_millis(100)).await; + accepted = accept() => match accepted { + Ok(stream) => { + let id = next_connection.fetch_add(1, Ordering::SeqCst); + connections.spawn(connection(shared.clone(), stream, id)); + } + Err(err) => { + served(Err(err)); + tokio::time::sleep(Duration::from_millis(100)).await; + } + }, + } + } + // Every connection stops reading, and its requests end before the processes are + // waited for: none of them overlaps the stop. + let mut answers = Vec::new(); + while let Some(ended) = connections.join_next().await { + if let Ok(ended) = ended { + if let Some(ending) = ended.served { + served(Ok(ending)); } + answers.extend(ended.stop_answer); } } + let terminated = shared.end_processes().await; + for (request_id, write) in answers { + let response = + WireEnvelope::response(request_id, Ok(RunnerOpResult::Stopped { terminated })); + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + let _ = writer.shutdown().await; + } + served(Ok(Served::Stopped { terminated })); + terminated } +} - /// Serve one connection until it detaches, is lost or stops the Runner. Every request it - /// sent has ended when this returns, its `Registration`, `Prepare` and `Launch` requests - /// included, so the next connection starts after them. - pub async fn serve(&mut self, stream: S) -> Served - where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, - { - let previous_controller = self.controlled; - let hello = hello( - &self.registrar, - RunnerLifecycle::Independent, - previous_controller, - self.runner.live_processes(), - ); - let (mut read, write) = tokio::io::split(stream); - let write = Arc::new(Mutex::new(write)); - let (stop_events, events_stopped) = oneshot::channel(); - let forwarder = tokio::spawn(forward_events( - self.events.clone(), - write.clone(), - events_stopped, - )); - let mut beside: JoinSet<()> = JoinSet::new(); - let mut cache: VecDeque<(String, WireEnvelope)> = VecDeque::new(); - let stop = self.stop.clone(); - let ending = loop { - let frame = tokio::select! { - biased; - () = stop.requested() => break Ending::Stop(None), - frame = read_frame(&mut read) => frame, - }; - let Ok(Some(payload)) = frame else { - break Ending::Lost; - }; - let (request_id, op) = match parse_request(&payload) { - Ok(request) => request, - Err(response) => { - let mut writer = write.lock().await; - let _ = write_frame(&mut *writer, &response).await; - break Ending::Lost; - } - }; - let response = match op { - Some(RunnerOp::Detach) => break Ending::Detach(request_id), - Some(RunnerOp::Stop) => { - self.stop.request(); - break Ending::Stop(Some(request_id)); +/// Which connection controls an independent Runner, and under which epoch (P1R-C03). +#[derive(Default)] +struct Control { + /// The epoch of the last Claim; 0 before the first. It never decreases. + epoch: u64, + controller: Option, + /// The connection whose Claim waits for the controller before it. + claiming: Option, + /// The operations database of the first accepted Claim. + store_id: Option, + /// A controller started work (P1R-C01). + started_work: bool, +} + +struct Controller { + connection: u64, + epoch: u64, + /// The connection reads no further request: its stream ended, or it detached; its + /// requests may still be running. + ended: Arc, + /// An earlier controller started work: this one starts nothing (P1R-C01). + barrier: bool, +} + +/// What an independent Runner's connections share. +struct Shared { + runner: InProcessRunner, + registrar: Registrar, + events: Arc>, + /// The number of the last exit event. + numbered: Arc>, + stop: StopHandle, + identity: RunnerIdentity, + owner_uid: u32, + claim_wait: Duration, + observer: Option, + control: std::sync::Mutex, + /// Bumped whenever a controller lets go. + released: watch::Sender, +} + +/// How one connection of an independent Runner ended. +struct Ended { + /// How it ended, when it controlled the Runner and did not stop it. + served: Option, + /// A `Stop` it sent, to be answered once the processes ended. + stop_answer: Option<(String, Arc>)>, +} + +/// Why an independent Runner's connection stopped reading. +enum Ending { + Detach(String), + Lost, + /// With the request's id when `Stop` asked for it. + Stop(Option), +} + +/// The peer of a connection, from its credentials (P1R-C03). +struct Peer { + uid: std::io::Result, + pid: Option, +} + +impl Peer { + fn of(stream: &UnixStream) -> Self { + match stream.peer_cred() { + Ok(credentials) => Self { + uid: Ok(credentials.uid()), + pid: credentials.pid(), + }, + Err(err) => Self { + uid: Err(err), + pid: None, + }, + } + } +} + +impl Shared { + fn new(independent: IndependentRunner) -> Self { + let IndependentRunner { + runner, + events, + registrar, + stop, + identity, + owner_uid, + claim_wait, + observer, + } = independent; + Self { + runner, + registrar, + events: Arc::new(Mutex::new(EventHub { + events: events.receiver, + undelivered: None, + })), + numbered: events.numbered, + stop, + identity, + owner_uid, + claim_wait, + observer, + control: std::sync::Mutex::new(Control::default()), + released: watch::Sender::new(0), + } + } + + fn control(&self) -> std::sync::MutexGuard<'_, Control> { + self.control.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Answer connection `connection`'s Claim for the operations database `store_id`, and tell + /// the observer. + async fn claim( + &self, + connection: u64, + peer: &Peer, + ended: &Arc, + store_id: &str, + ) -> RunnerOpResult { + match self.take_control(connection, peer, ended, store_id).await { + Ok(claim) => { + self.note(ClaimNote::Claimed { + epoch: claim.epoch, + previous_controller: claim.previous_controller, + peer_pid: peer.pid, + }); + RunnerOpResult::Claimed(claim) + } + Err((refusal, message)) => { + self.note(ClaimNote::Refused { + refusal, + message: message.clone(), + peer_pid: peer.pid, + }); + RunnerOpResult::ClaimRefused { refusal, message } + } + } + } + + fn note(&self, note: ClaimNote) { + if let Some(observer) = &self.observer { + observer(note); + } + } + + async fn take_control( + &self, + connection: u64, + peer: &Peer, + ended: &Arc, + store_id: &str, + ) -> Result { + if self.stop.is_requested() { + return Err((ClaimRefusal::Stopping, "the Runner is stopping".into())); + } + match &peer.uid { + Ok(uid) if *uid == self.owner_uid => {} + Ok(uid) => { + return Err(( + ClaimRefusal::PeerUid, + format!( + "the connection's peer runs as uid {uid}, and this Runner as uid {}", + self.owner_uid + ), + )) + } + Err(err) => { + return Err(( + ClaimRefusal::PeerUid, + format!("the connection's peer credentials could not be read: {err}"), + )) + } + } + if store_id.is_empty() { + return Err(( + ClaimRefusal::StoreMismatch, + "a Claim names the operations database of its gateway".into(), + )); + } + { + let mut control = self.control(); + if let Some(known) = control + .store_id + .as_deref() + .filter(|known| *known != store_id) + { + return Err(( + ClaimRefusal::StoreMismatch, + format!( + "this Runner was claimed for the operations database {known}, not \ + {store_id}" + ), + )); + } + if let Some(controller) = &control.controller { + if controller.connection == connection { + return Err(( + ClaimRefusal::ControllerActive, + format!( + "this connection already controls the Runner, under epoch {}", + controller.epoch + ), + )); } - Some(op) if starts_work(&op) && previous_controller => { - WireEnvelope::response(request_id, Err(recovery_pending())) + if !controller.ended.load(Ordering::SeqCst) { + return Err(( + ClaimRefusal::ControllerActive, + "another connection controls this Runner and is still open".into(), + )); } - op => { - if op.as_ref().is_some_and(starts_work) { - self.controlled = true; + } + if control.claiming.is_some() { + return Err(( + ClaimRefusal::ControllerActive, + "another connection is claiming this Runner".into(), + )); + } + control.claiming = Some(connection); + } + let mut released = self.released.subscribe(); + let deadline = tokio::time::Instant::now() + self.claim_wait; + loop { + { + let mut control = self.control(); + if control.controller.is_none() { + control.claiming = None; + control.epoch += 1; + let epoch = control.epoch; + let previous_controller = control.started_work; + control.controller = Some(Controller { + connection, + epoch, + ended: ended.clone(), + barrier: previous_controller, + }); + if control.store_id.is_none() { + control.store_id = Some(store_id.to_string()); } - #[cfg(feature = "devguard")] - let op = match op { - Some(op) if answered_beside(&op) => { - let (write, registrar, runner) = - (write.clone(), self.registrar.clone(), self.runner.clone()); - beside.spawn(async move { - let result = answer_beside(op, registrar, runner).await; - let response = WireEnvelope::response(request_id, result); - let mut writer = write.lock().await; - let _ = write_frame(&mut *writer, &response).await; - }); - continue; - } - op => op, - }; - answer(&mut cache, &self.runner, request_id, op, &hello).await + drop(control); + return Ok(self.claimed(epoch, previous_controller)); + } + let refusal = if self.stop.is_requested() { + Some((ClaimRefusal::Stopping, "the Runner is stopping".to_string())) + } else if ended.load(Ordering::SeqCst) { + Some(( + ClaimRefusal::RequestsRunning, + "the connection ended while its Claim waited".to_string(), + )) + } else if tokio::time::Instant::now() >= deadline { + Some(( + ClaimRefusal::RequestsRunning, + format!( + "the requests of the connection that controlled this Runner did \ + not end within {:?}", + self.claim_wait + ), + )) + } else { + None + }; + if let Some(refusal) = refusal { + control.claiming = None; + return Err(refusal); } - }; - let mut writer = write.lock().await; - if write_frame(&mut *writer, &response).await.is_err() { - break Ending::Lost; } + tokio::select! { + _ = released.changed() => {} + () = tokio::time::sleep(CLAIM_POLL) => {} + () = self.stop.requested() => {} + } + } + } + + /// The answer to an accepted Claim: the events numbered so far, then every process, so an + /// exit numbered up to `events_through` is in the inventory. + fn claimed(&self, epoch: u64, previous_controller: bool) -> RunnerClaim { + let events_through = *self.numbered.lock().unwrap_or_else(PoisonError::into_inner); + let processes = self + .runner + .inventory() + .into_iter() + .map(|(workspace_id, status)| RunnerInventoryEntry { + workspace_id, + status, + }) + .collect(); + RunnerClaim { + epoch, + previous_controller, + processes, + events_through, + } + } + + /// Admit connection `connection`'s request `op`, sent under `epoch`: only the controller's, + /// with the current epoch when it changes a process or the workspace (P1R-C03), and nothing + /// that starts work from a controller after one that did (P1R-C01). + fn admit(&self, connection: u64, epoch: Option, op: &RunnerOp) -> Result<(), ErrorBody> { + let mut control = self.control(); + let Some((current, barrier)) = control + .controller + .as_ref() + .filter(|controller| controller.connection == connection) + .map(|controller| (controller.epoch, controller.barrier)) + else { + return Err(stale_controller( + "this connection does not control the independent Runner: another connection \ + claimed it, or this one never did; nothing was started or changed", + )); }; - if let Ending::Detach(request_id) = &ending { - let response = WireEnvelope::response(request_id.clone(), Ok(RunnerOpResult::Detached)); - let mut writer = write.lock().await; - let _ = write_frame(&mut *writer, &response).await; + if fenced(op) && epoch != Some(current) { + return Err(stale_controller(format!( + "the request was sent under controller epoch {}, and the Runner is controlled \ + under epoch {current}: nothing was started or changed", + epoch.map_or_else(|| "none".to_string(), |epoch| epoch.to_string()), + ))); } - // The connection's requests end before anything else: no later connection, and no - // stop, overlaps them. - while beside.join_next().await.is_some() {} - let served = match ending { - Ending::Detach(_) => Served::Detached, - Ending::Lost => Served::Lost, - Ending::Stop(request_id) => { - let terminated = self.end_processes().await; - if let Some(request_id) = request_id { - let response = WireEnvelope::response( - request_id, - Ok(RunnerOpResult::Stopped { terminated }), - ); - let mut writer = write.lock().await; - let _ = write_frame(&mut *writer, &response).await; - } - Served::Stopped { terminated } + if starts_work(op) { + if barrier { + return Err(recovery_pending()); } - }; - let _ = stop_events.send(()); - let _ = forwarder.await; - let _ = write.lock().await.shutdown().await; - served + control.started_work = true; + } + Ok(()) } - /// Stop with no connection open (P1R-C01): terminate every process and wait for them, as - /// `Stop` does. How many were still running. - pub async fn stop(&self) -> u32 { - self.stop.request(); - self.end_processes().await + /// Connection `connection` lets go of the Runner, if it controls it. Whether it did. + fn release(&self, connection: u64) -> bool { + let mut control = self.control(); + if !control + .controller + .as_ref() + .is_some_and(|controller| controller.connection == connection) + { + return false; + } + control.controller = None; + drop(control); + self.released.send_modify(|released| *released += 1); + true } /// Terminate every process, those a request started after the stop request included, and @@ -921,49 +1368,194 @@ impl IndependentRunner { } } -/// Why an independent Runner's connection stopped reading. -enum Ending { - Detach(String), - Lost, - /// With the request's id when `Stop` asked for it. - Stop(Option), -} - -pub async fn write_frame( - write: &mut W, - envelope: &WireEnvelope, -) -> Result<(), std::io::Error> { - let payload = serde_json::to_vec(envelope).map_err(std::io::Error::other)?; - let len = u32::try_from(payload.len()).map_err(std::io::Error::other)?; - write.write_all(&len.to_be_bytes()).await?; - write.write_all(&payload).await?; - write.flush().await -} - -pub async fn read_frame( - read: &mut R, -) -> Result>, std::io::Error> { - let mut len_buf = [0u8; 4]; - match read.read_exact(&mut len_buf).await { - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), - Err(err) => return Err(err), - } - let len = u32::from_be_bytes(len_buf) as usize; - if len == 0 || len > MAX_FRAME_BYTES { - return Err(std::io::Error::other(format!( - "invalid runner frame length {len}" - ))); +/// Read a connection's requests ahead of their answers, so the connection's end is seen while a +/// request runs: `ended` is set once the stream ends or cannot be read. +async fn read_ahead( + mut read: tokio::net::unix::OwnedReadHalf, + frames: mpsc::Sender>, + ended: Arc, +) { + while let Ok(Some(payload)) = read_frame(&mut read).await { + if frames.send(payload).await.is_err() { + return; + } } - let mut payload = vec![0u8; len]; - read.read_exact(&mut payload).await?; - Ok(Some(payload)) + ended.store(true, Ordering::SeqCst); } -async fn dispatch(runner: &InProcessRunner, op: RunnerOp) -> Result { - let result = match op { - RunnerOp::Read { - workspace, +/// One connection of an independent Runner, from its accept to its end. Every request it sent +/// has ended when this returns, its `Registration`, `Prepare` and `Launch` requests included, +/// and it controls the Runner no longer. +async fn connection(shared: Arc, stream: UnixStream, id: u64) -> Ended { + let peer = Peer::of(&stream); + let (read, write) = stream.into_split(); + let write = Arc::new(Mutex::new(write)); + let ended = Arc::new(AtomicBool::new(false)); + let (frames_in, mut frames) = mpsc::channel(READ_AHEAD); + let reader = tokio::spawn(read_ahead(read, frames_in, ended.clone())); + let hello = hello( + &shared.registrar, + RunnerLifecycle::Independent, + &shared.identity, + ); + let mut forwarder: Option<(oneshot::Sender<()>, tokio::task::JoinHandle<()>)> = None; + let mut beside: JoinSet<()> = JoinSet::new(); + let mut cache: VecDeque<(String, WireEnvelope)> = VecDeque::new(); + let stop = shared.stop.clone(); + let ending = loop { + let frame = tokio::select! { + biased; + () = stop.requested() => break Ending::Stop(None), + frame = frames.recv() => frame, + }; + let Some(payload) = frame else { + break Ending::Lost; + }; + let (request_id, epoch, op) = match parse_request(&payload) { + Ok(request) => request, + Err(response) => { + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + break Ending::Lost; + } + }; + let response = match op { + Some(RunnerOp::Detach) => break Ending::Detach(request_id), + Some(RunnerOp::Stop) => match shared.admit(id, epoch, &RunnerOp::Stop) { + Ok(()) => { + stop.request(); + break Ending::Stop(Some(request_id)); + } + Err(refusal) => WireEnvelope::response(request_id, Err(refusal)), + }, + Some(RunnerOp::Claim { store_id }) => { + let result = shared.claim(id, &peer, &ended, &store_id).await; + let claimed = matches!(result, RunnerOpResult::Claimed(_)); + let response = WireEnvelope::response(request_id, Ok(result)); + { + let mut writer = write.lock().await; + if write_frame(&mut *writer, &response).await.is_err() { + break Ending::Lost; + } + } + // The controller's exit events follow its Claim's answer. + if claimed { + let (stop_events, events_stopped) = oneshot::channel(); + forwarder = Some(( + stop_events, + tokio::spawn(forward_events( + shared.events.clone(), + write.clone(), + events_stopped, + )), + )); + } + continue; + } + op @ (Some(RunnerOp::Hello | RunnerOp::Replay { .. }) | None) => { + answer(&mut cache, &shared.runner, request_id, op, &hello).await + } + Some(op) => match shared.admit(id, epoch, &op) { + Err(refusal) => WireEnvelope::response(request_id, Err(refusal)), + Ok(()) => { + #[cfg(feature = "devguard")] + let op = match op { + op if answered_beside(&op) => { + let (write, registrar, runner) = ( + write.clone(), + shared.registrar.clone(), + shared.runner.clone(), + ); + beside.spawn(async move { + let result = answer_beside(op, registrar, runner).await; + let response = WireEnvelope::response(request_id, result); + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + }); + continue; + } + op => op, + }; + answer(&mut cache, &shared.runner, request_id, Some(op), &hello).await + } + }, + }; + let mut writer = write.lock().await; + if write_frame(&mut *writer, &response).await.is_err() { + break Ending::Lost; + } + }; + reader.abort(); + // It reads no further request: a Claim waits for its requests from now on, rather than + // being refused as one made while it controls the Runner. + ended.store(true, Ordering::SeqCst); + if let Ending::Detach(request_id) = &ending { + let response = WireEnvelope::response(request_id.clone(), Ok(RunnerOpResult::Detached)); + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + } + // The connection's requests end before anything else: no later Claim, and no stop, + // overlaps them. + while beside.join_next().await.is_some() {} + if let Some((stop_events, forwarding)) = forwarder { + let _ = stop_events.send(()); + let _ = forwarding.await; + } + let controlled = shared.release(id); + match ending { + Ending::Stop(Some(request_id)) => Ended { + served: None, + stop_answer: Some((request_id, write)), + }, + ending => { + let _ = write.lock().await.shutdown().await; + Ended { + served: match ending { + Ending::Detach(_) if controlled => Some(Served::Detached), + Ending::Lost if controlled => Some(Served::Lost), + _ => None, + }, + stop_answer: None, + } + } + } +} + +pub async fn write_frame( + write: &mut W, + envelope: &WireEnvelope, +) -> Result<(), std::io::Error> { + let payload = serde_json::to_vec(envelope).map_err(std::io::Error::other)?; + let len = u32::try_from(payload.len()).map_err(std::io::Error::other)?; + write.write_all(&len.to_be_bytes()).await?; + write.write_all(&payload).await?; + write.flush().await +} + +pub async fn read_frame( + read: &mut R, +) -> Result>, std::io::Error> { + let mut len_buf = [0u8; 4]; + match read.read_exact(&mut len_buf).await { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), + Err(err) => return Err(err), + } + let len = u32::from_be_bytes(len_buf) as usize; + if len == 0 || len > MAX_FRAME_BYTES { + return Err(std::io::Error::other(format!( + "invalid runner frame length {len}" + ))); + } + let mut payload = vec![0u8; len]; + read.read_exact(&mut payload).await?; + Ok(Some(payload)) +} + +async fn dispatch(runner: &InProcessRunner, op: RunnerOp) -> Result { + let result = match op { + RunnerOp::Read { + workspace, path, offset, limit, @@ -1025,8 +1617,8 @@ async fn dispatch(runner: &InProcessRunner, op: RunnerOp) -> Result unreachable!("hello is answered before dispatch"), RunnerOp::Replay { .. } => unreachable!("replay is handled before dispatch"), - RunnerOp::Detach | RunnerOp::Stop => { - unreachable!("detach and stop are handled before dispatch") + RunnerOp::Claim { .. } | RunnerOp::Detach | RunnerOp::Stop => { + unreachable!("claim, detach and stop are handled before dispatch") } #[cfg(feature = "devguard")] RunnerOp::Registration => unreachable!("registration is handled before dispatch"), @@ -1049,8 +1641,8 @@ mod tests { } #[test] - fn wire_protocol_is_v9() { - assert_eq!(WIRE_PROTOCOL, 9); + fn wire_protocol_is_v10() { + assert_eq!(WIRE_PROTOCOL, 10); } #[tokio::test] @@ -1382,8 +1974,6 @@ mod tests { RunnerOpResult::Hello { lifecycle: RunnerLifecycle::Managed, identity: None, - previous_controller: false, - inherited_processes: 0, registration: false, admission: false, launch: false, @@ -1637,11 +2227,14 @@ mod tests { assert!(registering.await.unwrap() > 0); } - /// An independent Runner keeps its registration across its connections and states it in - /// each `Hello`, and ends a connection only once its `Registration` requests were - /// answered, so the next connection starts after them (P1R-C01). + /// An independent Runner keeps its registration across its controllers and states it in + /// each `Hello`, and a Claim made once its controller detached waits for that + /// controller's `Registration` requests, which run beside its others, so the next + /// controller starts after them (P1R-C01, P1R-C03). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn an_independent_runner_answers_registration_before_the_next_connection() { + async fn an_independent_runner_answers_registration_before_the_next_claim() { + use super::independent::{Host, STORE}; + use std::time::Instant; let dir = private_directory(); let credential = dir.path().join("consumer.secret"); std::fs::OpenOptions::new() @@ -1673,49 +2266,58 @@ mod tests { }, OwnerCredential::File(credential), ); - let (worker, events) = host_worker(); - let mut independent = IndependentRunner::new(worker, events).with_registration( - Some(OwnerRegistration::new(owner, ResourceOwner::Worker)), - None, - ); - for connection in 0..2 { - let (client, server) = tokio::net::UnixStream::pair().unwrap(); - let serving = tokio::spawn(async move { - let served = independent.serve(server).await; - (served, independent) - }); - let (mut read, mut write) = client.into_split(); - request(&mut write, "rrpc-hello", RunnerOp::Hello).await; - match response(&mut read).await.result { - Some(RunnerOpResult::Hello { - lifecycle, - registration, - admission, - launch, - .. - }) => assert_eq!( - (lifecycle, registration, admission, launch), - (RunnerLifecycle::Independent, true, true, false), - "connection {connection}" - ), - other => panic!("unexpected {other:?}"), - } - request(&mut write, "rrpc-reg", RunnerOp::Registration).await; - request(&mut write, "rrpc-detach", RunnerOp::Detach).await; - // Both are answered before the connection ends. - let mut answered = Vec::new(); - while let Some(frame) = read_frame(&mut read).await.unwrap() { - let envelope: WireEnvelope = serde_json::from_slice(&frame).unwrap(); - if envelope.kind == WireKind::Response { - answered.push(envelope.request_id.unwrap()); - } + let registration = OwnerRegistration::new(owner, ResourceOwner::Worker); + let host = Host::start_with(move |independent| { + independent.with_registration(Some(registration), None) + }); + let stated = |hello| match hello { + Ok(RunnerOpResult::Hello { + lifecycle, + registration, + admission, + launch, + .. + }) => (lifecycle, registration, admission, launch), + other => panic!("unexpected {other:?}"), + }; + let mut first = host.connect_raw().await; + assert_eq!( + stated(first.call(RunnerOp::Hello, None).await), + (RunnerLifecycle::Independent, true, true, false) + ); + assert_eq!(first.claim(STORE).await.unwrap().epoch, 1); + let registering = first.send(RunnerOp::Registration, Some(1)).await; + let detaching = first.send(RunnerOp::Detach, None).await; + // Both are answered before the connection ends; the next connection claims the + // Runner as soon as the detach is answered. + let mut answered = Vec::new(); + let mut claiming = None; + while let Some(reply) = first.next_response().await { + let id = reply.request_id.unwrap(); + if id == detaching { + let mut next = host.connect_raw().await; + claiming = Some(tokio::spawn(async move { + let hello = next.call(RunnerOp::Hello, None).await; + let claim = next.claim(STORE).await; + (hello, claim, Instant::now()) + })); } - answered.sort(); - assert_eq!(answered, ["rrpc-detach", "rrpc-reg"]); - let (served, returned) = serving.await.unwrap(); - assert_eq!(served, Served::Detached); - independent = returned; + answered.push((id, Instant::now())); } + let registered_at = answered + .iter() + .find(|(id, _)| *id == registering) + .expect("the registration is answered") + .1; + let (hello, claim, claimed_at) = + claiming.expect("the detach is answered").await.unwrap(); + assert_eq!( + stated(hello), + (RunnerLifecycle::Independent, true, true, false) + ); + let claim = claim.unwrap(); + assert_eq!((claim.epoch, claim.previous_controller), (2, false)); + assert!(registered_at <= claimed_at); } } @@ -1744,10 +2346,10 @@ mod tests { } } - /// A managed worker refuses `Detach` and `Stop` and keeps serving; its `Hello` states its - /// lifecycle and identity (P1R-C01). + /// A managed worker refuses `Claim`, `Detach` and `Stop` and keeps serving; its `Hello` + /// states its lifecycle and identity (P1R-C01, P1R-C03). #[tokio::test] - async fn a_managed_worker_refuses_detach_and_stop() { + async fn a_managed_worker_refuses_claim_detach_and_stop() { let (client, server) = tokio::net::UnixStream::pair().unwrap(); let (worker, events) = host_worker(); tokio::spawn(async move { @@ -1757,6 +2359,12 @@ mod tests { }); let (mut read, mut write) = client.into_split(); for (id, op) in [ + ( + "rrpc-claim", + RunnerOp::Claim { + store_id: "store-1".into(), + }, + ), ("rrpc-detach", RunnerOp::Detach), ("rrpc-stop", RunnerOp::Stop), ("rrpc-hello", RunnerOp::Hello), @@ -1771,14 +2379,10 @@ mod tests { Some(RunnerOpResult::Hello { lifecycle, identity, - previous_controller, - inherited_processes, .. }) => { assert_eq!(lifecycle, RunnerLifecycle::Managed); assert_eq!(identity, Some(RunnerIdentity::this_process())); - assert!(!previous_controller); - assert_eq!(inherited_processes, 0); } None => assert_eq!(reply.error.unwrap().code, ErrorCode::InvalidCommand), other => panic!("unexpected {other:?}"), @@ -1786,42 +2390,149 @@ mod tests { } } - /// An independent Runner across its connections, through the gateway's client (P1R-C01). - mod independent { + /// A request's epoch, an event's number and a Claim cross the wire; a request sent under no + /// epoch states none (P1R-C03). + #[test] + fn an_epoch_an_event_number_and_a_claim_cross_the_wire() { + let request = WireEnvelope::request( + "rrpc-1".into(), + RunnerOp::Terminate { + process_id: ProcessId("proc-1".into()), + }, + ) + .with_epoch(Some(7)); + let json = serde_json::to_value(&request).unwrap(); + assert_eq!(json["epoch"], 7); + let parsed: WireEnvelope = serde_json::from_value(json).unwrap(); + assert_eq!(parsed.epoch, Some(7)); + let bare = + serde_json::to_value(WireEnvelope::request("rrpc-2".into(), RunnerOp::Hello)).unwrap(); + assert!(bare.get("epoch").is_none(), "{bare}"); + let event = WireEnvelope::event( + 3, + RunnerEvent::ProcessExited { + process_id: ProcessId("proc-1".into()), + }, + ); + let parsed: WireEnvelope = + serde_json::from_slice(&serde_json::to_vec(&event).unwrap()).unwrap(); + assert_eq!(parsed.seq, Some(3)); + assert_eq!( + serde_json::to_value(RunnerOp::Claim { + store_id: "store-1".into() + }) + .unwrap(), + serde_json::json!({ "op": "claim", "data": { "store_id": "store-1" } }) + ); + let claimed = RunnerOpResult::Claimed(RunnerClaim { + epoch: 2, + previous_controller: true, + processes: vec![RunnerInventoryEntry { + workspace_id: "demo".into(), + status: RunnerProcessStatus { + process_id: ProcessId("proc-1".into()), + state: codespace_domain::ProcessState::Running, + exit_code: None, + termination: None, + signal: None, + output_total: 4, + output_retained_from: 0, + eof: false, + }, + }], + events_through: 5, + }); + let json = serde_json::to_value(&claimed).unwrap(); + assert_eq!(json["type"], "claimed"); + assert_eq!(json["data"]["epoch"], 2); + assert_eq!(json["data"]["events_through"], 5); + assert_eq!(json["data"]["processes"][0]["workspace_id"], "demo"); + match serde_json::from_value(json).unwrap() { + RunnerOpResult::Claimed(claim) => assert_eq!(claim.processes.len(), 1), + other => panic!("unexpected {other:?}"), + } + let refused = RunnerOpResult::ClaimRefused { + refusal: ClaimRefusal::ControllerActive, + message: "another connection controls this Runner".into(), + }; + assert_eq!( + serde_json::to_value(&refused).unwrap()["data"]["refusal"], + "controller_active" + ); + for refusal in [ + ClaimRefusal::PeerUid, + ClaimRefusal::StoreMismatch, + ClaimRefusal::ControllerActive, + ClaimRefusal::RequestsRunning, + ClaimRefusal::Stopping, + ] { + assert_eq!( + serde_json::to_value(refusal).unwrap(), + serde_json::json!(refusal.as_str()) + ); + } + } + + /// An independent Runner across its connections, through the gateway's client and through + /// connections that send frames as no gateway's would (P1R-C01, P1R-C03). + pub(super) mod independent { use super::*; - use crate::{RunnerApplyPatchRequest, RunnerReadProcess, RunnerWriteStdin, UdsRunner}; + use crate::{ + LinkEvent, LinkObserver, RunnerApplyPatchRequest, RunnerReadProcess, RunnerWriteStdin, + UdsRunner, CLAIM_START_BOUND, + }; use codespace_domain::{ProcessState, ProcessTermination, Profile, WorkspaceId}; - use std::path::Path; + use std::path::{Path, PathBuf}; use std::time::Instant; + use tokio::task::AbortHandle; + + /// The operations database the gateways here claim the Runner for. + pub(in crate::wire::tests) const STORE: &str = "store-test"; + + /// A short private directory: a socket's path must fit `sun_path` on macOS. + fn short_directory() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("cs-") + .tempdir_in("/tmp") + .unwrap() + } - /// An independent Runner that serves the streams sent to it in turn, as the worker binary - /// serves each connection it accepts. Dropping it stops the Runner. - struct Host { - streams: mpsc::UnboundedSender, + /// An independent Runner listening on a socket, as the worker binary does. Dropping it + /// stops the Runner. + pub(in crate::wire::tests) struct Host { + pub(in crate::wire::tests) socket: PathBuf, + _dir: tempfile::TempDir, served: mpsc::UnboundedReceiver, - stop: StopHandle, - runner: InProcessRunner, + pub(in crate::wire::tests) stop: StopHandle, + pub(in crate::wire::tests) runner: InProcessRunner, + claims: Arc>>, } impl Host { - fn start() -> Self { + pub(in crate::wire::tests) fn start() -> Self { + Self::start_with(|independent| independent) + } + + pub(in crate::wire::tests) fn start_with( + configure: impl FnOnce(IndependentRunner) -> IndependentRunner, + ) -> Self { + let dir = short_directory(); + let socket = dir.path().join("runner.sock"); + let listener = Arc::new(tokio::net::UnixListener::bind(&socket).unwrap()); let (runner, events) = host_worker(); - let independent = IndependentRunner::new(runner.clone(), events); + let claims = Arc::new(std::sync::Mutex::new(Vec::new())); + let noted = claims.clone(); + let independent = configure( + IndependentRunner::new(runner.clone(), events).with_claim_observer(Arc::new( + move |note| noted.lock().unwrap().push(note), + )), + ); let stop = independent.stop_handle(); - let (streams, incoming) = mpsc::unbounded_channel::(); - let incoming = Arc::new(Mutex::new(incoming)); let (report, served) = mpsc::unbounded_channel(); tokio::spawn(independent.run( move || { - let incoming = incoming.clone(); - async move { - incoming - .lock() - .await - .recv() - .await - .ok_or_else(|| std::io::Error::other("no more connections")) - } + let listener = listener.clone(); + async move { listener.accept().await.map(|(stream, _)| stream) } }, move |ending| { if let Ok(ending) = ending { @@ -1830,40 +2541,48 @@ mod tests { }, )); Self { - streams, + socket, + _dir: dir, served, stop, runner, + claims, } } - /// A new gateway's connection, after its handshake. + /// A gateway's client after its handshake, which has not claimed the Runner. + async fn connect_unclaimed(&self, on_exit: ShellRelease) -> UdsRunner { + UdsRunner::connect(&self.socket, on_exit).await.unwrap() + } + + /// A gateway's client that claimed the Runner. async fn connect(&self) -> UdsRunner { self.connect_reporting_exits(Arc::new(|_: &str| {})).await } async fn connect_reporting_exits(&self, on_exit: ShellRelease) -> UdsRunner { - let (client, server) = tokio::net::UnixStream::pair().unwrap(); - self.streams.send(server).unwrap(); - let gateway = UdsRunner::from_stream(client, on_exit); - gateway.handshake().await.unwrap(); + let gateway = self.connect_unclaimed(on_exit).await; + gateway.claim(STORE).await.unwrap(); gateway } /// A connection the gateway's client does not speak for. - fn connect_raw(&self) -> tokio::net::UnixStream { - let (client, server) = tokio::net::UnixStream::pair().unwrap(); - self.streams.send(server).unwrap(); - client + pub(in crate::wire::tests) async fn connect_raw(&self) -> Raw { + Raw::new(tokio::net::UnixStream::connect(&self.socket).await.unwrap()) } - /// How the connection being served ended. + /// How the next controlling connection ended, or the stop. async fn served(&mut self) -> Served { tokio::time::timeout(Duration::from_secs(20), self.served.recv()) .await .expect("the connection ends") .expect("the Runner reports it") } + + /// What the Runner noted of each Claim so far. + fn claims(&self) -> Vec { + self.claims.lock().unwrap().clone() + } } impl Drop for Host { @@ -1872,6 +2591,212 @@ mod tests { } } + /// A connection that sends requests frame by frame. + pub(in crate::wire::tests) struct Raw { + read: tokio::net::unix::OwnedReadHalf, + write: tokio::net::unix::OwnedWriteHalf, + next: u64, + } + + impl Raw { + fn new(stream: tokio::net::UnixStream) -> Self { + let (read, write) = stream.into_split(); + Self { + read, + write, + next: 1, + } + } + + /// Send `op` under `epoch`; the request's id. + pub(in crate::wire::tests) async fn send( + &mut self, + op: RunnerOp, + epoch: Option, + ) -> String { + let id = format!("rrpc-raw-{}", self.next); + self.next += 1; + write_frame( + &mut self.write, + &WireEnvelope::request(id.clone(), op).with_epoch(epoch), + ) + .await + .unwrap(); + id + } + + /// The next response, past any event, or none once the connection ended. + pub(in crate::wire::tests) async fn next_response(&mut self) -> Option { + loop { + let frame = + tokio::time::timeout(Duration::from_secs(20), read_frame(&mut self.read)) + .await + .expect("a response or the end") + .ok()??; + let envelope: WireEnvelope = serde_json::from_slice(&frame).unwrap(); + if envelope.kind == WireKind::Response { + return Some(envelope); + } + } + } + + /// Send `op` under `epoch` and return its answer. + pub(in crate::wire::tests) async fn call( + &mut self, + op: RunnerOp, + epoch: Option, + ) -> Result { + let id = self.send(op, epoch).await; + let reply = self.next_response().await.expect("an answer"); + assert_eq!(reply.request_id.as_deref(), Some(id.as_str())); + match (reply.result, reply.error) { + (Some(result), None) => Ok(result), + (None, Some(error)) => Err(error), + other => panic!("unexpected {other:?}"), + } + } + + /// Claim the Runner for the operations database `store_id`. + pub(in crate::wire::tests) async fn claim( + &mut self, + store_id: &str, + ) -> Result { + match self + .call( + RunnerOp::Claim { + store_id: store_id.into(), + }, + None, + ) + .await + { + Ok(RunnerOpResult::Claimed(claim)) => Ok(claim), + Ok(RunnerOpResult::ClaimRefused { refusal, message }) => { + Err((refusal, message)) + } + other => panic!("unexpected {other:?}"), + } + } + } + + /// A relay between clients and a Runner's socket, which cuts its connections, holds + /// new ones, or loses one answer of the next one, as a connection between them can + /// fail (P1R-C03). + struct Relay { + socket: PathBuf, + _dir: tempfile::TempDir, + target: Arc>, + links: Arc>>, + held: Arc>, + /// The next connection is cut at this answer of the Runner, which is lost. + lose: Arc>>, + accepting: AbortHandle, + } + + impl Relay { + fn start(target: &Path) -> Self { + let dir = short_directory(); + let socket = dir.path().join("relay.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let target = Arc::new(std::sync::Mutex::new(target.to_path_buf())); + let links: Arc>> = Arc::default(); + let held = Arc::new(watch::Sender::new(false)); + let lose: Arc>> = Arc::default(); + let accepting = tokio::spawn({ + let (target, links, held, lose) = + (target.clone(), links.clone(), held.clone(), lose.clone()); + async move { + while let Ok((client, _)) = listener.accept().await { + let mut released = held.subscribe(); + let target = target.lock().unwrap().clone(); + let lose = lose.lock().unwrap().take(); + let link = tokio::spawn(async move { + let _ = released.wait_for(|held| !*held).await; + if let Ok(runner) = tokio::net::UnixStream::connect(&target).await { + relay_link(client, runner, lose).await; + } + }); + links.lock().unwrap().push(link.abort_handle()); + } + } + }) + .abort_handle(); + Self { + socket, + _dir: dir, + target, + links, + held, + lose, + accepting, + } + } + + /// Cut every connection: both of its ends see it end. + fn cut(&self) { + for link in self.links.lock().unwrap().drain(..) { + link.abort(); + } + } + + /// Hold each new connection, unanswered, until let go. + fn hold(&self, held: bool) { + self.held.send_replace(held); + } + + /// Lose the next connection at the Runner's `answer`th answer, counted from 1. + fn lose_answer(&self, answer: usize) { + *self.lose.lock().unwrap() = Some(answer); + } + + /// Relay new connections to the socket `target`. + fn retarget(&self, target: &Path) { + *self.target.lock().unwrap() = target.to_path_buf(); + } + } + + impl Drop for Relay { + fn drop(&mut self) { + self.accepting.abort(); + self.cut(); + } + } + + /// One relayed connection, until the Runner ends it or the relay loses an answer. + async fn relay_link( + client: tokio::net::UnixStream, + runner: tokio::net::UnixStream, + lose: Option, + ) { + let (mut client_read, mut client_write) = client.into_split(); + let (mut runner_read, mut runner_write) = runner.into_split(); + let upstream = async move { + let _ = tokio::io::copy(&mut client_read, &mut runner_write).await; + let _ = runner_write.shutdown().await; + std::future::pending::<()>().await + }; + let downstream = async move { + let mut answers = 0; + while let Ok(Some(frame)) = read_frame(&mut runner_read).await { + answers += 1; + if Some(answers) == lose { + return; + } + let length = u32::try_from(frame.len()).unwrap().to_be_bytes(); + if client_write.write_all(&length).await.is_err() + || client_write.write_all(&frame).await.is_err() + { + return; + } + } + let _ = client_write.shutdown().await; + }; + tokio::select! { + () = upstream => {} + () = downstream => {} + } + } + fn workspace(root: &Path) -> Workspace { Workspace::new( WorkspaceId("demo".into()), @@ -1934,20 +2859,63 @@ mod tests { } } + fn exits() -> (ShellRelease, mpsc::UnboundedReceiver) { + let (exits, exited_ids) = mpsc::unbounded_channel(); + ( + Arc::new(move |process_id: &str| { + let _ = exits.send(process_id.to_string()); + }), + exited_ids, + ) + } + async fn next_exit(exited_ids: &mut mpsc::UnboundedReceiver) -> Option { tokio::time::timeout(Duration::from_secs(10), exited_ids.recv()) .await .expect("an exit event") } - async fn disconnected(gateway: &UdsRunner, process_id: &ProcessId) { + /// No further exit reaches the client. + async fn no_more_exits(exited_ids: &mut mpsc::UnboundedReceiver) { + if let Ok(exit) = + tokio::time::timeout(Duration::from_millis(300), exited_ids.recv()).await + { + assert_eq!(exit, None, "an exit delivered again"); + } + } + + fn link_events() -> (LinkObserver, mpsc::UnboundedReceiver) { + let (noted, events) = mpsc::unbounded_channel(); + ( + Arc::new(move |event| { + let _ = noted.send(event); + }), + events, + ) + } + + /// The next event of a client's connection, past any failed try. + async fn next_link_event(events: &mut mpsc::UnboundedReceiver) -> LinkEvent { + loop { + match tokio::time::timeout(Duration::from_secs(20), events.recv()) + .await + .expect("a link event") + .expect("the client reports it") + { + LinkEvent::Retrying { .. } => {} + event => return event, + } + } + } + + async fn disconnected(gateway: &UdsRunner, process_id: &ProcessId) -> ErrorBody { let deadline = Instant::now() + Duration::from_secs(10); loop { match gateway.process_status(process_id).await { Err(RunnerError::Execution(body)) if body.code == ErrorCode::RunnerDisconnected => { - return + return body } other => assert!(Instant::now() < deadline, "{other:?}"), } @@ -1955,21 +2923,38 @@ mod tests { } } - /// After a detach the next gateway reads the output and status of the same process_id, + fn recovery(gateway: &UdsRunner) -> Option { + gateway.new_work_refusal().map(|refusal| refusal.code) + } + + /// After a detach the next gateway claims the Runner under the next epoch, is told + /// every process it inherits, reads the output and status of the same process_id, /// writes to it and resizes it, and the process times out at its original deadline. - /// That gateway starts nothing new, also once the inherited processes ended. + /// That gateway starts nothing new, also once the inherited processes ended. The + /// handover's latency is measured. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn the_next_gateway_inherits_the_processes_but_starts_nothing() { let mut host = Host::start(); let root = tempfile::tempdir().unwrap(); let ws = workspace(root.path()); - let first = host.connect().await; + let first = host.connect_unclaimed(Arc::new(|_: &str| {})).await; let hello = first.hello(); assert_eq!(hello.lifecycle, RunnerLifecycle::Independent); assert_eq!(hello.identity, Some(RunnerIdentity::this_process())); - assert!(!hello.previous_controller); - assert_eq!(hello.inherited_processes, 0); - assert!(!first.refuses_new_work()); + // Nothing is sent under a Claim before there is one. + assert_eq!(first.dispatch_epoch().unwrap(), None); + let claim = first.claim(STORE).await.unwrap(); + assert_eq!( + claim, + RunnerClaim { + epoch: 1, + previous_controller: false, + processes: Vec::new(), + events_through: 0, + } + ); + assert_eq!(first.claimed(), Some(claim)); + assert_eq!(first.dispatch_epoch().unwrap(), Some(1)); let pipe = ProcessId("proc-pipe".into()); let tty = ProcessId("proc-tty".into()); let started = Instant::now(); @@ -2010,12 +2995,42 @@ mod tests { ErrorCode::RunnerDisconnected ); - let second = host.connect().await; - let inherited = second.hello(); - assert_eq!(inherited.identity, hello.identity); - assert!(inherited.previous_controller); - assert_eq!(inherited.inherited_processes, 2); - assert!(second.refuses_new_work()); + let (on_exit, mut exited_ids) = exits(); + let second = host.connect_unclaimed(on_exit).await; + assert_eq!(second.hello().identity, hello.identity); + let asked = Instant::now(); + let inherited = second.claim(STORE).await.unwrap(); + eprintln!( + "p1r-c03 claim latency: a handover after a detach was granted in {:?}", + asked.elapsed() + ); + assert_eq!( + ( + inherited.epoch, + inherited.previous_controller, + inherited.events_through + ), + (2, true, 0) + ); + let held: Vec<_> = inherited + .processes + .iter() + .map(|entry| { + ( + entry.workspace_id.as_str(), + entry.status.process_id.0.as_str(), + entry.status.state, + ) + }) + .collect(); + assert_eq!( + held, + [ + ("demo", "proc-pipe", ProcessState::Running), + ("demo", "proc-tty", ProcessState::Running) + ] + ); + assert_eq!(recovery(&second), Some(ErrorCode::RecoveryPending)); output_with(&second, &pipe, "ready").await; second .write_stdin(RunnerWriteStdin { @@ -2071,13 +3086,30 @@ mod tests { exited(&second, &tty).await.termination, Some(ProcessTermination::Terminated) ); + // Both exits reached this controller. + let mut ended = vec![ + next_exit(&mut exited_ids).await.unwrap(), + next_exit(&mut exited_ids).await.unwrap(), + ]; + ended.sort(); + assert_eq!(ended, ["proc-pipe", "proc-tty"]); // The barrier stays once the inherited processes ended. second.detach().await.unwrap(); assert_eq!(host.served().await, Served::Detached); let third = host.connect().await; - assert!(third.hello().previous_controller); - assert_eq!(third.hello().inherited_processes, 0); - assert!(third.refuses_new_work()); + let claim = third.claimed().unwrap(); + assert_eq!( + (claim.epoch, claim.previous_controller, claim.events_through), + (3, true, 2) + ); + assert!( + claim + .processes + .iter() + .all(|entry| entry.status.state == ProcessState::Exited), + "{claim:?}" + ); + assert_eq!(recovery(&third), Some(ErrorCode::RecoveryPending)); assert_eq!( code(third.exec(&ws, touch).await.unwrap_err()), ErrorCode::RecoveryPending @@ -2090,12 +3122,31 @@ mod tests { assert!(!root.path().join("added.txt").exists()); assert_eq!(third.stop_runner().await.unwrap(), 0); assert_eq!(host.served().await, Served::Stopped { terminated: 0 }); - } + let claimed: Vec<_> = host + .claims() + .into_iter() + .map(|note| match note { + ClaimNote::Claimed { + epoch, + previous_controller, + peer_pid, + } => { + assert!( + peer_pid.is_none_or(|pid| pid == std::process::id() as i32), + "{peer_pid:?}" + ); + (epoch, previous_controller) + } + other => panic!("unexpected {other:?}"), + }) + .collect(); + assert_eq!(claimed, [(1, false), (2, true), (3, true)]); + } - /// Only a connection that starts work becomes the controller: one that only read leaves - /// the next gateway free to start work. + /// Only a controller that starts work bars the next one from starting work: one that + /// only read leaves the next gateway free to. #[tokio::test] - async fn a_gateway_that_starts_nothing_leaves_no_controller() { + async fn a_controller_that_starts_nothing_leaves_no_barrier() { let mut host = Host::start(); let root = tempfile::tempdir().unwrap(); let ws = workspace(root.path()); @@ -2104,8 +3155,8 @@ mod tests { reader.detach().await.unwrap(); assert_eq!(host.served().await, Served::Detached); let worker = host.connect().await; - assert!(!worker.hello().previous_controller); - assert!(!worker.refuses_new_work()); + assert!(!worker.claimed().unwrap().previous_controller); + assert_eq!(recovery(&worker), None); worker .exec( &ws, @@ -2115,7 +3166,687 @@ mod tests { .unwrap(); worker.close().await; assert_eq!(host.served().await, Served::Lost); - assert!(host.connect().await.hello().previous_controller); + assert!(host.connect().await.claimed().unwrap().previous_controller); + } + + /// A Claim's answer lists every process the Runner holds, ended ones included, and the + /// number of the last exit it reflects; an exit an earlier controller received is not + /// sent again, and later ones follow (P1R-C03). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_claim_states_the_inventory_and_the_exits_it_reflects() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let (on_exit, mut exited_ids) = exits(); + let first = host.connect_reporting_exits(on_exit).await; + first + .exec( + &ws, + command(&["/bin/sh", "-c", "exit 3"], "proc-a", 60_000, false), + ) + .await + .unwrap(); + assert_eq!(next_exit(&mut exited_ids).await.as_deref(), Some("proc-a")); + first + .exec(&ws, command(&["/bin/sleep", "30"], "proc-b", 60_000, false)) + .await + .unwrap(); + first.detach().await.unwrap(); + assert_eq!(host.served().await, Served::Detached); + + let (on_exit, mut exited_ids) = exits(); + let second = host.connect_reporting_exits(on_exit).await; + let claim = second.claimed().unwrap(); + assert_eq!((claim.epoch, claim.events_through), (2, 1)); + let listed: Vec<_> = claim + .processes + .iter() + .map(|entry| { + ( + entry.workspace_id.as_str(), + entry.status.process_id.0.as_str(), + entry.status.state, + entry.status.exit_code, + ) + }) + .collect(); + assert_eq!( + listed, + [ + ("demo", "proc-a", ProcessState::Exited, Some(3)), + ("demo", "proc-b", ProcessState::Running, None) + ] + ); + second.terminate(&ProcessId("proc-b".into())).await.unwrap(); + assert_eq!(next_exit(&mut exited_ids).await.as_deref(), Some("proc-b")); + no_more_exits(&mut exited_ids).await; + } + + /// A Claim is refused while the connection that controls the Runner is open, however + /// long it says nothing, and on that connection itself; once it ends, the next Claim + /// takes over (D6, P1R-C03). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_claim_is_refused_while_the_controller_is_open() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let controller = host.connect().await; + let mut other = host.connect_raw().await; + let (refusal, message) = other.claim(STORE).await.unwrap_err(); + assert_eq!(refusal, ClaimRefusal::ControllerActive); + assert!(message.contains("still open"), "{message}"); + match controller.claim(STORE).await.unwrap_err() { + crate::ClaimError::Refused { refusal, message } => { + assert_eq!(refusal, ClaimRefusal::ControllerActive); + assert!(message.contains("already controls"), "{message}"); + } + other => panic!("unexpected {other}"), + } + // The controller goes on under its Claim. + assert_eq!(controller.dispatch_epoch().unwrap(), Some(1)); + controller.find(&ws, None, None, None).await.unwrap(); + controller.close().await; + assert_eq!(host.served().await, Served::Lost); + let claim = other.claim(STORE).await.unwrap(); + assert_eq!((claim.epoch, claim.previous_controller), (2, false)); + assert!(matches!( + host.claims().as_slice(), + [ + ClaimNote::Claimed { epoch: 1, .. }, + ClaimNote::Refused { + refusal: ClaimRefusal::ControllerActive, + .. + }, + ClaimNote::Refused { + refusal: ClaimRefusal::ControllerActive, + .. + }, + ClaimNote::Claimed { epoch: 2, .. }, + ] + )); + } + + /// Of connections that claim the Runner at once, one controls it: the others are + /// refused, and their changes too, until it ends (P1R-C03). + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn of_claims_made_at_once_one_wins() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let mut claiming = JoinSet::new(); + for _ in 0..8 { + let mut raw = host.connect_raw().await; + claiming.spawn(async move { + let claim = raw.claim(STORE).await; + (raw, claim) + }); + } + let (mut winners, mut losers) = (Vec::new(), Vec::new()); + while let Some(joined) = claiming.join_next().await { + match joined.unwrap() { + (raw, Ok(claim)) => winners.push((raw, claim)), + (raw, Err((refusal, message))) => { + assert_eq!(refusal, ClaimRefusal::ControllerActive, "{message}"); + losers.push(raw); + } + } + } + assert_eq!(winners.len(), 1); + let (mut winner, claim) = winners.pop().unwrap(); + assert_eq!(claim.epoch, 1); + let sleep = |id: &str| RunnerOp::Exec { + workspace: ws.clone(), + request: command(&["/bin/sleep", "30"], id, 60_000, false), + }; + assert!(matches!( + winner.call(sleep("proc-winner"), Some(1)).await, + Ok(RunnerOpResult::Exec(_)) + )); + for (index, loser) in losers.iter_mut().enumerate() { + let refusal = loser + .call(sleep(&format!("proc-loser-{index}")), Some(1)) + .await + .unwrap_err(); + assert_eq!(refusal.code, ErrorCode::StaleController, "{refusal:?}"); + } + assert_eq!(host.runner.inventory().len(), 1); + assert!(matches!( + winner.call(RunnerOp::Detach, None).await, + Ok(RunnerOpResult::Detached) + )); + assert_eq!(host.served().await, Served::Detached); + let claim = losers[0].claim(STORE).await.unwrap(); + assert_eq!((claim.epoch, claim.previous_controller), (2, true)); + } + + /// A Claim from a peer of another user is refused, as the Runner reads the user from the + /// connection's credentials, and the connection controls nothing (P1R-C03). + #[tokio::test] + async fn a_claim_from_another_user_is_refused() { + // SAFETY: geteuid has no preconditions and cannot fail. + let uid = unsafe { libc::geteuid() }; + let host = Host::start_with(move |independent| independent.with_owner_uid(uid ^ 1)); + let mut raw = host.connect_raw().await; + let (refusal, message) = raw.claim(STORE).await.unwrap_err(); + assert_eq!(refusal, ClaimRefusal::PeerUid); + assert!( + message.contains(&format!("uid {uid}")) + && message.contains(&format!("uid {}", uid ^ 1)), + "{message}" + ); + let refusal = raw + .call( + RunnerOp::ProcessStatus { + process_id: ProcessId("proc-1".into()), + }, + None, + ) + .await + .unwrap_err(); + assert_eq!(refusal.code, ErrorCode::StaleController); + match host.claims().as_slice() { + [ClaimNote::Refused { + refusal: ClaimRefusal::PeerUid, + peer_pid, + .. + }] => assert!( + peer_pid.is_none_or(|pid| pid == std::process::id() as i32), + "{peer_pid:?}" + ), + other => panic!("unexpected {other:?}"), + } + } + + /// The Runner keeps the operations database it was first claimed for and refuses a + /// Claim for another, or for none; that database's gateway claims it again (P1R-C03). + #[tokio::test] + async fn a_claim_for_another_operations_database_is_refused() { + let mut host = Host::start(); + let mut first = host.connect_raw().await; + assert_eq!( + first.claim("").await.unwrap_err().0, + ClaimRefusal::StoreMismatch + ); + assert_eq!(first.claim(STORE).await.unwrap().epoch, 1); + assert!(matches!( + first.call(RunnerOp::Detach, None).await, + Ok(RunnerOpResult::Detached) + )); + assert_eq!(host.served().await, Served::Detached); + let mut other = host.connect_raw().await; + let (refusal, message) = other.claim("store-other").await.unwrap_err(); + assert_eq!(refusal, ClaimRefusal::StoreMismatch); + assert!( + message.contains(STORE) && message.contains("store-other"), + "{message}" + ); + let mut again = host.connect_raw().await; + assert_eq!(again.claim(STORE).await.unwrap().epoch, 2); + } + + /// Only the controller's requests are answered beyond `Hello`, `Claim` and `Detach`, + /// and each that changes a process or the workspace must carry the epoch of its Claim: + /// one under no epoch or another, and one prepared under an earlier Claim that arrives + /// after the next one, are refused with STALE_CONTROLLER, and nothing starts or changes + /// (P1R-C03). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn requests_outside_the_current_claim_are_refused() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let marker = root.path().join("started"); + let touch = |id: &str| RunnerOp::Exec { + workspace: ws.clone(), + request: command( + &["/usr/bin/touch", &marker.display().to_string()], + id, + 5_000, + false, + ), + }; + let patch = RunnerOp::ApplyPatch { + workspace: ws.clone(), + request: RunnerApplyPatchRequest { + patch: "*** Begin Patch\n*** Add File: added.txt\n+added\n*** End Patch\n" + .into(), + expected_versions: Default::default(), + check_only: false, + }, + }; + let mut unclaimed = host.connect_raw().await; + assert!(matches!( + unclaimed.call(RunnerOp::Hello, None).await, + Ok(RunnerOpResult::Hello { .. }) + )); + let mut controller = host.connect_raw().await; + assert_eq!(controller.claim(STORE).await.unwrap().epoch, 1); + // A connection that never claimed the Runner, whatever epoch it names. + for op in [ + touch("proc-a"), + patch.clone(), + RunnerOp::ProcessStatus { + process_id: ProcessId("proc-a".into()), + }, + RunnerOp::Find { + workspace: ws.clone(), + glob: None, + offset: None, + limit: None, + }, + RunnerOp::TerminateWorkspace { + workspace_id: "demo".into(), + }, + RunnerOp::Stop, + ] { + let refusal = unclaimed.call(op, Some(1)).await.unwrap_err(); + assert_eq!(refusal.code, ErrorCode::StaleController, "{refusal:?}"); + } + // The controller's change under no epoch, or another. + for epoch in [None, Some(0), Some(2)] { + for op in [touch("proc-b"), patch.clone(), RunnerOp::Stop] { + let refusal = controller.call(op, epoch).await.unwrap_err(); + assert_eq!(refusal.code, ErrorCode::StaleController, "{refusal:?}"); + } + } + // A query needs no epoch. + assert!(matches!( + controller + .call( + RunnerOp::Find { + workspace: ws.clone(), + glob: None, + offset: None, + limit: None, + }, + None + ) + .await, + Ok(RunnerOpResult::Find(_)) + )); + assert!(host.runner.inventory().is_empty()); + assert!(!marker.exists() && !root.path().join("added.txt").exists()); + assert!(!host.stop.is_requested()); + // Under its epoch, the controller's change is made. + assert!(matches!( + controller + .call( + RunnerOp::Exec { + workspace: ws.clone(), + request: command(&["/bin/sleep", "30"], "proc-c", 60_000, false), + }, + Some(1) + ) + .await, + Ok(RunnerOpResult::Exec(_)) + )); + assert!(matches!( + controller.call(RunnerOp::Detach, None).await, + Ok(RunnerOpResult::Detached) + )); + assert_eq!(host.served().await, Served::Detached); + // A change prepared under the first Claim, sent after the second. + let mut next = host.connect_raw().await; + assert_eq!(next.claim(STORE).await.unwrap().epoch, 2); + for op in [ + RunnerOp::Terminate { + process_id: ProcessId("proc-c".into()), + }, + touch("proc-d"), + ] { + let refusal = next.call(op, Some(1)).await.unwrap_err(); + assert_eq!(refusal.code, ErrorCode::StaleController, "{refusal:?}"); + } + assert_eq!( + host.runner + .process_status(&ProcessId("proc-c".into())) + .await + .unwrap() + .state, + ProcessState::Running + ); + assert_eq!(host.runner.inventory().len(), 1); + assert!(!marker.exists()); + } + + /// A Claim made once its controller's connection ended waits for that connection's + /// requests, which may still change a process, and is granted once they end (P1R-C03, + /// risk 12). The wait is measured. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_claim_waits_for_the_requests_of_the_controller_before_it() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let first = host + .connect() + .await + .with_call_deadline(Duration::from_millis(200)); + // A process that never reads its stdin: a large write to it waits for its timeout, + // which runs from its spawn. + let stuck = ProcessId("proc-stuck".into()); + let stuck_since = Instant::now(); + first + .exec(&ws, command(&["/bin/sleep", "30"], &stuck.0, 1_500, false)) + .await + .unwrap(); + let err = first + .write_stdin(RunnerWriteStdin { + process_id: stuck.clone(), + data: "x".repeat(4 * 1024 * 1024), + }) + .await + .unwrap_err(); + assert!( + matches!(err, RunnerError::TransportAmbiguous { .. }), + "{err:?}" + ); + let mut next = host.connect_raw().await; + let asked = Instant::now(); + let claim = next.claim(STORE).await.unwrap(); + let waited = asked.elapsed(); + eprintln!( + "p1r-c03 claim latency: waited {waited:?} for the earlier controller's write to \ + end with its process's 1.5 s timeout" + ); + assert_eq!(claim.epoch, 2); + assert!( + stuck_since.elapsed() >= Duration::from_millis(1_500), + "granted before the write ended: {:?}", + stuck_since.elapsed() + ); + assert!(waited < CLAIM_WAIT_BOUND, "{waited:?}"); + assert_eq!(host.served().await, Served::Lost); + } + + /// Past its bound, a Claim that waits for the requests of the controller before it is + /// refused and changes nothing: the next Claim, once they ended, is the Runner's + /// second (P1R-C03, risk 12). The wait is measured. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_claim_past_its_wait_bound_is_refused_and_changes_nothing() { + let mut host = Host::start_with(|independent| { + independent.with_claim_wait(Duration::from_millis(300)) + }); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let first = host + .connect() + .await + .with_call_deadline(Duration::from_millis(200)); + let stuck = ProcessId("proc-stuck".into()); + first + .exec(&ws, command(&["/bin/sleep", "30"], &stuck.0, 3_000, false)) + .await + .unwrap(); + assert!(first + .write_stdin(RunnerWriteStdin { + process_id: stuck.clone(), + data: "x".repeat(4 * 1024 * 1024), + }) + .await + .is_err()); + let mut next = host.connect_raw().await; + let asked = Instant::now(); + let (refusal, message) = next.claim(STORE).await.unwrap_err(); + let waited = asked.elapsed(); + eprintln!( + "p1r-c03 claim latency: refused after {waited:?} past a wait bound of 300 ms" + ); + assert_eq!(refusal, ClaimRefusal::RequestsRunning, "{message}"); + assert!(message.contains("did not end within"), "{message}"); + assert!( + waited >= Duration::from_millis(300) && waited < Duration::from_secs(2), + "{waited:?}" + ); + assert_eq!(host.served().await, Served::Lost); + let claim = next.claim(STORE).await.unwrap(); + assert_eq!((claim.epoch, claim.previous_controller), (2, true)); + } + + /// A client that claimed its Runner claims it again once its connection is lost, + /// refusing every request with RUNNER_RECONNECTING meanwhile. The processes keep + /// running, nothing runs twice, each exit reaches the client once, the Claim again is + /// the next epoch, and a request prepared under the lost Claim is refused (P1R-C03). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_client_claims_its_runner_again_after_a_lost_connection() { + let host = Host::start(); + let relay = Relay::start(&host.socket); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let (on_exit, mut exited_ids) = exits(); + let (observer, mut events) = link_events(); + let (gateway, claim) = + UdsRunner::claim_independent(&relay.socket, STORE, on_exit, observer) + .await + .unwrap(); + assert_eq!((claim.epoch, claim.previous_controller), (1, false)); + let long = ProcessId("proc-long".into()); + let gated = ProcessId("proc-gated".into()); + gateway + .exec(&ws, command(&["/bin/sleep", "30"], &long.0, 60_000, false)) + .await + .unwrap(); + gateway + .exec( + &ws, + command( + &["/bin/sh", "-c", "while [ ! -e go ]; do sleep 0.02; done"], + &gated.0, + 60_000, + false, + ), + ) + .await + .unwrap(); + relay.hold(true); + relay.cut(); + assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); + // Nothing is sent meanwhile. + assert_eq!( + code(gateway.process_status(&long).await.unwrap_err()), + ErrorCode::RunnerReconnecting + ); + assert_eq!( + gateway.dispatch_epoch().unwrap_err().code, + ErrorCode::RunnerReconnecting + ); + let touch = command(&["/usr/bin/touch", "marker"], "proc-new", 5_000, false); + assert_eq!( + code(gateway.exec(&ws, touch.clone()).await.unwrap_err()), + ErrorCode::RunnerReconnecting + ); + // A process that ends while the client reconnects. + std::fs::write(root.path().join("go"), "").unwrap(); + exited(&host.runner, &gated).await; + relay.hold(false); + match next_link_event(&mut events).await { + LinkEvent::Reclaimed(claim) => { + assert_eq!((claim.epoch, claim.previous_controller), (2, true)); + assert_eq!(claim.processes.len(), 2, "{claim:?}"); + } + other => panic!("unexpected {other:?}"), + } + assert_eq!(gateway.claimed().map(|claim| claim.epoch), Some(2)); + assert_eq!( + gateway.process_status(&long).await.unwrap().state, + ProcessState::Running + ); + assert_eq!( + next_exit(&mut exited_ids).await.as_deref(), + Some("proc-gated") + ); + // A request prepared under the lost Claim is refused, and nothing new starts. + assert_eq!( + code( + gateway + .at_epoch(1) + .exec(&ws, touch.clone()) + .await + .unwrap_err() + ), + ErrorCode::StaleController + ); + assert_eq!(recovery(&gateway), Some(ErrorCode::RecoveryPending)); + assert_eq!( + code(gateway.exec(&ws, touch).await.unwrap_err()), + ErrorCode::RecoveryPending + ); + gateway.terminate(&long).await.unwrap(); + assert_eq!( + next_exit(&mut exited_ids).await.as_deref(), + Some("proc-long") + ); + no_more_exits(&mut exited_ids).await; + assert_eq!(host.runner.inventory().len(), 2); + assert!(!root.path().join("marker").exists()); + } + + /// A Claim whose answer is lost is asked again over a new connection: the Runner let go + /// of the lost connection it had granted the Claim to, and grants the next one + /// (P1R-C03). The time to the second grant is measured. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_claim_whose_answer_is_lost_is_asked_again() { + let host = Host::start(); + let relay = Relay::start(&host.socket); + // The `Hello` is answered; the Claim's answer is lost with its connection. + relay.lose_answer(2); + let (observer, mut events) = link_events(); + let asked = Instant::now(); + let (gateway, claim) = UdsRunner::claim_independent( + &relay.socket, + STORE, + Arc::new(|_: &str| {}), + observer, + ) + .await + .unwrap(); + let elapsed = asked.elapsed(); + eprintln!( + "p1r-c03 claim latency: a Claim whose answer was lost was granted again after \ + {elapsed:?}" + ); + assert_eq!( + claim.epoch, 2, + "the lost connection held the first: {claim:?}" + ); + assert!(elapsed < CLAIM_START_BOUND, "{elapsed:?}"); + assert!(matches!(events.try_recv(), Ok(LinkEvent::Retrying { .. }))); + assert_eq!(gateway.dispatch_epoch().unwrap(), Some(2)); + let granted = host + .claims() + .into_iter() + .filter(|note| matches!(note, ClaimNote::Claimed { .. })) + .count(); + assert_eq!(granted, 2); + } + + /// While a client reconnects, another process of the same user can claim the Runner + /// (D6): the client's Claims are refused while that connection is open, and it claims + /// the Runner once that connection ends (P1R-C03). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_runner_claimed_by_another_connection_meanwhile_is_claimed_once_it_ends() { + let host = Host::start(); + let relay = Relay::start(&host.socket); + let (observer, mut events) = link_events(); + let (gateway, _) = UdsRunner::claim_independent( + &relay.socket, + STORE, + Arc::new(|_: &str| {}), + observer, + ) + .await + .unwrap(); + relay.hold(true); + relay.cut(); + assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); + let mut other = host.connect_raw().await; + assert_eq!(other.claim(STORE).await.unwrap().epoch, 2); + relay.hold(false); + let refused = match tokio::time::timeout(Duration::from_secs(20), events.recv()) + .await + .unwrap() + .unwrap() + { + LinkEvent::Retrying { reason, .. } => reason, + other => panic!("unexpected {other:?}"), + }; + assert!(refused.contains("still open"), "{refused}"); + assert_eq!( + code( + gateway + .process_status(&ProcessId("proc-1".into())) + .await + .unwrap_err() + ), + ErrorCode::RunnerReconnecting + ); + drop(other); + match next_link_event(&mut events).await { + LinkEvent::Reclaimed(claim) => assert_eq!(claim.epoch, 3), + other => panic!("unexpected {other:?}"), + } + assert_eq!(gateway.dispatch_epoch().unwrap(), Some(3)); + } + + /// A client stops claiming its Runner again once another Runner answers at the socket + /// or nothing accepts connections there, and refuses every request with + /// RUNNER_DISCONNECTED from then on, saying why (P1R-C03). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_client_that_cannot_claim_its_runner_again_is_disconnected() { + let host = Host::start(); + let other = Host::start_with(|independent| { + independent.with_identity(RunnerIdentity { + boot_id: None, + pid: 1, + started_at_ms: 1, + nonce: "another-runner".into(), + }) + }); + let relay = Relay::start(&host.socket); + let (observer, mut events) = link_events(); + let (gateway, _) = UdsRunner::claim_independent( + &relay.socket, + STORE, + Arc::new(|_: &str| {}), + observer, + ) + .await + .unwrap(); + relay.retarget(&other.socket); + relay.cut(); + assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); + match next_link_event(&mut events).await { + LinkEvent::Gone(reason) => { + assert!(reason.contains("another Runner answers"), "{reason}") + } + other => panic!("unexpected {other:?}"), + } + let refusal = disconnected(&gateway, &ProcessId("proc-1".into())).await; + assert!( + refusal.message.contains("another Runner answers"), + "{refusal:?}" + ); + // The other Runner was asked nothing but its `Hello`. + assert!(other.claims().is_empty()); + + let mut stopping = Host::start(); + let (observer, mut events) = link_events(); + let (gateway, _) = UdsRunner::claim_independent( + &stopping.socket, + STORE, + Arc::new(|_: &str| {}), + observer, + ) + .await + .unwrap(); + stopping.stop.request(); + assert_eq!(stopping.served().await, Served::Stopped { terminated: 0 }); + assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); + match next_link_event(&mut events).await { + LinkEvent::Gone(reason) => { + assert!(reason.contains("nothing accepts connections"), "{reason}") + } + other => panic!("unexpected {other:?}"), + } + disconnected(&gateway, &ProcessId("proc-1".into())).await; } /// An explicit stop ends the processes, a tty's process group included, and the Runner. @@ -2167,14 +3898,16 @@ mod tests { ); } - /// A stop request, as the Runner's own SIGTERM or SIGINT makes, ends the connection being - /// served and the processes; the gateway's later requests are refused. + /// A stop request, as the Runner's own SIGTERM or SIGINT makes, ends the controller's + /// connection and the processes; the gateway's later requests are refused, and a Claim + /// meanwhile is refused. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_stop_request_ends_the_connection_and_the_processes() { let mut host = Host::start(); let root = tempfile::tempdir().unwrap(); let ws = workspace(root.path()); let gateway = host.connect().await; + let mut other = host.connect_raw().await; let pipe = ProcessId("proc-pipe".into()); gateway .exec(&ws, command(&["/bin/sleep", "30"], &pipe.0, 60_000, false)) @@ -2187,10 +3920,12 @@ mod tests { Some(ProcessTermination::Terminated) ); disconnected(&gateway, &pipe).await; + // The other connection ended with the stop. + assert!(other.next_response().await.is_none()); } /// A connection that ends without a word is a loss: the processes keep running, and an - /// exit while no gateway is connected reaches the next one. + /// exit while no gateway controls the Runner reaches the next controller. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_lost_connection_keeps_the_processes_and_their_exit_events() { let mut host = Host::start(); @@ -2213,14 +3948,11 @@ mod tests { exited(&host.runner, &quick).await.termination, Some(ProcessTermination::Exited) ); - let (exits, mut exited_ids) = mpsc::unbounded_channel(); - let second = host - .connect_reporting_exits(Arc::new(move |process_id: &str| { - let _ = exits.send(process_id.to_string()); - })) - .await; - assert!(second.hello().previous_controller); - assert_eq!(second.hello().inherited_processes, 1); + let (on_exit, mut exited_ids) = exits(); + let second = host.connect_reporting_exits(on_exit).await; + let claim = second.claimed().unwrap(); + assert!(claim.previous_controller); + assert_eq!(claim.processes.len(), 2); assert_eq!( next_exit(&mut exited_ids).await.as_deref(), Some("proc-quick") @@ -2234,10 +3966,11 @@ mod tests { next_exit(&mut exited_ids).await.as_deref(), Some("proc-long") ); + no_more_exits(&mut exited_ids).await; } - /// A gateway whose connection ends right after its `Detach` or `Stop` still detaches or - /// stops: the Runner acts on what it read, though its answer is lost. + /// A controller whose connection ends right after its `Detach` or `Stop` still detaches + /// or stops: the Runner acts on what it read, though its answer is lost. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_detach_or_stop_whose_answer_is_lost_still_holds() { let mut host = Host::start(); @@ -2251,15 +3984,14 @@ mod tests { .unwrap(); gateway.close().await; assert_eq!(host.served().await, Served::Lost); - for (op, ending) in [ - (RunnerOp::Detach, Served::Detached), - (RunnerOp::Stop, Served::Stopped { terminated: 1 }), + for (epoch, op, ending) in [ + (2, RunnerOp::Detach, Served::Detached), + (3, RunnerOp::Stop, Served::Stopped { terminated: 1 }), ] { - let (_, mut write) = host.connect_raw().into_split(); - write_frame(&mut write, &WireEnvelope::request("rrpc-end".into(), op)) - .await - .unwrap(); - drop(write); + let mut raw = host.connect_raw().await; + assert_eq!(raw.claim(STORE).await.unwrap().epoch, epoch); + raw.send(op, Some(epoch)).await; + drop(raw); assert_eq!(host.served().await, ending); assert_eq!( host.runner.process_status(&long).await.unwrap().state == ProcessState::Running, diff --git a/crates/runner/tests/uds_runner.rs b/crates/runner/tests/uds_runner.rs index 9d79120..82ebf44 100644 --- a/crates/runner/tests/uds_runner.rs +++ b/crates/runner/tests/uds_runner.rs @@ -121,7 +121,7 @@ async fn uds_process_status_echo_and_protocol_6_hello() { .expect("serve runner"); }); let runner = UdsRunner::from_stream(client, Arc::new(|_| {})); - runner.handshake().await.expect("hello protocol 9"); + runner.handshake().await.expect("hello protocol 10"); let dir = tempdir().unwrap(); let ws = workspace(dir.path()); let process_id = ProcessId("proc-uds-status".into()); diff --git a/crates/server/src/devguard.rs b/crates/server/src/devguard.rs index 2f7eca8..537a10e 100644 --- a/crates/server/src/devguard.rs +++ b/crates/server/src/devguard.rs @@ -1914,8 +1914,6 @@ pub(crate) mod tests { protocol: WIRE_PROTOCOL, lifecycle: RunnerLifecycle::Managed, identity: None, - previous_controller: false, - inherited_processes: 0, registration, admission: registration, launch: false, diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 598eeb2..2b6d7d8 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -1,4 +1,3 @@ -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -136,8 +135,6 @@ struct StartedRunner { runner: RuntimeBackend, process: Option, lifecycle: GatewayLifecycle, - /// Set before the gateway ends its connection to an independent Runner itself. - leaving: Arc, /// The records of the executions sent to an independent Runner (P1R-C02). records: Option>, } @@ -151,13 +148,11 @@ impl StartedRunner { runner, process, lifecycle, - leaving, records: _, } = self; if let (GatewayLifecycle::Independent { exit_intent }, RuntimeBackend::Uds(runner)) = (lifecycle, &runner) { - leaving.store(true, Ordering::SeqCst); let intent = match ending { Ending::Restart => RunnerExitIntent::Detach, Ending::Shutdown => exit_intent, @@ -206,13 +201,11 @@ async fn start_runner( store_for_lease.release_process(process_id); }) }; - let leaving = Arc::new(AtomicBool::new(false)); match cli.runner { RunnerMode::InProcess => Ok(StartedRunner { runner: RuntimeBackend::in_process(on_release), process: None, lifecycle, - leaving, records: None, }), RunnerMode::Uds => { @@ -221,11 +214,15 @@ async fn start_runner( { // Each exit releases its lease and is recorded (P1R-C02). let (on_release, exits) = ProcessRecords::release_hook(store.clone()); - // The gateway only connects; the Runner registers itself, from its own - // credential file, and nothing is handed to it (P1R-C01). - let runner = - RuntimeProcess::connect_independent(socket, on_release, leaving.clone()) - .await?; + // The Runner remembers the operations database it was first claimed for + // (P1R-C03). + let store_id = store.store_id().map_err(|err| { + anyhow::anyhow!("the operations database's identity: {}", err.message) + })?; + // The gateway only connects and claims; the Runner registers itself, from its + // own credential file, and nothing is handed to it (P1R-C01). + let (runner, _claim) = + RuntimeProcess::connect_independent(socket, &store_id, on_release).await?; #[cfg(feature = "devguard")] if cli.devguard.worker().is_some() { codespace_server::runtime::require_registration(&runner, socket)?; @@ -240,7 +237,6 @@ async fn start_runner( runner, process: None, lifecycle, - leaving, records: Some(records), }); } @@ -260,7 +256,6 @@ async fn start_runner( runner: RuntimeBackend::Uds(runner), process: Some(process), lifecycle, - leaving, records: None, }); } @@ -271,7 +266,6 @@ async fn start_runner( runner: RuntimeBackend::Uds(runner), process: Some(process), lifecycle, - leaving, records: None, }); } @@ -285,7 +279,6 @@ async fn start_runner( runner: RuntimeBackend::Uds(runner), process: None, lifecycle, - leaving, records: None, }) } diff --git a/crates/server/src/mcp.rs b/crates/server/src/mcp.rs index 569a4dc..7e2f311 100644 --- a/crates/server/src/mcp.rs +++ b/crates/server/src/mcp.rs @@ -158,23 +158,35 @@ subtree and drops handles. Restart does not recover process_id. If you lose \ the spawn response before receiving process_id, do not search for the lost \ handle and do not start a duplicate command."; -/// The lifetime paragraph for an independent Runner (P1R-C01), after what the gateway's -/// shutdown does. It states only what the Runner guarantees: a pipe process has no process -/// group of its own, so terminating one signals that process alone. +/// The lifetime paragraph for an independent Runner (P1R-C01, P1R-C03), after what the +/// gateway's shutdown does. It states only what the Runner guarantees: a pipe process has no +/// process group of its own, so terminating one signals that process alone; and one gateway at a +/// time controls the Runner, but nothing keeps another process of the same user from claiming it +/// while none does. const INDEPENDENT_LIFETIME: &str = "\ Managed process lifetime is owned by an independent Runner that this \ gateway did not start, not by the MCP session or the gateway. Client or \ -HTTP disconnect keeps the process running. {shutdown} If this gateway \ -loses its Runner connection, the processes keep running, but this gateway \ -refuses every later request that needs the Runner with RUNNER_DISCONNECTED \ -until it is restarted. A restarted gateway can read the output and status \ -of, write to, resize and terminate the processes started before, with the \ -same process_id, but it refuses new exec_command, apply_patch and \ +HTTP disconnect keeps the process running. {shutdown} One gateway at a \ +time controls the Runner. If this gateway loses its Runner connection, the \ +processes keep running and the gateway claims the Runner again; meanwhile \ +every request that needs the Runner returns RUNNER_RECONNECTING and can be \ +retried once the gateway has reconnected. If the gateway cannot claim the \ +Runner again, those requests return RUNNER_DISCONNECTED until it is \ +restarted. STALE_CONTROLLER means that the Runner refused a request sent \ +under an earlier claim; nothing was started or changed. After a reconnect \ +or a restart, the gateway can read the output and status of, write to, \ +resize and terminate the processes started before, with the same \ +process_id, but if work was started on the Runner under an earlier claim, \ +this gateway's own included, it refuses new exec_command, apply_patch and \ operation_resume with RECOVERY_PENDING until the Runner is stopped and a \ -new one is started. Terminating a pipe process signals that process only, \ -not its process group, so processes it started may outlive it. If you lose \ -the spawn response before receiving process_id, do not search for the lost \ -handle and do not start a duplicate command."; +new one is started. A request refused with one of these codes consumes no \ +approval, and a patch refused with one keeps no result for its \ +operation_key. While no gateway controls the Runner, any process of the \ +same user that can reach its socket, such as a command run without a \ +sandbox, can take control of it. Terminating a pipe process signals that \ +process only, not its process group, so processes it started may outlive \ +it. If you lose the spawn response before receiving process_id, do not \ +search for the lost handle and do not start a duplicate command."; const DETACH_SHUTDOWN: &str = "\ When this gateway shuts down it detaches: the Runner keeps the processes, \ @@ -261,14 +273,30 @@ impl CodeSpace { self } - /// New work is refused when the runner is an independent Runner that an earlier gateway - /// controlled (P1R-C01), before any approval, lease or claim: what that gateway left is not - /// recovered yet. + /// New work is refused before any approval, lease or claim when the runner is an + /// independent Runner that this gateway cannot send it to (P1R-C01, P1R-C03): an earlier + /// controller started work on it, which is not recovered yet, or the connection to it is + /// lost or being made again. fn refuse_new_work(&self) -> Result<(), ErrorBody> { - if self.runner.refuses_new_work() { - return Err(codespace_runner::recovery_pending()); + match self.runner.new_work_refusal() { + Some(refusal) => Err(refusal), + None => Ok(()), + } + } + + /// Give back the approval being resumed, unconsumed, after a refusal that proves its + /// execution or patch never started (CSRG-U5, P1R-C03). + fn give_back(&self, resume: Option<&ApprovalId>) { + let Some(approval_id) = resume else { + return; + }; + if let Err(err) = self.store.return_resume(approval_id) { + tracing::warn!( + approval_id = %approval_id.0, + error = %err.message, + "the approval could not be given back; its resume consumes it" + ); } - Ok(()) } /// `info` with the process lifetime of the gateway's runner (P1R-C01). @@ -668,6 +696,9 @@ impl CodeSpace { codespace_policy::allow(ws, Action::Write, &ClientClaims::default())?; ws.require_file_write()?; let _lease = self.store.acquire_write(¶ms.workspace_id.0).await?; + // Checked again once the lease is held, before the approval being resumed is marked + // (P1R-C03): a refusal here leaves it queued. + let epoch = self.runner.dispatch_epoch()?; if let Some(id) = resume { self.store.mark_resuming(id)?; } @@ -685,6 +716,7 @@ impl CodeSpace { Begin::Fresh(operation_id) => { let result = match self .runner + .at_epoch(epoch) .apply_patch( ws, RunnerApplyPatchRequest { @@ -704,6 +736,19 @@ impl CodeSpace { work_id: None, coordination: None, }, + // No Runner acted on it (P1R-C03): the operation is forgotten, so its + // operation_key begins afresh, and the approval is given back. + Err(RunnerError::Execution(err)) if never_sent(&err) => { + if let Err(forgotten) = self.store.abandon(&operation_id) { + tracing::warn!( + operation_id = %operation_id.0, + error = %forgotten.message, + "the operation that no Runner acted on could not be forgotten" + ); + } + self.give_back(resume); + return Err(err); + } Err(RunnerError::Execution(err)) => { let failed = ApplyPatchResult::new(PatchStatus::Rejected, operation_id.clone()); @@ -997,14 +1042,20 @@ impl CodeSpace { .store .acquire_shell_busy(¶ms.workspace_id.0, &process_id.0) .await?; + // Checked again once the workspace is reserved, before the approval being resumed is + // marked (P1R-C03): a refusal here leaves it queued and releases the reservation. + let epoch = self.runner.dispatch_epoch()?; + let runner = self.runner.at_epoch(epoch); // A governed execution's DevGuard attempt, minted before its record names it. let attempt_id = participation.attempt_id(); match &self.records { - // Recorded before it is sent to an independent Runner, in the transaction that marks - // the approval it resumes (P1R-C02). + // Recorded before it is sent to an independent Runner, under the controller epoch it + // is sent with, in the transaction that marks the approval it resumes (P1R-C02, + // P1R-C03). Some(records) => records.dispatching( &process_id, ¶ms, + epoch, participation.recorded(), attempt_id.as_deref(), resume, @@ -1030,14 +1081,14 @@ impl CodeSpace { { // The workspace lease is held while the owner prepares; nothing is started, // so it is released with the answer whatever the attempt's own state. - let refused = self.prepare_governed(ws, req, attempt_id).await; + let refused = self.prepare_governed(&runner, ws, req, attempt_id).await; self.record_not_started(&process_id); reservation.abort(); return Err(self.refused_governed(resume, refused)); } // The owner may create a process from here on (CSRG-U4). reservation.arm_dispatch(); - return match self.launch_governed(ws, req, attempt_id).await { + return match self.launch_governed(&runner, ws, req, attempt_id).await { Ok(launch) => { reservation.confirm(); self.record_launch(&launch); @@ -1065,7 +1116,7 @@ impl CodeSpace { } } reservation.arm_dispatch(); - match self.runner.exec(ws, req).await { + match runner.exec(ws, req).await { Ok(result) => { reservation.confirm(); if let Some(records) = &self.records { @@ -1093,7 +1144,12 @@ impl CodeSpace { Err(err) => { reservation.abort(); self.record_not_started(&process_id); - Err(err.into_error_body()) + let err = err.into_error_body(); + // No Runner acted on it (P1R-C03). + if never_sent(&err) { + self.give_back(resume); + } + Err(err) } } } @@ -1147,6 +1203,27 @@ impl NotStarted { proven: false, } } + + /// The owner's refusal of a request, which proves that nothing started when no Runner acted + /// on the request (P1R-C03). + fn refused(body: ErrorBody) -> Self { + Self { + proven: never_sent(&body), + body, + } + } +} + +/// A refusal of a request that no independent Runner acted on, so nothing it asked for started +/// (P1R-C01, P1R-C03): the gateway could not send it, or the Runner refused it before acting. +fn never_sent(err: &ErrorBody) -> bool { + matches!( + err.code, + ErrorCode::StaleController + | ErrorCode::RecoveryPending + | ErrorCode::RunnerReconnecting + | ErrorCode::RunnerDisconnected + ) } /// How a new execution in a workspace takes part in the resource authority. @@ -1216,6 +1293,7 @@ impl CodeSpace { #[cfg(feature = "devguard")] async fn prepare_governed( &self, + runner: &RuntimeBackend, ws: &Workspace, req: RunnerExecRequest, attempt_id: String, @@ -1226,12 +1304,9 @@ impl CodeSpace { "no execution owner registration", )); }; - match registration - .prepare(&self.runner, ws, req, attempt_id) - .await - { + match registration.prepare(runner, ws, req, attempt_id).await { Ok(report) => NotStarted::report(report, &ws.id.0), - Err(err) => NotStarted::unproven(err), + Err(err) => NotStarted::refused(err), } } @@ -1240,6 +1315,7 @@ impl CodeSpace { #[cfg(feature = "devguard")] async fn launch_governed( &self, + runner: &RuntimeBackend, ws: &Workspace, req: RunnerExecRequest, attempt_id: String, @@ -1251,9 +1327,9 @@ impl CodeSpace { ))); }; match registration - .launch(&self.runner, ws, req, attempt_id) + .launch(runner, ws, req, attempt_id) .await - .map_err(NotStarted::unproven)? + .map_err(NotStarted::refused)? { codespace_runner::launch::LaunchReport::Launched(launch) => Ok(launch), codespace_runner::launch::LaunchReport::NotLaunched(report) => { @@ -1263,18 +1339,12 @@ impl CodeSpace { } /// The refusal of a governed execution. An approval being resumed gets its hold back when - /// the authority proves that the command never started (CSRG-U5); otherwise the resume - /// consumes it as before. + /// the authority proves that the command never started (CSRG-U5), or no Runner acted on it + /// (P1R-C03); otherwise the resume consumes it as before. #[cfg(feature = "devguard")] fn refused_governed(&self, resume: Option<&ApprovalId>, refused: NotStarted) -> ErrorBody { - if let (Some(approval_id), true) = (resume, refused.proven) { - if let Err(err) = self.store.return_resume(approval_id) { - tracing::warn!( - approval_id = %approval_id.0, - error = %err.message, - "the approval could not be given back; its resume consumes it" - ); - } + if refused.proven { + self.give_back(resume); } refused.body } @@ -3017,8 +3087,6 @@ mod devguard_tests { protocol: WIRE_PROTOCOL, lifecycle: RunnerLifecycle::Managed, identity: None, - previous_controller: false, - inherited_processes: 0, registration: true, admission: true, launch: false, @@ -3118,8 +3186,6 @@ mod devguard_tests { protocol: WIRE_PROTOCOL, lifecycle: RunnerLifecycle::Managed, identity: None, - previous_controller: false, - inherited_processes: 0, registration: true, admission: true, launch: true, @@ -4397,10 +4463,13 @@ mod independent_tests { use crate::config::{GatewayLifecycle, RunnerExitIntent}; use crate::records::ProcessRecords; use codespace_domain::{ - ApprovalDecision, ProcessDisconnectAction, ProcessState, ProcessStatusParams, + ApprovalDecision, OperationKey, ProcessDisconnectAction, ProcessState, ProcessStatusParams, ProcessTermination, TerminateProcessParams, WorkspaceId, }; - use codespace_runner::{host_worker, IndependentRunner, RunnerIdentity, StopHandle, UdsRunner}; + use codespace_runner::{ + host_worker, IndependentRunner, LinkEvent, RunnerIdentity, RunnerOp, RunnerOpResult, + StopHandle, UdsRunner, + }; use codespace_store::{ ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, RunnerRef, }; @@ -4453,8 +4522,8 @@ mod independent_tests { } } - /// A gateway connected to an independent Runner through `stream`, recording its executions - /// in `database`. + /// A gateway connected to an independent Runner through `stream`, which it claims for + /// `database` (P1R-C03), recording its executions there. async fn recording_gateway( root: &std::path::Path, database: &std::path::Path, @@ -4464,6 +4533,7 @@ mod independent_tests { let (release, exits) = ProcessRecords::release_hook(store.clone()); let runner = UdsRunner::from_stream(stream, release); runner.handshake().await.unwrap(); + runner.claim(&store.store_id().unwrap()).await.unwrap(); let identity = runner .hello() .identity @@ -4573,16 +4643,35 @@ mod independent_tests { } } - /// A Runner that answers `Hello` as the independent Runner `identity`, hands the first - /// execution it is sent to `on_exec`, and then is lost without answering it. + /// A Runner that answers `Hello` as the independent Runner `identity` and a `Claim` with + /// epoch 1, hands the first execution it is sent to `on_exec`, and then is lost without + /// answering it. async fn fake_runner( - mut stream: UnixStream, + stream: UnixStream, identity: RunnerIdentity, on_exec: impl FnOnce(RunnerExecRequest), + ) { + let mut on_exec = Some(on_exec); + fake_runner_answering(stream, identity, |op| match (op, on_exec.take()) { + (RunnerOp::Exec { request, .. }, Some(on_exec)) => { + on_exec(request); + None + } + (other, _) => panic!("a request the fake Runner does not answer: {other:?}"), + }) + .await + } + + /// A Runner that answers `Hello` as the independent Runner `identity` and a `Claim` with + /// epoch 1, and any other request as `answer` says, until it returns none: the Runner is + /// then lost without answering. + async fn fake_runner_answering( + mut stream: UnixStream, + identity: RunnerIdentity, + mut answer: impl FnMut(RunnerOp) -> Option>, ) { use codespace_runner::{ - read_frame, write_frame, RunnerLifecycle, RunnerOp, RunnerOpResult, WireEnvelope, - WIRE_PROTOCOL, + read_frame, write_frame, RunnerClaim, RunnerLifecycle, WireEnvelope, WIRE_PROTOCOL, }; while let Ok(Some(frame)) = read_frame(&mut stream).await { let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); @@ -4601,12 +4690,163 @@ mod independent_tests { let reply = WireEnvelope::response(request.request_id.unwrap(), Ok(hello)); write_frame(&mut stream, &reply).await.unwrap(); } - Some(RunnerOp::Exec { request, .. }) => return on_exec(request), - other => panic!("a request the fake Runner does not answer: {other:?}"), + Some(RunnerOp::Claim { .. }) => { + let claimed = RunnerOpResult::Claimed(RunnerClaim { + epoch: 1, + previous_controller: false, + processes: Vec::new(), + events_through: 0, + }); + let reply = WireEnvelope::response(request.request_id.unwrap(), Ok(claimed)); + write_frame(&mut stream, &reply).await.unwrap(); + } + Some(op) => match answer(op) { + Some(result) => { + let reply = WireEnvelope::response(request.request_id.unwrap(), result); + write_frame(&mut stream, &reply).await.unwrap(); + } + None => return, + }, + None => panic!("a request without an operation"), + } + } + } + + /// A relay between gateways and the Runner in this process, listening on a socket, which + /// cuts its connections or holds new ones unanswered, as a lost connection does (P1R-C03). + struct Relay { + socket: std::path::PathBuf, + _dir: tempfile::TempDir, + links: Arc>>, + held: Arc>, + accepting: tokio::task::AbortHandle, + } + + impl Relay { + fn start(runner: &Runner) -> Self { + // A short path: a socket's path must fit `sun_path` on macOS. + let dir = tempfile::Builder::new() + .prefix("cs-") + .tempdir_in("/tmp") + .unwrap(); + let socket = dir.path().join("relay.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let links: Arc>> = Arc::default(); + let held = Arc::new(tokio::sync::watch::Sender::new(false)); + let streams = runner.streams.clone(); + let accepting = tokio::spawn({ + let (links, held) = (links.clone(), held.clone()); + async move { + while let Ok((mut gateway, _)) = listener.accept().await { + let mut released = held.subscribe(); + let streams = streams.clone(); + let link = tokio::spawn(async move { + let _ = released.wait_for(|held| !*held).await; + let (mut runner, served) = UnixStream::pair().unwrap(); + if streams.send(served).is_ok() { + let _ = + tokio::io::copy_bidirectional(&mut gateway, &mut runner).await; + } + }); + links.lock().unwrap().push(link.abort_handle()); + } + } + }) + .abort_handle(); + Self { + socket, + _dir: dir, + links, + held, + accepting, + } + } + + /// Cut every connection: both of its ends see it end. + fn cut(&self) { + for link in self.links.lock().unwrap().drain(..) { + link.abort(); + } + } + + /// Hold each new connection, unanswered, until let go. + fn hold(&self, held: bool) { + self.held.send_replace(held); + } + } + + impl Drop for Relay { + fn drop(&mut self) { + self.accepting.abort(); + self.cut(); + } + } + + /// A gateway that connects to the independent Runner at `socket` and claims it for + /// `database` as `codespace-mcp` does (P1R-C03), claiming it again when its connection is + /// lost and recording its executions there, with what becomes of its connection. + async fn claiming_gateway( + root: &std::path::Path, + database: &std::path::Path, + socket: &std::path::Path, + ) -> (CodeSpace, mpsc::UnboundedReceiver) { + let store = Arc::new(Store::open(database).unwrap()); + let (release, exits) = ProcessRecords::release_hook(store.clone()); + let (noted, events) = mpsc::unbounded_channel(); + let (runner, claim) = UdsRunner::claim_independent( + socket, + &store.store_id().unwrap(), + release, + Arc::new(move |event| { + let _ = noted.send(event); + }), + ) + .await + .unwrap(); + assert_eq!((claim.epoch, claim.previous_controller), (1, false)); + let identity = runner.hello().identity.unwrap(); + let runner = RuntimeBackend::Uds(runner); + let records = ProcessRecords::start(store.clone(), &identity, runner.clone(), exits); + let gateway = + CodeSpace::with_store_and_runner(participation_registry(root, "off"), store, runner) + .with_lifecycle(DETACHING) + .with_records(records); + (gateway, events) + } + + /// What next becomes of the gateway's connection, past its failed tries. + async fn next_link_event(events: &mut mpsc::UnboundedReceiver) -> LinkEvent { + loop { + match tokio::time::timeout(Duration::from_secs(20), events.recv()) + .await + .expect("a link event") + .expect("the client") + { + LinkEvent::Retrying { .. } => {} + event => return event, } } } + /// How many executions the operations database records. + fn recorded_executions(database: &std::path::Path) -> u64 { + rusqlite::Connection::open(database) + .unwrap() + .query_row("SELECT COUNT(*) FROM processes", [], |row| row.get(0)) + .unwrap() + } + + fn patch(workspace_id: &str, file: &str, operation_key: Option<&str>) -> ApplyPatchParams { + ApplyPatchParams { + workspace_id: WorkspaceId(workspace_id.into()), + patch: format!("*** Begin Patch\n*** Add File: {file}\n+added\n*** End Patch\n"), + expected_versions: BTreeMap::new(), + operation_key: operation_key.map(|key| OperationKey(key.into())), + check_only: false, + work_id: None, + } + } + /// After a detach the next gateway reads and controls the processes it inherits, but new /// executions, patches and approval resumes are refused before anything is claimed, so a /// granted approval stays granted. @@ -4618,7 +4858,7 @@ mod independent_tests { let database = dir.path().join("operations.db"); let runner = Runner::start(); let first = runner.gateway(&root, &database).await; - assert!(!first.runner.refuses_new_work()); + assert_eq!(first.runner.new_work_refusal(), None); let touch = format!("{}/resumed", root.display()); let held = refused( first @@ -4650,7 +4890,10 @@ mod independent_tests { detach(&first).await; let next = runner.gateway(&root, &database).await; - assert!(next.runner.refuses_new_work()); + assert_eq!( + next.runner.new_work_refusal().map(|refusal| refusal.code), + Some(ErrorCode::RecoveryPending) + ); let info = next .workspace_info(Parameters(WorkspaceInfoParams { workspace_id: Some("demo".into()), @@ -4809,7 +5052,8 @@ mod independent_tests { let output = output_of(&gateway, &started.process_id).await; let record = record_once(&database, &started.process_id, terminal).await; assert_eq!(record.runner, runner_ref(RunnerIdentity::this_process())); - assert_eq!(record.controller_epoch, None); + // Sent under the gateway's Claim, the Runner's first (P1R-C03). + assert_eq!(record.controller_epoch, Some(1)); assert_eq!(record.workspace_id, "demo"); assert_eq!(record.approval_id, None); assert_eq!(record.attempt_id, None); @@ -4914,6 +5158,7 @@ mod independent_tests { drop(lost); let uds = UdsRunner::from_stream(client, release); uds.handshake().await.unwrap(); + uds.claim(&store.store_id().unwrap()).await.unwrap(); let identity = uds.hello().identity.unwrap(); let backend = RuntimeBackend::Uds(uds); let (_, none) = ProcessRecords::release_hook(store.clone()); @@ -5219,6 +5464,313 @@ mod independent_tests { ); } + /// A gateway whose connection to its independent Runner is lost refuses new executions, + /// patches and approval resumes with RUNNER_RECONNECTING while it claims the Runner again + /// (P1R-C03): nothing is reserved or recorded for them, and the approval stays granted. Once + /// it claimed the Runner again, a gateway that had started nothing works as before, under + /// the new epoch. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_gateway_refuses_new_work_while_it_claims_its_runner_again() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let relay = Relay::start(&runner); + let (gateway, mut events) = claiming_gateway(&root, &database, &relay.socket).await; + let touch = format!("{}/resumed", root.display()); + let held = refused( + gateway + .exec_command(Parameters(exec_params( + "gov-confirm", + &["/usr/bin/touch", &touch], + false, + ))) + .await, + ); + assert_eq!(held.code, ErrorCode::ApprovalRequired, "{held:?}"); + let approval_id = ApprovalId(held.approval_id.unwrap()); + gateway + .approval_resolve(Parameters(ApprovalResolveParams { + approval_id: approval_id.clone(), + decision: ApprovalDecision::Grant, + })) + .await + .unwrap(); + + relay.hold(true); + relay.cut(); + assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RunnerReconnecting, "{refusal:?}"); + let refusal = refused( + gateway + .apply_patch(Parameters(patch("demo", "added.txt", Some("patch-1")))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RunnerReconnecting, "{refusal:?}"); + let refusal = refused( + gateway + .operation_resume(Parameters(OperationResumeParams { + approval_id: approval_id.clone(), + })) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RunnerReconnecting, "{refusal:?}"); + assert_eq!( + gateway.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Granted + ); + for workspace_id in ["demo", "gov-confirm"] { + drop( + gateway + .store + .try_acquire_write(workspace_id) + .expect("nothing holds the workspace"), + ); + } + assert_eq!(recorded_executions(&database), 0); + assert_eq!( + gateway + .store + .status_lookup(None, Some(&OperationKey("patch-1".into()))) + .unwrap_err() + .code, + ErrorCode::OperationNotFound + ); + + relay.hold(false); + match next_link_event(&mut events).await { + LinkEvent::Reclaimed(claim) => { + assert_eq!((claim.epoch, claim.previous_controller), (2, false)) + } + other => panic!("unexpected {other:?}"), + } + assert_eq!(gateway.runner.new_work_refusal(), None); + let resumed = gateway + .operation_resume(Parameters(OperationResumeParams { + approval_id: approval_id.clone(), + })) + .await + .unwrap() + .0; + let started = resumed.exec_command.expect("the execution"); + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!(record.controller_epoch, Some(2)); + assert_eq!(record.approval_id.as_ref(), Some(&approval_id)); + assert_eq!(record.exit_code, Some(0)); + assert!(root.join("resumed").exists()); + assert_eq!( + gateway.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Consumed + ); + } + + /// A gateway that started work and claimed its Runner again after its connection was lost + /// is a controller after one that started work, its own earlier connection, so it starts + /// nothing more until recovery is built (P1R-C01, P1R-C03). It reads and ends the process it + /// started, under the new epoch, and records how that process ended. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_gateway_that_started_work_claims_its_runner_again_and_starts_nothing() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let relay = Relay::start(&runner); + let (gateway, mut events) = claiming_gateway(&root, &database, &relay.socket).await; + let started = gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/sh", "-c", "echo before; exec sleep 30"], + false, + ))) + .await + .unwrap() + .0; + + relay.cut(); + assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); + match next_link_event(&mut events).await { + LinkEvent::Reclaimed(claim) => { + assert_eq!((claim.epoch, claim.previous_controller), (2, true)); + assert!( + claim + .processes + .iter() + .any(|entry| entry.status.process_id == started.process_id + && entry.status.state == ProcessState::Running), + "{claim:?}" + ); + } + other => panic!("unexpected {other:?}"), + } + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let read = gateway + .read_process(Parameters(ReadProcessParams { + process_id: started.process_id.clone(), + cursor: 0, + })) + .await + .unwrap() + .0; + if read.chunk.contains("before") { + break; + } + assert!(Instant::now() < deadline, "{read:?}"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + gateway + .terminate_process(Parameters(TerminateProcessParams { + process_id: started.process_id.clone(), + })) + .await + .unwrap(); + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!(record.controller_epoch, Some(1)); + assert_eq!( + record.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Terminated)), + "{record:?}" + ); + released(&gateway, "demo").await; + } + + /// An execution, a patch or an approval resume that the Runner refuses as sent under an + /// earlier Claim never ran (P1R-C03): the approval is given back granted and can be resumed + /// again, the patch's operation_key begins afresh, the workspace is free again, and the + /// execution is recorded as not started. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_request_refused_as_stale_never_ran_and_gives_its_approval_back() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let (client, server) = UnixStream::pair().unwrap(); + let sent = Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = sent.clone(); + tokio::spawn(fake_runner_answering(server, fake_identity(), move |op| { + seen.lock().unwrap().push(match op { + RunnerOp::Exec { request, .. } => request.process_id.0, + RunnerOp::ApplyPatch { .. } => "apply_patch".into(), + other => panic!("a request the fake Runner does not answer: {other:?}"), + }); + Some(Err(ErrorBody::new( + ErrorCode::StaleController, + "this connection does not control the Runner under epoch 1: it was claimed \ + again, under epoch 2", + ))) + })); + let gateway = recording_gateway(&root, &database, client).await; + let touch = format!("{}/resumed", root.display()); + let held = refused( + gateway + .exec_command(Parameters(exec_params( + "gov-confirm", + &["/usr/bin/touch", &touch], + false, + ))) + .await, + ); + let exec_approval = ApprovalId(held.approval_id.unwrap()); + let held = refused( + gateway + .apply_patch(Parameters(patch( + "gov-confirm", + "held.txt", + Some("patch-1"), + ))) + .await, + ); + let patch_approval = ApprovalId(held.approval_id.unwrap()); + for approval_id in [&exec_approval, &patch_approval] { + gateway + .approval_resolve(Parameters(ApprovalResolveParams { + approval_id: approval_id.clone(), + decision: ApprovalDecision::Grant, + })) + .await + .unwrap(); + } + + for (approval_id, resumes) in [(&exec_approval, 2), (&patch_approval, 2)] { + for _ in 0..resumes { + let refusal = refused( + gateway + .operation_resume(Parameters(OperationResumeParams { + approval_id: approval_id.clone(), + })) + .await, + ); + assert_eq!(refusal.code, ErrorCode::StaleController, "{refusal:?}"); + assert_eq!( + gateway.store.inspect_approval(approval_id).unwrap().0, + ApprovalState::Granted + ); + } + } + assert_eq!( + gateway + .store + .status_lookup(None, Some(&OperationKey("patch-1".into()))) + .unwrap_err() + .code, + ErrorCode::OperationNotFound + ); + for _ in 0..2 { + let refusal = refused( + gateway + .apply_patch(Parameters(patch("demo", "added.txt", Some("patch-2")))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::StaleController, "{refusal:?}"); + } + let sent = sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 6, "{sent:?}"); + assert_eq!(&sent[2..], ["apply_patch"; 4]); + for process_id in &sent[..2] { + let record = gateway + .store + .process_record(process_id) + .unwrap() + .expect("the record"); + assert_eq!( + record.outcome, + Some(ProcessOutcome::NotStarted), + "{record:?}" + ); + assert_eq!(record.approval_id.as_ref(), Some(&exec_approval)); + } + for workspace_id in ["demo", "gov-confirm"] { + drop( + gateway + .store + .try_acquire_write(workspace_id) + .expect("nothing holds the workspace"), + ); + } + assert!(!root.join("resumed").exists()); + } + /// Each independent mode states only what its Runner guarantees, and the rest of the /// instructions stay as they are. #[test] @@ -5235,8 +5787,13 @@ mod independent_tests { for expected in [ "owned by an independent Runner", "Client or HTTP disconnect keeps the process running", + "One gateway at a time controls the Runner", + "RUNNER_RECONNECTING", "RUNNER_DISCONNECTED", + "STALE_CONTROLLER", "RECOVERY_PENDING", + "consumes no approval", + "any process of the same user", "same process_id", "signals that process only", says, diff --git a/crates/server/src/records.rs b/crates/server/src/records.rs index c7e8194..dd1907c 100644 --- a/crates/server/src/records.rs +++ b/crates/server/src/records.rs @@ -79,12 +79,14 @@ impl ProcessRecords { records } - /// Record an execution before it is sent to the Runner, and mark the approval it resumes - /// `resuming` with it. When the record cannot be written, nothing is sent. + /// Record an execution before it is sent to the Runner under the controller epoch + /// `controller_epoch` (P1R-C03), and mark the approval it resumes `resuming` with it. When + /// the record cannot be written, nothing is sent. pub(crate) fn dispatching( &self, process_id: &ProcessId, params: &ExecCommandParams, + controller_epoch: Option, participation: RecordedParticipation, attempt_id: Option<&str>, resume: Option<&ApprovalId>, @@ -93,6 +95,7 @@ impl ProcessRecords { .record_dispatch(&NewProcessRecord { process_id: &process_id.0, runner: &self.runner, + controller_epoch, workspace_id: ¶ms.workspace_id.0, approval_id: resume, attempt_id, diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index bd61b83..b4474d7 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -7,7 +7,8 @@ use std::time::Duration; use anyhow::{anyhow, Context, Result}; use codespace_runner::{ - allocate_private_runner_dir, runner_socket_path, RunnerLifecycle, ShellRelease, UdsRunner, + allocate_private_runner_dir, runner_socket_path, ClaimError, LinkEvent, LinkObserver, + RunnerClaim, RunnerLifecycle, ShellRelease, UdsRunner, }; use codespace_store::Store; use tokio::net::UnixStream; @@ -139,66 +140,47 @@ impl RuntimeProcess { Ok(runner) } - /// Connect to an independent Runner that an operator or a harness started (P1R-C01). Its - /// processes outlive this connection: a lost connection releases nothing, and every later - /// request is refused until the gateway is restarted. A worker that is not an independent - /// Runner is refused. `leaving` is set before the gateway itself ends the connection. + /// Connect to an independent Runner that an operator or a harness started (P1R-C01) and + /// claim it for the operations database `store_id` (P1R-C03). Its processes outlive the + /// connection: a lost connection releases nothing, and the gateway claims the Runner again, + /// refusing every request that needs it meanwhile. A worker that is not an independent + /// Runner stating its identity, and a Runner that refuses the Claim for any reason but + /// another connection controlling it, are refused; one that another connection controls is + /// asked again for up to [`codespace_runner::CLAIM_START_BOUND`]. pub async fn connect_independent( socket: &Path, + store_id: &str, on_process_exit: ShellRelease, - leaving: Arc, - ) -> Result { - let stream = UnixStream::connect(socket) - .await - .with_context(|| format!("connect {}", socket.display()))?; - let runner = UdsRunner::from_stream_with_disconnect( - stream, - on_process_exit, - Arc::new(move || { - if !leaving.load(std::sync::atomic::Ordering::SeqCst) { - tracing::error!( - "lost the connection to the independent Runner: its processes keep \ - running, and this gateway refuses every request that needs the Runner \ - until it is restarted" - ); - } - }), - ); - runner - .handshake() - .await - .map_err(|err| anyhow!("runner hello: {err}"))?; + ) -> Result<(UdsRunner, RunnerClaim)> { + let (runner, claim) = + UdsRunner::claim_independent(socket, store_id, on_process_exit, link_observer()) + .await + .map_err(|err| match err { + ClaimError::Refused { refusal, message } => anyhow!( + "--runner-lifecycle independent: the Runner at {} refused this \ + gateway's Claim ({}): {message}", + socket.display(), + refusal.as_str() + ), + ClaimError::NotIndependent(message) => { + anyhow!("--runner-lifecycle independent: {message}") + } + ClaimError::Transport(err) => anyhow!( + "--runner-lifecycle independent: claim the Runner at {}: {err}", + socket.display() + ), + })?; let hello = runner.hello(); - if hello.lifecycle != RunnerLifecycle::Independent { - runner.close().await; - return Err(anyhow!( - "--runner-lifecycle independent: the worker at {} is {}, not an independent Runner", - socket.display(), - hello.lifecycle.as_str() - )); - } - // Its executions are recorded under its identity (P1R-C02). - if hello.identity.is_none() { - runner.close().await; - return Err(anyhow!( - "--runner-lifecycle independent: the Runner at {} did not state its identity", - socket.display() - )); - } tracing::info!( runner_pid = hello.identity.as_ref().map(|identity| identity.pid), - previous_controller = hello.previous_controller, - inherited_processes = hello.inherited_processes, - "connected to an independent Runner; it is not a supported mode yet" + epoch = claim.epoch, + previous_controller = claim.previous_controller, + processes = claim.processes.len(), + events_through = claim.events_through, + "claimed an independent Runner; it is not a supported mode yet" ); - if hello.previous_controller { - tracing::warn!( - "an earlier gateway controlled this independent Runner: its processes can be \ - queried, read, written to, resized and terminated, but new executions, patches \ - and approval resumes are refused with RECOVERY_PENDING" - ); - } - Ok(runner) + warn_recovery_pending(&claim); + Ok((runner, claim)) } pub fn socket(&self) -> &Path { @@ -235,6 +217,66 @@ impl Drop for RuntimeProcess { } } +/// After a Claim that found work an earlier controller started, this one included before it +/// lost its connection: nothing new starts (P1R-C01, P1R-C03). +fn warn_recovery_pending(claim: &RunnerClaim) { + if claim.previous_controller { + tracing::warn!( + epoch = claim.epoch, + "an earlier controller started work on this independent Runner: its processes can be \ + queried, read, written to, resized and terminated, but new executions, patches and \ + approval resumes are refused with RECOVERY_PENDING" + ); + } +} + +/// What the gateway logs of its connection to the independent Runner it claimed (P1R-C03): a +/// failed try to claim it again is logged once for each new reason. +fn link_observer() -> LinkObserver { + let last_reason = std::sync::Mutex::new(None::); + Arc::new(move |event| match event { + LinkEvent::Lost => tracing::warn!( + "lost the connection to the independent Runner: its processes keep running, and \ + this gateway claims it again, refusing requests that need it with \ + RUNNER_RECONNECTING meanwhile" + ), + LinkEvent::Retrying { reason, retry } => { + let mut last = last_reason + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if last.as_deref() == Some(reason.as_str()) { + tracing::debug!(%reason, ?retry, "claiming the independent Runner again"); + } else { + tracing::warn!( + %reason, + ?retry, + "could not claim the independent Runner; trying again" + ); + *last = Some(reason); + } + } + LinkEvent::Reclaimed(claim) => { + *last_reason + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + tracing::info!( + epoch = claim.epoch, + previous_controller = claim.previous_controller, + processes = claim.processes.len(), + events_through = claim.events_through, + "claimed the independent Runner again" + ); + warn_recovery_pending(&claim); + } + LinkEvent::Gone(reason) => tracing::error!( + %reason, + "cannot claim the independent Runner again: its processes keep running if it did not \ + stop, and this gateway refuses every request that needs the Runner with \ + RUNNER_DISCONNECTED until it is restarted" + ), + }) +} + /// A worker whose `Hello` did not state that it registers was built without DevGuard. #[cfg(feature = "devguard")] pub fn require_registration(runner: &UdsRunner, bin: &Path) -> Result<()> { diff --git a/crates/server/tests/independent_runner.rs b/crates/server/tests/independent_runner.rs index 2556640..40a3447 100644 --- a/crates/server/tests/independent_runner.rs +++ b/crates/server/tests/independent_runner.rs @@ -1,17 +1,20 @@ //! The gateway with an independent Runner (P1R-C01), each started as an operator starts it: the //! Runner outlives a gateway, the next gateway inherits its processes but starts nothing, and a //! gateway that ends leaves or stops the Runner as its exit intent says. Each gateway records -//! its executions in the operations database, which it is the only writer of (P1R-C02). The -//! tests with a Runner need the worker binary (`CODESPACE_RUNTIME_BIN`) and return early -//! without it, as the gateway's other worker tests do. +//! its executions in the operations database, which it is the only writer of (P1R-C02), and +//! takes control of the Runner with a Claim for that database (P1R-C03). The tests with a +//! Runner need the worker binary (`CODESPACE_RUNTIME_BIN`) and return early without it, as the +//! gateway's other worker tests do. use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Stdio}; +use std::sync::Arc; use std::time::Duration; use codespace_domain::{ - ProcessTermination, TOOL_EXEC_COMMAND, TOOL_PROCESS_STATUS, TOOL_READ_PROCESS, + ProcessState, ProcessTermination, TOOL_EXEC_COMMAND, TOOL_PROCESS_STATUS, TOOL_READ_PROCESS, }; +use codespace_runner::{ClaimError, ClaimRefusal, UdsRunner}; use codespace_store::{ProcessOutcome, ProcessPhase, Store, WriterLock}; use rmcp::model::{CallToolRequestParams, CallToolResult}; use rmcp::service::{RunningService, ServiceError}; @@ -69,7 +72,10 @@ impl Setup { std::fs::write( &cfg, serde_json::json!({ - "workspaces": { "demo": { "root": ws, "profile": "workspace-write" } } + "workspaces": { + "demo": { "root": ws, "profile": "workspace-write" }, + "online": { "root": ws, "profile": "workspace-write", "network": "enabled" } + } }) .to_string(), ) @@ -216,10 +222,18 @@ struct Gateway { impl Gateway { async fn exec(&self, command: &[&str]) -> Result { + self.exec_in("demo", command).await + } + + async fn exec_in( + &self, + workspace_id: &str, + command: &[&str], + ) -> Result { self.client .call_tool( CallToolRequestParams::new(TOOL_EXEC_COMMAND).with_arguments(object!({ - "workspace_id": "demo", + "workspace_id": workspace_id, "command": command })), ) @@ -228,7 +242,11 @@ impl Gateway { /// Start `command` and return its process id. async fn started(&self, command: &[&str]) -> String { - let started = self.exec(command).await.expect("exec"); + self.started_in("demo", command).await + } + + async fn started_in(&self, workspace_id: &str, command: &[&str]) -> String { + let started = self.exec_in(workspace_id, command).await.expect("exec"); let body = payload(&started); assert_eq!(body["dispatch_status"], "confirmed", "{body}"); body["process_id"].as_str().unwrap().to_string() @@ -655,3 +673,177 @@ async fn a_restarted_gateway_waits_for_the_one_before_it_and_finds_its_records() let status = exited(&mut runner).await; assert!(status.success(), "{status:?}"); } + +/// While a gateway controls the independent Runner, even one stopped with SIGSTOP whose +/// connection stays open, another Claim is refused: nothing pushes a live controller out +/// (P1R-C03). Once that gateway ends, the next Claim is granted after it, states the processes +/// it inherits, and lets a later gateway take over in turn. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_claim_is_refused_while_a_stopped_gateway_holds_the_runner_and_granted_once_it_ends() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let first = setup.gateway("detach").await; + let process_id = first + .started(&["/bin/sh", "-c", "echo started; exec sleep 30"]) + .await; + first.output(&process_id, "started").await; + // The store id is no secret from the same user: it guards against a mistaken gateway. + let store_id = Store::open(&setup.database()).unwrap().store_id().unwrap(); + let stopped = std::process::Command::new("kill") + .args(["-STOP", &first.process.id().unwrap().to_string()]) + .status() + .unwrap(); + assert!(stopped.success()); + let client = UdsRunner::connect(&setup.socket, Arc::new(|_: &str| {})) + .await + .unwrap(); + match client.claim(&store_id).await { + Err(ClaimError::Refused { refusal, message }) => { + assert_eq!(refusal, ClaimRefusal::ControllerActive, "{message}"); + assert!(message.contains("still open"), "{message}"); + } + other => panic!("unexpected {other:?}"), + } + let killed = Instant::now(); + first.kill().await; + let claim = loop { + match client.claim(&store_id).await { + Ok(claim) => break claim, + // Until the Runner reads the end of the killed gateway's connection. + Err(ClaimError::Refused { + refusal: ClaimRefusal::ControllerActive, + message, + }) => { + assert!(killed.elapsed() < Duration::from_secs(10), "{message}"); + sleep(Duration::from_millis(20)).await; + } + other => panic!("unexpected {other:?}"), + } + }; + eprintln!( + "p1r-c03 claim latency: granted {:?} after the stopped gateway was killed", + killed.elapsed() + ); + assert_eq!((claim.epoch, claim.previous_controller), (2, true)); + assert!( + claim + .processes + .iter() + .any(|entry| entry.status.process_id.0 == process_id + && entry.status.state == ProcessState::Running), + "{claim:?}" + ); + client.detach().await.unwrap(); + + let next = setup.gateway("stop").await; + assert_eq!(next.status(&process_id).await["state"], "running"); + let refused = next.exec(&["/bin/echo", "new"]).await; + assert_eq!(error_body(&refused)["code"], "RECOVERY_PENDING"); + let ended = next.shut_down().await; + assert!(ended.success(), "{ended:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); +} + +/// A gateway whose operations database is not the one the independent Runner was first +/// claimed for is refused, and leaves the Runner as it was (P1R-C03): the gateway of the +/// Runner's own database then starts work normally, as nothing was started before it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_gateway_with_another_operations_database_is_refused() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let first = setup.gateway("detach").await; + let ended = first.shut_down().await; + assert!(ended.success(), "{ended:?}"); + + let args = setup.independent("stop"); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + let mut other = setup.gateway_command(&args); + other.env( + "CODESPACE_OPERATIONS_DB", + setup.root.path().join("other.sqlite"), + ); + let refusal = refused(other).await; + assert!( + refusal.contains("refused this gateway's Claim (store_mismatch)"), + "{refusal}" + ); + let next = setup.gateway("stop").await; + let process_id = next.started(&["/bin/echo", "after"]).await; + next.output(&process_id, "after").await; + let ended = next.shut_down().await; + assert!(ended.success(), "{ended:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); +} + +/// Connects to the Unix socket named by its argument and says how far it got: whether it sees +/// the socket, then `connected`, or the call that failed and its errno. +const CONNECT: &str = r#"use Socket; +my $path = shift; +print -S $path ? "visible\n" : "invisible\n"; +socket(my $s, AF_UNIX, SOCK_STREAM, 0) or do { printf "socket %d\n", $! + 0; exit 0 }; +connect($s, pack_sockaddr_un($path)) or do { printf "connect %d\n", $! + 0; exit 0 }; +print "connected\n"; +"#; + +/// A command that the Linux sandbox runs cannot connect to the independent Runner's socket, +/// whether its workspace's network is restricted or enabled, so it cannot take control of the +/// Runner (P1R-C03); the same command run by the test connects. Without the sandbox, which an +/// enabled network needs, a command connects as any process of the same user does: the +/// exposure that the operator documents name. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_sandboxed_command_cannot_reach_the_runner_socket() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let socket = setup.socket.display().to_string(); + let direct = std::process::Command::new("/usr/bin/perl") + .args(["-e", CONNECT, &socket]) + .output() + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&direct.stdout), + "visible\nconnected\n", + "{direct:?}" + ); + let sandboxed = codespace_runner::linux_sandbox_available(); + let workspaces: &[&str] = if sandboxed { + &["demo", "online"] + } else { + &["demo"] + }; + let gateway = setup.gateway("stop").await; + for &workspace_id in workspaces { + let process_id = gateway + .started_in(workspace_id, &["/usr/bin/perl", "-e", CONNECT, &socket]) + .await; + let status = gateway.ended(&process_id).await; + assert_eq!(status["exit_code"], 0, "{workspace_id}: {status}"); + let output = gateway.output(&process_id, "\n").await; + let reached = output.lines().last().unwrap_or_default().to_string(); + if sandboxed { + assert!( + ["socket 1", "socket 13", "connect 1", "connect 13"].contains(&reached.as_str()), + "{workspace_id}: {output:?}" + ); + } else { + assert_eq!(reached, "connected", "{workspace_id}: {output:?}"); + } + } + // The gateway still controls the Runner. + let process_id = gateway.started(&["/bin/echo", "after"]).await; + gateway.output(&process_id, "after").await; + let ended = gateway.shut_down().await; + assert!(ended.success(), "{ended:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); +} diff --git a/crates/store/src/approvals.rs b/crates/store/src/approvals.rs index 6b65dd2..973bd3f 100644 --- a/crates/store/src/approvals.rs +++ b/crates/store/src/approvals.rs @@ -266,6 +266,9 @@ impl Store { if state == ApprovalState::Granted.as_str() { return Ok(()); } + // A resume refused before its approval was marked `resuming` stays queued: by the + // scheduler, or because the independent Runner could not be sent the execution + // (P1R-C01, P1R-C03). let queued = state == ApprovalState::Queued.as_str(); if queued && matches!( @@ -273,7 +276,12 @@ impl Store { Err(err) if matches!( err.code, - ErrorCode::WorkspaceBusy | ErrorCode::ResourceQueueFull + ErrorCode::WorkspaceBusy + | ErrorCode::ResourceQueueFull + | ErrorCode::RecoveryPending + | ErrorCode::RunnerReconnecting + | ErrorCode::RunnerDisconnected + | ErrorCode::StaleController ) ) { @@ -829,9 +837,17 @@ mod tests { #[test] fn queued_scheduler_errors_do_not_consume_approval() { - for (index, code) in [ErrorCode::WorkspaceBusy, ErrorCode::ResourceQueueFull] - .into_iter() - .enumerate() + for (index, code) in [ + ErrorCode::WorkspaceBusy, + ErrorCode::ResourceQueueFull, + // An independent Runner that could not be sent the execution (P1R-C03). + ErrorCode::RecoveryPending, + ErrorCode::RunnerReconnecting, + ErrorCode::RunnerDisconnected, + ErrorCode::StaleController, + ] + .into_iter() + .enumerate() { let store = Store::memory().unwrap(); let created = store diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index a0764f1..5f61904 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -40,6 +40,15 @@ pub use writer::WriterLock; /// without one (0) predates it and is brought up to it; a newer one is refused. pub const SCHEMA_VERSION: i64 = 1; +/// The database's own identity (P1R-C03): one row, written the first time it is asked for. A +/// table that an older CodeSpace does not know, which it leaves alone, so the schema version +/// stays. +const STORE_IDENTITY_SCHEMA: &str = "CREATE TABLE IF NOT EXISTS store_identity ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + store_id TEXT NOT NULL, + created_at_ms INTEGER NOT NULL +);"; + #[derive(Debug, Clone)] pub struct StoredOperation { pub operation_id: OperationId, @@ -238,6 +247,8 @@ impl Store { .map_err(|e| e.to_string())?; conn.execute_batch(processes::SCHEMA) .map_err(|e| e.to_string())?; + conn.execute_batch(STORE_IDENTITY_SCHEMA) + .map_err(|e| e.to_string())?; migrate_operations(&conn)?; migrate_approvals(&conn)?; // Last, so a database whose upgrade was cut short is upgraded again. @@ -482,6 +493,44 @@ impl Store { Ok(Begin::Fresh(operation_id)) } + /// Forget an operation begun here that no runner acted on (P1R-C03): its row is removed + /// while it has no result, so its `operation_key` begins afresh. Returns whether a row was + /// removed. + pub fn abandon(&self, operation_id: &OperationId) -> Result { + let conn = self.conn.lock().expect("sqlite mutex"); + let removed = conn + .execute( + "DELETE FROM operations WHERE operation_id = ?1 AND finished_at IS NULL", + params![operation_id.0], + ) + .map_err(sql_err)?; + Ok(removed == 1) + } + + /// This operations database's identity (P1R-C03), created the first time it is asked for + /// and the same from then on. An independent Runner remembers the identity of the first + /// gateway that claims it and refuses a gateway with another one: a guard against a + /// mistaken database, not an authentication. + pub fn store_id(&self) -> Result { + let conn = self.conn.lock().expect("sqlite mutex"); + let created_at_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0); + conn.execute( + "INSERT OR IGNORE INTO store_identity (singleton, store_id, created_at_ms) + VALUES (1, ?1, ?2)", + params![format!("store-{}", uuid::Uuid::new_v4()), created_at_ms], + ) + .map_err(sql_err)?; + conn.query_row( + "SELECT store_id FROM store_identity WHERE singleton = 1", + [], + |row| row.get(0), + ) + .map_err(sql_err) + } + pub fn finish( &self, operation_id: &OperationId, @@ -1176,6 +1225,7 @@ mod tests { .record_dispatch(&NewProcessRecord { process_id, runner: &runner(), + controller_epoch: None, workspace_id: "demo", approval_id: None, attempt_id: None, @@ -1367,4 +1417,51 @@ mod tests { ] ); } + + /// A database keeps its identity across reopenings, and another database has its own + /// (P1R-C03). The schema version stays. + #[test] + fn an_operations_database_keeps_its_identity() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ops.sqlite"); + let first = Store::open(&path).unwrap().store_id().unwrap(); + assert!(first.starts_with("store-"), "{first}"); + let store = Store::open(&path).unwrap(); + assert_eq!(store.store_id().unwrap(), first); + assert_eq!(store.store_id().unwrap(), first); + assert_eq!(user_version(&path), SCHEMA_VERSION); + let other = Store::open(&dir.path().join("other.sqlite")).unwrap(); + assert_ne!(other.store_id().unwrap(), first); + } + + /// An operation that never reached a runner is forgotten, so its key begins afresh; one + /// with a result is kept (P1R-C03). + #[test] + fn an_abandoned_operation_begins_afresh() { + let store = Store::memory().unwrap(); + let key = OperationKey("patch-key".into()); + let Begin::Fresh(abandoned) = store.begin(Some(&key), "demo", "fp").unwrap() else { + panic!("a fresh operation"); + }; + assert!(store.abandon(&abandoned).unwrap()); + assert_eq!( + store.get(&abandoned).unwrap_err().code, + ErrorCode::OperationNotFound + ); + let Begin::Fresh(finished) = store.begin(Some(&key), "demo", "fp").unwrap() else { + panic!("the key begins afresh"); + }; + assert_ne!(finished, abandoned); + store + .finish( + &finished, + &ApplyPatchResult::new(PatchStatus::Applied, finished.clone()), + ) + .unwrap(); + assert!(!store.abandon(&finished).unwrap()); + assert!(matches!( + store.begin(Some(&key), "demo", "fp").unwrap(), + Begin::Replayed(_) + )); + } } diff --git a/crates/store/src/processes.rs b/crates/store/src/processes.rs index d350298..ce4859b 100644 --- a/crates/store/src/processes.rs +++ b/crates/store/src/processes.rs @@ -111,6 +111,8 @@ impl RecordedParticipation { pub struct NewProcessRecord<'a> { pub process_id: &'a str, pub runner: &'a RunnerRef, + /// The controller epoch of the Claim the execution is sent under (P1R-C03). + pub controller_epoch: Option, pub workspace_id: &'a str, /// The approval this execution resumes: it is marked `resuming` with the record, in one /// transaction. @@ -193,7 +195,8 @@ impl ProcessOutcome { pub struct ProcessRecord { pub process_id: String, pub runner: RunnerRef, - /// The controller epoch the execution was sent under (P1R-C03). Nothing issues one yet. + /// The controller epoch of the Claim the execution was sent under (P1R-C03); none for a + /// record a gateway before it wrote. pub controller_epoch: Option, pub workspace_id: String, pub approval_id: Option, @@ -286,13 +289,14 @@ impl Store { process_id, runner_boot_id, runner_pid, runner_started_at_ms, runner_nonce, controller_epoch, workspace_id, approval_id, attempt_id, participation, tty, argv_digest, phase, created_at_ms, updated_at_ms) - VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?13)", + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?14)", params![ record.process_id, record.runner.boot_id, record.runner.pid, to_sql(record.runner.started_at_ms), record.runner.nonce, + record.controller_epoch.map(to_sql), record.workspace_id, record.approval_id.map(|id| id.0.as_str()), record.attempt_id, @@ -535,6 +539,7 @@ mod tests { store.record_dispatch(&NewProcessRecord { process_id, runner, + controller_epoch: Some(3), workspace_id: "demo", approval_id, attempt_id: Some("cs-attempt-1"), @@ -601,7 +606,7 @@ mod tests { let sent = record(&store, "proc-1"); assert_eq!(sent.phase, ProcessPhase::Dispatching); assert_eq!(sent.runner, runner); - assert_eq!(sent.controller_epoch, None); + assert_eq!(sent.controller_epoch, Some(3)); assert_eq!(sent.workspace_id, "demo"); assert_eq!(sent.approval_id, None); assert_eq!(sent.attempt_id.as_deref(), Some("cs-attempt-1")); @@ -914,6 +919,7 @@ mod tests { .record_dispatch(&NewProcessRecord { process_id: "proc-secret", runner: &runner, + controller_epoch: None, workspace_id: "demo", approval_id: None, attempt_id: None, diff --git a/docs/agent-integration.md b/docs/agent-integration.md index ca8ca41..94c402c 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -150,7 +150,7 @@ Interactive input and cancellation use the same handle: A live command occupies the workspace. Wait for it to end or terminate it before applying a patch or starting another command. Reads and searches remain available. Long-lived development servers therefore require a workflow that stops them before edits. -`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. +`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. While that gateway claims its Runner again after a lost connection, it refuses requests with `RUNNER_RECONNECTING`, which sent nothing and consumed no approval: retry after a pause. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. ## Confirm a held mutation diff --git a/docs/architecture.md b/docs/architecture.md index 3f79183..4eea997 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,7 +51,7 @@ If helper application fails, the Runner attempts per-file snapshot restoration. ## Process lifetime -MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. +MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead, and a gateway that lost its connection to it claims it again. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. ## Planned execution coordination diff --git a/docs/codex-reuse.md b/docs/codex-reuse.md index 05e9982..29d5e4c 100644 --- a/docs/codex-reuse.md +++ b/docs/codex-reuse.md @@ -16,7 +16,7 @@ CodeSpace uses selected libraries from a pinned Codex checkout to implement exec | Adapter | Codex components | Current role | | --- | --- | --- | | `crates/patch` | `codex-apply-patch`, `codex-exec-server::LOCAL_FS`, path utilities, process hardening | V4A parsing/application in `codespace-patch` | -| `crates/codex-runtime` | `codex-process-hardening`, `codex-uds` | Optional hardened worker and Unix socket | +| `crates/codex-runtime` | `codex-process-hardening`, `codex-uds` | Optional hardened worker and Unix socket. An [independent Runner](runner-isolation.md#claim) binds Tokio's listener, which `codex-uds` wraps on Unix, to read each connection's peer credentials | | `crates/pty` | `codex-utils-pty` | Terminal-backed execution for `tty: true` | | `crates/file-system` | `codex-file-system`, `LOCAL_FS`, path utilities | Runner file I/O and bounded walks with no-follow handling | | `crates/linux-sandbox` | `codex-linux-sandbox`, `codex-sandboxing`, `codex-protocol`, `codex-network-proxy` | Binary-only command sandbox helper and enabled-network proxy | @@ -37,7 +37,7 @@ Agent → CodeSpace MCP/policy/store → Runner contract Core crates have no direct Codex dependencies. Adapter crates are separate Cargo workspaces to accommodate the pinned upstream workspace dependencies. The filesystem and PTY adapters are library dependencies of the Runner; patch and Linux sandbox operations use helper processes. An isolated Cargo workspace alone does not create a process or security boundary. -The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 9 and sandbox-helper protocol version 1 are separate contracts. +The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 10 and sandbox-helper protocol version 1 are separate contracts. diff --git a/docs/devguard-integration.md b/docs/devguard-integration.md index a12fdb7..a3bf312 100644 --- a/docs/devguard-integration.md +++ b/docs/devguard-integration.md @@ -338,7 +338,7 @@ Default participation remains `off`; `required` is an explicit operator choice i ## Initial Gateway recovery scope -**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. A gateway does not reconnect yet, and a gateway that connects after another one started work refuses new executions, patches and approval resumes with `RECOVERY_PENDING` until the Runner is stopped and a new one is started. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. Reconnection, reconciliation and Runner loss come in the later units, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. +**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. The third unit, P1R-C03, adds the [Claim](runner-isolation.md#claim): one connection at a time controls the Runner, under a controller epoch that each request changing a process or the workspace carries, and a gateway that lost its connection claims the Runner again instead of being restarted. A gateway that controls the Runner after one that started work, the same gateway after a lost connection included, refuses new executions, patches and approval resumes with `RECOVERY_PENDING` until the Runner is stopped and a new one is started. While no gateway controls the Runner, any process of the same user that can reach its socket can claim it, an exposure the [Runner's documentation](runner-isolation.md#claim-exposure) states. Reconciliation and Runner loss come in the later units, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. P1-RECOVERY introduces an operator-selected independent Runner mode and explicit capability. Existing modes keep their current shutdown/disconnect contracts. InProcess shares the Gateway PID and is excluded from live Gateway restart recovery. diff --git a/docs/error-codes.md b/docs/error-codes.md index 5cf4a39..cf91408 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -54,8 +54,10 @@ Errors use uppercase identifiers and a message. An `operation_id` may be present | `APPROVAL_NOT_FOUND` | Unknown confirmation-hold id | | `APPROVAL_CONFLICT` | Hold is still pending, already decided, or a resume is already in progress | | `APPROVAL_AMBIGUOUS` | Resume was interrupted and the terminal result is not on disk. Includes `approval_id`; patch cases may also include `operation_id` | -| `RECOVERY_PENDING` | An earlier gateway started work on this gateway's [independent Runner](runner-isolation.md#independent-runner). The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts until the Runner is stopped and a new one is started. The approval stays as it was. Not occupancy; retrying does not help | -| `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and does not reconnect. The request was not sent, and the Runner's processes keep running; restart the gateway to reach them | +| `RECOVERY_PENDING` | A controller before this gateway's Claim started work on its [independent Runner](runner-isolation.md#claim), this gateway's own connection before a lost connection included. The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts until the Runner is stopped and a new one is started. The approval stays as it was. Not occupancy; retrying does not help | +| `RUNNER_RECONNECTING` | The gateway's connection to its [independent Runner](runner-isolation.md#independent-runner) was lost, and the gateway is claiming the Runner again. The request was not sent: nothing was started, reserved or recorded for it, and an approval it would resume is not consumed. The Runner's processes keep running; retry after a pause | +| `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and cannot claim it again: nothing accepts connections at its socket, another Runner answers there, or the Runner refused the Claim for a reason that waiting cannot end. The request was not sent, and the Runner's processes keep running; restart the gateway to reach them | +| `STALE_CONTROLLER` | The [independent Runner](runner-isolation.md#claim) refused a request sent under a Claim that another Claim replaced, or from a connection that does not control it. Nothing the request asked for was done, and an approval it resumed is granted again. Retrying sends the request under the gateway's current Claim | | `INTERNAL` | Scheduler or store invariant failed, or a gateway with an [independent Runner](runner-isolation.md#process-records) could not record an execution, which it then did not start. Not occupancy; do not retry as `WORKSPACE_BUSY` | ## Dispatch and completion diff --git a/docs/ko/agent-integration.md b/docs/ko/agent-integration.md index 4aac13e..c48f371 100644 --- a/docs/ko/agent-integration.md +++ b/docs/ko/agent-integration.md @@ -150,7 +150,7 @@ MCP 클라이언트 SDK로 stdio 또는 Streamable HTTP를 초기화하고, 초 실행 중인 명령은 작업 공간을 점유합니다. 패치를 적용하거나 다른 명령을 시작하려면 기존 명령이 끝날 때까지 기다리거나 종료하세요. 읽기와 검색은 계속 가능합니다. 개발 서버를 오래 실행하는 작업 흐름이라면 수정 전에 서버를 멈추는 절차가 필요합니다. -`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)에서는 `runner_disconnect`가 `keep_running`이고, `gateway_shutdown`은 게이트웨이가 detach하면 `keep_running`, Runner를 중지하면 `terminate`입니다. 그 게이트웨이가 하는 일은 `initialize.instructions`에서 확인하세요. spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. +`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)에서는 `runner_disconnect`가 `keep_running`이고, `gateway_shutdown`은 게이트웨이가 detach하면 `keep_running`, Runner를 중지하면 `terminate`입니다. 그 게이트웨이가 하는 일은 `initialize.instructions`에서 확인하세요. 그 게이트웨이는 연결을 잃은 뒤 Runner를 다시 Claim하는 동안 요청을 `RUNNER_RECONNECTING`으로 거부합니다. 이 거부는 아무것도 보내지 않았고 승인도 소모하지 않았으므로, 잠시 뒤 재시도하세요. spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. ## 보류된 변경 확인 diff --git a/docs/ko/architecture.md b/docs/ko/architecture.md index 5fd7f18..7e65a31 100644 --- a/docs/ko/architecture.md +++ b/docs/ko/architecture.md @@ -58,7 +58,7 @@ UDS worker와 기본 러너는 모두 같은 호스트에서 실행합니다. ## 프로세스 수명 -MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)는 이와 달리 게이트웨이보다 오래 삽니다. [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. +MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)는 이와 달리 게이트웨이보다 오래 살고, Runner와의 연결을 잃은 게이트웨이는 Runner를 다시 Claim합니다. [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. ## 계획된 실행 조정 구조 diff --git a/docs/ko/codex-reuse.md b/docs/ko/codex-reuse.md index 72f89e5..35a2ad3 100644 --- a/docs/ko/codex-reuse.md +++ b/docs/ko/codex-reuse.md @@ -19,7 +19,7 @@ CodeSpace는 특정 버전에 고정한 Codex 소스의 실행 라이브러리 | 어댑터 | Codex 구성 요소 | 현재 역할 | | --- | --- | --- | | `crates/patch` | `codex-apply-patch`, `codex-exec-server::LOCAL_FS`, 경로 도구, 프로세스 보호 | `codespace-patch` 안에서 V4A 파싱·적용 | -| `crates/codex-runtime` | `codex-process-hardening`, `codex-uds` | 선택적 worker의 보호 설정과 Unix 소켓 | +| `crates/codex-runtime` | `codex-process-hardening`, `codex-uds` | 선택적 worker의 보호 설정과 Unix 소켓. [독립 Runner](runner-isolation.md#claim)는 연결마다 peer credential을 읽으려고, `codex-uds`가 Unix에서 감싸는 Tokio listener를 직접 엽니다 | | `crates/pty` | `codex-utils-pty` | `tty: true`의 터미널 실행 | | `crates/file-system` | `codex-file-system`, `LOCAL_FS`, 경로 도구 | 심볼릭 링크를 따라가지 않는 Runner 파일 I/O와 범위가 제한된 탐색 | | `crates/linux-sandbox` | `codex-linux-sandbox`, `codex-sandboxing`, `codex-protocol`, `codex-network-proxy` | 실행 파일 전용 명령 샌드박스 도우미와 enabled 네트워크 프록시 | @@ -42,7 +42,7 @@ CodeSpace는 특정 버전에 고정한 Codex 소스의 실행 라이브러리 핵심 crate에는 직접적인 Codex 의존성이 없습니다. 어댑터는 고정된 업스트림의 workspace 의존성을 수용하기 위해 별도의 Cargo workspace로 구성합니다. 파일 시스템·PTY 어댑터는 Runner의 라이브러리 의존성이며, 패치와 Linux 샌드박스는 도우미 프로세스를 사용합니다. Cargo workspace를 분리하는 것만으로 프로세스나 보안 경계가 생기지는 않습니다. -Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 9와 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. +Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 10과 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. diff --git a/docs/ko/devguard-integration.md b/docs/ko/devguard-integration.md index bdecd99..1222cd9 100644 --- a/docs/ko/devguard-integration.md +++ b/docs/ko/devguard-integration.md @@ -338,7 +338,7 @@ contract/client pin, daemon/helper artifact와 CodeSpace Runner wire는 각각 ## 최초 Gateway 복구 범위 -**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. gateway는 아직 다시 연결하지 않으며, 다른 gateway가 작업을 시작한 뒤에 연결한 gateway는 Runner를 중지하고 새로 시작할 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. 두 번째 단위 P1R-C02는 [실행 기록](runner-isolation.md#process-records)을 더한다. gateway는 작업 데이터베이스에 쓰는 유일한 gateway로서 각 실행을 Runner에 보내기 전에 기록하고 끝날 때까지 따라간다. 재연결, 대조, Runner 손실 처리는 이후 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. +**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. 두 번째 단위 P1R-C02는 [실행 기록](runner-isolation.md#process-records)을 더한다. gateway는 작업 데이터베이스에 쓰는 유일한 gateway로서 각 실행을 Runner에 보내기 전에 기록하고 끝날 때까지 따라간다. 세 번째 단위 P1R-C03은 [Claim](runner-isolation.md#claim)을 더한다. 한 번에 한 연결만 controller epoch 아래에서 Runner를 제어하고, 프로세스나 workspace를 바꾸는 요청은 모두 그 epoch를 함께 보낸다. 연결을 잃은 gateway는 재시작하지 않고 Runner를 다시 Claim한다. 작업을 시작한 gateway 뒤에 Runner를 제어하는 gateway는, 연결을 잃었다가 다시 Claim한 같은 gateway를 포함해, Runner를 중지하고 새로 시작할 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. Runner를 제어하는 gateway가 없는 동안에는 Runner의 소켓에 닿을 수 있는 같은 사용자의 프로세스라면 무엇이든 Runner를 Claim할 수 있으며, 이 노출은 [Runner 문서](runner-isolation.md#claim-exposure)에 적었다. 대조와 Runner 손실 처리는 이후 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. P1-RECOVERY는 운영자가 선택하는 독립 Runner 모드와 명시적인 capability를 도입한다. 기존 모드의 종료·연결 소실 계약은 유지한다. InProcess는 Gateway와 PID를 공유하므로 Gateway 재시작 중 live 복구 대상에서 제외한다. diff --git a/docs/ko/error-codes.md b/docs/ko/error-codes.md index b225875..20482dc 100644 --- a/docs/ko/error-codes.md +++ b/docs/ko/error-codes.md @@ -57,8 +57,10 @@ | `APPROVAL_NOT_FOUND` | 알 수 없는 확인 홀드 ID | | `APPROVAL_CONFLICT` | 홀드가 아직 대기 중이거나 이미 결정되었거나, 재개가 이미 진행 중임 | | `APPROVAL_AMBIGUOUS` | 재개가 중단되어 단말 결과가 디스크에 없음. `approval_id` 포함. 패치는 `operation_id`도 있을 수 있음 | -| `RECOVERY_PENDING` | 이 게이트웨이의 [독립 Runner](runner-isolation.md#independent-runner)에서 이전 게이트웨이가 작업을 시작함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, Runner를 중지하고 새로 시작하기 전에는 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 점유가 아니므로 재시도해도 소용없음 | -| `RUNNER_DISCONNECTED` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 다시 연결하지 않음. 요청은 보내지 않았고 Runner의 프로세스는 계속 실행됨. 그 프로세스에 닿으려면 게이트웨이를 재시작할 것 | +| `RECOVERY_PENDING` | 이 게이트웨이가 Claim하기 전의 controller가 [독립 Runner](runner-isolation.md#claim)에서 작업을 시작함. 연결을 잃기 전의 이 게이트웨이 자신도 포함함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, Runner를 중지하고 새로 시작하기 전에는 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 점유가 아니므로 재시도해도 소용없음 | +| `RUNNER_RECONNECTING` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 Runner를 다시 Claim하는 중임. 요청은 보내지 않았음. 그 요청을 위해 시작하거나 예약하거나 기록한 것은 없고, 재개하려던 승인은 소비되지 않음. Runner의 프로세스는 계속 실행됨. 잠시 뒤 재시도할 것 | +| `RUNNER_DISCONNECTED` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 다시 Claim할 수 없음. 소켓에서 연결을 받지 않거나, 다른 Runner가 응답하거나, 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부함. 요청은 보내지 않았고 Runner의 프로세스는 계속 실행됨. 그 프로세스에 닿으려면 게이트웨이를 재시작할 것 | +| `STALE_CONTROLLER` | [독립 Runner](runner-isolation.md#claim)가 다른 Claim으로 바뀐 Claim 아래에서 보낸 요청, 또는 Runner를 제어하지 않는 연결의 요청을 거부함. 요청이 요구한 일은 하나도 하지 않았고, 요청이 재개한 승인은 다시 승인된 상태가 됨. 재시도하면 게이트웨이의 현재 Claim 아래에서 요청을 보냄 | | `INTERNAL` | 스케줄러·스토어 불변조건 실패, 또는 [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이가 실행을 기록하지 못해 시작하지 않음. 점유가 아니므로 `WORKSPACE_BUSY`로 재시도하지 말 것 | ## 실행 요청과 완료의 구분 diff --git a/docs/ko/operations.md b/docs/ko/operations.md index c22b858..9f8d25a 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -115,7 +115,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 살고 게이트웨이가 그 프로세스를 작업 데이터베이스에 기록하는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. +worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 살고, 게이트웨이가 그 프로세스를 작업 데이터베이스에 기록하며, 연결을 잃은 게이트웨이가 다시 Claim하는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. @@ -137,7 +137,7 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 -**스키마 버전과 이전 버전.** 작업 데이터베이스에는 스키마 버전이 적혀 있습니다. CodeSpace는 자기보다 새 스키마 버전의 데이터베이스를 아무것도 쓰기 전에 거부하고 그대로 둡니다. 그런 데이터베이스는 그것을 쓴 CodeSpace 버전으로 여세요. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 이 데이터베이스에 기록하며, 실행 중에는 그 옆의 lock 파일 `.gateway.lock`을 잡고 있습니다. 실행 기록이 생기기 전의 CodeSpace 버전도 이런 데이터베이스를 열 수 있고 기록은 건드리지 않지만, 스키마 버전을 확인하지도 lock을 걸지도 않습니다. 그러니 독립 Runner의 게이트웨이가 쓰는 데이터베이스에서는 이전 버전을 실행하지 마세요. 독립 모드에서 되돌아갈 때는 그 모드를 쓰지 않고 데이터베이스는 그대로 둡니다. 이전 사본을 복원하면 그 뒤에 쓴 패치 중복 실행 방지 기록, 확인 홀드, 실행 기록이 사라집니다. +**스키마 버전과 이전 버전.** 작업 데이터베이스에는 스키마 버전이 적혀 있습니다. CodeSpace는 자기보다 새 스키마 버전의 데이터베이스를 아무것도 쓰기 전에 거부하고 그대로 둡니다. 그런 데이터베이스는 그것을 쓴 CodeSpace 버전으로 여세요. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 이 데이터베이스에 기록하고, [Runner를 Claim](runner-isolation.md#claim)할 때 밝히는 데이터베이스의 임의 식별자를 이 데이터베이스에 두며, 실행 중에는 그 옆의 lock 파일 `.gateway.lock`을 잡고 있습니다. 실행 기록이 생기기 전의 CodeSpace 버전도 이런 데이터베이스를 열 수 있고 기록은 건드리지 않지만, 스키마 버전을 확인하지도 lock을 걸지도 않습니다. 그러니 독립 Runner의 게이트웨이가 쓰는 데이터베이스에서는 이전 버전을 실행하지 마세요. 독립 모드에서 되돌아갈 때는 그 모드를 쓰지 않고 데이터베이스는 그대로 둡니다. 이전 사본을 복원하면 그 뒤에 쓴 패치 중복 실행 방지 기록, 확인 홀드, 실행 기록이 사라집니다. 패치 응답을 받지 못했다면 `operation_id` 또는 `operation_key` 중 하나만 지정해 `operation_status`를 조회합니다. 재시작 후 미완료 기록은 `unknown`이 되므로 파일을 확인한 뒤 다음 행동을 결정하세요. 프로세스는 `process_id`로 관리하며 `operation_status`로 복구할 수 없습니다. 프로세스가 죽을 때 `queued`이던 확인 홀드는 다시 acquire합니다. `resuming`이던 홀드는 패치 원장에서 복구하거나, 저장된 단말 결과를 재현하거나, `APPROVAL_AMBIGUOUS`를 반환할 수 있습니다. exec는 다시 spawn하지 않습니다. [재시도와 복구 규칙](agent-integration.md)을 참고하세요. diff --git a/docs/ko/runner-isolation.md b/docs/ko/runner-isolation.md index bbb4d74..74d7741 100644 --- a/docs/ko/runner-isolation.md +++ b/docs/ko/runner-isolation.md @@ -13,7 +13,7 @@ `in-process`는 `codespace-mcp` 안에서 프로세스를 관리합니다. `uds`(Unix domain socket, Unix 도메인 소켓)는 같은 호스트의 `codespace-codex-runtime` 안에서 같은 관리 코드를 실행합니다. worker는 시작 시 Codex의 프로세스 보호 설정을 적용하고 전용 Unix 소켓을 엽니다. worker 자체를 보호하는 것과 실행할 명령에 샌드박스를 적용하는 것은 별개입니다. -게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 9 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. +게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 10 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. 관리 프로세스의 수명은 MCP 연결이 아니라 러너 인스턴스가 소유합니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지합니다. 게이트웨이와 worker의 UDS 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 관리 중인 worker와 자식 프로세스가 종료되고 핸들이 사라집니다. 종료할 때 게이트웨이는 worker와의 연결을 닫으므로 worker는 종료하면서 자기 프로세스를 끝내고, 5초 뒤에도 실행 중인 worker는 강제로 종료됩니다. 재시작은 `process_id`를 복구하지 않습니다. [독립 Runner](#independent-runner)는 예외로, 그 프로세스는 게이트웨이보다 오래 삽니다. @@ -23,7 +23,7 @@ ## 독립 Runner -> **아직 지원하는 모드가 아닙니다.** 독립 Runner와 그 실행의 기록은 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 독립 Runner에 다시 연결하지 않으며, 다른 게이트웨이 뒤에 연결한 게이트웨이는 새 작업을 시작할 수 없습니다. 재연결과 상태 대조는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. +> **아직 지원하는 모드가 아닙니다.** 독립 Runner와 그 실행의 기록, 그리고 한 게이트웨이에서 다음 게이트웨이로 제어권을 넘기는 Claim은 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 연결을 잃으면 Runner를 다시 Claim합니다. 그러나 작업을 시작한 게이트웨이 뒤에 Runner를 제어하는 게이트웨이는, 연결을 잃었다가 다시 Claim한 같은 게이트웨이를 포함해 새 작업을 시작할 수 없습니다. 상태 대조는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. 독립 Runner는 자신에게 연결하는 게이트웨이보다 오래 삽니다. 게이트웨이가 종료되거나 재시작되거나 강제로 종료되어도 프로세스와 그 출력, timeout은 계속됩니다. 운영자나 하네스가 독립 Runner를 시작하고, 게이트웨이는 연결만 합니다. @@ -34,7 +34,7 @@ codespace-mcp --runner uds --runner-socket "$RUNNER_DIR/runner.sock" \ --operations-db "$STATE_DIR/operations.sqlite" ``` -Runner는 소켓 디렉터리를 권한 0700으로 직접 만듭니다. 이미 있는 디렉터리는 같은 사용자의 디렉터리이고 권한이 0700이며 심볼릭 링크가 아닐 때만 받아들입니다. 이미 있는 디렉터리를 바꾸지 않으며 `/`, `/tmp`, `/var/tmp`, `$HOME`은 거부합니다. 게이트웨이 연결은 한 번에 하나씩 받고, 앞 연결의 요청이 모두 끝난 뒤에 다음 연결을 받습니다. 중지할 때는 소켓을 지우고, 디렉터리도 직접 만든 것이면 지웁니다. +Runner는 소켓 디렉터리를 권한 0700으로 직접 만듭니다. 이미 있는 디렉터리는 같은 사용자의 디렉터리이고 권한이 0700이며 심볼릭 링크가 아닐 때만 받아들입니다. 이미 있는 디렉터리를 바꾸지 않으며 `/`, `/tmp`, `/var/tmp`, `$HOME`은 거부합니다. 연결은 여러 개를 동시에 받지만, 한 번에 한 연결만 Runner를 [제어](#claim)합니다. 중지할 때는 소켓을 지우고, 디렉터리도 직접 만든 것이면 지웁니다. 게이트웨이의 `--runner-lifecycle`이 Runner의 수명 모드와 다르거나, Runner가 자기 식별 정보를 알리지 않거나, `--runner-lifecycle independent`에 `--runner uds`, `--runner-socket`, `--runner-exit-intent`, `--operations-db` 중 하나가 없거나, `--runtime-bin`이나 `--runner-dir`가 함께 있으면 게이트웨이는 시작하지 않습니다. `--runner-exit-intent`는 `--runner-lifecycle independent`와 함께일 때만 받습니다. @@ -42,22 +42,34 @@ Runner는 소켓 디렉터리를 권한 0700으로 직접 만듭니다. 이미 | --- | --- | --- | | 게이트웨이 종료: stdin이 끝나거나 SIGTERM이나 SIGINT를 받음 | worker가 프로세스를 끝내고 종료합니다 | `--runner-exit-intent detach`면 게이트웨이가 detach하고 프로세스는 계속됩니다. `stop`이면 Runner가 프로세스를 끝내고 종료합니다 | | 재시작하려고 게이트웨이에 SIGUSR1을 보냄 | 처리하지 않으므로 게이트웨이가 강제 종료된 경우와 같습니다 | 종료 의도와 관계없이 게이트웨이가 detach하고 프로세스는 계속됩니다 | -| 게이트웨이가 강제 종료되거나, detach나 stop 없이 연결이 끝남 | worker가 프로세스를 끝내고 종료합니다 | 중지가 아니라 연결 손실입니다. 프로세스는 계속되고 Runner는 다음 게이트웨이를 기다립니다 | -| Runner 호출이 120초 deadline을 넘김 | 게이트웨이가 연결을 닫고, worker는 프로세스를 끝내고 종료합니다 | 게이트웨이가 연결을 닫고, 프로세스는 계속됩니다 | +| 게이트웨이가 강제 종료되거나, detach나 stop 없이 연결이 끝남 | worker가 프로세스를 끝내고 종료합니다 | 중지가 아니라 연결 손실입니다. 프로세스는 계속되고 Runner는 다음 Claim을 기다립니다. 연결만 잃었다면 같은 게이트웨이가 다시 Claim합니다 | +| Runner 호출이 120초 deadline을 넘김 | 게이트웨이가 연결을 닫고, worker는 프로세스를 끝내고 종료합니다 | 게이트웨이가 연결을 닫고 Runner를 다시 Claim하며, 프로세스는 계속됩니다 | | 게이트웨이가 Runner에 중지를 요청하거나, Runner가 SIGTERM이나 SIGINT를 받음 | 관리 worker는 요청을 거부하며 이 신호를 처리하지 않습니다 | Runner가 프로세스를 끝내고 최대 5초 동안 기다린 뒤 종료합니다 | | 프로세스가 timeout에 도달함 | 프로세스를 강제로 종료합니다 | 게이트웨이 연결 여부와 관계없이 원래 deadline에 프로세스를 강제로 종료합니다 | 프로세스를 끝낼 때 tty 프로세스는 그 프로세스 그룹에 신호를 보내지만, 파이프 프로세스는 그 프로세스에만 보냅니다. 파이프 프로세스에는 자기 프로세스 그룹이 없으므로, 그 프로세스가 시작한 프로세스는 중지할 때를 포함해 더 오래 남을 수 있습니다. -**연결을 잃었을 때.** 게이트웨이는 다시 연결하지 않습니다. Runner와의 연결이 끝나면 Runner가 필요한 요청을 모두 `RUNNER_DISCONNECTED`로 거부하며, Runner의 프로세스에 다시 닿으려면 게이트웨이를 재시작해야 합니다. 연결이 끝날 때 이미 보낸 요청은 지금처럼 불확실한 결과를 받습니다. 실행이라면 `dispatch_status: unknown`입니다. + -**이전 게이트웨이가 있었을 때.** 새로 시작한 Runner에 처음 연결한 게이트웨이는 관리 worker와 같이 동작합니다. 한 게이트웨이가 그 Runner에서 작업(실행, 패치, DevGuard를 쓰면 준비나 launch)을 시작한 뒤에는, 다음 게이트웨이들이 Runner의 프로세스를 넘겨받습니다. 그 게이트웨이는 `process_id`로 프로세스를 조회하고, 출력을 읽고, stdin에 쓰고, 크기를 바꾸고, 종료할 수 있지만, 새 실행, 패치, 승인 재개는 `RECOVERY_PENDING`으로 거부하고 승인은 그대로 둡니다. 넘겨받은 프로세스가 모두 끝난 뒤에도 마찬가지이므로, 작업을 다시 시작하려면 Runner를 중지하고 새로 시작합니다. 게이트웨이는 Runner의 식별 정보(프로세스 ID, 부팅과 시작 시각), 이전에 다른 게이트웨이가 제어했는지, 넘겨받은 프로세스 수를 로그에 남깁니다. +**Claim과 controller epoch.** 게이트웨이는 자기 작업 데이터베이스를 밝히는 `Claim`으로 Runner의 제어권을 얻습니다. Runner는 소켓의 peer credential로 확인한 연결 상대가 Runner와 같은 사용자로 실행 중이고, Runner가 다른 작업 데이터베이스를 위해 Claim된 적이 없을 때만 Claim을 받아들입니다. Runner는 처음 Claim한 데이터베이스의 식별자를 기억하고, 다른 데이터베이스를 쓰는 게이트웨이를 거부합니다. 이 식별자는 잘못된 데이터베이스로 시작한 게이트웨이를 막는 장치일 뿐 자격 증명이 아닙니다. 데이터베이스를 소유한 사용자는 이 식별자를 읽을 수 있기 때문입니다. 다른 연결이 Runner를 제어하고 있고 그 연결이 아직 열려 있으면, 그 게이트웨이가 멈춰 있더라도 Runner는 Claim을 거부합니다. 살아 있는 controller를 밀어내는 방법은 없습니다. 그 연결이 끝난 뒤에는 Claim이 그 연결이 보낸 요청을 준비와 launch까지 포함해 최대 10초 동안 기다립니다. 요청이 그보다 오래 걸리면 Claim은 거부되고 아무것도 바뀌지 않습니다. + +Claim을 받아들일 때마다 Runner의 controller epoch가 하나 올라갑니다. epoch는 Runner가 살아 있는 동안 줄지 않으며, 새 Runner는 다시 1부터 시작합니다. Claim의 응답은 Runner의 프로세스를 각자의 workspace, 상태와 함께 나열하고, 이 목록에 반영된 마지막 종료 이벤트의 번호를 알립니다. Runner를 제어하지 않는 연결은 `Hello`, `Claim`, `Detach`에만 응답을 받고, 종료 이벤트는 제어하는 연결에만 갑니다. 프로세스나 workspace를 바꾸거나 Runner를 중지하는 요청은 그 요청을 보낸 Claim의 epoch를 함께 보냅니다. 실행, 패치, stdin 쓰기, 크기 변경, 프로세스나 workspace 프로세스의 종료, 중지, DevGuard를 쓰면 준비와 launch가 여기에 해당합니다. Runner는 이전 epoch로 온 이런 요청을 `STALE_CONTROLLER`로 거부하며, 그 요청이 요구한 일은 하나도 하지 않습니다. 게이트웨이는 각 실행의 기록에 epoch를 남기고, Runner는 받아들이거나 거부한 Claim마다 Claim한 상대의 프로세스 ID를 로그에 남깁니다. + +다른 연결이 Runner를 제어하는 동안이나 그 연결의 요청이 처리되는 동안 시작한 게이트웨이는 간격을 늘려 가며 최대 30초 동안 다시 요청하고, 그래도 안 되면 시작을 거부합니다. 다른 작업 데이터베이스를 쓰는 경우처럼 다른 이유로 거부된 게이트웨이는 바로 시작을 거부하며, Runner는 그대로 남습니다. + +**연결을 잃었을 때.** Runner와의 연결을 잃었거나, 호출이 deadline을 넘겨 연결을 닫은 게이트웨이는 다시 연결해 같은 Runner를 Claim합니다. 먼저 소켓에서 응답하는 Runner가 시작할 때 찾은 Runner와 같은 식별 정보를 가졌는지 확인합니다. 바로 한 번 시도하고, 실패하면 100밀리초를 기다린 뒤 다시 시도하며, 실패할 때마다 기다리는 시간을 두 배씩 최대 5초까지 늘립니다. 그동안에는 Runner가 필요한 요청을 모두 `RUNNER_RECONNECTING`으로 거부합니다. 그 요청을 위해서는 아무것도 보내거나 예약하거나 기록하지 않으며, 재개하려던 승인은 소비되지 않습니다. 소켓에서 더 이상 연결을 받지 않거나, 다른 Runner가 응답하거나, 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부하면 게이트웨이는 시도를 멈추고 그런 요청을 모두 `RUNNER_DISCONNECTED`로 거부합니다. 그러면 Runner의 프로세스에 다시 닿으려면 게이트웨이를 재시작해야 합니다. 연결을 잃을 때 이미 보낸 요청은 지금처럼 불확실한 결과를 받습니다. 실행이라면 `dispatch_status: unknown`입니다. Runner가 전하지 못한 종료 이벤트는 다음 controller에게 가고, 게이트웨이는 각 종료를 한 번만 전달합니다. + +**이전 controller가 있었을 때.** 새로 시작한 Runner를 처음 Claim한 게이트웨이는 관리 worker와 같이 동작합니다. 한 controller가 그 Runner에서 작업(실행, 패치, DevGuard를 쓰면 준비나 launch)을 시작한 뒤에는, 그 뒤의 controller들이 Runner의 프로세스를 넘겨받습니다. 연결을 잃었다가 Runner를 다시 Claim한 같은 게이트웨이도 마찬가지입니다. 그 게이트웨이는 `process_id`로 프로세스를 조회하고, 출력을 읽고, stdin에 쓰고, 크기를 바꾸고, 종료할 수 있지만, 새 실행, 패치, 승인 재개는 `RECOVERY_PENDING`으로 거부하고 승인은 그대로 둡니다. 넘겨받은 프로세스가 모두 끝난 뒤에도 마찬가지이므로, 작업을 다시 시작하려면 Runner를 중지하고 새로 시작합니다. 게이트웨이는 Runner의 식별 정보(프로세스 ID, 부팅과 시작 시각), Claim의 epoch, 이전 controller가 작업을 시작했는지, 넘겨받은 프로세스 수, 그 목록에 반영된 마지막 종료 이벤트의 번호를 로그에 남깁니다. + + + +**누가 제어권을 얻을 수 있는가.** Runner는 작업 데이터베이스를 밝히는 같은 사용자의 프로세스라면 무엇이든 믿습니다. 게이트웨이가 Runner를 제어하는 동안에는 다른 프로세스가 제어권을 얻을 수 없습니다. 그러나 게이트웨이가 제어하지 않는 동안, 곧 한 게이트웨이가 끝나고 다음 게이트웨이가 Claim하기 전이나 게이트웨이가 다시 연결하는 동안에는 소켓에 닿을 수 있는 같은 사용자의 프로세스라면 무엇이든 Runner를 Claim할 수 있습니다. 그러면 그 프로세스가 Runner의 프로세스와 그 입력, 출력, 종료를 갖게 되고, 앞선 controller가 작업을 시작한 적이 없다면 작업을 시작할 수도 있습니다. 게이트웨이는 그 연결이 끝난 뒤에야 제어권을 되찾으며, 그동안 계속 다시 요청합니다. 이런 프로세스는 게이트웨이가 샌드박스 없이 실행한 명령일 수 있습니다. macOS에서, 그리고 Linux에서 샌드박스 도우미가 없을 때가 그렇습니다. Linux 샌드박스가 실행한 명령은 소켓에 연결할 수 없습니다. 네트워크가 restricted이면 샌드박스가 연결을 막고, enabled이면 Unix 소켓 자체를 막기 때문입니다. 이런 프로세스를 막으려면 믿지 않는 다른 프로그램을 실행하지 않는 사용자로 게이트웨이와 Runner를 실행하고, Linux에서는 샌드박스 도우미를 두어 게이트웨이의 명령이 샌드박스 안에서 실행되게 합니다. -**실행 기록.** 게이트웨이는 각 실행을 Runner에 보내기 전에 작업 데이터베이스에 기록하고, 실행이 끝날 때까지 따라갑니다. 그래서 메모리를 잃은 게이트웨이도 같은 실행을 다시 알아볼 수 있습니다. 기록에는 `process_id`, 실행을 보낸 Runner의 식별 정보(프로세스 ID, 부팅, 시작 시각, 임의의 nonce), workspace, 실행이 재개하는 승인, DevGuard attempt, DevGuard 참여 여부, tty 여부, argv의 SHA-256 digest가 들어가며, argv 자체와 그 환경, secret은 들어가지 않습니다. 기록의 단계는 Runner가 실행을 시작했다고 답할 때까지 `dispatching`이고, 그 뒤로는 `running`입니다. Runner가 실행이 끝났다고 알리면 Runner가 보고한 종료 방식, 종료 코드, 신호와 함께 `terminal`이 되고, Runner가 실행을 시작하지 않았다고 답할 때도 `terminal`이 됩니다. 기록에는 Runner가 정한 deadline, Runner가 만든 루트 프로세스, 게이트웨이가 출력을 어디까지 읽었는지도 남습니다. 루트 프로세스는 프로세스 ID와, 나중에 같은 ID를 받은 프로세스와 구별해 주는 시작 시각으로 남기며, `tty: true` 실행의 루트는 남기지 않습니다. +**실행 기록.** 게이트웨이는 각 실행을 Runner에 보내기 전에 작업 데이터베이스에 기록하고, 실행이 끝날 때까지 따라갑니다. 그래서 메모리를 잃은 게이트웨이도 같은 실행을 다시 알아볼 수 있습니다. 기록에는 `process_id`, 실행을 보낸 Runner의 식별 정보(프로세스 ID, 부팅, 시작 시각, 임의의 nonce), 실행을 보낸 [controller epoch](#claim), workspace, 실행이 재개하는 승인, DevGuard attempt, DevGuard 참여 여부, tty 여부, argv의 SHA-256 digest가 들어가며, argv 자체와 그 환경, secret은 들어가지 않습니다. 기록의 단계는 Runner가 실행을 시작했다고 답할 때까지 `dispatching`이고, 그 뒤로는 `running`입니다. Runner가 실행이 끝났다고 알리면 Runner가 보고한 종료 방식, 종료 코드, 신호와 함께 `terminal`이 되고, Runner가 실행을 시작하지 않았다고 답할 때도 `terminal`이 됩니다. 기록에는 Runner가 정한 deadline, Runner가 만든 루트 프로세스, 게이트웨이가 출력을 어디까지 읽었는지도 남습니다. 루트 프로세스는 프로세스 ID와, 나중에 같은 ID를 받은 프로세스와 구별해 주는 시작 시각으로 남기며, `tty: true` 실행의 루트는 남기지 않습니다. -승인을 재개하는 실행은 그 승인을 resuming으로 바꾸는 transaction 안에서 기록합니다. 기록을 쓸 수 없으면 실행을 `INTERNAL`로 거부하고 아무것도 보내지 않습니다. 연결된 게이트웨이가 없는 동안 끝난 실행은 Runner가 보관했다가 다음 게이트웨이에 알리고, 그 게이트웨이가 기록합니다. `process_status`와 `read_process`도 Runner가 보고한 내용을 기록합니다. 기록이 핸들을 되살리는 일은 없으며, 기록만으로 실행이 시작되었는지나 어떻게 끝났는지가 증명되지도 않습니다. 기준은 Runner이며, 응답을 잃은 실행의 기록은 `dispatching`으로 남습니다. 기록으로 작업을 복구하는 일은 다음 단계에서 더합니다. +승인을 재개하는 실행은 그 승인을 resuming으로 바꾸는 transaction 안에서 기록합니다. 기록을 쓸 수 없으면 실행을 `INTERNAL`로 거부하고 아무것도 보내지 않습니다. Runner를 제어하는 게이트웨이가 없는 동안 끝난 실행은 Runner가 보관했다가 다음 controller에 알리고, 그 게이트웨이가 기록합니다. `process_status`와 `read_process`도 Runner가 보고한 내용을 기록합니다. 기록이 핸들을 되살리는 일은 없으며, 기록만으로 실행이 시작되었는지나 어떻게 끝났는지가 증명되지도 않습니다. 기준은 Runner이며, 응답을 잃은 실행의 기록은 `dispatching`으로 남습니다. 기록으로 작업을 복구하는 일은 다음 단계에서 더합니다. **writer는 하나.** 독립 Runner에 연결한 게이트웨이는 자기 작업 데이터베이스에 쓰는 유일한 게이트웨이입니다. 데이터베이스를 열기 전에 그 옆의 파일 `.gateway.lock`에 배타 lock을 걸고, 어떤 식으로 종료하든 종료할 때까지 lock을 유지합니다. 이 파일에는 lock을 가진 게이트웨이의 프로세스 ID가 적힙니다. 앞 게이트웨이가 아직 detach하는 동안 새 게이트웨이를 시작했을 때처럼 lock이 이미 잡혀 있으면, 새 게이트웨이는 기다린다는 로그를 남기고 최대 15초 동안 기다린 뒤 시작을 거부합니다. 이 lock은 이 모드에서만 겁니다. 데이터베이스의 스키마 버전과 이전 CodeSpace 버전은 [운영](operations.md#operations-database)을 참고하세요. diff --git a/docs/ko/security-model.md b/docs/ko/security-model.md index 1d6f2b7..9ea5918 100644 --- a/docs/ko/security-model.md +++ b/docs/ko/security-model.md @@ -20,6 +20,8 @@ Runner는 허용된 요청을 실행합니다. `PathSandbox`는 논리적인 파 선택적 UDS worker는 같은 호스트에서 프로세스를 분리합니다. 소켓을 전용 디렉터리에 두지만 프로세스 분리 자체가 명령 샌드박스는 아닙니다. Linux 도우미의 사용 가능 검사가 성공하면 명령 샌드박스를 사용합니다. 조건과 실패 시 동작은 [러너 격리](runner-isolation.md)에 설명합니다. +아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#claim)는 같은 사용자의 연결을 모두 받고, Claim한 연결 하나에만 한 번에 제어를 맡깁니다. 게이트웨이가 Runner를 제어하지 않는 동안에는 소켓에 닿을 수 있는 그 사용자의 프로세스라면 무엇이든 Runner를 Claim할 수 있으며, Linux 샌드박스 없이 실행한 명령도 여기에 포함됩니다. [누가 제어권을 얻을 수 있는가](runner-isolation.md#claim-exposure)를 참고하세요. + @@ -44,7 +46,7 @@ Linux 도우미가 없으면 restricted 정책의 작업 공간에서도 비격 버전 검사는 예상하지 못한 파일 버전에 패치를 적용하는 것을 막습니다. 작업 키는 패치 중복 요청을 구분하며 인증 토큰이 아닙니다. 작업 공간 점유는 파일을 변경할 수 있는 패치·명령 실행을 직렬화합니다. 요청 소유 충돌은 `acquire()`에서 시작하는 인메모리 FIFO에서 기다리고, 라이브 프로세스는 뒤 대기자와 새 요청에 `WORKSPACE_BUSY`를 반환합니다. 점유 상태는 재시작 후에도 남지 않습니다. 영속적인 프로세스 복구는 없습니다. -`CODESPACE_OPERATIONS_DB`의 `approvals` 확인 홀드는 행이 `pending`·`granted`·`queued`·`resuming`인 동안 V4A 패치나 exec argv를 보관합니다. 패치 원장은 패치 본문이 아니라 해시를 보관합니다. `denied` 또는 `consumed` 뒤에는 홀드 본문을 digest 메타(도구, 작업 공간, fingerprint)로 바꿉니다. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 여기에 기록하는데(`processes`), argv는 SHA-256 digest로만 남기고 argv 자체와 그 환경, secret은 남기지 않습니다. 이 데이터베이스는 토큰·게이트웨이 설정과 같이 작업 공간 루트 밖에 두세요. 홀드는 워크플로 일시정지이며 같은 MCP 호출자가 grant할 수 있습니다. 격리 경계가 아닙니다. +`CODESPACE_OPERATIONS_DB`의 `approvals` 확인 홀드는 행이 `pending`·`granted`·`queued`·`resuming`인 동안 V4A 패치나 exec argv를 보관합니다. 패치 원장은 패치 본문이 아니라 해시를 보관합니다. `denied` 또는 `consumed` 뒤에는 홀드 본문을 digest 메타(도구, 작업 공간, fingerprint)로 바꿉니다. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 여기에 기록하는데(`processes`), argv는 SHA-256 digest로만 남기고 argv 자체와 그 환경, secret은 남기지 않습니다. 또 Runner를 Claim할 때 밝히는 데이터베이스의 임의 식별자(`store_identity`)를 여기에 둡니다. 이 식별자는 데이터베이스를 서로 구별할 뿐 secret이 아닙니다. 이 데이터베이스는 토큰·게이트웨이 설정과 같이 작업 공간 루트 밖에 두세요. 홀드는 워크플로 일시정지이며 같은 MCP 호출자가 grant할 수 있습니다. 격리 경계가 아닙니다. 패치 스냅샷 복원은 가능한 범위에서 수행하며 모든 실패의 롤백을 보장하지 않습니다. `unknown`, 부분 실패, 적용 후 검증 오류가 발생하면 해당 파일을 확인하세요. [패치 동작](behavior-differences.md)과 [연동 복구 규칙](agent-integration.md)을 참고하세요. diff --git a/docs/operations.md b/docs/operations.md index 9d46fff..7eb489a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -109,7 +109,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway and whose gateway records them in the operations database, is not a supported mode yet. +The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, whose gateway records them in the operations database, and which a gateway claims again after a lost connection, is not a supported mode yet. @@ -129,7 +129,7 @@ Store logs and the database outside the managed workspace, with tokens and gatew -**Schema version and earlier versions.** The operations database states its schema version. CodeSpace refuses a database with a newer schema version than its own before writing anything, and leaves it as it is: open it with the CodeSpace version that wrote it. A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions in the database, and holds the lock file `.gateway.lock` beside it while it runs. A CodeSpace version from before these records still opens such a database and leaves the records alone, but it neither checks the schema version nor takes the lock, so do not run one on a database that a gateway with an independent Runner uses. To go back from the independent mode, stop using it and keep the database: restoring an earlier copy would drop the patch idempotency records, confirmation holds and process records written since. +**Schema version and earlier versions.** The operations database states its schema version. CodeSpace refuses a database with a newer schema version than its own before writing anything, and leaves it as it is: open it with the CodeSpace version that wrote it. A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions in the database, keeps there the random identity of the database that it names when it [claims the Runner](runner-isolation.md#claim), and holds the lock file `.gateway.lock` beside it while it runs. A CodeSpace version from before these records still opens such a database and leaves the records alone, but it neither checks the schema version nor takes the lock, so do not run one on a database that a gateway with an independent Runner uses. To go back from the independent mode, stop using it and keep the database: restoring an earlier copy would drop the patch idempotency records, confirmation holds and process records written since. After losing a patch response, query `operation_status` with exactly one of `operation_id` or `operation_key`. An unfinished record becomes `unknown` after restart; inspect files before deciding what to do. Processes use `process_id` and cannot be recovered through `operation_status`. A confirmation hold that was `queued` when the process died can be resumed and will reacquire. A hold that was `resuming` may recover a patch from that ledger, replay a stored terminal result, or return `APPROVAL_AMBIGUOUS`; exec is not respawned. See [retry and recovery rules](agent-integration.md). diff --git a/docs/runner-isolation.md b/docs/runner-isolation.md index 6b6a7aa..6541726 100644 --- a/docs/runner-isolation.md +++ b/docs/runner-isolation.md @@ -10,7 +10,7 @@ There are three separate questions: which process runs the work, whether command `in-process` runs the supervisor inside `codespace-mcp`. `uds` (Unix domain socket) runs that supervisor inside `codespace-codex-runtime` on the same host. The worker starts with Codex process hardening and binds a private Unix socket. Hardening the worker does not sandbox its commands. -The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 9, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. +The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 10, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. Managed process lifetime is owned by the runner instance, not the MCP connection. Streamable HTTP or MCP client disconnect keeps processes running. Gateway/worker UDS disconnect or gateway shutdown (including stdio EOF) ends the owned worker and its children; process handles are lost. At shutdown the gateway closes the worker's connection, so the worker ends its processes as it exits; a worker still running 5 seconds later is killed. Restart does not restore `process_id`. An [independent Runner](#independent-runner) is the exception: its processes outlive the gateway. @@ -20,7 +20,7 @@ Managed process lifetime is owned by the runner instance, not the MCP connection ## Independent Runner -> **Not a supported mode yet.** The independent Runner and the records of its executions are the first steps of Gateway recovery. A gateway does not reconnect to it, and a gateway that connects after another one cannot start new work. Later steps add reconnection and reconciliation, and the mode is supported only once it is qualified. +> **Not a supported mode yet.** The independent Runner, the records of its executions and the Claim that hands its control from one gateway to the next are the first steps of Gateway recovery. A gateway claims the Runner again when its connection is lost, but a gateway that controls it after one that started work, the same gateway after a lost connection included, cannot start new work. A later step adds reconciliation, and the mode is supported only once it is qualified. An independent Runner outlives the gateways that connect to it: its processes, their output and their timeouts carry on when a gateway exits, restarts or is killed. An operator or a harness starts it, and a gateway only connects to it: @@ -31,7 +31,7 @@ codespace-mcp --runner uds --runner-socket "$RUNNER_DIR/runner.sock" \ --operations-db "$STATE_DIR/operations.sqlite" ``` -The Runner creates its socket's directory with mode 0700, or accepts an existing one only when it is a directory of the same user with mode 0700 and not a symbolic link. It never changes an existing directory, and it refuses `/`, `/tmp`, `/var/tmp` and `$HOME`. It serves one gateway connection at a time, and accepts the next only after every request of the previous one has ended. When it stops, it removes its socket, and the directory if it created it. +The Runner creates its socket's directory with mode 0700, or accepts an existing one only when it is a directory of the same user with mode 0700 and not a symbolic link. It never changes an existing directory, and it refuses `/`, `/tmp`, `/var/tmp` and `$HOME`. It serves any number of connections at once, and one of them at a time [controls it](#claim). When it stops, it removes its socket, and the directory if it created it. A gateway does not start when its `--runner-lifecycle` differs from the Runner's lifecycle, when the Runner does not state its identity, or when `--runner-lifecycle independent` comes without `--runner uds`, `--runner-socket`, `--runner-exit-intent` or `--operations-db`, or with `--runtime-bin` or `--runner-dir`. `--runner-exit-intent` is accepted only with `--runner-lifecycle independent`. @@ -39,22 +39,34 @@ A gateway does not start when its `--runner-lifecycle` differs from the Runner's | --- | --- | --- | | The gateway shuts down: its stdin ends, or it receives SIGTERM or SIGINT | The worker ends its processes and exits | With `--runner-exit-intent detach` the gateway detaches, and the processes carry on. With `stop` the Runner ends its processes and exits | | The gateway receives SIGUSR1 to be restarted | Not handled: as when the gateway is killed | The gateway detaches whatever its exit intent, and the processes carry on | -| The gateway is killed, or its connection ends without a detach or a stop | The worker ends its processes and exits | A loss, never a stop: the processes carry on, and the Runner waits for the next gateway | -| A call to the Runner passes its 120-second deadline | The gateway closes the connection, and the worker ends its processes and exits | The gateway closes the connection, and the processes carry on | +| The gateway is killed, or its connection ends without a detach or a stop | The worker ends its processes and exits | A loss, never a stop: the processes carry on, and the Runner waits for the next Claim, which comes from the same gateway when only its connection was lost | +| A call to the Runner passes its 120-second deadline | The gateway closes the connection, and the worker ends its processes and exits | The gateway closes the connection and claims the Runner again, and the processes carry on | | A gateway asks the Runner to stop, or the Runner receives SIGTERM or SIGINT | A managed worker refuses the request and does not handle these signals | The Runner ends its processes, waits up to 5 seconds for them, and exits | | A process reaches its timeout | The process is killed | The process is killed at its original deadline, whether or not a gateway is connected | Ending a process signals a tty process's process group, but a pipe process alone: a pipe process has no process group of its own, so the processes it started may outlive it, a stop included. -**A lost connection.** A gateway does not reconnect. Once its connection to the Runner has ended, it refuses every request that needs the Runner with `RUNNER_DISCONNECTED`, and it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection ended keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. + -**After an earlier gateway.** The first gateway that connects to a newly started Runner works as it does with a managed worker. Once a gateway has started work there (an execution, a patch, or with DevGuard a preparation or a launch), every later gateway inherits the Runner's processes: it can query, read, write to, resize and terminate them with their `process_id`, but it refuses new executions, patches and approval resumes with `RECOVERY_PENDING`, and leaves the approvals as they are. This holds after the inherited processes have ended too; to start work again, stop the Runner and start a new one. The gateway logs the Runner's identity (its process ID, boot and start time), whether a gateway controlled it before and how many processes it inherited. +**Claim and controller epoch.** A gateway takes control of the Runner with a `Claim` that names its operations database. The Runner grants it only when the connection's peer runs as the Runner's own user, as the socket's peer credentials state, and when the Runner was not claimed before for another operations database: it keeps the identity of the database of its first Claim and refuses a gateway of another one. That identity guards against a gateway started with the wrong database. It is not a credential, since the user who owns the database can read it. The Runner refuses a Claim while another connection controls it and is still open, even one whose gateway is stopped: nothing pushes a live controller out. Once that connection has ended, a Claim waits up to 10 seconds for the requests it sent, preparations and launches included. When they run longer, the Claim is refused and changes nothing. + +Each granted Claim raises the Runner's controller epoch by one. The epoch never goes down while the Runner lives; a new Runner starts again from 1. The Claim's answer lists the Runner's processes, each with its workspace and status, and states the number of the last exit event that the list reflects. A connection that does not control the Runner gets answers to `Hello`, `Claim` and `Detach` only, and the Runner sends exit events only to its controlling connection. Each request that changes a process or the workspace, or stops the Runner, carries the epoch of the Claim it was sent under: an execution, a patch, writing to stdin, resizing, terminating a process or a workspace's processes, a stop, and with DevGuard a preparation or a launch. The Runner refuses such a request from an earlier epoch with `STALE_CONTROLLER`, and nothing it asked for is done. The gateway records the epoch with each execution, and the Runner logs each Claim that it grants or refuses with the process ID of the peer that made it. + +A gateway that starts while another connection controls the Runner, or while that connection's requests run, asks again with a growing pause for up to 30 seconds, and then refuses to start. A gateway refused for another reason, such as another operations database, refuses to start at once and leaves the Runner as it was. + +**A lost connection.** A gateway whose connection to the Runner is lost, or that closed it after a call passed its deadline, connects and claims the same Runner again. It first checks that the Runner at the socket has the identity it found at its start. It tries at once, waits 100 milliseconds after the first failed try, and twice as long after each further one, up to 5 seconds. Meanwhile it refuses each request that needs the Runner with `RUNNER_RECONNECTING`: nothing is sent, reserved or recorded for the request, and an approval it would resume is not consumed. When nothing accepts connections at the socket any more, another Runner answers there, or the Runner refuses the Claim for a reason that waiting cannot end, the gateway stops trying and refuses every such request with `RUNNER_DISCONNECTED`; it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection was lost keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. An exit event that the Runner could not deliver goes to its next controller, and the gateway hands each exit on once. + +**After an earlier controller.** The first gateway that claims a newly started Runner works as it does with a managed worker. Once a controller has started work there (an execution, a patch, or with DevGuard a preparation or a launch), every later controller inherits the Runner's processes, the same gateway included once it claimed the Runner again after a lost connection. It can query, read, write to, resize and terminate them with their `process_id`, but it refuses new executions, patches and approval resumes with `RECOVERY_PENDING`, and leaves the approvals as they are. This holds after the inherited processes have ended too; to start work again, stop the Runner and start a new one. The gateway logs the Runner's identity (its process ID, boot and start time), the epoch of its Claim, whether a controller before it started work, how many processes it inherited, and the number of the last exit event that list reflects. + + + +**Who can take control.** The Runner trusts any process of its own user that names the operations database. No other process can take control while a gateway controls the Runner. While no gateway does, between one gateway's end and the next one's Claim or while a gateway reconnects, any process of the same user that can reach the socket can claim the Runner. That process then holds the Runner's processes, their input, output and termination, and it can start work when no controller before it did. The gateway takes control again only once that connection ends, and asks again meanwhile. Such a process may be a command that a gateway runs without a sandbox: on macOS, and on Linux without the sandbox helper. A command that the Linux sandbox runs cannot connect to the socket: the sandbox denies it the connection with a restricted network, and Unix sockets with an enabled network. To keep such processes out, run the gateway and the Runner as a user that runs nothing else you do not trust, and on Linux keep the sandbox helper available, so that the gateway's commands run sandboxed. -**Process records.** The gateway records each execution in its operations database before it sends the execution to the Runner, and follows it to its end, so that a gateway that lost its memory can identify the same execution again. A record holds the `process_id`, the identity of the Runner it was sent to (its process ID, boot, start time and a random nonce), the workspace, the approval it resumes, its DevGuard attempt, whether it takes part in DevGuard, whether it has a tty, and the SHA-256 digest of its argv, never the argv, its environment or a secret. Its phase is `dispatching` until the Runner answers that it started the execution, and `running` after that. It becomes `terminal` once the Runner reports that the execution ended, with the termination, exit code and signal the Runner reported, or once the Runner answers that it did not start it. The record also keeps the deadline the Runner set, the root process the Runner created, by its process ID and the start time that tells it apart from a later process with the same ID (a `tty: true` root is not named), and how far a gateway read the output. +**Process records.** The gateway records each execution in its operations database before it sends the execution to the Runner, and follows it to its end, so that a gateway that lost its memory can identify the same execution again. A record holds the `process_id`, the identity of the Runner it was sent to (its process ID, boot, start time and a random nonce), the [controller epoch](#claim) it was sent under, the workspace, the approval it resumes, its DevGuard attempt, whether it takes part in DevGuard, whether it has a tty, and the SHA-256 digest of its argv, never the argv, its environment or a secret. Its phase is `dispatching` until the Runner answers that it started the execution, and `running` after that. It becomes `terminal` once the Runner reports that the execution ended, with the termination, exit code and signal the Runner reported, or once the Runner answers that it did not start it. The record also keeps the deadline the Runner set, the root process the Runner created, by its process ID and the start time that tells it apart from a later process with the same ID (a `tty: true` root is not named), and how far a gateway read the output. -An execution that resumes an approval is recorded in the transaction that marks the approval resuming. When the record cannot be written, the execution is refused with `INTERNAL` and nothing is sent. The next gateway records the exits that happened while no gateway was connected, which the Runner keeps for it, and `process_status` and `read_process` record what the Runner reports. A record never revives a handle, and never by itself proves that an execution started or how it ended: the Runner is the authority, and the record of an execution whose answer was lost stays `dispatching`. Recovering work from the records comes in later steps. +An execution that resumes an approval is recorded in the transaction that marks the approval resuming. When the record cannot be written, the execution is refused with `INTERNAL` and nothing is sent. The next controller records the exits that happened while no gateway controlled the Runner, which the Runner keeps for it, and `process_status` and `read_process` record what the Runner reports. A record never revives a handle, and never by itself proves that an execution started or how it ended: the Runner is the authority, and the record of an execution whose answer was lost stays `dispatching`. Recovering work from the records comes in later steps. **One writer.** A gateway with an independent Runner is its operations database's only writer. Before it opens the database it takes an exclusive lock on a file beside it, `.gateway.lock`, which names the process ID of the gateway that holds it, and it keeps the lock until it exits, however it exits. A gateway that finds the lock held, such as one started while the gateway before it still detaches, logs that it waits, waits up to 15 seconds, and then refuses to start. Only this mode takes the lock. See [Operations](operations.md#operations-database) for the database's schema version and earlier CodeSpace versions. diff --git a/docs/security-model.md b/docs/security-model.md index de4d951..f94b509 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -15,6 +15,8 @@ The Runner executes approved requests. `PathSandbox` checks logical file scope; The optional UDS worker separates processes on the same host. Its socket lives in a private directory, but process separation is not an OS command sandbox. The Linux helper supplies the command sandbox when its probe succeeds. [Runner isolation](runner-isolation.md) defines those conditions and failure behavior. +An [independent Runner](runner-isolation.md#claim), not a supported mode yet, serves every connection of its own user and lets one at a time control it after a Claim. While no gateway controls it, any process of that user that can reach its socket can claim it, a command run without the Linux sandbox included; see [who can take control](runner-isolation.md#claim-exposure). + @@ -34,7 +36,7 @@ The operator-registered root is a trust anchor. Keep tokens, the operations data Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery. -Confirmation holds in `CODESPACE_OPERATIONS_DB` (`approvals`) store the V4A patch or exec argv while the row is `pending`, `granted`, `queued`, or `resuming`. The patch operations ledger stores hashes, not patch text. After `denied` or `consumed`, the hold body is replaced with digest metadata (tool, workspace, fingerprint). A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions there (`processes`), each with the SHA-256 digest of its argv, never the argv, its environment or a secret. Keep this database outside the workspace root, with tokens and gateway configuration. The hold is a workflow pause: the same MCP caller can grant it. It is not an isolation boundary. +Confirmation holds in `CODESPACE_OPERATIONS_DB` (`approvals`) store the V4A patch or exec argv while the row is `pending`, `granted`, `queued`, or `resuming`. The patch operations ledger stores hashes, not patch text. After `denied` or `consumed`, the hold body is replaced with digest metadata (tool, workspace, fingerprint). A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions there (`processes`), each with the SHA-256 digest of its argv, never the argv, its environment or a secret, and keeps there the database's random identity (`store_identity`), which it names when it claims the Runner: it tells one database from another and is not a secret. Keep this database outside the workspace root, with tokens and gateway configuration. The hold is a workflow pause: the same MCP caller can grant it. It is not an isolation boundary. Patch snapshot restoration is best effort and is not an all-failure rollback guarantee. On `unknown`, partial failure, or post-apply verification error, inspect affected files. See [patch behavior](behavior-differences.md) and [integration recovery rules](agent-integration.md). diff --git a/docs/translations.json b/docs/translations.json index c0e8e7f..d5f6bfa 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -78,8 +78,8 @@ "최소-도입-조건", "최초-gateway-복구-범위" ], - "source_sha256": "e5cfae184723094b5cbc3b271c3d704cf755fbdd0a2866be544017accbab5647", - "translation_sha256": "b566cc59248d0913b80cf337a4809202de48e3d251a628c0873dac4ee69c2649" + "source_sha256": "ef6902babea74a960118cef8b12204606c8707a3c13b5f28bb0cec113f633222", + "translation_sha256": "b7562abebafea29eb2da52ea4f421c41a8f2ed5af475a3be49a0937e215a9ae0" }, { "id": "cs-rg-qualification", @@ -257,8 +257,8 @@ "재시도와-복구", "파일-읽기와-패치" ], - "source_sha256": "a0ac8251ae13700936d1f2011aeb709100b1c3ea5cfd60ec9053575f463c4909", - "translation_sha256": "ea48623f485e9b683065e990a1d3cca15f05b83f315384fbb8792116a8df7e80" + "source_sha256": "0e413aee000ab6f986dd428388789defbf866d86f9cf1376bf9c7bee2b21eab3", + "translation_sha256": "5acba3bbfa22171eb019ede270d3c4f04587454759a1e7af728e5205b567014a" }, { "id": "operations", @@ -310,8 +310,8 @@ "작업-공간-등록", "첫-연결-확인" ], - "source_sha256": "6771c8a61ac3106aec333a32dbb72d7dd6a9f28f1d79fe43fff4a36d066592d7", - "translation_sha256": "179443bfeaf66b429399e7f199977544833958add18416c1266cd06f85f7ad0b" + "source_sha256": "dc741a1830193cedbf1a5a49327bed25d7d4ba59782e00e24fc3e55b34f872df", + "translation_sha256": "debd98914cb9bdf0f84e1b4ea409695ec5a049a19ad785708ee0ba6f94fbfcb8" }, { "id": "chatgpt-connector", @@ -400,8 +400,8 @@ "현재-실행-구조", "확장-시-유지할-경계" ], - "source_sha256": "1e7fb7ea6df71ceb47c00c43948c1d82bd99461c4dc5798974cd9508ad4ab690", - "translation_sha256": "d46c5e472191b3071cd0e27c08d37195f637eb15c28644653f0292057458daea" + "source_sha256": "be5e155f374697eb4aedd3cb5eb08db8b00a67ec0c91b91f7b4503f3cf143220", + "translation_sha256": "9bd1f5d137a1cbb5d1ed97be05129f276420927664c337c80e53b9e4283b6477" }, { "id": "execution-substrate", @@ -576,8 +576,8 @@ "프로세스-정직성", "프로필-mvp" ], - "source_sha256": "c500143209c56fc7949beb8fd3d46ccb5dffa59d34ef987d4095388d8965f69b", - "translation_sha256": "262d567a19dad4f1f1ab617ac24f215bb45e6578b41b71dd48b11729445dbd86" + "source_sha256": "e3e7395c07e2660aae172e256231cea660d464c1e6f464d7e27f96b56e9c720e", + "translation_sha256": "2e812460939e5716d4ca20d5a097346cab81c0c39062f0e48e9b29118e5ad313" }, { "id": "runner-isolation", @@ -587,6 +587,8 @@ "source": "docs/runner-isolation.md", "translation": "docs/ko/runner-isolation.md", "anchors": [ + "claim", + "claim-exposure", "compose-픽스처가-하는-일", "compose-픽스처가-하는-일", "container-fixture-and-verification", @@ -610,8 +612,8 @@ "컨테이너-실험-구성과-검증", "호스트와-worker-실행" ], - "source_sha256": "4afc4358cc0ae2c5437e27c8530ab69de89d8b9b5a92447896cb8aff18359b94", - "translation_sha256": "f1eb75de13e8037baddc37b256d9e89e77a44489296996e4ef84e161d9f12c36" + "source_sha256": "b8ca7eaf43a128d33f41ae39999349c275dd677177589f8873d6997dbfb71b76", + "translation_sha256": "004b9bb59acd29d188e81b46d7f1fc9b32f3ca145faeb20e55d579a0bce2bc69" }, { "id": "error-codes", @@ -639,8 +641,8 @@ "전송-실패", "전송-실패-작업-없음" ], - "source_sha256": "6ef6e724e3cf3b46d259fcf7fb60fa2d7912d7e2b7ca438eb7a230ac822d022f", - "translation_sha256": "9ccea1fedaf5129cf948c5d4671d54369753b923e9ca016a6dcce11e1d27661b" + "source_sha256": "9dec383f2ff5d3c470603d03f6bac2cee684b87ac775c13ab229d99a73c97936", + "translation_sha256": "70569634b14f9b6419c3eb1c90228eb9b3ef364c6b33a6b61ec8c5db75c3bc26" }, { "id": "codex-reuse", @@ -721,8 +723,8 @@ "핀-6b9826e의-후보", "핵심-대-어댑터" ], - "source_sha256": "d2d099d4d51afb70810a3512a2463eeaf2d94efc272697dd278b82fac7c0d9c3", - "translation_sha256": "0db076e7693509c777c47d3aa9819f81463f763be63705ad0eb426c5f62121e1" + "source_sha256": "aae68b25a95333d12733ae14054ed304a24ce90931cf29c8cafd0200c4238a93", + "translation_sha256": "c9df1c720035bf7138a79d0f748972cf226d8c8728242bad526ab1fa0bd6df19" }, { "id": "upstream-lock", diff --git a/scripts/validate-upstream.py b/scripts/validate-upstream.py index 0dd9c92..f9e138f 100644 --- a/scripts/validate-upstream.py +++ b/scripts/validate-upstream.py @@ -86,9 +86,10 @@ def stages(): # preparation of governed executions, CSRG-U3, and their launch, CSRG-U4), the worker # and the gateway where socket creation is not atomically close-on-exec, the gateway's # against the worker, the fixture authority and the helper built here. With them, the - # independent Runner (P1R-C01): its identity, socket directory and wire, the gateway's - # settings and refusals, then the worker binary and the gateway binary against it, - # with the default worker built again over the DevGuard one. + # independent Runner (P1R-C01): its identity, socket directory and wire, its Claim with + # the peer credentials that macOS reads through `getpeereid` (P1R-C03), the gateway's + # settings, refusals and reconnect, then the worker binary and the gateway binary + # against it, with the default worker built again over the DevGuard one. + devguard_binaries() + [cargo('test', 'devguard'), cargo('test', 'codex-runtime', '--features', 'devguard', '--bin', 'codespace-codex-runtime'),