diff --git a/crates/codex-runtime/src/bin/codespace-codex-runtime.rs b/crates/codex-runtime/src/bin/codespace-codex-runtime.rs index 85cd208..7ff0ff2 100644 --- a/crates/codex-runtime/src/bin/codespace-codex-runtime.rs +++ b/crates/codex-runtime/src/bin/codespace-codex-runtime.rs @@ -148,8 +148,8 @@ async fn serve() { } } -/// What an independent Runner logs of each Claim (P1R-C03): any process of its user that can -/// reach its socket may claim it while no connection controls it. +/// What an independent Runner logs of each Claim (P1R-C03, P1R-C04): any process of its user +/// that can reach its socket may claim it while no connection controls it. fn log_claim(note: ClaimNote) { let peer = |pid: Option| pid.map_or_else(|| "unknown".into(), |pid| pid.to_string()); match note { @@ -161,7 +161,8 @@ fn log_claim(note: ClaimNote) { "codespace-codex-runtime: process {} claimed the Runner under epoch {epoch}{}", peer(peer_pid), if previous_controller { - "; an earlier controller started work, so it starts nothing" + "; an earlier controller started work, so it starts nothing where that \ + controller did until it reconciled it" } else { "" } diff --git a/crates/codex-runtime/tests/runtime_binary.rs b/crates/codex-runtime/tests/runtime_binary.rs index bc98943..6d9458c 100644 --- a/crates/codex-runtime/tests/runtime_binary.rs +++ b/crates/codex-runtime/tests/runtime_binary.rs @@ -6,7 +6,9 @@ use std::process::Command; use std::sync::Arc; use std::time::Duration; -use codespace_domain::{ErrorCode, ProcessId, ProcessState, Profile, WorkspaceId}; +use codespace_domain::{ + ErrorCode, ProcessId, ProcessState, ProcessTermination, Profile, WorkspaceId, +}; use codespace_policy::Workspace; use codespace_runner::{ read_frame, write_frame, Runner, RunnerClaim, RunnerError, RunnerExecRequest, RunnerLifecycle, @@ -146,7 +148,7 @@ async fn hello_on_live_socket() { match parsed.result { Some(RunnerOpResult::Hello { protocol, .. }) => { assert_eq!(protocol, WIRE_PROTOCOL); - assert_eq!(protocol, 10); + assert_eq!(protocol, 11); } other => panic!("unexpected hello {other:?}"), } @@ -369,6 +371,7 @@ async fn an_independent_runner_outlives_its_gateways_until_one_stops_it() { let (next, inherited) = gateway(&socket).await; assert_eq!(next.hello().identity, hello.identity); assert_eq!((inherited.epoch, inherited.previous_controller), (2, true)); + assert_eq!(inherited.barrier, ["demo"]); let mut processes: Vec<_> = inherited .processes .iter() @@ -404,7 +407,35 @@ async fn an_independent_runner_outlives_its_gateways_until_one_stops_it() { Err(RunnerError::Execution(body)) => assert_eq!(body.code, ErrorCode::RecoveryPending), other => panic!("unexpected {other:?}"), } - assert_eq!(next.stop_runner().await.unwrap(), 2); + let stopped = next.stop_runner().await.unwrap(); + assert_eq!(stopped.terminated, 2); + let mut ended: Vec<_> = stopped + .processes + .iter() + .map(|entry| { + ( + entry.status.process_id.0.as_str(), + entry.status.state, + entry.status.termination, + ) + }) + .collect(); + ended.sort_by_key(|(process_id, ..)| *process_id); + assert_eq!( + ended, + [ + ( + "proc-pipe", + ProcessState::Exited, + Some(ProcessTermination::Terminated) + ), + ( + "proc-tty", + ProcessState::Exited, + Some(ProcessTermination::Terminated) + ) + ] + ); let status = exited(&mut runner).await; assert!(status.success(), "{status:?}"); assert!(!still_beating(&pipe_beat).await); diff --git a/crates/runner/src/admission.rs b/crates/runner/src/admission.rs index 27e52f7..cbd20b7 100644 --- a/crates/runner/src/admission.rs +++ b/crates/runner/src/admission.rs @@ -29,7 +29,7 @@ //! Cancelling is the one settlement used, and only where its precondition holds: this owner //! never asked DevGuard to launch the attempt, so a cancellation finds it prepared or settled. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::ffi::OsString; use std::path::Path; use std::sync::Arc; @@ -564,6 +564,95 @@ fn wire_name(value: &impl Serialize) -> String { .unwrap_or_default() } +/// The admissions an independent Runner asked the authority for, by attempt ID, each with the +/// process it was for (P1R-C04): what it needs to look its attempts up again, since the +/// authority checks a lookup against the attempt's meaning. Kept for the Runner's lifetime. +#[derive(Clone, Default)] +pub struct Admissions(Arc>>); + +impl Admissions { + fn remember(&self, process_id: &ProcessId, admission: &Admission) { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + admission.attempt_id.clone(), + (process_id.clone(), admission.clone()), + ); + } + + fn admitted(&self, attempt_id: &str, process_id: &ProcessId) -> Option { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(attempt_id) + .filter(|(admitted_for, _)| admitted_for == process_id) + .map(|(_, admission)| admission.clone()) + } +} + +/// A governed execution's attempt, which a gateway asks its independent Runner to look up +/// (P1R-C04). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AttemptRef { + pub attempt_id: String, + pub process_id: ProcessId, +} + +/// What the authority states of an attempt that an independent Runner looked up (P1R-C04). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AttemptLookup { + pub attempt_id: String, + pub process_id: ProcessId, + pub found: LookedUp, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "lookup", content = "attempt", rename_all = "snake_case")] +pub enum LookedUp { + /// This Runner never asked the authority to admit this attempt for this process. + NotAdmitted, + /// The authority has no record of the attempt. + NotFound, + /// The authority's record of the attempt. + Found(AttemptState), + /// The authority could not be asked, or gave no usable answer: ask again later. + Failed, +} + +/// Look up each of `attempts` with `authority`, as this owner admitted it (P1R-C04). An attempt +/// that this owner never admitted for its process is not asked about. +pub async fn look_up( + authority: Option>, + admissions: &Admissions, + attempts: Vec, +) -> Vec { + let mut found = Vec::with_capacity(attempts.len()); + for AttemptRef { + attempt_id, + process_id, + } in attempts + { + let looked_up = match (admissions.admitted(&attempt_id, &process_id), &authority) { + (None, _) => LookedUp::NotAdmitted, + (Some(_), None) => LookedUp::Failed, + (Some(admission), Some(authority)) => { + match call(authority, &admission, Call::Lookup).await { + Answer::Attempt(attempt) => LookedUp::Found(attempt_state(&attempt)), + Answer::Refused(devguard::ErrorCode::NotFound) => LookedUp::NotFound, + Answer::Refused(_) | Answer::NotSent(_) | Answer::Unknown => LookedUp::Failed, + } + } + }; + found.push(AttemptLookup { + attempt_id, + process_id, + found: looked_up, + }); + } + found +} + /// Prepares and launches governed executions for the owner that holds both the process slots /// and the registered identity. #[derive(Clone)] @@ -572,6 +661,7 @@ pub struct Preparer { runner: InProcessRunner, ttl_bound: Duration, launcher: Option, + admissions: Option, } impl Preparer { @@ -581,9 +671,16 @@ impl Preparer { runner, ttl_bound: Duration::from_millis(devguard_prepared_ttl()), launcher: None, + admissions: None, } } + /// Remember each admission asked for in `admissions`, to look it up again (P1R-C04). + pub fn with_admissions(mut self, admissions: Option) -> Self { + self.admissions = admissions; + self + } + /// Hold prepared attempts for at most `bound` here, whatever the authority grants. pub fn with_ttl_bound(mut self, bound: Duration) -> Self { self.ttl_bound = bound; @@ -662,6 +759,9 @@ impl Preparer { reserved: None, applied: Applied::NotLaunched, }; + if let Some(admissions) = &self.admissions { + admissions.remember(&req.process_id, &admission); + } let started = Instant::now(); match call(&self.authority, &admission, Call::Admit).await { Answer::Attempt(attempt) => { @@ -1874,4 +1974,71 @@ pub(crate) mod tests { assert_ne!(meaning(ws, req).digest().unwrap(), digest); } } + + /// An independent Runner looks up only the attempts it admitted, each for the process it + /// admitted it for and as it admitted it. An authority that cannot answer, or a Runner + /// without its registration, leaves the attempt to be asked again (P1R-C04). + #[tokio::test] + async fn an_independent_runner_looks_up_only_the_attempts_it_admitted() { + let dir = tempfile::tempdir().unwrap(); + let runner = runner(); + let admissions = Admissions::default(); + let fake = Fake::admitting([prepared()]) + .then_lookup(Answer::Attempt(record(dg::Phase::LaunchCommitted, None))) + .then_lookup(Answer::Attempt(record( + dg::Phase::Released, + Some(dg::Release::NoHelperCreated), + ))) + .then_lookup(Answer::Refused(devguard::ErrorCode::NotFound)) + .then_lookup(Answer::Unknown) + .then_lookup(Answer::NotSent(unreachable_registration())); + let report = Preparer::new(fake.clone(), runner.clone()) + .with_admissions(Some(admissions.clone())) + .prepare_and_dispose( + governed(dir.path()), + touch(dir.path(), "proc-l"), + "cs-l".into(), + ) + .await; + assert_eq!(report.attempt, AttemptState::Cancelled); + let authority: Arc = fake.clone(); + let attempt = |attempt_id: &str, process_id: &str| AttemptRef { + attempt_id: attempt_id.into(), + process_id: ProcessId(process_id.into()), + }; + let found = |lookups: Vec| -> Vec { + lookups.into_iter().map(|lookup| lookup.found).collect() + }; + + let elsewhere = look_up( + Some(authority.clone()), + &admissions, + vec![attempt("cs-other", "proc-l"), attempt("cs-l", "proc-other")], + ) + .await; + assert_eq!(found(elsewhere), [LookedUp::NotAdmitted; 2]); + assert_eq!(fake.count(LOOKUP), 0); + for expected in [ + LookedUp::Found(AttemptState::Launched), + LookedUp::Found(AttemptState::Released), + LookedUp::NotFound, + LookedUp::Failed, + LookedUp::Failed, + ] { + let lookups = look_up( + Some(authority.clone()), + &admissions, + vec![attempt("cs-l", "proc-l")], + ) + .await; + assert_eq!(found(lookups), [expected]); + } + let unregistered = look_up(None, &admissions, vec![attempt("cs-l", "proc-l")]).await; + assert_eq!(found(unregistered), [LookedUp::Failed]); + assert_eq!(fake.count(LOOKUP), 5); + // Each lookup asked about the admission itself, under its digest. + let seen = fake.seen.lock().unwrap(); + assert!(seen.iter().all(|admission| *admission == seen[0])); + assert_nothing_started(dir.path(), &runner); + } } diff --git a/crates/runner/src/api.rs b/crates/runner/src/api.rs index e64d25b..fbe380d 100644 --- a/crates/runner/src/api.rs +++ b/crates/runner/src/api.rs @@ -153,6 +153,25 @@ pub struct RunnerProcessStatus { pub eof: bool, } +/// What an independent Runner knows of a process that it was asked to start (P1R-C04). It +/// remembers, for its whole lifetime, whether it created a process for each `process_id`, and +/// nothing else of a process it no longer holds. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "recalled", rename_all = "snake_case")] +pub enum Recalled { + /// No request to start it reached this Runner. + NotReceived, + /// A request to start it reached this Runner, which created no process for it. + Refused, + /// This Runner created it and holds it, running or ended. + Held { + workspace_id: String, + status: RunnerProcessStatus, + }, + /// This Runner created it, and no longer holds it: it ended, and how is not known any more. + Dropped, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RunnerResizeResult { pub rows: u16, diff --git a/crates/runner/src/lib.rs b/crates/runner/src/lib.rs index f17982c..37b833a 100644 --- a/crates/runner/src/lib.rs +++ b/crates/runner/src/lib.rs @@ -122,7 +122,7 @@ mod watch; mod wire; pub use api::{ - default_exec_timeout_ms, runner_local_exec_env, RunnerApplyPatchRequest, + default_exec_timeout_ms, runner_local_exec_env, Recalled, RunnerApplyPatchRequest, RunnerApplyPatchResult, RunnerCwd, RunnerError, RunnerExecEnv, RunnerExecPolicy, RunnerExecRequest, RunnerExecResult, RunnerProcessRoot, RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, DEFAULT_TIMEOUT_MS, MAX_OUTPUT_BYTES, @@ -134,7 +134,7 @@ pub use fork_handlers::prepare_fork_spawns; pub use patch_helper::ensure_helper_for_tests; pub use process::{ InProcessRunner, RetentionPolicy, ShellRelease, SlotReservation, DEFAULT_COMPLETED_TTL, - DEFAULT_MAX_COMPLETED, DEFAULT_MAX_PROCESSES, DEFAULT_TIMEOUT, + DEFAULT_MAX_COMPLETED, DEFAULT_MAX_PROCESSES, DEFAULT_MAX_UNACKNOWLEDGED, DEFAULT_TIMEOUT, }; /// True when this process probed a working Linux sandbox helper. @@ -152,16 +152,17 @@ pub use socket::{ }; pub use uds::{ runner_disconnected, runner_reconnecting, ClaimError, DisconnectHook, LinkEvent, LinkObserver, - RunnerHello, UdsRunner, CLAIM_START_BOUND, RECLAIM_BACKOFF_MAX, RUNNER_CALL_DEADLINE, + RunnerHello, RunnerStopped, UdsRunner, CLAIM_START_BOUND, RECLAIM_BACKOFF_MAX, + RUNNER_CALL_DEADLINE, }; pub use watch::{FsWatchEvent, FsWatchKind, WatchClosed, WatchSubscription}; #[cfg(feature = "devguard")] pub use wire::serve_runner_connection_with_registration; pub use wire::{ host_worker, read_frame, recovery_pending, serve_runner_connection, write_frame, ClaimNote, - ClaimObserver, ClaimRefusal, IndependentRunner, RunnerClaim, RunnerEvent, RunnerEvents, - RunnerInventoryEntry, RunnerOp, RunnerOpResult, Served, StopHandle, WireEnvelope, WireKind, - CLAIM_WAIT_BOUND, STOP_DRAIN_BOUND, WIRE_PROTOCOL, + ClaimObserver, ClaimRefusal, IndependentRunner, RecalledProcess, RunnerClaim, RunnerEvent, + RunnerEvents, RunnerInventoryEntry, RunnerOp, RunnerOpResult, Served, StopHandle, WireEnvelope, + WireKind, CLAIM_WAIT_BOUND, STOP_DRAIN_BOUND, WIRE_PROTOCOL, }; /// Execution-plane API. Control-plane fields (`work_id`, coordination, @@ -342,6 +343,18 @@ impl RuntimeBackend { self.dispatch_epoch().err() } + /// Tell an independent Runner that this gateway recorded how these processes ended + /// (P1R-C04): see [`UdsRunner::acknowledge`]. Any other runner keeps nothing for a + /// controller, so it is told nothing. + pub async fn acknowledge(&self, process_ids: &[ProcessId]) -> Result<(), RunnerError> { + match self { + Self::Uds(runner) if runner.hello().lifecycle == RunnerLifecycle::Independent => { + runner.acknowledge(process_ids).await + } + _ => Ok(()), + } + } + /// This backend, sending its requests under the controller epoch `epoch` when there is one /// (P1R-C03): see [`UdsRunner::at_epoch`]. pub fn at_epoch(&self, epoch: Option) -> Self { diff --git a/crates/runner/src/process.rs b/crates/runner/src/process.rs index c98116a..5d05ba8 100644 --- a/crates/runner/src/process.rs +++ b/crates/runner/src/process.rs @@ -32,14 +32,18 @@ use tokio::task::JoinHandle; use crate::lifecycle::process_start_identity; use crate::{ - runner_local_exec_env, RunnerCwd, RunnerExecRequest, RunnerExecResult, RunnerProcessRoot, - RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, + runner_local_exec_env, Recalled, RunnerCwd, RunnerExecRequest, RunnerExecResult, + RunnerProcessRoot, RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, + RunnerResizeResult, RunnerWriteStdin, }; pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); pub const DEFAULT_MAX_PROCESSES: usize = 8; pub const DEFAULT_COMPLETED_TTL: Duration = Duration::from_secs(15 * 60); pub const DEFAULT_MAX_COMPLETED: usize = 64; +/// How many ended processes an independent Runner keeps, after their handles are released, +/// for a controller that has not acknowledged them yet (P1R-C04). +pub const DEFAULT_MAX_UNACKNOWLEDGED: usize = 64; /// Called with a server-minted `process_id` when that process exits. pub type ShellRelease = Arc; @@ -72,6 +76,8 @@ pub struct InProcessRunner { inner: Arc>>, /// Slots taken for processes not yet created. Locked after `inner`, never before it. reserved: Arc>>, + /// What an independent Runner remembers for its controllers. Locked last. + memory: Arc>, /// The live process limit, when set here instead of by `CODESPACE_MAX_PROCESSES`. max_processes: Option, on_release: ShellRelease, @@ -79,6 +85,24 @@ pub struct InProcessRunner { pub(crate) watches: crate::watch::WatchSet, } +/// What an independent Runner remembers for its controllers (P1R-C04), once +/// [`InProcessRunner::keep_until_acknowledged`] turned it on: for its whole lifetime, whether it +/// created a process for each `process_id` it was asked to start, by its ID alone; and how many +/// ended processes it keeps, after their handles are released, until a controller +/// acknowledges them. +#[derive(Default)] +struct Memory { + keep: Option, + received: HashMap, +} + +/// Whether a runner created the process it was asked to start (P1R-C04). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Received { + Executed, + Refused, +} + /// A process slot, taken before the process is created (CSRG-U3). A slot counts against the /// live process limit from the moment it is taken, so no process is created without one. It /// is freed when it is dropped unfilled; once a created process fills it, the process's slot @@ -116,6 +140,8 @@ struct Slot { completed_at: Arc>>, lifecycle: Arc>, on_kill: Option, + /// When a controller acknowledged how the ended process ended (P1R-C04). + acknowledged: Option, } enum SessionIo { @@ -127,6 +153,9 @@ enum SessionIo { session: Arc, writer: mpsc::Sender>, }, + /// The ended process's handle was released when it would have been evicted, and its + /// status and output are kept until a controller acknowledges them (P1R-C04). + Released, } /// Lets a test act inside a managed PTY spawn's window, while the launch's two descriptors are @@ -465,6 +494,7 @@ impl InProcessRunner { Self { inner: Arc::new(Mutex::new(HashMap::new())), reserved: Arc::new(Mutex::new(HashSet::new())), + memory: Arc::new(Mutex::new(Memory::default())), max_processes: None, on_release, retention, @@ -516,32 +546,140 @@ impl InProcessRunner { live_count(&map) + reserved } + /// Evict the ended processes past the retention policy: those that ended more than `ttl` + /// ago, and the oldest beyond `max_completed`. An independent Runner releases the handle of + /// such a process at the same moment, but keeps its status and output until a controller + /// acknowledges them (P1R-C04): releasing the handle earlier would change what its drop does + /// to the process group of a tty process, so it is never released earlier (D11). Once + /// acknowledged, a released process stays `ttl` longer; beyond `keep` released processes, + /// the oldest goes, and the runner remembers only that it ended. fn evict_completed(&self, map: &mut HashMap) { let now = Instant::now(); let ttl = self.retention.ttl; + let keep = self.memory.lock().expect("memory").keep; + let mut ended: Vec<(String, Instant)> = map + .iter() + .filter(|(_, slot)| !matches!(slot.io, SessionIo::Released)) + .filter_map(|(id, slot)| completed_at(slot).map(|at| (id.clone(), at))) + .collect(); + ended.sort_by_key(|(_, at)| *at); + let over = ended.len().saturating_sub(self.retention.max_completed); + for (index, (id, at)) in ended.into_iter().enumerate() { + if index >= over && now.duration_since(at) <= ttl { + continue; + } + match map.get_mut(&id) { + Some(slot) if keep.is_some() && slot.acknowledged.is_none() => { + slot.io = SessionIo::Released; + slot.on_kill = None; + } + _ => { + map.remove(&id); + } + } + } + let Some(keep) = keep else { + return; + }; map.retain(|_, slot| { - let completed = slot.completed_at.lock().ok().and_then(|guard| *guard); - !matches!(completed, Some(at) if now.duration_since(at) > ttl) + !(matches!(slot.io, SessionIo::Released) + && slot + .acknowledged + .is_some_and(|at| now.duration_since(at) > ttl)) }); - let mut completed: Vec<(String, Instant)> = map + let mut released: Vec<(String, Instant)> = map .iter() - .filter_map(|(id, slot)| { - slot.completed_at - .lock() - .ok() - .and_then(|guard| guard.map(|at| (id.clone(), at))) - }) + .filter(|(_, slot)| matches!(slot.io, SessionIo::Released)) + .filter_map(|(id, slot)| completed_at(slot).map(|at| (id.clone(), at))) .collect(); - if completed.len() <= self.retention.max_completed { + if released.len() <= keep { return; } - completed.sort_by_key(|(_, at)| *at); - let drop_n = completed.len() - self.retention.max_completed; - for (id, _) in completed.into_iter().take(drop_n) { + released.sort_by_key(|(_, at)| *at); + let drop_n = released.len() - keep; + for (id, _) in released.into_iter().take(drop_n) { map.remove(&id); } } + /// Hand `reservation`'s slot to the process created for it, under the runner's lock: the + /// runner remembers that it created it (P1R-C04). + fn fill(&self, reservation: SlotReservation, slot: Slot) { + let mut map = self.inner.lock().expect("runner"); + self.evict_completed(&mut map); + let process_id = reservation.process_id.clone(); + reservation.fill(&mut map, slot); + self.remember(&process_id, Received::Executed); + } + + fn remember(&self, process_id: &str, received: Received) { + let mut memory = self.memory.lock().expect("memory"); + if memory.keep.is_none() { + return; + } + let known = memory + .received + .entry(process_id.to_string()) + .or_insert(received); + if received == Received::Executed { + *known = Received::Executed; + } + } + + /// Keep what an independent Runner's controllers need from now on (P1R-C04): whether a + /// process was created for each `process_id` this runner is asked to start, for its whole + /// lifetime, and each ended process's status and output until a controller acknowledges + /// them, at most `keep` of them once their handles are released. + pub fn keep_until_acknowledged(&self, keep: usize) { + self.memory.lock().expect("memory").keep = Some(keep.max(1)); + } + + /// A request to start `process_id` reached this runner. Unless it created the process, it + /// remembers that it refused it (P1R-C04). + pub(crate) fn note_received(&self, process_id: &str) { + self.remember(process_id, Received::Refused); + } + + /// What this runner knows of `process_id` (P1R-C04). Only an independent Runner that keeps + /// what its controllers need remembers a process it no longer holds. + pub fn recall(&self, process_id: &ProcessId) -> Recalled { + let mut map = self.inner.lock().expect("runner"); + self.evict_completed(&mut map); + if let Some(slot) = map.get(&process_id.0) { + return Recalled::Held { + workspace_id: slot.workspace_id.clone(), + status: slot_status(process_id, slot), + }; + } + match self + .memory + .lock() + .expect("memory") + .received + .get(&process_id.0) + { + Some(Received::Executed) => Recalled::Dropped, + Some(Received::Refused) => Recalled::Refused, + None => Recalled::NotReceived, + } + } + + /// A controller recorded how these processes ended (P1R-C04): each is evicted as any + /// other ended process from now on, and one whose handle was already released stays + /// `ttl` longer. A process that has not ended is left as it is. + pub fn acknowledge(&self, process_ids: &[ProcessId]) { + let mut map = self.inner.lock().expect("runner"); + let now = Instant::now(); + for process_id in process_ids { + if let Some(slot) = map.get_mut(&process_id.0) { + if slot.acknowledged.is_none() && completed_at(slot).is_some() { + slot.acknowledged = Some(now); + } + } + } + self.evict_completed(&mut map); + } + pub async fn spawn_host( &self, ws: &Workspace, @@ -700,12 +838,9 @@ impl InProcessRunner { completed_at: completed_at.clone(), lifecycle: lifecycle.clone(), on_kill: on_kill.clone(), + acknowledged: None, }; - { - let mut map = self.inner.lock().expect("runner"); - self.evict_completed(&mut map); - reservation.fill(&mut map, slot); - } + self.fill(reservation, slot); let out_handle = stdout.map(|out| { let buf = output.clone(); @@ -885,12 +1020,9 @@ impl InProcessRunner { completed_at: completed_at.clone(), lifecycle: lifecycle.clone(), on_kill: on_kill.clone(), + acknowledged: None, }; - { - let mut map = self.inner.lock().expect("runner"); - self.evict_completed(&mut map); - reservation.fill(&mut map, slot); - } + self.fill(reservation, slot); if let Some(mut watch) = watch { let (observer, life) = (observer.clone(), lifecycle.clone()); @@ -979,6 +1111,12 @@ impl InProcessRunner { match &slot.io { SessionIo::Pipe { stdin, .. } => WriteTarget::Pipe(stdin.clone()), SessionIo::Pty { writer, .. } => WriteTarget::Pty(writer.clone()), + SessionIo::Released => { + return Err(ErrorBody::new( + ErrorCode::ProcessNotFound, + "stdin is closed", + )) + } } }; match target { @@ -1094,6 +1232,10 @@ impl InProcessRunner { ErrorCode::ProcessNotTty, "process is not attached to a PTY", )), + SessionIo::Released => Err(ErrorBody::new( + ErrorCode::ProcessNotRunning, + "process is not running", + )), SessionIo::Pty { session, .. } => { if session.has_exited() { return Err(ErrorBody::new( @@ -1288,6 +1430,8 @@ fn request_kill(slot: &Slot) -> Result { after_kill(&slot.on_kill); Ok(true) } + // Only an ended process's handle is released. + SessionIo::Released => Ok(false), } } @@ -1328,6 +1472,11 @@ async fn pump_reader( } } +/// When the process of `slot` ended: it was reaped and its output ended. +fn completed_at(slot: &Slot) -> Option { + slot.completed_at.lock().ok().and_then(|guard| *guard) +} + /// A process's status, from its slot. fn slot_status(process_id: &ProcessId, slot: &Slot) -> RunnerProcessStatus { let snap = slot.lifecycle.lock().expect("lifecycle").snapshot; @@ -2311,7 +2460,7 @@ mod tests { let map = runner.inner.lock().unwrap(); match &map[&process_id.0].io { SessionIo::Pipe { child, .. } => exited_unreaped(&child.lock().unwrap()), - SessionIo::Pty { .. } => panic!("{} is not a pipe child", process_id.0), + _ => panic!("{} is not a pipe child", process_id.0), } }; if exited { @@ -2381,7 +2530,7 @@ mod tests { let child = child.lock().unwrap(); child.id().is_some() && exited_unreaped(&child) } - SessionIo::Pty { .. } => panic!("not a pipe child"), + _ => panic!("not a pipe child"), } }; seen.lock().unwrap().push((point, unreaped)); @@ -2463,7 +2612,7 @@ mod tests { let map = runner.inner.lock().unwrap(); match &map[&process_id.0].io { SessionIo::Pty { session, .. } => session.has_exited(), - SessionIo::Pipe { .. } => panic!("{} is not a PTY child", process_id.0), + _ => panic!("{} is not a PTY child", process_id.0), } } @@ -2962,7 +3111,7 @@ mod tests { let pid = child.lock().unwrap().id().expect("running child"); libc::pid_t::try_from(pid).unwrap() } - SessionIo::Pty { .. } => panic!("{} is not a pipe child", process_id.0), + _ => panic!("{} is not a pipe child", process_id.0), } } @@ -3119,4 +3268,214 @@ mod tests { } runner.kill_host_workspace("demo").unwrap(); } + + /// A quick process `process_id` that ended, run by `runner`. + async fn ended_process( + runner: &InProcessRunner, + ws: &Workspace, + process_id: &ProcessId, + tty: bool, + ) { + let mut req = RunnerExecRequest::for_host( + vec!["/bin/sh".into(), "-c".into(), "echo done".into()], + process_id.clone(), + Profile::WorkspaceWrite, + ); + req.tty = tty; + runner.spawn_host(ws, req).await.unwrap(); + wait_exited(runner, process_id).await; + } + + fn released(runner: &InProcessRunner, process_id: &ProcessId) -> bool { + matches!( + runner + .inner + .lock() + .unwrap() + .get(&process_id.0) + .map(|slot| &slot.io), + Some(SessionIo::Released) + ) + } + + /// An independent Runner releases an ended process's handle when it would have evicted it, + /// a tty one included, and no earlier (D11), but keeps its status and output until a + /// controller acknowledges them; acknowledged, it goes after the retention time, and the + /// runner remembers only that it created it (P1R-C04). + #[tokio::test] + async fn an_independent_runner_keeps_an_ended_process_until_it_is_acknowledged() { + let dir = tempdir().unwrap(); + let ws = workspace(dir.path()); + let ttl = Duration::from_millis(300); + let runner = InProcessRunner::with_retention( + Arc::new(|_| {}), + RetentionPolicy { + ttl, + max_completed: 64, + }, + ); + runner.keep_until_acknowledged(DEFAULT_MAX_UNACKNOWLEDGED); + for (process_id, tty) in [("proc-pipe", false), ("proc-tty", true)] { + let process_id = ProcessId(process_id.into()); + ended_process(&runner, &ws, &process_id, tty).await; + // Within the retention time the handle is held, as always. + assert!(!released(&runner, &process_id)); + tokio::time::sleep(ttl + Duration::from_millis(100)).await; + // Past it the handle goes, and the status and the output stay. + assert_eq!(runner.occupied_slots(), 0); + assert!(released(&runner, &process_id), "{process_id:?}"); + let status = runner.process_status(&process_id).await.unwrap(); + assert_eq!( + (status.state, status.termination, status.eof), + (ProcessState::Exited, Some(ProcessTermination::Exited), true) + ); + assert!(wait_chunk(&runner, &process_id).await.contains("done")); + match runner.recall(&process_id) { + Recalled::Held { + workspace_id, + status: recalled, + } => assert_eq!((workspace_id.as_str(), recalled), ("demo", status)), + other => panic!("unexpected {other:?}"), + } + assert_eq!( + runner + .write_stdin(RunnerWriteStdin { + process_id: process_id.clone(), + data: "x".into(), + }) + .await + .unwrap_err() + .as_execution() + .map(|body| body.code), + Some(ErrorCode::ProcessNotFound) + ); + assert_eq!( + runner + .resize(&process_id, 30, 90) + .await + .unwrap_err() + .as_execution() + .map(|body| body.code), + Some(ErrorCode::ProcessNotRunning) + ); + runner.terminate(&process_id).await.unwrap(); + // Acknowledged, it stays the retention time once more, then goes. + runner.acknowledge(std::slice::from_ref(&process_id)); + assert!(runner.process_status(&process_id).await.is_ok()); + tokio::time::sleep(ttl + Duration::from_millis(100)).await; + assert_eq!( + runner + .process_status(&process_id) + .await + .unwrap_err() + .as_execution() + .map(|body| body.code), + Some(ErrorCode::ProcessNotFound) + ); + assert_eq!(runner.recall(&process_id), Recalled::Dropped); + } + } + + /// An acknowledged process is evicted as any ended process always was: its handle is not + /// kept past the retention time (P1R-C04). + #[tokio::test] + async fn an_independent_runner_evicts_an_acknowledged_process_as_before() { + let dir = tempdir().unwrap(); + let ws = workspace(dir.path()); + let ttl = Duration::from_millis(300); + let runner = InProcessRunner::with_retention( + Arc::new(|_| {}), + RetentionPolicy { + ttl, + max_completed: 64, + }, + ); + runner.keep_until_acknowledged(DEFAULT_MAX_UNACKNOWLEDGED); + let process_id = ProcessId("proc-acked".into()); + ended_process(&runner, &ws, &process_id, false).await; + runner.acknowledge(std::slice::from_ref(&process_id)); + assert!(runner.process_status(&process_id).await.is_ok()); + tokio::time::sleep(ttl + Duration::from_millis(100)).await; + assert!(runner.process_status(&process_id).await.is_err()); + assert_eq!(runner.recall(&process_id), Recalled::Dropped); + } + + /// Beyond its bound of released processes an independent Runner drops the oldest first, + /// remembering that it created it (P1R-C04). + #[tokio::test] + async fn an_independent_runner_drops_the_oldest_released_process_beyond_its_bound() { + let dir = tempdir().unwrap(); + let ws = workspace(dir.path()); + let runner = InProcessRunner::with_retention( + Arc::new(|_| {}), + RetentionPolicy { + ttl: Duration::from_secs(60), + max_completed: 1, + }, + ); + runner.keep_until_acknowledged(1); + let ids: Vec<_> = (1..=3).map(|n| ProcessId(format!("proc-{n}"))).collect(); + for process_id in &ids { + ended_process(&runner, &ws, process_id, false).await; + } + // The newest holds its handle; the next one is released and kept; the oldest went. + assert_eq!(runner.occupied_slots(), 0); + assert!(!released(&runner, &ids[2])); + assert!(released(&runner, &ids[1])); + assert_eq!(runner.recall(&ids[0]), Recalled::Dropped); + assert!(matches!(runner.recall(&ids[1]), Recalled::Held { .. })); + assert!(matches!(runner.recall(&ids[2]), Recalled::Held { .. })); + assert_eq!( + runner + .read_process(RunnerReadProcess { + process_id: ids[0].clone(), + cursor: 0, + }) + .await + .unwrap_err() + .as_execution() + .map(|body| body.code), + Some(ErrorCode::ProcessNotFound) + ); + } + + /// An independent Runner remembers, for each `process_id` it was asked to start, whether it + /// created the process; a runner that keeps nothing for a controller remembers nothing of a + /// process it does not hold (P1R-C04). + #[tokio::test] + async fn an_independent_runner_remembers_each_process_it_was_asked_to_start() { + let dir = tempdir().unwrap(); + let ws = workspace(dir.path()); + let runner = InProcessRunner::new(Arc::new(|_| {})); + runner.keep_until_acknowledged(DEFAULT_MAX_UNACKNOWLEDGED); + let refused = ProcessId("proc-refused".into()); + let created = ProcessId("proc-created".into()); + let never = ProcessId("proc-never".into()); + // Refused before any process is created, with or without the Linux sandbox, under + // which a missing executable is a sandbox helper that fails. + let no_program = RunnerExecRequest::for_host( + vec![String::new()], + refused.clone(), + Profile::WorkspaceWrite, + ); + assert_eq!( + runner.spawn_host(&ws, no_program).await.unwrap_err().code, + ErrorCode::InvalidCommand + ); + runner.note_received(&refused.0); + ended_process(&runner, &ws, &created, false).await; + // Noting the request after the answer never hides a created process. + runner.note_received(&created.0); + assert_eq!(runner.recall(&refused), Recalled::Refused); + assert!(matches!( + runner.recall(&created), + Recalled::Held { ref workspace_id, ref status } + if workspace_id == "demo" && status.state == ProcessState::Exited + )); + assert_eq!(runner.recall(&never), Recalled::NotReceived); + + let managed = InProcessRunner::new(Arc::new(|_| {})); + managed.note_received(&refused.0); + assert_eq!(managed.recall(&refused), Recalled::NotReceived); + } } diff --git a/crates/runner/src/uds.rs b/crates/runner/src/uds.rs index 2e302f9..bf80232 100644 --- a/crates/runner/src/uds.rs +++ b/crates/runner/src/uds.rs @@ -12,6 +12,10 @@ //! refuses requests with `RUNNER_RECONNECTING` meanwhile; once it cannot claim that Runner //! again, it refuses them with `RUNNER_DISCONNECTED`. Any other client refuses every request //! with `RUNNER_DISCONNECTED` once its connection to an independent Runner ended. +//! +//! The Runner's answer to a Claim holds every process it holds and the number of the last exit +//! event before it took that inventory: the client delivers only the exit events after it +//! (P1R-C04), as the inventory accounts for the earlier ones. use std::collections::HashMap; use std::io; @@ -26,12 +30,12 @@ use codespace_domain::{ErrorBody, ErrorCode, FindResult, ProcessId, ReadResult}; use codespace_policy::Workspace; use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio::net::UnixStream; -use tokio::sync::{oneshot, Mutex}; +use tokio::sync::{oneshot, watch, Mutex}; use tokio::time::timeout; use crate::wire::{ - read_frame, recovery_pending, write_frame, ClaimRefusal, RunnerClaim, RunnerEvent, RunnerOp, - RunnerOpResult, WireEnvelope, WireKind, WIRE_PROTOCOL, + read_frame, write_frame, ClaimRefusal, RecalledProcess, RunnerClaim, RunnerEvent, + RunnerInventoryEntry, RunnerOp, RunnerOpResult, WireEnvelope, WireKind, WIRE_PROTOCOL, }; use crate::{ Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, RunnerError, RunnerExecRequest, @@ -101,6 +105,15 @@ pub enum LinkEvent { pub type LinkObserver = Arc; +/// How a stopped independent Runner left its processes (P1R-C01, P1R-C04). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RunnerStopped { + /// How many processes were still running when the Runner terminated them. + pub terminated: u32, + /// Every process the Runner held when it answered, with its last status. + pub processes: Vec, +} + /// Why a client does not control an independent Runner (P1R-C03). #[derive(Debug, Clone)] pub enum ClaimError { @@ -185,8 +198,11 @@ struct Client { reclaim: OnceLock, /// The gateway leaves the Runner: no connection is made again. leaving: AtomicBool, - /// The number of the last exit event delivered (P1R-C03). + /// The number of the last exit event delivered, or accounted for by a Claim's inventory + /// (P1R-C03, P1R-C04). last_event: AtomicU64, + /// The Claim of the client's current connection, once it has one (P1R-C04). + claims: watch::Sender>, } pub struct UdsRunner { @@ -316,9 +332,18 @@ impl UdsRunner { /// database `store_id` (P1R-C03). The connection's later requests carry the Claim's epoch. pub async fn claim(&self, store_id: &str) -> Result { let link = self.client.current().map_err(ClaimError::Transport)?; - self.client + let claim = self + .client .claim_on(&link, store_id, self.call_deadline) - .await + .await?; + self.client.claims.send_replace(Some(claim.clone())); + Ok(claim) + } + + /// The Claim of this client's current connection, and each later one as the client claims + /// its Runner again (P1R-C04). + pub fn claims(&self) -> watch::Receiver> { + self.client.claims.subscribe() } /// Connect to the independent Runner at `socket` and claim it for the operations database @@ -450,8 +475,9 @@ impl UdsRunner { /// The controller epoch under which this client sends a new execution, patch or approval /// resume, none for a worker that is not claimed, or why it starts none (P1R-C01, - /// P1R-C03): the connection to its independent Runner is lost or being made again, or an - /// earlier controller started work on the Runner. + /// P1R-C03): the connection to its independent Runner is lost or being made again. Where + /// an earlier controller started work, the Runner itself refuses new work until the + /// gateway reconciled that workspace (P1R-C04). pub fn dispatch_epoch(&self) -> Result, ErrorBody> { if !self.client.independent() { return Ok(None); @@ -461,7 +487,6 @@ impl UdsRunner { .current() .map_err(RunnerError::into_error_body)?; match link.claim.get() { - Some(claim) if claim.previous_controller => Err(recovery_pending()), Some(claim) => Ok(Some(self.epoch.unwrap_or(claim.epoch))), None => Ok(self.epoch), } @@ -487,11 +512,18 @@ impl UdsRunner { } /// Stop the independent Runner: it ends every process and exits (P1R-C01). How many - /// processes were still running. No connection is made again. - pub async fn stop_runner(&self) -> Result { + /// processes were still running, and how each process it held ended (P1R-C04). No + /// connection is made again. + pub async fn stop_runner(&self) -> Result { self.client.leaving.store(true, Ordering::SeqCst); let stopped = match self.call(RunnerOp::Stop).await { - Ok(RunnerOpResult::Stopped { terminated }) => Ok(terminated), + Ok(RunnerOpResult::Stopped { + terminated, + processes, + }) => Ok(RunnerStopped { + terminated, + processes, + }), Ok(other) => Err(unexpected(other)), Err(err) => Err(err), }; @@ -499,6 +531,66 @@ impl UdsRunner { stopped } + /// What the independent Runner knows of each of `process_ids` (P1R-C04), in that order. + pub async fn recall( + &self, + process_ids: &[ProcessId], + ) -> Result, RunnerError> { + match self + .call(RunnerOp::Recall { + process_ids: process_ids.to_vec(), + }) + .await? + { + RunnerOpResult::Recalled(recalled) if recalled.len() == process_ids.len() => { + Ok(recalled) + } + other => Err(unexpected(other)), + } + } + + /// Tell the independent Runner that this gateway reconciled these workspaces (P1R-C04): it + /// starts work there for this gateway again. + pub async fn reconciled(&self, workspace_ids: &[String]) -> Result<(), RunnerError> { + match self + .call(RunnerOp::Reconciled { + workspace_ids: workspace_ids.to_vec(), + }) + .await? + { + RunnerOpResult::Reconciled => Ok(()), + other => Err(unexpected(other)), + } + } + + /// Tell the independent Runner that this gateway recorded how these processes ended + /// (P1R-C04): it keeps them only as long as any other ended process from now on. + pub async fn acknowledge(&self, process_ids: &[ProcessId]) -> Result<(), RunnerError> { + match self + .call(RunnerOp::Acknowledge { + process_ids: process_ids.to_vec(), + }) + .await? + { + RunnerOpResult::Acknowledged => Ok(()), + other => Err(unexpected(other)), + } + } + + /// Have the independent Runner look up, with the resource authority, the attempts it + /// admitted for these processes (P1R-C04), in that order. + #[cfg(feature = "devguard")] + pub async fn look_up_attempts( + &self, + attempts: Vec, + ) -> Result, RunnerError> { + let asked = attempts.len(); + match self.call(RunnerOp::Lookup { attempts }).await? { + RunnerOpResult::LookedUp(found) if found.len() == asked => Ok(found), + other => Err(unexpected(other)), + } + } + /// Whether the worker's `Hello` stated that it answers `Registration` (CSRG-U2). #[cfg(feature = "devguard")] pub fn states_registration(&self) -> bool { @@ -698,6 +790,7 @@ impl Client { reclaim: OnceLock::new(), leaving: AtomicBool::new(false), last_event: AtomicU64::new(0), + claims: watch::Sender::new(None), } } @@ -767,7 +860,7 @@ impl Client { if let Err(err) = write_result { link.pending.lock().await.remove(&request_id); if matches!(err, RunnerError::TransportAmbiguous { .. }) { - self.close_link(link).await; + self.stop_writing(link).await; } return Err(err); } @@ -805,6 +898,16 @@ impl Client { } } + /// A write over `link` failed, most often because the Runner had closed the connection: no + /// request goes over it again. Answers the Runner wrote before it closed are still read, a + /// `Stop`'s among them, and the requests still waiting end once the stream does, as when + /// the Runner closes it first. + async fn stop_writing(&self, link: &Link) { + link.alive.store(false, Ordering::SeqCst); + let mut writer = link.writer.lock().await; + let _ = writer.shutdown().await; + } + /// End `link`: no request goes over it again, and those waiting on it end ambiguous. A /// client that claims its Runner again does so when `link` is its connection. async fn close_link(self: &Arc, link: &Arc) { @@ -858,6 +961,7 @@ impl Client { if !self.leaving.load(Ordering::SeqCst) { *state = State::Linked(link); drop(state); + self.claims.send_replace(Some(claim.clone())); (reclaim.observer)(LinkEvent::Reclaimed(claim)); return; } @@ -999,6 +1103,14 @@ async fn read_loop( } } WireKind::Response => { + // The Claim's inventory accounts for every exit numbered up to its + // `events_through`, and the controller's events follow its answer + // (P1R-C04). + if let Some(RunnerOpResult::Claimed(claim)) = &envelope.result { + client + .last_event + .fetch_max(claim.events_through, Ordering::SeqCst); + } if let Some(id) = envelope.request_id.clone() { if let Some(tx) = link.pending.lock().await.remove(&id) { let _ = tx.send(Ok(envelope)); @@ -1411,4 +1523,101 @@ mod tests { let _ = answering.await.unwrap(); assert_eq!(*delivered.lock().unwrap(), ["proc-1", "proc-3", "proc-4"]); } + + /// The exits that a Claim's inventory accounts for, numbered up to its `events_through`, + /// are not delivered after its answer; later ones are, and the Claim is told to whoever + /// watches the client's Claims (P1R-C04). + #[tokio::test] + async fn an_independent_claim_accounts_for_the_exits_before_it() { + let (client, server) = UnixStream::pair().unwrap(); + let delivered = Arc::new(std::sync::Mutex::new(Vec::new())); + let hook = delivered.clone(); + let runner = UdsRunner::from_stream( + client, + Arc::new(move |process_id: &str| hook.lock().unwrap().push(process_id.to_string())), + ); + let mut claims = runner.claims(); + assert!(claims.borrow().is_none()); + let (mut read, mut write) = server.into_split(); + let exited = |process_id: &str| RunnerEvent::ProcessExited { + process_id: ProcessId(process_id.into()), + }; + let answering = tokio::spawn(async move { + let frame = read_frame(&mut read).await.unwrap().unwrap(); + let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); + assert!(matches!(request.op, Some(RunnerOp::Claim { .. }))); + let claim = RunnerClaim { + epoch: 4, + previous_controller: true, + barrier: vec!["demo".into()], + processes: Vec::new(), + events_through: 3, + }; + write_frame( + &mut write, + &WireEnvelope::response( + request.request_id.unwrap(), + Ok(RunnerOpResult::Claimed(claim)), + ), + ) + .await + .unwrap(); + for (seq, process_id) in [(2, "proc-2"), (3, "proc-3"), (4, "proc-4")] { + write_frame(&mut write, &WireEnvelope::event(seq, exited(process_id))) + .await + .unwrap(); + } + let frame = read_frame(&mut read).await.unwrap().unwrap(); + let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); + assert_eq!(request.epoch, Some(4)); + write_frame( + &mut write, + &WireEnvelope::response(request.request_id.unwrap(), Ok(RunnerOpResult::Terminate)), + ) + .await + .unwrap(); + (read, write) + }); + let claim = runner.claim("store-test").await.unwrap(); + assert_eq!(claim.barrier, ["demo"]); + claims.changed().await.unwrap(); + assert_eq!(claims.borrow().as_ref(), Some(&claim)); + runner.terminate(&ProcessId("proc-5".into())).await.unwrap(); + let _ = answering.await.unwrap(); + assert_eq!(*delivered.lock().unwrap(), ["proc-4"]); + } + + /// A Runner that answers a request and then closes the connection, as one that stops does, + /// fails the write of a request sent after that, but the answer it wrote before it closed + /// is still read (P1R-C04): a late request does not cost a `Stop` its answer. + #[tokio::test] + async fn an_answer_written_before_the_runner_closed_outlives_a_later_failed_write() { + let (client, server) = UnixStream::pair().unwrap(); + let runner = UdsRunner::from_stream(client, Arc::new(|_: &str| {})); + let (mut read, mut write) = server.into_split(); + let first = tokio::spawn({ + let runner = runner.clone(); + async move { runner.terminate(&ProcessId("proc-1".into())).await } + }); + let frame = read_frame(&mut read).await.unwrap().unwrap(); + let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); + write_frame( + &mut write, + &WireEnvelope::response(request.request_id.unwrap(), Ok(RunnerOpResult::Terminate)), + ) + .await + .unwrap(); + drop((read, write)); + // This test's runtime runs one task at a time: the client has not read the answer yet. + let late = runner.terminate(&ProcessId("proc-2".into())).await; + assert!( + matches!(late, Err(RunnerError::TransportAmbiguous { .. })), + "{late:?}" + ); + first.await.unwrap().unwrap(); + assert!(matches!( + runner.terminate(&ProcessId("proc-3".into())).await, + Err(RunnerError::TransportBeforeDispatch { .. }) + )); + } } diff --git a/crates/runner/src/wire.rs b/crates/runner/src/wire.rs index 9b0cbff..087b2e2 100644 --- a/crates/runner/src/wire.rs +++ b/crates/runner/src/wire.rs @@ -9,8 +9,10 @@ //! processes, their output, their timeouts and their exit events outlive each connection. A //! connection controls the Runner only after its `Claim` (P1R-C03), and every request that //! changes a process or the workspace carries the epoch of that Claim, which the Runner checks. +//! A controller after one that started work in a workspace starts nothing there until it +//! reconciled that workspace (P1R-C04). -use std::collections::VecDeque; +use std::collections::{BTreeSet, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, PoisonError}; use std::time::Duration; @@ -26,12 +28,13 @@ use tokio::task::JoinSet; use crate::lifecycle::{RunnerIdentity, RunnerLifecycle}; use crate::{ - InProcessRunner, Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, RunnerError, - RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, - RunnerResizeResult, RunnerWriteStdin, ShellRelease, + InProcessRunner, Recalled, Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, + RunnerError, RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, RunnerReadProcess, + RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, ShellRelease, + DEFAULT_MAX_UNACKNOWLEDGED, }; -pub const WIRE_PROTOCOL: u32 = 10; +pub const WIRE_PROTOCOL: u32 = 11; const MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; const MAX_REPLAY: usize = 32; /// How long a stopping independent Runner waits for its terminated processes to be reaped and @@ -215,6 +218,24 @@ pub enum RunnerOp { /// progress end, terminates every process, waits for them, answers `Stopped` and exits. /// Only its controller stops it (P1R-C03). A managed worker refuses it. Stop, + /// What an independent Runner knows of each of these processes (P1R-C04), answered + /// `Recalled`: whether a request to start it reached the Runner, whether the Runner created + /// it, and its status while the Runner holds it. Only its controller asks. A managed + /// worker refuses it. + Recall { + process_ids: Vec, + }, + /// The controller reconciled these workspaces after its Claim (P1R-C04): the Runner starts + /// work there for it again, answering `Reconciled`. A managed worker refuses it. + Reconciled { + workspace_ids: Vec, + }, + /// The controller recorded how these processes ended (P1R-C04), answered `Acknowledged`: + /// the Runner evicts them as any other ended process from now on, instead of keeping them + /// for a controller. A managed worker refuses it. + Acknowledge { + process_ids: Vec, + }, /// The execution owner's DevGuard registration (CSRG-U2). Sent only to a worker whose /// `Hello` states `registration`; a worker without it would close the connection. #[cfg(feature = "devguard")] @@ -239,6 +260,13 @@ pub enum RunnerOp { request: RunnerExecRequest, attempt_id: String, }, + /// Look up, with DevGuard, the attempts this independent Runner admitted for these + /// processes (P1R-C04), answered `LookedUp`. Only its controller asks. A managed worker + /// refuses it. + #[cfg(feature = "devguard")] + Lookup { + attempts: Vec, + }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -292,16 +320,27 @@ pub enum RunnerOpResult { Detached, /// An independent Runner's answer to `Stop`, sent once its processes ended or /// [`STOP_DRAIN_BOUND`] passed (P1R-C01): how many processes were still running when it - /// terminated them. + /// terminated them, and every process it held then, with its last status (P1R-C04). Stopped { terminated: u32, + #[serde(default)] + processes: Vec, }, + /// An independent Runner's answer to `Recall` (P1R-C04), in the order asked. + Recalled(Vec), + /// An independent Runner's answer to `Reconciled` (P1R-C04). + Reconciled, + /// An independent Runner's answer to `Acknowledge` (P1R-C04). + Acknowledged, #[cfg(feature = "devguard")] Registration(codespace_domain::ResourceAuthorityInfo), #[cfg(feature = "devguard")] Prepared(crate::admission::ManagedPreparation), #[cfg(feature = "devguard")] Launched(crate::launch::LaunchReport), + /// An independent Runner's answer to `Lookup` (P1R-C04), in the order asked. + #[cfg(feature = "devguard")] + LookedUp(Vec), } /// What a connection that claimed an independent Runner controls (P1R-C03). @@ -310,9 +349,12 @@ pub struct RunnerClaim { /// The controller epoch of this Claim, one more than the one before. Every request of this /// connection that changes a process or the workspace carries it. pub epoch: u64, - /// An earlier controller started work, so this connection may observe and control the - /// processes it inherits but starts nothing (P1R-C01). + /// An earlier controller started work: in each workspace of `barrier` (P1R-C01). pub previous_controller: bool, + /// The workspaces in which an earlier controller started work (P1R-C04). This connection + /// starts nothing in one until it states that it reconciled it. + #[serde(default)] + pub barrier: Vec, /// Every process the Runner holds when the Claim is answered, those that ended included. pub processes: Vec, /// The number of the last exit event before the inventory was taken: an event numbered up @@ -328,6 +370,13 @@ pub struct RunnerInventoryEntry { pub status: RunnerProcessStatus, } +/// What an independent Runner knows of one process (P1R-C04). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecalledProcess { + pub process_id: ProcessId, + pub recalled: Recalled, +} + /// Why an independent Runner refused a `Claim` (P1R-C03). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -368,15 +417,16 @@ fn is_false(value: &bool) -> bool { !*value } -/// The refusal of new work on a connection to an independent Runner that an earlier controller -/// started work on (P1R-C01): recovering what that controller left is not available yet. -pub fn recovery_pending() -> ErrorBody { +/// The refusal of new work in a workspace where an earlier controller of an independent Runner +/// started work, which this controller has not reconciled yet (P1R-C01, P1R-C04). +pub fn recovery_pending(workspace_id: &str) -> ErrorBody { ErrorBody::new( ErrorCode::RecoveryPending, - "an earlier gateway controlled this independent Runner, and recovering what it left is \ - not available yet: its processes can be queried, read, written to, resized and \ - terminated, but nothing new is started. Stop the Runner and start a new one to start \ - work again", + format!( + "an earlier gateway started work in workspace `{workspace_id}` on this independent \ + Runner, and this gateway has not reconciled it yet: its processes can be queried, \ + read, written to, resized and terminated, but nothing new is started there" + ), ) } @@ -392,6 +442,8 @@ fn stale_controller(message: impl Into) -> ErrorBody { struct Registrar { registration: Option>, launcher: Option, + /// The admissions an independent Runner asked for, to look them up again (P1R-C04). + admissions: Option, } #[cfg(not(feature = "devguard"))] type Registrar = (); @@ -493,6 +545,7 @@ where Registrar { registration, launcher, + admissions: None, }, ) .await @@ -545,7 +598,7 @@ where }; #[cfg(feature = "devguard")] let op = match op { - Some(op) if answered_beside(&op) => { + Some(op) if answered_beside(&op) && !independent_only(&op) => { // Beside the other requests, and never cached: asking again opens a new // session, and a preparation or a launch is never replayed. A launch is its own // task, so it settles even if the gateway stops waiting. @@ -561,16 +614,14 @@ where op => op, }; let response = match op { - Some(RunnerOp::Claim { .. } | RunnerOp::Detach | RunnerOp::Stop) => { - WireEnvelope::response( - request_id, - Err(ErrorBody::new( - ErrorCode::InvalidCommand, - "this worker belongs to the gateway that started it; only an \ - independent Runner is claimed, detaches or stops on request", - )), - ) - } + Some(op) if independent_only(&op) => WireEnvelope::response( + request_id, + Err(ErrorBody::new( + ErrorCode::InvalidCommand, + "this worker belongs to the gateway that started it; only an independent \ + Runner is claimed, detaches, stops on request or is reconciled", + )), + ), op => answer(&mut cache, &runner, request_id, op, &hello).await, }; let mut writer = write.lock().await; @@ -578,6 +629,21 @@ where } } +/// A request that only an independent Runner answers (P1R-C01, P1R-C03, P1R-C04). +fn independent_only(op: &RunnerOp) -> bool { + match op { + RunnerOp::Claim { .. } + | RunnerOp::Detach + | RunnerOp::Stop + | RunnerOp::Recall { .. } + | RunnerOp::Reconciled { .. } + | RunnerOp::Acknowledge { .. } => true, + #[cfg(feature = "devguard")] + RunnerOp::Lookup { .. } => true, + _ => false, + } +} + /// A request's id, epoch and operation, or the response that refuses it and ends the /// connection. fn parse_request( @@ -667,7 +733,10 @@ async fn answer( fn answered_beside(op: &RunnerOp) -> bool { matches!( op, - RunnerOp::Registration | RunnerOp::Prepare { .. } | RunnerOp::Launch { .. } + RunnerOp::Registration + | RunnerOp::Prepare { .. } + | RunnerOp::Launch { .. } + | RunnerOp::Lookup { .. } ) } @@ -692,6 +761,7 @@ async fn answer_beside( } => match registrar.registration { Some(registration) => Ok(RunnerOpResult::Prepared( crate::admission::Preparer::new(registration.owner(), runner) + .with_admissions(registrar.admissions) .prepare_and_dispose(workspace, request, attempt_id) .await, )), @@ -708,6 +778,7 @@ async fn answer_beside( (Some(registration), Some(launcher)) => Ok(RunnerOpResult::Launched( crate::admission::Preparer::new(registration.owner(), runner) .with_launcher(Some(launcher)) + .with_admissions(registrar.admissions) .prepare_and_launch(workspace, request, attempt_id) .await, )), @@ -716,19 +787,49 @@ async fn answer_beside( "this worker was started without DevGuard settings or a launch helper", )), }, + // An attempt this Runner cannot ask about is reported as a failed lookup. + RunnerOp::Lookup { attempts } => Ok(RunnerOpResult::LookedUp( + crate::admission::look_up( + registrar.registration.map( + |registration| -> Arc { + registration.owner() + }, + ), + ®istrar.admissions.unwrap_or_default(), + attempts, + ) + .await, + )), other => unreachable!("{other:?} is answered in turn"), } } -/// A request that starts work an earlier connection's successor could not know about: a -/// process, a change to the workspace or an admission (P1R-C01). The first one accepted from a -/// controller keeps every later controller from starting work. -fn starts_work(op: &RunnerOp) -> bool { +/// The workspace of a request that starts work an earlier connection's successor could not +/// know about: a process, a change to the workspace or an admission (P1R-C01). The first one +/// accepted from a controller keeps every later controller from starting work in that +/// workspace until it reconciled it (P1R-C04). +fn starts_work(op: &RunnerOp) -> Option<&str> { match op { - RunnerOp::Exec { .. } | RunnerOp::ApplyPatch { .. } => true, + RunnerOp::Exec { workspace, .. } | RunnerOp::ApplyPatch { workspace, .. } => { + Some(&workspace.id.0) + } #[cfg(feature = "devguard")] - RunnerOp::Prepare { .. } | RunnerOp::Launch { .. } => true, - _ => false, + RunnerOp::Prepare { workspace, .. } | RunnerOp::Launch { workspace, .. } => { + Some(&workspace.id.0) + } + _ => None, + } +} + +/// The process that a request asks the Runner to start (P1R-C04). +fn started_process(op: &RunnerOp) -> Option<&ProcessId> { + match op { + RunnerOp::Exec { request, .. } => Some(&request.process_id), + #[cfg(feature = "devguard")] + RunnerOp::Prepare { request, .. } | RunnerOp::Launch { request, .. } => { + Some(&request.process_id) + } + _ => None, } } @@ -742,7 +843,9 @@ fn fenced(op: &RunnerOp) -> bool { | RunnerOp::Resize { .. } | RunnerOp::Terminate { .. } | RunnerOp::TerminateWorkspace { .. } - | RunnerOp::Stop => true, + | RunnerOp::Stop + | RunnerOp::Reconciled { .. } + | RunnerOp::Acknowledge { .. } => true, #[cfg(feature = "devguard")] RunnerOp::Prepare { .. } | RunnerOp::Launch { .. } => true, _ => false, @@ -862,8 +965,10 @@ pub type ClaimObserver = Arc; /// process or the workspace must carry the epoch of its Claim. A Claim is refused while /// another connection controls the Runner and is open; once that connection ends, the Claim /// waits up to [`CLAIM_WAIT_BOUND`] for its requests. A controller after one that started work -/// may observe and control the processes it inherits but starts nothing (P1R-C01); recovering -/// what an earlier controller left is later work (P1R-C04). +/// in a workspace may observe and control the processes it inherits, but starts nothing in +/// that workspace until it states that it reconciled it (P1R-C04). The Runner keeps what its +/// controllers need for that: whether it created a process for each `process_id` it was asked +/// to start, and each ended process until a controller acknowledges it. pub struct IndependentRunner { runner: InProcessRunner, events: RunnerEvents, @@ -877,6 +982,7 @@ pub struct IndependentRunner { impl IndependentRunner { pub fn new(runner: InProcessRunner, events: RunnerEvents) -> Self { + runner.keep_until_acknowledged(DEFAULT_MAX_UNACKNOWLEDGED); let stop = StopHandle { requested: Arc::new(watch::Sender::new(false)), terminated: Arc::new(AtomicU32::new(0)), @@ -909,6 +1015,7 @@ impl IndependentRunner { self.registrar = Registrar { registration, launcher, + admissions: Some(crate::admission::Admissions::default()), }; self } @@ -998,9 +1105,15 @@ impl IndependentRunner { } } let terminated = shared.end_processes().await; + let processes = shared.inventory(); for (request_id, write) in answers { - let response = - WireEnvelope::response(request_id, Ok(RunnerOpResult::Stopped { terminated })); + let response = WireEnvelope::response( + request_id, + Ok(RunnerOpResult::Stopped { + terminated, + processes: processes.clone(), + }), + ); let mut writer = write.lock().await; let _ = write_frame(&mut *writer, &response).await; let _ = writer.shutdown().await; @@ -1020,8 +1133,8 @@ struct Control { claiming: Option, /// The operations database of the first accepted Claim. store_id: Option, - /// A controller started work (P1R-C01). - started_work: bool, + /// The workspaces in which a controller started work (P1R-C01, P1R-C04). + started_in: BTreeSet, } struct Controller { @@ -1030,8 +1143,9 @@ struct Controller { /// The connection reads no further request: its stream ended, or it detached; its /// requests may still be running. ended: Arc, - /// An earlier controller started work: this one starts nothing (P1R-C01). - barrier: bool, + /// The workspaces in which an earlier controller started work, and which this one has not + /// reconciled yet: it starts nothing there (P1R-C04). + barrier: BTreeSet, } /// What an independent Runner's connections share. @@ -1240,18 +1354,18 @@ impl Shared { control.claiming = None; control.epoch += 1; let epoch = control.epoch; - let previous_controller = control.started_work; + let barrier = control.started_in.clone(); control.controller = Some(Controller { connection, epoch, ended: ended.clone(), - barrier: previous_controller, + barrier: barrier.clone(), }); if control.store_id.is_none() { control.store_id = Some(store_id.to_string()); } drop(control); - return Ok(self.claimed(epoch, previous_controller)); + return Ok(self.claimed(epoch, barrier)); } let refusal = if self.stop.is_requested() { Some((ClaimRefusal::Stopping, "the Runner is stopping".to_string())) @@ -1287,35 +1401,40 @@ impl Shared { /// The answer to an accepted Claim: the events numbered so far, then every process, so an /// exit numbered up to `events_through` is in the inventory. - fn claimed(&self, epoch: u64, previous_controller: bool) -> RunnerClaim { + fn claimed(&self, epoch: u64, barrier: BTreeSet) -> RunnerClaim { let events_through = *self.numbered.lock().unwrap_or_else(PoisonError::into_inner); - let processes = self - .runner + RunnerClaim { + epoch, + previous_controller: !barrier.is_empty(), + barrier: barrier.into_iter().collect(), + processes: self.inventory(), + events_through, + } + } + + /// Every process the Runner holds, those that ended included. + fn inventory(&self) -> Vec { + self.runner .inventory() .into_iter() .map(|(workspace_id, status)| RunnerInventoryEntry { workspace_id, status, }) - .collect(); - RunnerClaim { - epoch, - previous_controller, - processes, - events_through, - } + .collect() } /// Admit connection `connection`'s request `op`, sent under `epoch`: only the controller's, /// with the current epoch when it changes a process or the workspace (P1R-C03), and nothing - /// that starts work from a controller after one that did (P1R-C01). + /// that starts work in a workspace where an earlier controller did, until this one + /// reconciled it (P1R-C04). fn admit(&self, connection: u64, epoch: Option, op: &RunnerOp) -> Result<(), ErrorBody> { let mut control = self.control(); - let Some((current, barrier)) = control + let Some(current) = control .controller .as_ref() .filter(|controller| controller.connection == connection) - .map(|controller| (controller.epoch, controller.barrier)) + .map(|controller| controller.epoch) else { return Err(stale_controller( "this connection does not control the independent Runner: another connection \ @@ -1329,15 +1448,45 @@ impl Shared { epoch.map_or_else(|| "none".to_string(), |epoch| epoch.to_string()), ))); } - if starts_work(op) { - if barrier { - return Err(recovery_pending()); + if let Some(workspace_id) = starts_work(op) { + if control + .controller + .as_ref() + .is_some_and(|controller| controller.barrier.contains(workspace_id)) + { + return Err(recovery_pending(workspace_id)); } - control.started_work = true; + control.started_in.insert(workspace_id.to_string()); } Ok(()) } + /// The controller `connection` reconciled these workspaces (P1R-C04): it starts work there + /// from now on. Admitted as a fenced request first. + fn reconciled(&self, connection: u64, workspace_ids: &[String]) { + let mut control = self.control(); + if let Some(controller) = control + .controller + .as_mut() + .filter(|controller| controller.connection == connection) + { + for workspace_id in workspace_ids { + controller.barrier.remove(workspace_id); + } + } + } + + /// What the Runner knows of each of `process_ids` (P1R-C04). + fn recall(&self, process_ids: &[ProcessId]) -> Vec { + process_ids + .iter() + .map(|process_id| RecalledProcess { + process_id: process_id.clone(), + recalled: self.runner.recall(process_id), + }) + .collect() + } + /// Connection `connection` lets go of the Runner, if it controls it. Whether it did. fn release(&self, connection: u64) -> bool { let mut control = self.control(); @@ -1455,11 +1604,26 @@ async fn connection(shared: Arc, stream: UnixStream, id: u64) -> Ended { op @ (Some(RunnerOp::Hello | RunnerOp::Replay { .. }) | None) => { answer(&mut cache, &shared.runner, request_id, op, &hello).await } - Some(op) => match shared.admit(id, epoch, &op) { - Err(refusal) => WireEnvelope::response(request_id, Err(refusal)), - Ok(()) => { - #[cfg(feature = "devguard")] - let op = match op { + Some(op) => { + // A request to start a process that created none is remembered as refused, once + // it is answered (P1R-C04). + let started = started_process(&op).cloned(); + let response = match shared.admit(id, epoch, &op) { + Err(refusal) => WireEnvelope::response(request_id, Err(refusal)), + Ok(()) => match op { + RunnerOp::Recall { process_ids } => WireEnvelope::response( + request_id, + Ok(RunnerOpResult::Recalled(shared.recall(&process_ids))), + ), + RunnerOp::Reconciled { workspace_ids } => { + shared.reconciled(id, &workspace_ids); + WireEnvelope::response(request_id, Ok(RunnerOpResult::Reconciled)) + } + RunnerOp::Acknowledge { process_ids } => { + shared.runner.acknowledge(&process_ids); + WireEnvelope::response(request_id, Ok(RunnerOpResult::Acknowledged)) + } + #[cfg(feature = "devguard")] op if answered_beside(&op) => { let (write, registrar, runner) = ( write.clone(), @@ -1467,18 +1631,26 @@ async fn connection(shared: Arc, stream: UnixStream, id: u64) -> Ended { shared.runner.clone(), ); beside.spawn(async move { - let result = answer_beside(op, registrar, runner).await; + let result = answer_beside(op, registrar, runner.clone()).await; + if let Some(process_id) = started { + runner.note_received(&process_id.0); + } let response = WireEnvelope::response(request_id, result); let mut writer = write.lock().await; let _ = write_frame(&mut *writer, &response).await; }); continue; } - op => op, - }; - answer(&mut cache, &shared.runner, request_id, Some(op), &hello).await + op => { + answer(&mut cache, &shared.runner, request_id, Some(op), &hello).await + } + }, + }; + if let Some(process_id) = started { + shared.runner.note_received(&process_id.0); } - }, + response + } }; let mut writer = write.lock().await; if write_frame(&mut *writer, &response).await.is_err() { @@ -1617,8 +1789,13 @@ async fn dispatch(runner: &InProcessRunner, op: RunnerOp) -> Result unreachable!("hello is answered before dispatch"), RunnerOp::Replay { .. } => unreachable!("replay is handled before dispatch"), - RunnerOp::Claim { .. } | RunnerOp::Detach | RunnerOp::Stop => { - unreachable!("claim, detach and stop are handled before dispatch") + RunnerOp::Claim { .. } + | RunnerOp::Detach + | RunnerOp::Stop + | RunnerOp::Recall { .. } + | RunnerOp::Reconciled { .. } + | RunnerOp::Acknowledge { .. } => { + unreachable!("an independent Runner's requests are handled before dispatch") } #[cfg(feature = "devguard")] RunnerOp::Registration => unreachable!("registration is handled before dispatch"), @@ -1626,6 +1803,8 @@ async fn dispatch(runner: &InProcessRunner, op: RunnerOp) -> Result unreachable!("preparation is handled before dispatch"), #[cfg(feature = "devguard")] RunnerOp::Launch { .. } => unreachable!("launch is handled before dispatch"), + #[cfg(feature = "devguard")] + RunnerOp::Lookup { .. } => unreachable!("lookup is handled before dispatch"), }; result.map_err(RunnerError::into_error_body) } @@ -1641,8 +1820,8 @@ mod tests { } #[test] - fn wire_protocol_is_v10() { - assert_eq!(WIRE_PROTOCOL, 10); + fn wire_protocol_is_v11() { + assert_eq!(WIRE_PROTOCOL, 11); } #[tokio::test] @@ -2346,10 +2525,11 @@ mod tests { } } - /// A managed worker refuses `Claim`, `Detach` and `Stop` and keeps serving; its `Hello` - /// states its lifecycle and identity (P1R-C01, P1R-C03). + /// A managed worker refuses `Claim`, `Detach`, `Stop`, `Recall`, `Reconciled`, + /// `Acknowledge` and with DevGuard `Lookup`, and keeps serving; its `Hello` states its + /// lifecycle and identity (P1R-C01, P1R-C03, P1R-C04). #[tokio::test] - async fn a_managed_worker_refuses_claim_detach_and_stop() { + async fn a_managed_worker_refuses_what_only_an_independent_runner_answers() { let (client, server) = tokio::net::UnixStream::pair().unwrap(); let (worker, events) = host_worker(); tokio::spawn(async move { @@ -2358,7 +2538,7 @@ mod tests { .expect("serve"); }); let (mut read, mut write) = client.into_split(); - for (id, op) in [ + let mut requests = vec![ ( "rrpc-claim", RunnerOp::Claim { @@ -2367,8 +2547,37 @@ mod tests { ), ("rrpc-detach", RunnerOp::Detach), ("rrpc-stop", RunnerOp::Stop), - ("rrpc-hello", RunnerOp::Hello), - ] { + ( + "rrpc-recall", + RunnerOp::Recall { + process_ids: vec![ProcessId("proc-1".into())], + }, + ), + ( + "rrpc-reconciled", + RunnerOp::Reconciled { + workspace_ids: vec!["demo".into()], + }, + ), + ( + "rrpc-acknowledge", + RunnerOp::Acknowledge { + process_ids: vec![ProcessId("proc-1".into())], + }, + ), + ]; + #[cfg(feature = "devguard")] + requests.push(( + "rrpc-lookup", + RunnerOp::Lookup { + attempts: vec![crate::admission::AttemptRef { + attempt_id: "cs-1".into(), + process_id: ProcessId("proc-1".into()), + }], + }, + )); + requests.push(("rrpc-hello", RunnerOp::Hello)); + for (id, op) in requests { write_frame(&mut write, &WireEnvelope::request(id.into(), op)) .await .unwrap(); @@ -2427,6 +2636,7 @@ mod tests { let claimed = RunnerOpResult::Claimed(RunnerClaim { epoch: 2, previous_controller: true, + barrier: vec!["demo".into()], processes: vec![RunnerInventoryEntry { workspace_id: "demo".into(), status: RunnerProcessStatus { @@ -2446,11 +2656,19 @@ mod tests { assert_eq!(json["type"], "claimed"); assert_eq!(json["data"]["epoch"], 2); assert_eq!(json["data"]["events_through"], 5); + assert_eq!(json["data"]["barrier"], serde_json::json!(["demo"])); assert_eq!(json["data"]["processes"][0]["workspace_id"], "demo"); - match serde_json::from_value(json).unwrap() { + match serde_json::from_value(json.clone()).unwrap() { RunnerOpResult::Claimed(claim) => assert_eq!(claim.processes.len(), 1), other => panic!("unexpected {other:?}"), } + // A Claim's answer without a barrier states none (P1R-C04). + let mut without = json; + without["data"].as_object_mut().unwrap().remove("barrier"); + match serde_json::from_value(without).unwrap() { + RunnerOpResult::Claimed(claim) => assert!(claim.barrier.is_empty()), + other => panic!("unexpected {other:?}"), + } let refused = RunnerOpResult::ClaimRefused { refusal: ClaimRefusal::ControllerActive, message: "another connection controls this Runner".into(), @@ -2798,8 +3016,12 @@ mod tests { } fn workspace(root: &Path) -> Workspace { + workspace_named(root, "demo") + } + + fn workspace_named(root: &Path, id: &str) -> Workspace { Workspace::new( - WorkspaceId("demo".into()), + WorkspaceId(id.into()), root.to_path_buf(), Profile::WorkspaceWrite, ) @@ -2923,17 +3145,15 @@ mod tests { } } - fn recovery(gateway: &UdsRunner) -> Option { - gateway.new_work_refusal().map(|refusal| refusal.code) - } - /// After a detach the next gateway claims the Runner under the next epoch, is told - /// every process it inherits, reads the output and status of the same process_id, - /// writes to it and resizes it, and the process times out at its original deadline. - /// That gateway starts nothing new, also once the inherited processes ended. The - /// handover's latency is measured. + /// every process it inherits and each workspace where work was started, reads the + /// output and status of the same process_id, writes to it and resizes it, and the + /// process times out at its original deadline. That gateway starts nothing new in that + /// workspace, also once the inherited processes ended, until it states that it + /// reconciled it (P1R-C04); in a workspace where no work was started it starts work at + /// once. The handover's latency is measured. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn the_next_gateway_inherits_the_processes_but_starts_nothing() { + async fn the_next_gateway_inherits_the_processes_and_starts_work_once_it_reconciled() { let mut host = Host::start(); let root = tempfile::tempdir().unwrap(); let ws = workspace(root.path()); @@ -2949,6 +3169,7 @@ mod tests { RunnerClaim { epoch: 1, previous_controller: false, + barrier: Vec::new(), processes: Vec::new(), events_through: 0, } @@ -3012,6 +3233,7 @@ mod tests { ), (2, true, 0) ); + assert_eq!(inherited.barrier, ["demo"]); let held: Vec<_> = inherited .processes .iter() @@ -3030,7 +3252,8 @@ mod tests { ("demo", "proc-tty", ProcessState::Running) ] ); - assert_eq!(recovery(&second), Some(ErrorCode::RecoveryPending)); + // The client sends new work; the Runner refuses it in the workspace (P1R-C04). + assert_eq!(second.dispatch_epoch().unwrap(), Some(2)); output_with(&second, &pipe, "ready").await; second .write_stdin(RunnerWriteStdin { @@ -3059,7 +3282,7 @@ mod tests { elapsed >= Duration::from_millis(3_000) && elapsed < Duration::from_secs(10), "{elapsed:?}" ); - // Nothing new starts, and the workspace is not changed. + // Nothing new starts in the workspace, and it is not changed. let marker = root.path().join("started"); let touch = command( &["/usr/bin/touch", &marker.display().to_string()], @@ -3080,28 +3303,40 @@ mod tests { code(second.apply_patch(&ws, patch.clone()).await.unwrap_err()), ErrorCode::RecoveryPending ); + // Work starts at once in a workspace where none was started. + let elsewhere = tempfile::tempdir().unwrap(); + second + .exec( + &workspace_named(elsewhere.path(), "elsewhere"), + command(&["/bin/sh", "-c", "exit 0"], "proc-elsewhere", 5_000, false), + ) + .await + .unwrap(); // An inherited process still ends on request. second.terminate(&tty).await.unwrap(); assert_eq!( exited(&second, &tty).await.termination, Some(ProcessTermination::Terminated) ); - // Both exits reached this controller. + // The three exits reached this controller. let mut ended = vec![ next_exit(&mut exited_ids).await.unwrap(), next_exit(&mut exited_ids).await.unwrap(), + next_exit(&mut exited_ids).await.unwrap(), ]; ended.sort(); - assert_eq!(ended, ["proc-pipe", "proc-tty"]); - // The barrier stays once the inherited processes ended. + assert_eq!(ended, ["proc-elsewhere", "proc-pipe", "proc-tty"]); + // The barrier stays once the inherited processes ended, for every later controller, + // and covers the workspace this one worked in. second.detach().await.unwrap(); assert_eq!(host.served().await, Served::Detached); let third = host.connect().await; let claim = third.claimed().unwrap(); assert_eq!( (claim.epoch, claim.previous_controller, claim.events_through), - (3, true, 2) + (3, true, 3) ); + assert_eq!(claim.barrier, ["demo", "elsewhere"]); assert!( claim .processes @@ -3109,18 +3344,40 @@ mod tests { .all(|entry| entry.status.state == ProcessState::Exited), "{claim:?}" ); - assert_eq!(recovery(&third), Some(ErrorCode::RecoveryPending)); assert_eq!( - code(third.exec(&ws, touch).await.unwrap_err()), + code(third.exec(&ws, touch.clone()).await.unwrap_err()), ErrorCode::RecoveryPending ); assert_eq!( - code(third.apply_patch(&ws, patch).await.unwrap_err()), + code(third.apply_patch(&ws, patch.clone()).await.unwrap_err()), ErrorCode::RecoveryPending ); assert!(!marker.exists()); assert!(!root.path().join("added.txt").exists()); - assert_eq!(third.stop_runner().await.unwrap(), 0); + // Stating that it reconciled the workspace lifts the barrier there, for this + // controller, and only under its own epoch. + assert_eq!( + code( + third + .at_epoch(2) + .reconciled(&["demo".into()]) + .await + .unwrap_err() + ), + ErrorCode::StaleController + ); + third.reconciled(&["demo".into()]).await.unwrap(); + match third.apply_patch(&ws, patch).await { + Ok(_) => assert!(root.path().join("added.txt").exists()), + // Where the patch helper is not built, it refuses the patch, not the barrier. + Err(err) => assert_ne!(code(err), ErrorCode::RecoveryPending), + } + third.exec(&ws, touch).await.unwrap(); + exited(&third, &ProcessId("proc-new".into())).await; + assert!(marker.exists()); + let stopped = third.stop_runner().await.unwrap(); + assert_eq!(stopped.terminated, 0); + assert_eq!(stopped.processes.len(), 4, "{stopped:?}"); assert_eq!(host.served().await, Served::Stopped { terminated: 0 }); let claimed: Vec<_> = host .claims() @@ -3143,8 +3400,8 @@ mod tests { assert_eq!(claimed, [(1, false), (2, true), (3, true)]); } - /// Only a controller that starts work bars the next one from starting work: one that - /// only read leaves the next gateway free to. + /// Only a controller that starts work in a workspace bars the next one from starting + /// work there: one that only read leaves the next gateway free to. #[tokio::test] async fn a_controller_that_starts_nothing_leaves_no_barrier() { let mut host = Host::start(); @@ -3155,8 +3412,9 @@ mod tests { reader.detach().await.unwrap(); assert_eq!(host.served().await, Served::Detached); let worker = host.connect().await; - assert!(!worker.claimed().unwrap().previous_controller); - assert_eq!(recovery(&worker), None); + let claim = worker.claimed().unwrap(); + assert!(!claim.previous_controller); + assert!(claim.barrier.is_empty()); worker .exec( &ws, @@ -3166,7 +3424,142 @@ mod tests { .unwrap(); worker.close().await; assert_eq!(host.served().await, Served::Lost); - assert!(host.connect().await.claimed().unwrap().previous_controller); + let claim = host.connect().await.claimed().unwrap(); + assert!(claim.previous_controller); + assert_eq!(claim.barrier, ["demo"]); + } + + /// A Runner tells its controller, for each process asked about, whether a request to + /// start it arrived and whether it created it, with its status while it holds it; a + /// request that its barrier refused is one it received and created nothing for + /// (P1R-C04). Only its controller asks, and only under its own epoch does it + /// acknowledge an end. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_runner_recalls_whether_it_created_each_process() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let first = host.connect().await; + let running = ProcessId("proc-running".into()); + let failed = ProcessId("proc-failed".into()); + let never = ProcessId("proc-never".into()); + first + .exec( + &ws, + command(&["/bin/sleep", "30"], &running.0, 60_000, false), + ) + .await + .unwrap(); + // Refused before any process is created, with or without the Linux sandbox. + let no_program = command(&[""], &failed.0, 5_000, false); + assert_eq!( + code(first.exec(&ws, no_program).await.unwrap_err()), + ErrorCode::InvalidCommand + ); + first.detach().await.unwrap(); + assert_eq!(host.served().await, Served::Detached); + + let second = host.connect().await; + let barred = ProcessId("proc-barred".into()); + let quick = command(&["/bin/sh", "-c", "exit 0"], &barred.0, 5_000, false); + assert_eq!( + code(second.exec(&ws, quick).await.unwrap_err()), + ErrorCode::RecoveryPending + ); + let asked = [running.clone(), failed, never, barred]; + let recalled = second.recall(&asked).await.unwrap(); + assert_eq!( + recalled + .iter() + .map(|recalled| recalled.process_id.clone()) + .collect::>(), + asked + ); + match &recalled[0].recalled { + Recalled::Held { + workspace_id, + status, + } => assert_eq!( + (workspace_id.as_str(), status.state), + ("demo", ProcessState::Running) + ), + other => panic!("unexpected {other:?}"), + } + assert_eq!( + recalled[1..] + .iter() + .map(|recalled| recalled.recalled.clone()) + .collect::>(), + [Recalled::Refused, Recalled::NotReceived, Recalled::Refused] + ); + // A connection that does not control the Runner is not told. + let mut raw = host.connect_raw().await; + let refused = raw + .call( + RunnerOp::Recall { + process_ids: vec![running.clone()], + }, + None, + ) + .await + .unwrap_err(); + assert_eq!(refused.code, ErrorCode::StaleController); + second.terminate(&running).await.unwrap(); + exited(&second, &running).await; + match &second.recall(std::slice::from_ref(&running)).await.unwrap()[0].recalled { + Recalled::Held { status, .. } => assert_eq!(status.state, ProcessState::Exited), + other => panic!("unexpected {other:?}"), + } + assert_eq!( + code( + second + .at_epoch(1) + .acknowledge(std::slice::from_ref(&running)) + .await + .unwrap_err() + ), + ErrorCode::StaleController + ); + second + .acknowledge(std::slice::from_ref(&running)) + .await + .unwrap(); + } + + /// The controller has the Runner look up a governed execution's attempt with the + /// Runner's own DevGuard session: an attempt the Runner never admitted is not asked + /// about, and a connection that does not control the Runner is not told (P1R-C04). + #[cfg(feature = "devguard")] + #[tokio::test] + async fn an_independent_runner_looks_up_attempts_for_its_controller_only() { + use crate::admission::{AttemptLookup, AttemptRef, LookedUp}; + let host = Host::start(); + let mut controller = host.connect_raw().await; + let epoch = controller.claim(STORE).await.unwrap().epoch; + let attempts = vec![AttemptRef { + attempt_id: "cs-1".into(), + process_id: ProcessId("proc-1".into()), + }]; + let lookup = RunnerOp::Lookup { + attempts: attempts.clone(), + }; + match controller.call(lookup, Some(epoch)).await { + Ok(RunnerOpResult::LookedUp(found)) => assert_eq!( + found, + [AttemptLookup { + attempt_id: "cs-1".into(), + process_id: ProcessId("proc-1".into()), + found: LookedUp::NotAdmitted, + }] + ), + other => panic!("unexpected {other:?}"), + } + let mut other = host.connect_raw().await; + let refused = other + .call(RunnerOp::Lookup { attempts }, None) + .await + .unwrap_err(); + assert_eq!(refused.code, ErrorCode::StaleController); } /// A Claim's answer lists every process the Runner holds, ended ones included, and the @@ -3656,23 +4049,33 @@ mod tests { // A process that ends while the client reconnects. std::fs::write(root.path().join("go"), "").unwrap(); exited(&host.runner, &gated).await; + let mut claims = gateway.claims(); relay.hold(false); match next_link_event(&mut events).await { LinkEvent::Reclaimed(claim) => { assert_eq!((claim.epoch, claim.previous_controller), (2, true)); + assert_eq!(claim.barrier, ["demo"]); + // The new Claim's inventory accounts for the exit: its event, which the + // Runner sends again, is not delivered (P1R-C04). + assert_eq!(claim.events_through, 1); + let gated_status = claim + .processes + .iter() + .find(|entry| entry.status.process_id == gated) + .map(|entry| entry.status.state); + assert_eq!(gated_status, Some(ProcessState::Exited), "{claim:?}"); assert_eq!(claim.processes.len(), 2, "{claim:?}"); } other => panic!("unexpected {other:?}"), } assert_eq!(gateway.claimed().map(|claim| claim.epoch), Some(2)); + // Whoever reconciles after a Claim is told of each one. + claims.changed().await.unwrap(); + assert_eq!(claims.borrow().as_ref().map(|claim| claim.epoch), Some(2)); assert_eq!( gateway.process_status(&long).await.unwrap().state, ProcessState::Running ); - assert_eq!( - next_exit(&mut exited_ids).await.as_deref(), - Some("proc-gated") - ); // A request prepared under the lost Claim is refused, and nothing new starts. assert_eq!( code( @@ -3684,7 +4087,6 @@ mod tests { ), ErrorCode::StaleController ); - assert_eq!(recovery(&gateway), Some(ErrorCode::RecoveryPending)); assert_eq!( code(gateway.exec(&ws, touch).await.unwrap_err()), ErrorCode::RecoveryPending @@ -3876,12 +4278,40 @@ mod tests { .unwrap(); output_with(&gateway, &tty, "tty-ready").await; let started = Instant::now(); - assert_eq!(gateway.stop_runner().await.unwrap(), 2); + let stopped = gateway.stop_runner().await.unwrap(); assert!( started.elapsed() < STOP_DRAIN_BOUND, "{:?}", started.elapsed() ); + assert_eq!(stopped.terminated, 2); + // The answer states how each process ended (P1R-C04). + let ended: Vec<_> = stopped + .processes + .iter() + .map(|entry| { + ( + entry.status.process_id.0.as_str(), + entry.status.state, + entry.status.termination, + ) + }) + .collect(); + assert_eq!( + ended, + [ + ( + "proc-pipe", + ProcessState::Exited, + Some(ProcessTermination::Terminated) + ), + ( + "proc-tty", + ProcessState::Exited, + Some(ProcessTermination::Terminated) + ) + ] + ); assert_eq!(host.served().await, Served::Stopped { terminated: 2 }); assert_eq!(host.runner.occupied_slots(), 0); for process_id in [&pipe, &tty] { @@ -3925,9 +4355,10 @@ mod tests { } /// A connection that ends without a word is a loss: the processes keep running, and an - /// exit while no gateway controls the Runner reaches the next controller. + /// exit while no gateway controls the Runner reaches the next controller in its Claim's + /// inventory, not as an event (P1R-C04); later exits come as events. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn a_lost_connection_keeps_the_processes_and_their_exit_events() { + async fn a_lost_connection_keeps_the_processes_and_the_next_claim_accounts_for_exits() { let mut host = Host::start(); let root = tempfile::tempdir().unwrap(); let ws = workspace(root.path()); @@ -3952,10 +4383,18 @@ mod tests { let second = host.connect_reporting_exits(on_exit).await; let claim = second.claimed().unwrap(); assert!(claim.previous_controller); - assert_eq!(claim.processes.len(), 2); + assert_eq!(claim.events_through, 1); + let listed: Vec<_> = claim + .processes + .iter() + .map(|entry| (entry.status.process_id.0.as_str(), entry.status.state)) + .collect(); assert_eq!( - next_exit(&mut exited_ids).await.as_deref(), - Some("proc-quick") + listed, + [ + ("proc-long", ProcessState::Running), + ("proc-quick", ProcessState::Exited) + ] ); assert_eq!( second.process_status(&long).await.unwrap().state, diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 4df41ce..a0a52be 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -11,6 +11,7 @@ pub mod mcp; pub mod patch_helper; pub mod protocol; pub mod records; +pub mod recovery; pub mod runtime; pub mod stdio; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 2b6d7d8..6c74924 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -10,7 +10,8 @@ use codespace_server::config::{ }; use codespace_server::http::serve_http_handler; use codespace_server::logging; -use codespace_server::records::ProcessRecords; +use codespace_server::records::{Leases, ProcessRecords}; +use codespace_server::recovery::Recovery; use codespace_server::runtime::RuntimeProcess; use codespace_server::stdio; use codespace_server::CodeSpace; @@ -75,6 +76,9 @@ async fn serve(cli: Cli, lifecycle: GatewayLifecycle) -> Result<()> { if let Some(records) = &started.records { handler = handler.with_records(records.clone()); } + if let Some(recovery) = &started.recovery { + handler = handler.with_recovery(recovery.clone()); + } #[cfg(feature = "devguard")] handler.register_owner_now(); let (result, ending) = match lifecycle { @@ -137,6 +141,8 @@ struct StartedRunner { lifecycle: GatewayLifecycle, /// The records of the executions sent to an independent Runner (P1R-C02). records: Option>, + /// The reconciliation with an independent Runner after each Claim (P1R-C04). + recovery: Option>, } impl StartedRunner { @@ -148,7 +154,8 @@ impl StartedRunner { runner, process, lifecycle, - records: _, + records, + recovery: _, } = self; if let (GatewayLifecycle::Independent { exit_intent }, RuntimeBackend::Uds(runner)) = (lifecycle, &runner) @@ -169,8 +176,16 @@ impl StartedRunner { ), }, RunnerExitIntent::Stop => match runner.stop_runner().await { - Ok(terminated) => { - tracing::info!(terminated, "stopped the independent Runner") + Ok(stopped) => { + // How each process ended, as the Runner reported it as it stopped + // (P1R-C04). + if let Some(records) = &records { + records.stopped(&stopped.processes); + } + tracing::info!( + terminated = stopped.terminated, + "stopped the independent Runner" + ) } Err(err) => { return Err(anyhow::anyhow!( @@ -207,13 +222,15 @@ async fn start_runner( process: None, lifecycle, records: None, + recovery: None, }), RunnerMode::Uds => { if let (GatewayLifecycle::Independent { .. }, Some(socket)) = (lifecycle, &cli.runner_socket) { // Each exit releases its lease and is recorded (P1R-C02). - let (on_release, exits) = ProcessRecords::release_hook(store.clone()); + let leases = Leases::new(store.clone()); + let (on_release, exits) = ProcessRecords::release_hook(leases.clone()); // The Runner remembers the operations database it was first claimed for // (P1R-C03). let store_id = store.store_id().map_err(|err| { @@ -221,7 +238,7 @@ async fn start_runner( })?; // The gateway only connects and claims; the Runner registers itself, from its // own credential file, and nothing is handed to it (P1R-C01). - let (runner, _claim) = + let (runner, claim) = RuntimeProcess::connect_independent(socket, &store_id, on_release).await?; #[cfg(feature = "devguard")] if cli.devguard.worker().is_some() { @@ -231,13 +248,18 @@ async fn start_runner( .hello() .identity .ok_or_else(|| anyhow::anyhow!("the independent Runner has no identity"))?; - let runner = RuntimeBackend::Uds(runner); - let records = ProcessRecords::start(store, &identity, runner.clone(), exits); + let backend = RuntimeBackend::Uds(runner.clone()); + let records = + ProcessRecords::start(store.clone(), &identity, backend.clone(), exits); + // What earlier claims left is reconciled before new work starts where they + // started work, and again after each later Claim (P1R-C04). + let recovery = Recovery::start(store, runner, records.clone(), leases, &claim); return Ok(StartedRunner { - runner, + runner: backend, process: None, lifecycle, records: Some(records), + recovery: Some(recovery), }); } if let Some(bin) = &cli.runtime_bin { @@ -257,6 +279,7 @@ async fn start_runner( process: Some(process), lifecycle, records: None, + recovery: None, }); } let (process, runner) = @@ -267,6 +290,7 @@ async fn start_runner( process: Some(process), lifecycle, records: None, + recovery: None, }); } let socket = cli.runner_socket.as_ref().ok_or_else(|| { @@ -280,6 +304,7 @@ async fn start_runner( process: None, lifecycle, records: None, + recovery: None, }) } } diff --git a/crates/server/src/mcp.rs b/crates/server/src/mcp.rs index 7e2f311..c7b35a7 100644 --- a/crates/server/src/mcp.rs +++ b/crates/server/src/mcp.rs @@ -55,6 +55,8 @@ pub struct CodeSpace { pub(crate) lifecycle: crate::config::GatewayLifecycle, /// The records of the executions sent to an independent Runner (P1R-C02). pub(crate) records: Option>, + /// The reconciliation with an independent Runner after each Claim (P1R-C04). + pub(crate) recovery: Option>, } fn err_json(err: ErrorBody) -> String { @@ -158,8 +160,8 @@ subtree and drops handles. Restart does not recover process_id. If you lose \ the spawn response before receiving process_id, do not search for the lost \ handle and do not start a duplicate command."; -/// The lifetime paragraph for an independent Runner (P1R-C01, P1R-C03), after what the -/// gateway's shutdown does. It states only what the Runner guarantees: a pipe process has no +/// The lifetime paragraph for an independent Runner (P1R-C01, P1R-C03, P1R-C04), after what +/// the gateway's shutdown does. It states only what the Runner guarantees: a pipe process has no /// process group of its own, so terminating one signals that process alone; and one gateway at a /// time controls the Runner, but nothing keeps another process of the same user from claiming it /// while none does. @@ -176,17 +178,22 @@ restarted. STALE_CONTROLLER means that the Runner refused a request sent \ under an earlier claim; nothing was started or changed. After a reconnect \ or a restart, the gateway can read the output and status of, write to, \ resize and terminate the processes started before, with the same \ -process_id, but if work was started on the Runner under an earlier claim, \ -this gateway's own included, it refuses new exec_command, apply_patch and \ -operation_resume with RECOVERY_PENDING until the Runner is stopped and a \ -new one is started. A request refused with one of these codes consumes no \ +process_id. In a workspace where work was started under an earlier claim, \ +this gateway's own included, it first reconciles what that work left with \ +its records and the Runner, and refuses new exec_command, apply_patch and \ +operation_resume there with RECOVERY_PENDING meanwhile; a workspace that \ +cannot be reconciled keeps returning RECOVERY_PENDING until an operator \ +resolves it. A request refused with one of these codes consumes no \ approval, and a patch refused with one keeps no result for its \ -operation_key. While no gateway controls the Runner, any process of the \ -same user that can reach its socket, such as a command run without a \ -sandbox, can take control of it. Terminating a pipe process signals that \ -process only, not its process group, so processes it started may outlive \ -it. If you lose the spawn response before receiving process_id, do not \ -search for the lost handle and do not start a duplicate command."; +operation_key. A process found running when the gateway reconciles holds \ +its workspace again, so new work there returns WORKSPACE_BUSY until the \ +process exits or is terminated. While no gateway controls the Runner, any \ +process of the same user that can reach its socket, such as a command run \ +without a sandbox, can take control of it. Terminating a pipe process \ +signals that process only, not its process group, so processes it started \ +may outlive it. If you lose the spawn response before receiving \ +process_id, do not search for the lost handle and do not start a duplicate \ +command."; const DETACH_SHUTDOWN: &str = "\ When this gateway shuts down it detaches: the Runner keeps the processes, \ @@ -256,6 +263,7 @@ impl CodeSpace { registration: None, lifecycle: crate::config::GatewayLifecycle::Managed, records: None, + recovery: None, } } @@ -273,13 +281,33 @@ impl CodeSpace { self } - /// New work is refused before any approval, lease or claim when the runner is an - /// independent Runner that this gateway cannot send it to (P1R-C01, P1R-C03): an earlier - /// controller started work on it, which is not recovered yet, or the connection to it is - /// lost or being made again. - fn refuse_new_work(&self) -> Result<(), ErrorBody> { - match self.runner.new_work_refusal() { - Some(refusal) => Err(refusal), + /// The handler of a gateway that reconciles what earlier claims of its independent Runner + /// left before it starts new work in a workspace (P1R-C04). + pub fn with_recovery(mut self, recovery: Arc) -> Self { + self.recovery = Some(recovery); + self + } + + /// New work in `workspace_id` is refused before any approval, lease or claim when the + /// runner is an independent Runner that this gateway cannot send it to (P1R-C01, P1R-C03, + /// P1R-C04): the connection to it is lost or being made again, or the gateway has not + /// reconciled that workspace since its Claim. + fn refuse_new_work(&self, workspace_id: &str) -> Result<(), ErrorBody> { + self.dispatch_epoch(workspace_id).map(|_| ()) + } + + /// The controller epoch under which new work in `workspace_id` is sent, or why none is + /// (P1R-C03, P1R-C04): see [`Self::refuse_new_work`]. + fn dispatch_epoch(&self, workspace_id: &str) -> Result, ErrorBody> { + let epoch = self.runner.dispatch_epoch()?; + self.admit(workspace_id, epoch)?; + Ok(epoch) + } + + /// Whether this gateway reconciled `workspace_id` since the Claim of `epoch` (P1R-C04). + fn admit(&self, workspace_id: &str, epoch: Option) -> Result<(), ErrorBody> { + match &self.recovery { + Some(recovery) => recovery.admit(workspace_id, epoch), None => Ok(()), } } @@ -394,7 +422,8 @@ impl CodeSpace { .map_err(err_json)?; allow(ws, Action::Write, &ClientClaims::default()).map_err(err_json)?; ws.require_file_write().map_err(err_json)?; - self.refuse_new_work().map_err(err_json)?; + self.refuse_new_work(¶ms.workspace_id.0) + .map_err(err_json)?; if let Err(err) = self.maybe_hold_patch(ws, ¶ms) { return Err(err_json(err)); } @@ -435,7 +464,8 @@ impl CodeSpace { if params.command.is_empty() || params.command[0].is_empty() { return Err(err_json(invalid_argv())); } - self.refuse_new_work().map_err(err_json)?; + self.refuse_new_work(¶ms.workspace_id.0) + .map_err(err_json)?; // A workspace this gateway cannot govern is refused before any approval or lease. self.participation(ws).await.map_err(err_json)?; if let Err(err) = self.maybe_hold_exec(ws, ¶ms) { @@ -513,7 +543,9 @@ impl CodeSpace { .map_err(runner_err_json)?; if let Some(records) = &self.records { match result.state { - ProcessState::Exited => records.ended(&result), + ProcessState::Exited => { + records.ended(&result); + } ProcessState::Running => { records.observed_output(&result.process_id, result.output_total) } @@ -697,8 +729,8 @@ impl CodeSpace { ws.require_file_write()?; let _lease = self.store.acquire_write(¶ms.workspace_id.0).await?; // Checked again once the lease is held, before the approval being resumed is marked - // (P1R-C03): a refusal here leaves it queued. - let epoch = self.runner.dispatch_epoch()?; + // (P1R-C03, P1R-C04): a refusal here leaves it queued. + let epoch = self.dispatch_epoch(¶ms.workspace_id.0)?; if let Some(id) = resume { self.store.mark_resuming(id)?; } @@ -879,8 +911,17 @@ impl CodeSpace { &self, params: OperationResumeParams, ) -> Result { - // Before the claim, so the approval stays as it is (P1R-C01). - self.refuse_new_work()?; + // Before the claim, so the approval stays as it is (P1R-C01, P1R-C04): one that would + // start work waits while its workspace is not reconciled. + let epoch = self.runner.dispatch_epoch()?; + if let Some(approval) = self.store.approval(¶ms.approval_id)? { + if matches!( + approval.state, + ApprovalState::Granted | ApprovalState::Queued | ApprovalState::Resuming + ) { + self.admit(&approval.workspace_id, epoch)?; + } + } match self.store.claim_resume(¶ms.approval_id)? { ResumeClaim::ReplaySuccess(result) => Ok(result), ResumeClaim::ReplayError(err) => Err(err), @@ -955,6 +996,23 @@ impl CodeSpace { self.commit_resume(&record.approval_id, outcome) } ApprovalTargetTool::ExecCommand => { + // A record of the execution that proves it started consumes the approval with + // its process (P1R-C04); without one, whether it started is not known. + let started = self + .store + .records_of_approval(&record.approval_id)? + .into_iter() + .find(crate::recovery::proves_started); + if let Some(started) = started { + let mut result = crate::recovery::consumed_by( + &record.approval_id, + &ProcessId(started.process_id), + ); + if let Some(exec) = result.exec_command.as_mut() { + exec.coordination = self.hint(&record.workspace_id, None); + } + return self.commit_resume(&record.approval_id, Ok(result)); + } let err = ErrorBody::new( ErrorCode::ApprovalAmbiguous, "exec resume is ambiguous after interruption; the process was not restarted", @@ -1043,8 +1101,9 @@ impl CodeSpace { .acquire_shell_busy(¶ms.workspace_id.0, &process_id.0) .await?; // Checked again once the workspace is reserved, before the approval being resumed is - // marked (P1R-C03): a refusal here leaves it queued and releases the reservation. - let epoch = self.runner.dispatch_epoch()?; + // marked (P1R-C03, P1R-C04): a refusal here leaves it queued and releases the + // reservation. + let epoch = self.dispatch_epoch(¶ms.workspace_id.0)?; let runner = self.runner.at_epoch(epoch); // A governed execution's DevGuard attempt, minted before its record names it. let attempt_id = participation.attempt_id(); @@ -4461,7 +4520,8 @@ mod independent_tests { use super::tests::{exec_params, participation_registry}; use super::*; use crate::config::{GatewayLifecycle, RunnerExitIntent}; - use crate::records::ProcessRecords; + use crate::records::{Leases, ProcessRecords}; + use crate::recovery::Recovery; use codespace_domain::{ ApprovalDecision, OperationKey, ProcessDisconnectAction, ProcessState, ProcessStatusParams, ProcessTermination, TerminateProcessParams, WorkspaceId, @@ -4471,7 +4531,8 @@ mod independent_tests { StopHandle, UdsRunner, }; use codespace_store::{ - ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, RunnerRef, + NewProcessRecord, ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, + RunnerRef, }; use std::collections::BTreeMap; use std::time::{Duration, Instant}; @@ -4523,26 +4584,71 @@ mod independent_tests { } /// A gateway connected to an independent Runner through `stream`, which it claims for - /// `database` (P1R-C03), recording its executions there. + /// `database` (P1R-C03), recording its executions there, once it reconciled what earlier + /// claims left (P1R-C04). async fn recording_gateway( root: &std::path::Path, database: &std::path::Path, stream: UnixStream, ) -> CodeSpace { let store = Arc::new(Store::open(database).unwrap()); - let (release, exits) = ProcessRecords::release_hook(store.clone()); + let leases = Leases::new(store.clone()); + let (release, exits) = ProcessRecords::release_hook(leases.clone()); let runner = UdsRunner::from_stream(stream, release); runner.handshake().await.unwrap(); - runner.claim(&store.store_id().unwrap()).await.unwrap(); + let claim = runner.claim(&store.store_id().unwrap()).await.unwrap(); + let gateway = reconciling_gateway(root, store, runner, leases, exits, &claim); + reconciled(&gateway, 1).await; + gateway + } + + /// A gateway connected to the independent Runner `runner`, which it claimed with `claim`, + /// that records its executions and reconciles as `codespace-mcp` does (P1R-C02, P1R-C04). + fn reconciling_gateway( + root: &std::path::Path, + store: Arc, + runner: UdsRunner, + leases: Arc, + exits: crate::records::Exits, + claim: &codespace_runner::RunnerClaim, + ) -> CodeSpace { let identity = runner .hello() .identity .expect("an independent Runner's identity"); - let runner = RuntimeBackend::Uds(runner); - let records = ProcessRecords::start(store.clone(), &identity, runner.clone(), exits); - CodeSpace::with_store_and_runner(participation_registry(root, "off"), store, runner) + let backend = RuntimeBackend::Uds(runner.clone()); + let records = ProcessRecords::start(store.clone(), &identity, backend.clone(), exits); + let recovery = Recovery::start(store.clone(), runner, records.clone(), leases, claim); + CodeSpace::with_store_and_runner(participation_registry(root, "off"), store, backend) .with_lifecycle(DETACHING) .with_records(records) + .with_recovery(recovery) + } + + /// Wait until the gateway has reconciled `passes` times (P1R-C04). + async fn reconciled(gateway: &CodeSpace, passes: u64) { + let mut done = gateway + .recovery + .as_ref() + .expect("a gateway that reconciles") + .passes(); + tokio::time::timeout( + Duration::from_secs(20), + done.wait_for(|done| *done >= passes), + ) + .await + .expect("the reconciliation does not end") + .expect("the reconciliation"); + } + + /// Wait until new work in `workspace_id` is not refused while the gateway reconciles + /// (P1R-C04). + async fn admitted(gateway: &CodeSpace, workspace_id: &str) { + let deadline = Instant::now() + Duration::from_secs(20); + while let Err(err) = gateway.refuse_new_work(workspace_id) { + assert!(Instant::now() < deadline, "{err:?}"); + tokio::time::sleep(Duration::from_millis(10)).await; + } } impl Drop for Runner { @@ -4604,6 +4710,18 @@ mod independent_tests { } } + /// What `path` says once it says anything: a sandboxed command can take a while to start. + async fn written(path: &std::path::Path) -> String { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match std::fs::read_to_string(path) { + Ok(text) if !text.is_empty() => return text, + read => assert!(Instant::now() < deadline, "{}: {read:?}", path.display()), + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + /// The whole output of a process that ended. async fn output_of(gateway: &CodeSpace, process_id: &ProcessId) -> ReadProcessResult { let deadline = Instant::now() + Duration::from_secs(10); @@ -4666,12 +4784,31 @@ mod independent_tests { /// epoch 1, and any other request as `answer` says, until it returns none: the Runner is /// then lost without answering. async fn fake_runner_answering( + stream: UnixStream, + identity: RunnerIdentity, + answer: impl FnMut(RunnerOp) -> Option>, + ) { + let claim = codespace_runner::RunnerClaim { + epoch: 1, + previous_controller: false, + barrier: Vec::new(), + processes: Vec::new(), + events_through: 0, + }; + fake_runner_claimed(stream, identity, claim, answer).await + } + + /// A Runner that answers `Hello` as the independent Runner `identity`, a `Claim` with + /// `claim`, and any other request as `answer` says, until it returns none: the Runner is + /// then lost without answering. + async fn fake_runner_claimed( mut stream: UnixStream, identity: RunnerIdentity, + claim: codespace_runner::RunnerClaim, mut answer: impl FnMut(RunnerOp) -> Option>, ) { use codespace_runner::{ - read_frame, write_frame, RunnerClaim, RunnerLifecycle, WireEnvelope, WIRE_PROTOCOL, + read_frame, write_frame, RunnerLifecycle, WireEnvelope, WIRE_PROTOCOL, }; while let Ok(Some(frame)) = read_frame(&mut stream).await { let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); @@ -4691,12 +4828,7 @@ mod independent_tests { write_frame(&mut stream, &reply).await.unwrap(); } Some(RunnerOp::Claim { .. }) => { - let claimed = RunnerOpResult::Claimed(RunnerClaim { - epoch: 1, - previous_controller: false, - processes: Vec::new(), - events_through: 0, - }); + let claimed = RunnerOpResult::Claimed(claim.clone()); let reply = WireEnvelope::response(request.request_id.unwrap(), Ok(claimed)); write_frame(&mut stream, &reply).await.unwrap(); } @@ -4791,7 +4923,8 @@ mod independent_tests { socket: &std::path::Path, ) -> (CodeSpace, mpsc::UnboundedReceiver) { let store = Arc::new(Store::open(database).unwrap()); - let (release, exits) = ProcessRecords::release_hook(store.clone()); + let leases = Leases::new(store.clone()); + let (release, exits) = ProcessRecords::release_hook(leases.clone()); let (noted, events) = mpsc::unbounded_channel(); let (runner, claim) = UdsRunner::claim_independent( socket, @@ -4804,13 +4937,8 @@ mod independent_tests { .await .unwrap(); assert_eq!((claim.epoch, claim.previous_controller), (1, false)); - let identity = runner.hello().identity.unwrap(); - let runner = RuntimeBackend::Uds(runner); - let records = ProcessRecords::start(store.clone(), &identity, runner.clone(), exits); - let gateway = - CodeSpace::with_store_and_runner(participation_registry(root, "off"), store, runner) - .with_lifecycle(DETACHING) - .with_records(records); + let gateway = reconciling_gateway(root, store, runner, leases, exits, &claim); + reconciled(&gateway, 1).await; (gateway, events) } @@ -4847,11 +4975,13 @@ mod independent_tests { } } - /// After a detach the next gateway reads and controls the processes it inherits, but new - /// executions, patches and approval resumes are refused before anything is claimed, so a - /// granted approval stays granted. + /// After a detach the next gateway reconciles what the gateway before it left (P1R-C04): + /// the process it inherits running holds its workspace again, with the same process_id, so + /// new work there is WORKSPACE_BUSY as under the gateway that started it, until the + /// process is terminated. The gateway reads and controls that process, and records how it + /// ended under the record the gateway before it wrote. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn an_independent_gateway_after_a_detach_starts_nothing_and_keeps_the_approval() { + async fn an_independent_gateway_after_a_detach_holds_the_process_it_inherits() { let dir = tempfile::tempdir().unwrap(); let root = dir.path().join("ws"); std::fs::create_dir(&root).unwrap(); @@ -4859,25 +4989,6 @@ mod independent_tests { let runner = Runner::start(); let first = runner.gateway(&root, &database).await; assert_eq!(first.runner.new_work_refusal(), None); - let touch = format!("{}/resumed", root.display()); - let held = refused( - first - .exec_command(Parameters(exec_params( - "gov-confirm", - &["/usr/bin/touch", &touch], - false, - ))) - .await, - ); - assert_eq!(held.code, ErrorCode::ApprovalRequired, "{held:?}"); - let approval_id = ApprovalId(held.approval_id.unwrap()); - first - .approval_resolve(Parameters(ApprovalResolveParams { - approval_id: approval_id.clone(), - decision: ApprovalDecision::Grant, - })) - .await - .unwrap(); let started = first .exec_command(Parameters(exec_params( "demo", @@ -4890,10 +5001,7 @@ mod independent_tests { detach(&first).await; let next = runner.gateway(&root, &database).await; - assert_eq!( - next.runner.new_work_refusal().map(|refusal| refusal.code), - Some(ErrorCode::RecoveryPending) - ); + assert_eq!(next.runner.new_work_refusal(), None); let info = next .workspace_info(Parameters(WorkspaceInfoParams { workspace_id: Some("demo".into()), @@ -4953,41 +5061,47 @@ mod independent_tests { ))) .await, ); - assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); - let refusal = refused( - next.apply_patch(Parameters(ApplyPatchParams { - workspace_id: WorkspaceId("demo".into()), - patch: "*** Begin Patch\n*** Add File: added.txt\n+added\n*** End Patch\n".into(), - expected_versions: BTreeMap::new(), - operation_key: None, - check_only: false, - work_id: None, - })) - .await, - ); - assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert_eq!(refusal.code, ErrorCode::WorkspaceBusy, "{refusal:?}"); let refusal = refused( - next.operation_resume(Parameters(OperationResumeParams { - approval_id: approval_id.clone(), - })) - .await, + next.apply_patch(Parameters(patch("demo", "added.txt", None))) + .await, ); - assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert_eq!(refusal.code, ErrorCode::WorkspaceBusy, "{refusal:?}"); + assert_eq!(recorded_executions(&database), 1); + let record = Store::open(&database) + .unwrap() + .process_record(&started.process_id.0) + .unwrap() + .expect("the record the first gateway wrote"); assert_eq!( - next.store.inspect_approval(&approval_id).unwrap().0, - ApprovalState::Granted + (record.phase, record.controller_epoch), + (ProcessPhase::Running, Some(1)) ); + next.terminate_process(Parameters(TerminateProcessParams { process_id: started.process_id.clone(), })) .await .unwrap(); - let deadline = Instant::now() + Duration::from_secs(10); - while status(&next).await != ProcessState::Exited { - assert!(Instant::now() < deadline); - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!root.join("resumed").exists()); + let record = record_once(&database, &started.process_id, terminal).await; + assert_eq!( + record.outcome, + Some(ProcessOutcome::Ended(ProcessTermination::Terminated)), + "{record:?}" + ); + released(&next, "demo").await; + let after = next + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "after"], + false, + ))) + .await + .unwrap() + .0; + assert_eq!(after.dispatch_status, ExecDispatchStatus::Confirmed); + let record = record_once(&database, &after.process_id, terminal).await; + assert_eq!(record.controller_epoch, Some(2)); assert!(!root.join("added.txt").exists()); } @@ -5087,9 +5201,10 @@ mod independent_tests { assert!(before <= record.created_at_ms && record.created_at_ms <= record.updated_at_ms); } - /// An execution that ends while no gateway is connected is recorded by the next gateway, to - /// which the Runner gives the exit it kept (P1R-C02): that gateway finds the execution by - /// the record the gateway before it wrote. + /// An execution that ends while no gateway is connected is recorded by the next gateway as + /// it reconciles, from the ended process the Runner kept for it (P1R-C02, P1R-C04): that + /// gateway finds the execution by the record the gateway before it wrote, and new work + /// starts in its workspace. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn an_end_while_no_gateway_is_connected_is_recorded_by_the_next_gateway() { let dir = tempfile::tempdir().unwrap(); @@ -5128,7 +5243,7 @@ mod independent_tests { .expect("the record the first gateway wrote"); assert_eq!(left.phase, ProcessPhase::Running, "{left:?}"); - let _next = runner.gateway(&root, &database).await; + let next = runner.gateway(&root, &database).await; let record = record_once(&database, &started.process_id, terminal).await; assert_eq!( record.outcome, @@ -5136,13 +5251,23 @@ mod independent_tests { ); assert_eq!(record.exit_code, Some(7)); assert_eq!((&record.runner, &record.root), (&left.runner, &left.root)); + let after = next + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "after"], + false, + ))) + .await + .unwrap() + .0; + assert_eq!(after.dispatch_status, ExecDispatchStatus::Confirmed); } - /// An end that reached a gateway that went away before recording it is recorded once the - /// next gateway asks the Runner about the execution (P1R-C02): a record never says more - /// than the Runner reported, and follows what the Runner reports. + /// An end that reached a gateway that went away before recording it is recorded by the + /// next gateway as it reconciles, from what the Runner reports of the execution (P1R-C02, + /// P1R-C04): a record never says more than the Runner reported. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn an_end_a_gateway_missed_is_recorded_when_the_next_gateway_asks_the_runner() { + async fn an_end_a_gateway_missed_is_recorded_when_the_next_gateway_reconciles() { let dir = tempfile::tempdir().unwrap(); let root = dir.path().join("ws"); std::fs::create_dir(&root).unwrap(); @@ -5154,14 +5279,15 @@ mod independent_tests { let (client, server) = UnixStream::pair().unwrap(); runner.streams.send(server).unwrap(); let store = Arc::new(Store::open(&database).unwrap()); - let (release, lost) = ProcessRecords::release_hook(store.clone()); + let leases = Leases::new(store.clone()); + let (release, lost) = ProcessRecords::release_hook(leases.clone()); drop(lost); let uds = UdsRunner::from_stream(client, release); uds.handshake().await.unwrap(); uds.claim(&store.store_id().unwrap()).await.unwrap(); let identity = uds.hello().identity.unwrap(); let backend = RuntimeBackend::Uds(uds); - let (_, none) = ProcessRecords::release_hook(store.clone()); + let (_, none) = ProcessRecords::release_hook(leases); let records = ProcessRecords::start(store.clone(), &identity, backend.clone(), none); CodeSpace::with_store_and_runner(participation_registry(&root, "off"), store, backend) .with_lifecycle(DETACHING) @@ -5177,8 +5303,6 @@ mod independent_tests { .unwrap() .0; released(&first, "demo").await; - detach(&first).await; - let next = runner.gateway(&root, &database).await; tokio::time::sleep(Duration::from_millis(200)).await; let missed = Store::open(&database) .unwrap() @@ -5186,6 +5310,21 @@ mod independent_tests { .unwrap() .unwrap(); assert_eq!(missed.phase, ProcessPhase::Running, "{missed:?}"); + detach(&first).await; + let next = runner.gateway(&root, &database).await; + let reconciled = Store::open(&database) + .unwrap() + .process_record(&started.process_id.0) + .unwrap() + .unwrap(); + assert_eq!( + (reconciled.outcome, reconciled.exit_code), + ( + Some(ProcessOutcome::Ended(ProcessTermination::Exited)), + Some(9) + ), + "{reconciled:?}" + ); let status = next .process_status(Parameters(ProcessStatusParams { process_id: started.process_id.clone(), @@ -5575,11 +5714,14 @@ mod independent_tests { } /// A gateway that started work and claimed its Runner again after its connection was lost - /// is a controller after one that started work, its own earlier connection, so it starts - /// nothing more until recovery is built (P1R-C01, P1R-C03). It reads and ends the process it - /// started, under the new epoch, and records how that process ended. + /// is a controller after one that started work, its own earlier connection, so it + /// reconciles before it starts new work where that connection did (P1R-C01, P1R-C03, + /// P1R-C04). A workspace no controller started work in takes new work at once. An exit + /// while the connection was lost, which the gateway never heard, is recorded and releases + /// its workspace; a process still running holds its workspace until its exit, heard under + /// the new epoch, releases it. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn a_gateway_that_started_work_claims_its_runner_again_and_starts_nothing() { + async fn a_gateway_that_started_work_claims_its_runner_again_and_reconciles_it() { let dir = tempfile::tempdir().unwrap(); let root = dir.path().join("ws"); std::fs::create_dir(&root).unwrap(); @@ -5587,72 +5729,121 @@ mod independent_tests { let runner = Runner::start(); let relay = Relay::start(&runner); let (gateway, mut events) = claiming_gateway(&root, &database, &relay.socket).await; - let started = gateway + let ended = gateway .exec_command(Parameters(exec_params( "demo", - &["/bin/sh", "-c", "echo before; exec sleep 30"], + &[ + "/bin/sh", + "-c", + "echo before; while [ ! -e go ]; do sleep 0.02; done; touch gone; exit 4", + ], + false, + ))) + .await + .unwrap() + .0; + let running = gateway + .exec_command(Parameters(exec_params( + "gov", + &[ + "/bin/sh", + "-c", + "while [ ! -e stop ]; do sleep 0.02; done; exit 5", + ], false, ))) .await .unwrap() .0; + relay.hold(true); relay.cut(); assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); + std::fs::write(root.join("go"), "").unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !root.join("gone").exists() { + assert!(Instant::now() < deadline, "the process does not end"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RunnerReconnecting, "{refusal:?}"); + relay.hold(false); match next_link_event(&mut events).await { LinkEvent::Reclaimed(claim) => { assert_eq!((claim.epoch, claim.previous_controller), (2, true)); + assert_eq!(claim.barrier, ["demo", "gov"]); assert!( claim .processes .iter() - .any(|entry| entry.status.process_id == started.process_id + .any(|entry| entry.status.process_id == running.process_id && entry.status.state == ProcessState::Running), "{claim:?}" ); } other => panic!("unexpected {other:?}"), } - let refusal = refused( + // Nothing waits in a workspace where no controller started work. + let held = refused( gateway .exec_command(Parameters(exec_params( - "demo", + "gov-confirm", &["/bin/echo", "new"], false, ))) .await, ); - assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); - let deadline = Instant::now() + Duration::from_secs(10); - loop { - let read = gateway - .read_process(Parameters(ReadProcessParams { - process_id: started.process_id.clone(), - cursor: 0, - })) - .await - .unwrap() - .0; - if read.chunk.contains("before") { - break; - } - assert!(Instant::now() < deadline, "{read:?}"); - tokio::time::sleep(Duration::from_millis(10)).await; - } - gateway - .terminate_process(Parameters(TerminateProcessParams { - process_id: started.process_id.clone(), - })) - .await - .unwrap(); - let record = record_once(&database, &started.process_id, terminal).await; - assert_eq!(record.controller_epoch, Some(1)); + assert_eq!(held.code, ErrorCode::ApprovalRequired, "{held:?}"); + + admitted(&gateway, "demo").await; + let record = record_once(&database, &ended.process_id, terminal).await; assert_eq!( - record.outcome, - Some(ProcessOutcome::Ended(ProcessTermination::Terminated)), + (record.outcome, record.exit_code, record.controller_epoch), + ( + Some(ProcessOutcome::Ended(ProcessTermination::Exited)), + Some(4), + Some(1) + ), "{record:?}" ); - released(&gateway, "demo").await; + let read = output_of(&gateway, &ended.process_id).await; + assert!(read.chunk.contains("before"), "{read:?}"); + let after = gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "after"], + false, + ))) + .await + .unwrap() + .0; + assert_eq!(after.dispatch_status, ExecDispatchStatus::Confirmed); + assert_eq!( + record_once(&database, &after.process_id, terminal) + .await + .controller_epoch, + Some(2) + ); + + admitted(&gateway, "gov").await; + let refusal = refused( + gateway + .exec_command(Parameters(exec_params("gov", &["/bin/echo", "new"], false))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::WorkspaceBusy, "{refusal:?}"); + std::fs::write(root.join("stop"), "").unwrap(); + let record = record_once(&database, &running.process_id, terminal).await; + assert_eq!(record.exit_code, Some(5), "{record:?}"); + released(&gateway, "gov").await; } /// An execution, a patch or an approval resume that the Runner refuses as sent under an @@ -5771,6 +5962,586 @@ mod independent_tests { assert!(!root.join("resumed").exists()); } + /// An approval to run `command` in `workspace_id`, granted. + async fn granted(gateway: &CodeSpace, workspace_id: &str, command: &[&str]) -> ApprovalId { + let created = gateway + .approval_create(Parameters(ApprovalCreateParams { + tool: ApprovalTargetTool::ExecCommand, + arguments: serde_json::to_value(exec_params(workspace_id, command, false)).unwrap(), + })) + .await + .unwrap() + .0; + gateway + .approval_resolve(Parameters(ApprovalResolveParams { + approval_id: created.approval_id.clone(), + decision: ApprovalDecision::Grant, + })) + .await + .unwrap(); + created.approval_id + } + + async fn resume( + gateway: &CodeSpace, + approval_id: &ApprovalId, + ) -> Result { + gateway + .operation_resume(Parameters(OperationResumeParams { + approval_id: approval_id.clone(), + })) + .await + .map(|resumed| resumed.0) + } + + /// A record of an execution sent to `runner` under `epoch`, which it started, as a gateway + /// before this one left it. + fn left_running( + database: &std::path::Path, + runner: &RunnerRef, + process_id: &str, + epoch: Option, + workspace_id: &str, + ) { + let store = Store::open(database).unwrap(); + store + .record_dispatch(&NewProcessRecord { + process_id, + runner, + controller_epoch: epoch, + workspace_id, + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["/bin/sleep".to_string(), "30".to_string()], + }) + .unwrap(); + store + .record_running(runner, process_id, None, None) + .unwrap(); + } + + /// An exec approval whose resume could not record its result stays resuming, and resuming + /// it again consumes it with the process that the execution's record names (P1R-C04): the + /// command runs once. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_resume_left_resuming_is_consumed_with_the_process_it_started() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + let approval_id = granted( + &gateway, + "gov-confirm", + &["/bin/sh", "-c", "echo run >> runs"], + ) + .await; + gateway.store.fail_next_finish(); + let refusal = refused(resume(&gateway, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::ApprovalAmbiguous, "{refusal:?}"); + assert_eq!( + gateway.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Resuming + ); + let records = gateway.store.records_of_approval(&approval_id).unwrap(); + assert_eq!(records.len(), 1, "{records:?}"); + let process_id = ProcessId(records[0].process_id.clone()); + for _ in 0..2 { + let resumed = resume(&gateway, &approval_id).await.unwrap(); + assert_eq!(resumed.state, ApprovalState::Consumed); + let started = resumed.exec_command.expect("the execution"); + assert_eq!( + (started.process_id, started.dispatch_status), + (process_id.clone(), ExecDispatchStatus::Confirmed) + ); + } + record_once(&database, &process_id, terminal).await; + assert_eq!(std::fs::read_to_string(root.join("runs")).unwrap(), "run\n"); + assert_eq!(recorded_executions(&database), 1); + } + + /// An approval a gateway left resuming, whose execution the Runner started, is consumed + /// with that process by the next gateway as it reconciles (P1R-C04): resuming it answers + /// that process, and the command never runs twice. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_consumes_an_interrupted_resume_with_its_process() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let first = runner.gateway(&root, &database).await; + let approval_id = granted( + &first, + "gov-confirm", + &["/bin/sh", "-c", "echo run >> runs; exec sleep 30"], + ) + .await; + first.store.fail_next_finish(); + let refusal = refused(resume(&first, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::ApprovalAmbiguous, "{refusal:?}"); + let process_id = ProcessId( + first.store.records_of_approval(&approval_id).unwrap()[0] + .process_id + .clone(), + ); + detach(&first).await; + + let next = runner.gateway(&root, &database).await; + assert_eq!( + next.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Consumed + ); + for _ in 0..2 { + let resumed = resume(&next, &approval_id).await.unwrap(); + assert_eq!( + resumed.exec_command.expect("the execution").process_id, + process_id + ); + } + // The command ran before it is terminated, however long its sandbox took to start it. + assert_eq!(written(&root.join("runs")).await, "run\n"); + next.terminate_process(Parameters(TerminateProcessParams { + process_id: process_id.clone(), + })) + .await + .unwrap(); + record_once(&database, &process_id, terminal).await; + assert_eq!(std::fs::read_to_string(root.join("runs")).unwrap(), "run\n"); + assert_eq!(recorded_executions(&database), 1); + } + + /// An approval a gateway left resuming, whose execution never reached the Runner, is given + /// back by the next gateway as it reconciles (P1R-C04): the Runner proves that it never + /// received the execution, so its record says that it never started, and the approval is + /// resumed again, once. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_gives_back_an_approval_whose_execution_never_reached_the_runner( + ) { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let first = runner.gateway(&root, &database).await; + let command = ["/bin/sh", "-c", "echo run >> runs"]; + let approval_id = granted(&first, "gov-confirm", &command).await; + // The gateway records the execution and marks the approval resuming, and ends before it + // sends the execution, as one that crashes there does. + assert!(matches!( + first.store.claim_resume(&approval_id).unwrap(), + ResumeClaim::Execute(..) + )); + let lost = ProcessId("proc-never-sent".into()); + first + .records + .as_ref() + .unwrap() + .dispatching( + &lost, + &exec_params("gov-confirm", &command, false), + Some(1), + RecordedParticipation::Off, + None, + Some(&approval_id), + ) + .unwrap(); + assert_eq!( + first.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Resuming + ); + detach(&first).await; + + let next = runner.gateway(&root, &database).await; + assert_eq!( + next.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Granted + ); + let record = next.store.process_record(&lost.0).unwrap().unwrap(); + assert_eq!( + record.outcome, + Some(ProcessOutcome::NotStarted), + "{record:?}" + ); + let started = resume(&next, &approval_id) + .await + .unwrap() + .exec_command + .expect("the execution"); + assert_ne!(started.process_id, lost); + record_once(&database, &started.process_id, terminal).await; + let replayed = resume(&next, &approval_id).await.unwrap(); + assert_eq!( + replayed.exec_command.expect("the execution").process_id, + started.process_id + ); + assert_eq!(std::fs::read_to_string(root.join("runs")).unwrap(), "run\n"); + } + + /// A workspace where another Runner was sent an execution whose record does not say it + /// ended stays blocked (P1R-C04): only that Runner could prove what became of it. New work + /// there is refused with RECOVERY_PENDING, an approval resume before its approval is + /// claimed, so a granted approval stays granted; the record stays as it was, and the other + /// workspaces work. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_blocks_a_workspace_another_runner_left_work_in() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let elsewhere = runner_ref(fake_identity()); + left_running( + &database, + &elsewhere, + "proc-elsewhere", + Some(7), + "gov-confirm", + ); + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + let approval_id = granted( + &gateway, + "gov-confirm", + &["/bin/sh", "-c", "echo run >> runs"], + ) + .await; + for _ in 0..2 { + let refusal = refused(resume(&gateway, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert!( + refusal.message.contains("proc-elsewhere") + && refusal.message.contains("another Runner") + && refusal.message.contains("operator"), + "{refusal:?}" + ); + assert_eq!( + gateway.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Granted + ); + } + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "gov-confirm", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + let refusal = refused( + gateway + .apply_patch(Parameters(patch( + "gov-confirm", + "added.txt", + Some("patch-1"), + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert_eq!( + gateway + .store + .status_lookup(None, Some(&OperationKey("patch-1".into()))) + .unwrap_err() + .code, + ErrorCode::OperationNotFound + ); + + let started = gateway + .exec_command(Parameters(exec_params("demo", &["/bin/echo", "ok"], false))) + .await + .unwrap() + .0; + record_once(&database, &started.process_id, terminal).await; + // Reconciled again after that exit, the workspace stays blocked. + reconciled(&gateway, 2).await; + let refusal = refused(resume(&gateway, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + let record = gateway + .store + .process_record("proc-elsewhere") + .unwrap() + .unwrap(); + assert_eq!( + (record.phase, record.runner), + (ProcessPhase::Running, elsewhere) + ); + assert!(!root.join("runs").exists()); + assert!(!root.join("added.txt").exists()); + } + + /// A workspace whose record and Runner disagree stays blocked, and so does one where the + /// Runner runs a process that no record accounts for, until that process ends (P1R-C04): + /// the refusal names the process, which can be queried and terminated meanwhile. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_blocks_a_workspace_whose_records_and_runner_disagree() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + // A record that says this Runner started an execution it never received. + let this = runner_ref(RunnerIdentity::this_process()); + left_running(&database, &this, "proc-phantom", None, "gov"); + // A process that a controller started without recording it. + let unrecorded = ProcessId("proc-unrecorded".into()); + { + let (client, server) = UnixStream::pair().unwrap(); + runner.streams.send(server).unwrap(); + let raw = UdsRunner::from_stream(client, Arc::new(|_: &str| {})); + raw.handshake().await.unwrap(); + let store_id = Store::open(&database).unwrap().store_id().unwrap(); + raw.claim(&store_id).await.unwrap(); + let registry = participation_registry(&root, "off"); + let ws = registry.get("demo").unwrap(); + codespace_runner::Runner::exec( + &raw, + ws, + RunnerExecRequest::for_host( + vec!["/bin/sleep".into(), "30".into()], + unrecorded.clone(), + ws.profile, + ), + ) + .await + .unwrap(); + raw.detach().await.unwrap(); + } + + let gateway = runner.gateway(&root, &database).await; + let refusal = refused( + gateway + .exec_command(Parameters(exec_params("gov", &["/bin/echo", "new"], false))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert!( + refusal.message.contains("proc-phantom") && refusal.message.contains("never received"), + "{refusal:?}" + ); + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert!(refusal.message.contains(&unrecorded.0), "{refusal:?}"); + let held = refused( + gateway + .exec_command(Parameters(exec_params( + "gov-confirm", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(held.code, ErrorCode::ApprovalRequired, "{held:?}"); + let status = gateway + .process_status(Parameters(ProcessStatusParams { + process_id: unrecorded.clone(), + })) + .await + .unwrap() + .0; + assert_eq!(status.state, ProcessState::Running); + + gateway + .terminate_process(Parameters(TerminateProcessParams { + process_id: unrecorded.clone(), + })) + .await + .unwrap(); + admitted(&gateway, "demo").await; + let started = gateway + .exec_command(Parameters(exec_params("demo", &["/bin/echo", "ok"], false))) + .await + .unwrap() + .0; + assert_eq!(started.dispatch_status, ExecDispatchStatus::Confirmed); + let refusal = refused( + gateway + .exec_command(Parameters(exec_params("gov", &["/bin/echo", "new"], false))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + let record = gateway + .store + .process_record("proc-phantom") + .unwrap() + .unwrap(); + assert_eq!(record.phase, ProcessPhase::Running); + } + + /// A governed execution's workspace is reconciled only once the resource authority states + /// its attempt (P1R-C04): while the authority cannot be asked, new work there is refused + /// with RECOVERY_PENDING and the gateway asks again, and the process it inherits can be + /// queried meanwhile. Once the authority states that a launch helper claimed the attempt, + /// the running process holds its workspace again. + #[cfg(feature = "devguard")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_waits_for_the_resource_authority_before_it_reconciles() { + use codespace_runner::admission::{AttemptLookup, AttemptState, LookedUp}; + use codespace_runner::{ + Recalled, RecalledProcess, RunnerClaim, RunnerInventoryEntry, RunnerProcessStatus, + }; + use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let governed = ProcessId("proc-governed".into()); + { + let store = Store::open(&database).unwrap(); + let runner = runner_ref(fake_identity()); + store + .record_dispatch(&NewProcessRecord { + process_id: &governed.0, + runner: &runner, + controller_epoch: Some(1), + workspace_id: "demo", + approval_id: None, + attempt_id: Some("attempt-1"), + participation: RecordedParticipation::Required, + tty: false, + argv: &["/bin/sleep".to_string(), "30".to_string()], + }) + .unwrap(); + store + .record_running(&runner, &governed.0, None, None) + .unwrap(); + } + let running = RunnerProcessStatus { + process_id: governed.clone(), + state: ProcessState::Running, + exit_code: None, + termination: None, + signal: None, + output_total: 0, + output_retained_from: 0, + eof: false, + }; + let claim = RunnerClaim { + epoch: 2, + previous_controller: true, + barrier: vec!["demo".into()], + processes: vec![RunnerInventoryEntry { + workspace_id: "demo".into(), + status: running.clone(), + }], + events_through: 0, + }; + let available = Arc::new(AtomicBool::new(false)); + let lookups = Arc::new(AtomicU32::new(0)); + let told = Arc::new(std::sync::Mutex::new(Vec::::new())); + let (client, server) = UnixStream::pair().unwrap(); + tokio::spawn(fake_runner_claimed(server, fake_identity(), claim, { + let (available, lookups, told) = (available.clone(), lookups.clone(), told.clone()); + move |op| { + Some(Ok(match op { + RunnerOp::Recall { process_ids } => RunnerOpResult::Recalled( + process_ids + .into_iter() + .map(|process_id| RecalledProcess { + process_id, + recalled: Recalled::Held { + workspace_id: "demo".into(), + status: running.clone(), + }, + }) + .collect(), + ), + RunnerOp::Lookup { attempts } => { + lookups.fetch_add(1, Ordering::SeqCst); + let found = if available.load(Ordering::SeqCst) { + LookedUp::Found(AttemptState::Launched) + } else { + LookedUp::Failed + }; + RunnerOpResult::LookedUp( + attempts + .into_iter() + .map(|attempt| AttemptLookup { + attempt_id: attempt.attempt_id, + process_id: attempt.process_id, + found, + }) + .collect(), + ) + } + RunnerOp::Reconciled { workspace_ids } => { + told.lock().unwrap().extend(workspace_ids); + RunnerOpResult::Reconciled + } + RunnerOp::Acknowledge { .. } => RunnerOpResult::Acknowledged, + RunnerOp::ProcessStatus { .. } => { + RunnerOpResult::ProcessStatus(running.clone()) + } + RunnerOp::WorkspaceOf { .. } => { + RunnerOpResult::WorkspaceOf(Some("demo".into())) + } + other => panic!("a request the fake Runner does not answer: {other:?}"), + })) + } + })); + let gateway = recording_gateway(&root, &database, client).await; + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert!(refusal.message.contains("attempt-1"), "{refusal:?}"); + let status = gateway + .process_status(Parameters(ProcessStatusParams { + process_id: governed.clone(), + })) + .await + .unwrap() + .0; + assert_eq!(status.state, ProcessState::Running); + reconciled(&gateway, 2).await; + assert!(told.lock().unwrap().is_empty()); + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + + available.store(true, Ordering::SeqCst); + admitted(&gateway, "demo").await; + assert!(lookups.load(Ordering::SeqCst) >= 3); + assert_eq!(*told.lock().unwrap(), ["demo"]); + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::WorkspaceBusy, "{refusal:?}"); + let record = gateway.store.process_record(&governed.0).unwrap().unwrap(); + assert_eq!(record.phase, ProcessPhase::Running); + } + /// Each independent mode states only what its Runner guarantees, and the rest of the /// instructions stay as they are. #[test] @@ -5792,6 +6563,9 @@ mod independent_tests { "RUNNER_DISCONNECTED", "STALE_CONTROLLER", "RECOVERY_PENDING", + "reconciles what that work left", + "WORKSPACE_BUSY", + "until an operator resolves it", "consumes no approval", "any process of the same user", "same process_id", diff --git a/crates/server/src/records.rs b/crates/server/src/records.rs index dd1907c..6387472 100644 --- a/crates/server/src/records.rs +++ b/crates/server/src/records.rs @@ -2,45 +2,109 @@ //! recorded in the operations database before it is sent to the Runner, and followed to its //! end, so that a gateway that lost its memory can identify the same execution again. A record //! never revives a handle, and never by itself proves that an execution started or how it -//! ended: the Runner is the authority on live processes. +//! ended: the Runner is the authority on live processes. Once a record says how an execution +//! ended, the gateway acknowledges it to the Runner, which until then keeps the ended process +//! for a controller (P1R-C04). -use std::sync::Arc; +use std::collections::HashSet; +use std::sync::{Arc, Mutex, PoisonError}; +use codespace_domain::ProcessTermination; use codespace_domain::{ ApprovalId, ErrorBody, ErrorCode, ExecCommandParams, ProcessId, ProcessState, }; use codespace_runner::{ - Runner, RunnerIdentity, RunnerProcessRoot, RunnerProcessStatus, RuntimeBackend, ShellRelease, + Runner, RunnerIdentity, RunnerInventoryEntry, RunnerProcessRoot, RunnerProcessStatus, + RuntimeBackend, ShellRelease, }; use codespace_store::{ NewProcessRecord, ProcessEnd, RecordedParticipation, RootProcess, RunnerRef, Store, }; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, watch}; /// The exits that a gateway's release hook hands to its process records. pub struct Exits(mpsc::UnboundedReceiver); +/// The workspace leases of a gateway connected to an independent Runner (P1R-C04): each exit +/// releases its process's lease, and a reconciliation holds a workspace again for a process +/// that it found running. While a reconciliation runs, the exits are noted, so a process that +/// exits after the Runner reported it running is never held again. +pub struct Leases { + store: Arc, + /// The processes that exited since the reconciliation in progress began, if one is. + exited: Mutex>>, +} + +impl Leases { + pub fn new(store: Arc) -> Arc { + Arc::new(Self { + store, + exited: Mutex::new(None), + }) + } + + fn exited(&self) -> std::sync::MutexGuard<'_, Option>> { + self.exited.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Release the lease of `process_id`, which exited or never started. + pub fn release(&self, process_id: &str) { + let mut exited = self.exited(); + if let Some(exited) = exited.as_mut() { + exited.insert(process_id.to_string()); + } + self.store.release_process(process_id); + } + + /// Note the exits from now on, for a reconciliation. + pub(crate) fn note_exits(&self) { + *self.exited() = Some(HashSet::new()); + } + + /// Stop noting the exits: the reconciliation ended. + pub(crate) fn forget_exits(&self) { + *self.exited() = None; + } + + /// Hold `workspace_id` again for `process_id`, which the Runner reported running. Returns + /// whether it is held: not when the process exited since, as its exit released it. + pub(crate) fn restore(&self, workspace_id: &str, process_id: &str) -> Result { + let exited = self.exited(); + if exited + .as_ref() + .is_some_and(|exited| exited.contains(process_id)) + { + return Ok(false); + } + self.store.hold_process(workspace_id, process_id)?; + Ok(true) + } +} + /// What a gateway records of the executions it sends to one independent Runner. pub struct ProcessRecords { store: Arc, runner: RunnerRef, + /// Bumped after each end that the records follow from an exit event (P1R-C04). + ends: watch::Sender, } impl ProcessRecords { /// The release hook of a gateway connected to an independent Runner: it releases the lease /// of each process that exits, as every gateway's hook does, and hands the exit on to the /// records. An exit that comes before the records exist waits for them. - pub fn release_hook(store: Arc) -> (ShellRelease, Exits) { + pub fn release_hook(leases: Arc) -> (ShellRelease, Exits) { let (exits, exited) = mpsc::unbounded_channel(); let release: ShellRelease = Arc::new(move |process_id: &str| { - store.release_process(process_id); + leases.release(process_id); let _ = exits.send(ProcessId(process_id.to_string())); }); (release, Exits(exited)) } /// The records of the executions sent to the Runner with `identity`, which from now on - /// follow its exit events: each one is recorded as the Runner reports it. + /// follow its exit events: each one is recorded as the Runner reports it, then acknowledged + /// to the Runner (P1R-C04). pub fn start( store: Arc, identity: &RunnerIdentity, @@ -55,14 +119,15 @@ impl ProcessRecords { started_at_ms: identity.started_at_ms, nonce: identity.nonce.clone(), }, + ends: watch::Sender::new(0), }); let following = records.clone(); let Exits(mut exited) = exits; tokio::spawn(async move { while let Some(process_id) = exited.recv().await { - match runner.process_status(&process_id).await { + let recorded = match runner.process_status(&process_id).await { Ok(status) if status.state == ProcessState::Exited => following.ended(&status), - Ok(_) => {} + Ok(_) => false, // The Runner reported the exit; how it ended is not known. Err(_) => following.end( &process_id, @@ -73,12 +138,33 @@ impl ProcessRecords { output_total: None, }, ), + }; + if recorded { + // The Runner keeps an ended process for a controller until it is told. + if let Err(err) = runner.acknowledge(std::slice::from_ref(&process_id)).await { + tracing::debug!( + process_id = %process_id.0, + error = %err, + "the Runner was not told that the process's end is recorded" + ); + } } + following.ends.send_modify(|ends| *ends += 1); } }); records } + /// The Runner these records are of. + pub(crate) fn runner(&self) -> &RunnerRef { + &self.runner + } + + /// Each end that the records follow from an exit event, as a count (P1R-C04). + pub(crate) fn ends(&self) -> watch::Receiver { + self.ends.subscribe() + } + /// Record an execution before it is sent to the Runner under the controller epoch /// `controller_epoch` (P1R-C03), and mark the approval it resumes `resuming` with it. When /// the record cannot be written, nothing is sent. @@ -135,13 +221,15 @@ impl ProcessRecords { ); } - /// The Runner answered that it did not start the execution. - pub(crate) fn not_started(&self, process_id: &ProcessId) { - self.end(process_id, &ProcessEnd::NotStarted); + /// The Runner answered that it did not start the execution, or proved it when the gateway + /// reconciled (P1R-C04). Returns whether the record could be written. + pub(crate) fn not_started(&self, process_id: &ProcessId) -> bool { + self.end(process_id, &ProcessEnd::NotStarted) } - /// The Runner reported the execution's status after it exited. - pub(crate) fn ended(&self, status: &RunnerProcessStatus) { + /// The Runner reported the execution's status after it exited. Returns whether the record + /// could be written. + pub(crate) fn ended(&self, status: &RunnerProcessStatus) -> bool { self.end( &status.process_id, &ProcessEnd::Ended { @@ -150,7 +238,41 @@ impl ProcessRecords { signal: status.signal.as_ref().map(|signal| signal.number), output_total: Some(status.output_total), }, - ); + ) + } + + /// The Runner created the execution and no longer keeps it: it ended, and how is not known + /// any more (P1R-C04). Returns whether the record could be written. + pub(crate) fn dropped(&self, process_id: &ProcessId) -> bool { + self.end( + process_id, + &ProcessEnd::Ended { + termination: Some(ProcessTermination::Unknown), + exit_code: None, + signal: None, + output_total: None, + }, + ) + } + + /// A reconciliation found the execution running on the Runner (P1R-C04). Returns whether + /// the record could be written. + pub(crate) fn found_running(&self, process_id: &ProcessId) -> bool { + self.write( + process_id, + "running", + self.store + .record_running(&self.runner, &process_id.0, None, None), + ) + } + + /// The Runner stopped, and reported how each process it held ended (P1R-C04). + pub fn stopped(&self, processes: &[RunnerInventoryEntry]) { + for entry in processes { + if entry.status.state == ProcessState::Exited { + self.ended(&entry.status); + } + } } /// A gateway observed the execution's output up to `cursor`. @@ -163,24 +285,28 @@ impl ProcessRecords { ); } - fn end(&self, process_id: &ProcessId, end: &ProcessEnd) { + fn end(&self, process_id: &ProcessId, end: &ProcessEnd) -> bool { self.write( process_id, "end", self.store.record_end(&self.runner, &process_id.0, end), - ); + ) } /// A record that cannot be updated keeps what it said: the execution goes on, and the - /// Runner remains the authority on it. - fn write(&self, process_id: &ProcessId, what: &str, written: Result) { - if let Err(err) = written { - tracing::warn!( - process_id = %process_id.0, - record = what, - error = %err.message, - "the process record was not updated" - ); + /// Runner remains the authority on it. Returns whether nothing failed. + fn write(&self, process_id: &ProcessId, what: &str, written: Result) -> bool { + match written { + Ok(_) => true, + Err(err) => { + tracing::warn!( + process_id = %process_id.0, + record = what, + error = %err.message, + "the process record was not updated" + ); + false + } } } } diff --git a/crates/server/src/recovery.rs b/crates/server/src/recovery.rs new file mode 100644 index 0000000..43304dc --- /dev/null +++ b/crates/server/src/recovery.rs @@ -0,0 +1,939 @@ +//! Reconciliation of a gateway with its independent Runner after each Claim (P1R-C04). +//! +//! A gateway that claims an independent Runner, when it starts or again after it lost its +//! connection, compares three accounts of what earlier claims left: its process records, what +//! the Runner holds and remembers of each process it was asked to start, and, for an execution +//! that takes part in the resource authority, the authority's attempt, which the Runner looks +//! up. Until a workspace is reconciled, the gateway and the Runner both refuse new executions, +//! patches and approval resumes there with `RECOVERY_PENDING`; the processes the gateway +//! inherits can always be queried, read, written to, resized and terminated. +//! +//! | Record | The record's Runner | Attempt (`required` only) | Outcome | +//! | --- | --- | --- | --- | +//! | dispatching or running | holds it running | a helper claimed it | the workspace is held for the process again | +//! | dispatching or running | ended it, held or dropped | settled | recorded as ended, and its lease released | +//! | dispatching | never received it, or created nothing | none, or settled before it started | recorded as not started, and its approval given back | +//! | none | holds it running | any | the workspace stays blocked | +//! | anything else | | | the workspace stays blocked: a failed lookup is asked again, a mismatch waits for an operator | +//! +//! A Runner proves that it never received a request only for itself: a record sent to another +//! Runner keeps its workspace blocked. An approval left `resuming` is consumed with the process +//! the Runner created for it, given back when the Runner proved that it created none, and +//! otherwise left as it is. A workspace that waits is reconciled again after each exit, and +//! one that waits for an answer also after a growing pause. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak}; +use std::time::Duration; + +use codespace_domain::{ + ApprovalId, ApprovalState, ErrorBody, ErrorCode, ExecCommandResult, ExecDispatchStatus, + OperationResumeResult, ProcessId, ProcessState, +}; +use codespace_runner::{Recalled, RunnerClaim, RunnerProcessStatus, UdsRunner}; +use codespace_store::{ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, Store}; +use tokio::sync::watch; + +use crate::records::{Leases, ProcessRecords}; + +/// The pause before a reconciliation that left a workspace waiting for an answer is tried +/// again, doubled after each further try up to [`RETRY_MAX`]. +const RETRY_START: Duration = Duration::from_millis(500); +/// The longest pause between two tries. +const RETRY_MAX: Duration = Duration::from_secs(30); + +/// Why nothing new starts in a workspace yet. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Bar { + /// What its reconciliation needs is not known yet: it is tried again. + Pending(String), + /// The records, the Runner and the resource authority do not agree, or this gateway cannot + /// ask about it: an operator decides. + Blocked(String), +} + +/// What new work waits for. +#[derive(Default)] +struct Gate { + /// The controller epoch of the last Claim taken into account. + claimed: Option, + /// Why the process records could not be read, if they could not: nothing new starts. + unreadable: Option, + /// The workspaces where nothing new starts yet, and why. + barred: BTreeMap, +} + +/// What a reconciliation left. +#[derive(Debug, Default, Clone, Copy)] +struct Outcome { + /// Something waits for an answer that may come later. + pending: bool, + /// Some workspace waits. + barred: bool, +} + +/// The reconciliation of a gateway with the independent Runner it claimed (P1R-C04). +pub struct Recovery { + store: Arc, + runner: UdsRunner, + records: Arc, + leases: Arc, + gate: Mutex, + /// The number of reconciliations so far. + passes: watch::Sender, +} + +impl Recovery { + /// Reconcile what earlier claims of `runner` left, from its Claim `claim` on, and again + /// after each later Claim. Until a workspace is reconciled, [`Self::admit`] refuses new work + /// there. + pub fn start( + store: Arc, + runner: UdsRunner, + records: Arc, + leases: Arc, + claim: &RunnerClaim, + ) -> Arc { + let recovery = Arc::new(Self { + store, + runner, + records, + leases, + gate: Mutex::new(Gate::default()), + passes: watch::Sender::new(0), + }); + recovery.take_account(claim); + tokio::spawn(run( + Arc::downgrade(&recovery), + recovery.runner.claims(), + recovery.records.ends(), + )); + recovery + } + + fn gate(&self) -> MutexGuard<'_, Gate> { + self.gate.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Whether new work starts in `workspace_id` under the controller epoch `epoch`: refused + /// with `RECOVERY_PENDING` until the gateway reconciled that workspace after the Claim of + /// `epoch`. Work under no epoch, for a worker that is not claimed, is not reconciled. + pub fn admit(&self, workspace_id: &str, epoch: Option) -> Result<(), ErrorBody> { + let Some(epoch) = epoch else { + return Ok(()); + }; + if self.gate().claimed.is_none_or(|claimed| claimed < epoch) { + if let Some(claim) = self.runner.claimed().filter(|claim| claim.epoch == epoch) { + self.take_account(&claim); + } + } + let gate = self.gate(); + if gate.claimed != Some(epoch) { + return Err(ErrorBody::new( + ErrorCode::RecoveryPending, + "this gateway claimed its independent Runner again and has not taken account of \ + that Claim yet, so nothing new is started; its processes can be queried, read, \ + written to, resized and terminated. Retry shortly", + )); + } + if let Some(reason) = &gate.unreadable { + return Err(ErrorBody::new( + ErrorCode::RecoveryPending, + format!( + "this gateway could not read its process records to reconcile them with its \ + independent Runner ({reason}), so nothing new is started; its processes can \ + be queried, read, written to, resized and terminated, and it tries again by \ + itself" + ), + )); + } + match gate.barred.get(workspace_id) { + None => Ok(()), + Some(Bar::Pending(reason)) => Err(ErrorBody::new( + ErrorCode::RecoveryPending, + format!( + "nothing new is started in workspace `{workspace_id}` until this gateway \ + reconciled it with its independent Runner: {reason}. Its processes can be \ + queried, read, written to, resized and terminated, and the gateway tries \ + again by itself" + ), + )), + Some(Bar::Blocked(reason)) => Err(ErrorBody::new( + ErrorCode::RecoveryPending, + format!( + "nothing new is started in workspace `{workspace_id}`: {reason}, so this \ + gateway cannot reconcile it with its independent Runner until an operator \ + resolves it. Its processes can be queried, read, written to, resized and \ + terminated" + ), + )), + } + } + + /// The number of reconciliations so far, as each one ends. For tests. + #[doc(hidden)] + pub fn passes(&self) -> watch::Receiver { + self.passes.subscribe() + } + + /// Take account of the Claim `claim`, once: until it is reconciled, nothing new starts in a + /// workspace where an earlier controller started work, where the Runner holds a running + /// process, or that a record left to reconcile names. + fn take_account(&self, claim: &RunnerClaim) { + if self + .gate() + .claimed + .is_some_and(|claimed| claimed >= claim.epoch) + { + return; + } + let waiting = Bar::Pending("its reconciliation after the Claim has not finished".into()); + let mut barred: BTreeMap = claim + .barrier + .iter() + .map(|workspace_id| (workspace_id.clone(), waiting.clone())) + .collect(); + for entry in &claim.processes { + if entry.status.state == ProcessState::Running { + barred.insert(entry.workspace_id.clone(), waiting.clone()); + } + } + let unreadable = match self + .store + .records_to_reconcile(self.records.runner(), claim.epoch) + { + Ok(records) => { + for record in records { + barred.insert(record.workspace_id, waiting.clone()); + } + None + } + Err(err) => Some(err.message), + }; + let mut gate = self.gate(); + if gate.claimed.is_some_and(|claimed| claimed >= claim.epoch) { + return; + } + *gate = Gate { + claimed: Some(claim.epoch), + unreadable, + barred, + }; + } + + /// Reconcile what the claims before `claim` left, while `claim` is the current one. + async fn pass(&self, claim: &RunnerClaim) -> Outcome { + let epoch = claim.epoch; + if self.runner.claimed().map(|claimed| claimed.epoch) != Some(epoch) { + // The connection is lost or was claimed again: a later Claim is reconciled. + return Outcome { + pending: true, + barred: true, + }; + } + let records = match self + .store + .records_to_reconcile(self.records.runner(), epoch) + { + Ok(records) => records, + Err(err) => { + let mut gate = self.gate(); + if gate.claimed == Some(epoch) { + gate.unreadable = Some(err.message); + } + return Outcome { + pending: true, + barred: true, + }; + } + }; + // From before the Runner is asked: a process that exits after it was reported running + // is never held again. + self.leases.note_exits(); + let bars = self + .reconcile(&self.runner.at_epoch(epoch), claim, &records) + .await; + self.leases.forget_exits(); + let outcome = Outcome { + pending: bars.values().any(|bar| matches!(bar, Bar::Pending(_))), + barred: !bars.is_empty(), + }; + let mut gate = self.gate(); + if gate.claimed == Some(epoch) { + log_changes(&gate.barred, &bars); + gate.barred = bars; + gate.unreadable = None; + } + outcome + } + + /// Reconcile `records` with what `runner` holds and remembers, and with the resource + /// authority; then tell the Runner which workspaces of `claim`'s barrier are reconciled. + /// Returns the workspaces where nothing new starts yet. + async fn reconcile( + &self, + runner: &UdsRunner, + claim: &RunnerClaim, + records: &[ProcessRecord], + ) -> BTreeMap { + let mut bars = Bars::default(); + let ours = self.records.runner(); + let mut decisions: Vec> = Vec::with_capacity(records.len()); + let mut asked = Vec::new(); + for (index, record) in records.iter().enumerate() { + decisions.push(match record.phase { + ProcessPhase::Terminal => Some(Decision::Settled { + started: record.outcome != Some(ProcessOutcome::NotStarted), + }), + // Only the Runner that was sent it can prove what became of it (P1R-C05). + _ if record.runner != *ours => Some(Decision::Blocked(format!( + "process `{}` was sent to another Runner, which this gateway cannot ask \ + what became of it", + record.process_id + ))), + ProcessPhase::Unknown => Some(Decision::Blocked(format!( + "the record of process `{}` says that what became of it cannot be learned", + record.process_id + ))), + ProcessPhase::Dispatching | ProcessPhase::Running => { + asked.push(index); + None + } + }); + } + // A running process that no record left to reconcile accounts for. + let accounted: HashSet<&str> = records + .iter() + .filter(|record| record.phase != ProcessPhase::Terminal) + .map(|record| record.process_id.as_str()) + .collect(); + let suspects: Vec<_> = claim + .processes + .iter() + .filter(|entry| { + entry.status.state == ProcessState::Running + && !accounted.contains(entry.status.process_id.0.as_str()) + }) + .collect(); + let mut ids: Vec = asked + .iter() + .map(|&index| ProcessId(records[index].process_id.clone())) + .collect(); + ids.extend(suspects.iter().map(|entry| entry.status.process_id.clone())); + let recalled: Result, String> = if ids.is_empty() { + Ok(HashMap::new()) + } else { + runner + .recall(&ids) + .await + .map(|recalled| { + recalled + .into_iter() + .map(|process| (process.process_id.0, process.recalled)) + .collect() + }) + .map_err(|err| err.to_string()) + }; + match &recalled { + Ok(remembered) => { + let attempts = look_up(runner, records, &asked).await; + for &index in &asked { + let record = &records[index]; + decisions[index] = Some(match remembered.get(&record.process_id) { + None => Decision::Pending(format!( + "the Runner did not answer about process `{}`", + record.process_id + )), + Some(recalled) => match record.participation { + RecordedParticipation::Off => decide(record, recalled), + RecordedParticipation::Required => { + decide_required(record, recalled, &attempts) + } + }, + }); + } + } + Err(err) => { + for &index in &asked { + decisions[index] = Some(Decision::Pending(format!( + "the Runner could not be asked about process `{}`: {err}", + records[index].process_id + ))); + } + } + } + + // The executions of each approval being resumed, oldest first, with its workspace. + let mut resumed: BTreeMap<&str, (&ApprovalId, &str, Vec)> = BTreeMap::new(); + let mut acknowledged = Vec::new(); + for (record, decision) in records.iter().zip(decisions) { + let decision = decision.expect("every record is decided"); + let link = self.apply(record, decision, &mut bars, &mut acknowledged); + if let Some(approval_id) = &record.approval_id { + resumed + .entry(&approval_id.0) + .or_insert_with(|| (approval_id, &record.workspace_id, Vec::new())) + .2 + .push(link); + } + } + let reconciled: HashSet<&str> = records + .iter() + .map(|record| record.process_id.as_str()) + .collect(); + for (approval_id, workspace_id, links) in resumed.into_values() { + if let Err(err) = self.settle(approval_id, &links, &reconciled) { + bars.pend( + workspace_id, + format!( + "approval `{}` could not be settled: {}", + approval_id.0, err.message + ), + ); + } + } + + for entry in &suspects { + let process_id = &entry.status.process_id; + match &recalled { + Ok(remembered) => match remembered.get(&process_id.0) { + Some(Recalled::Held { + workspace_id, + status, + }) if status.state == ProcessState::Running => bars.block( + workspace_id, + format!( + "process `{}` runs there, and no record of this gateway accounts \ + for it", + process_id.0 + ), + ), + // It ended: nothing of it is left to record. + Some(Recalled::Held { .. }) => acknowledged.push(process_id.clone()), + _ => {} + }, + Err(err) => bars.pend( + &entry.workspace_id, + format!( + "the Runner could not be asked about process `{}`: {err}", + process_id.0 + ), + ), + } + } + // Ended processes the Runner keeps although their records say how they ended, or have + // none: an acknowledgement of an earlier gateway that was lost. + acknowledged.extend( + claim + .processes + .iter() + .filter(|entry| { + entry.status.state == ProcessState::Exited + && !accounted.contains(entry.status.process_id.0.as_str()) + }) + .map(|entry| entry.status.process_id.clone()), + ); + + let reconciled: Vec = claim + .barrier + .iter() + .filter(|workspace_id| !bars.0.contains_key(*workspace_id)) + .cloned() + .collect(); + if !reconciled.is_empty() { + if let Err(err) = runner.reconciled(&reconciled).await { + for workspace_id in &reconciled { + bars.pend( + workspace_id, + format!("the Runner could not be told that it is reconciled: {err}"), + ); + } + } + } + if !acknowledged.is_empty() { + if let Err(err) = runner.acknowledge(&acknowledged).await { + tracing::debug!( + error = %err, + "the Runner was not told that the ends of the processes it keeps are \ + recorded" + ); + } + } + bars.0 + } + + /// Act on `decision` for `record`, and return what it says of the execution's start. + fn apply( + &self, + record: &ProcessRecord, + decision: Decision, + bars: &mut Bars, + acknowledged: &mut Vec, + ) -> Link { + let process_id = ProcessId(record.process_id.clone()); + let unwritten = || { + format!( + "the record of process `{}` could not be updated", + record.process_id + ) + }; + match decision { + Decision::Running => { + if !self.records.found_running(&process_id) { + bars.pend(&record.workspace_id, unwritten()); + } + if let Err(err) = self + .leases + .restore(&record.workspace_id, &record.process_id) + { + bars.pend( + &record.workspace_id, + format!( + "process `{}` runs there, and the workspace could not be held for \ + it: {}", + record.process_id, err.message + ), + ); + } + Link::Started(process_id) + } + Decision::Ended(status) => { + let written = match &status { + Some(status) => self.records.ended(status), + None => self.records.dropped(&process_id), + }; + self.leases.release(&record.process_id); + if !written { + bars.pend(&record.workspace_id, unwritten()); + } else if status.is_some() { + acknowledged.push(process_id.clone()); + } + Link::Started(process_id) + } + Decision::NotStarted => { + let written = self.records.not_started(&process_id); + self.leases.release(&record.process_id); + if written { + Link::NotStarted + } else { + bars.pend(&record.workspace_id, unwritten()); + Link::Unknown + } + } + Decision::Settled { started: true } => Link::Started(process_id), + Decision::Settled { started: false } => Link::NotStarted, + Decision::Pending(reason) => { + bars.pend(&record.workspace_id, reason); + Link::Unknown + } + Decision::Blocked(reason) => { + bars.block(&record.workspace_id, reason); + Link::Unknown + } + } + } + + /// Settle the approval `approval_id`, which its executions in `links` resumed: consumed + /// with the first one that started, given back when none did, and left as it is when + /// whether one started is not known. An approval resumed by an execution that this + /// reconciliation did not take up, one under the current Claim, is left to that resume. + fn settle( + &self, + approval_id: &ApprovalId, + links: &[Link], + reconciled: &HashSet<&str>, + ) -> Result<(), ErrorBody> { + if self + .store + .records_of_approval(approval_id)? + .iter() + .any(|record| !reconciled.contains(record.process_id.as_str())) + { + return Ok(()); + } + let started = links.iter().find_map(|link| match link { + Link::Started(process_id) => Some(process_id), + _ => None, + }); + match started { + Some(process_id) => { + let consumed = consumed_by(approval_id, process_id); + self.store + .settle_interrupted_resume(approval_id, Some(&consumed))?; + } + None if links.iter().all(|link| matches!(link, Link::NotStarted)) => { + self.store.settle_interrupted_resume(approval_id, None)?; + } + None => {} + } + Ok(()) + } +} + +/// Reconcile after each Claim of the runner whose Claims `claims` reports, and again while a +/// workspace waits: after each end that `ends` counts, and after a growing pause while one +/// waits for an answer. Ends with the reconciliation. +async fn run( + recovery: Weak, + mut claims: watch::Receiver>, + mut ends: watch::Receiver, +) { + let mut claim = claims.borrow_and_update().clone(); + let mut retry = RETRY_START; + loop { + ends.borrow_and_update(); + let outcome = match &claim { + Some(claim) => { + let Some(recovery) = recovery.upgrade() else { + return; + }; + recovery.take_account(claim); + let outcome = recovery.pass(claim).await; + recovery.passes.send_modify(|passes| *passes += 1); + outcome + } + None => Outcome::default(), + }; + tokio::select! { + changed = claims.changed() => { + if changed.is_err() { + return; + } + let next = claims.borrow_and_update().clone(); + if next.as_ref().map(|next| next.epoch) > claim.as_ref().map(|claim| claim.epoch) { + claim = next; + retry = RETRY_START; + } + } + changed = ends.changed(), if outcome.barred => { + if changed.is_err() { + return; + } + } + () = tokio::time::sleep(retry), if outcome.pending => { + retry = (retry * 2).min(RETRY_MAX); + } + } + } +} + +/// What a reconciliation learned of an execution. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Decision { + /// The Runner holds it running: its workspace is held for it again. + Running, + /// The Runner created it, and it ended: with this status while the Runner holds it. + Ended(Option), + /// The Runner proved that it never created it, and the resource authority agrees. + NotStarted, + /// Its record already says how it ended, or that it never started. + Settled { started: bool }, + /// What became of it is not known yet. + Pending(String), + /// The record, the Runner and the resource authority do not agree, or this gateway cannot + /// ask about it. + Blocked(String), +} + +/// What a reconciliation learned of an approval's execution. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Link { + /// The Runner created this process for it. + Started(ProcessId), + /// It never started. + NotStarted, + /// Whether it started is not known. + Unknown, +} + +/// The workspaces where nothing new starts yet: one an operator decides on stays so whatever +/// else waits there. +#[derive(Default)] +struct Bars(BTreeMap); + +impl Bars { + fn pend(&mut self, workspace_id: &str, reason: String) { + self.0 + .entry(workspace_id.to_string()) + .or_insert(Bar::Pending(reason)); + } + + fn block(&mut self, workspace_id: &str, reason: String) { + match self.0.get(workspace_id) { + Some(Bar::Blocked(_)) => {} + _ => { + self.0 + .insert(workspace_id.to_string(), Bar::Blocked(reason)); + } + } + } +} + +/// What became of an execution that takes no part in the resource authority, as its Runner +/// recalled it. +fn decide(record: &ProcessRecord, recalled: &Recalled) -> Decision { + match recalled { + Recalled::Held { workspace_id, .. } if *workspace_id != record.workspace_id => { + Decision::Blocked(format!( + "the Runner holds process `{}` in workspace `{workspace_id}`, not where its \ + record says", + record.process_id + )) + } + Recalled::Held { status, .. } if status.state == ProcessState::Running => Decision::Running, + Recalled::Held { status, .. } => Decision::Ended(Some(status.clone())), + Recalled::Dropped => Decision::Ended(None), + Recalled::NotReceived | Recalled::Refused if record.phase == ProcessPhase::Dispatching => { + Decision::NotStarted + } + Recalled::NotReceived | Recalled::Refused => Decision::Blocked(format!( + "the record of process `{}` says the Runner started it, and the Runner {}", + record.process_id, + recalled_as(recalled) + )), + } +} + +/// How the Runner recalled a process, for a message. +fn recalled_as(recalled: &Recalled) -> &'static str { + match recalled { + Recalled::NotReceived => "never received it", + Recalled::Refused => "created nothing for it", + Recalled::Held { status, .. } if status.state == ProcessState::Running => { + "holds it running" + } + Recalled::Held { .. } => "holds it ended", + Recalled::Dropped => "created it and no longer holds it", + } +} + +/// The attempts of governed executions, as the Runner looked them up with the resource +/// authority, by process. +#[cfg(feature = "devguard")] +type Attempts = HashMap; + +/// A gateway without DevGuard looks up no attempt. +#[cfg(not(feature = "devguard"))] +struct Attempts; + +/// The attempts of the governed executions among `records` at `asked`. +#[cfg(feature = "devguard")] +async fn look_up(runner: &UdsRunner, records: &[ProcessRecord], asked: &[usize]) -> Attempts { + use codespace_runner::admission::AttemptRef; + let attempts: Vec = asked + .iter() + .map(|&index| &records[index]) + .filter(|record| record.participation == RecordedParticipation::Required) + .filter_map(|record| { + record.attempt_id.as_ref().map(|attempt_id| AttemptRef { + attempt_id: attempt_id.clone(), + process_id: ProcessId(record.process_id.clone()), + }) + }) + .collect(); + if attempts.is_empty() { + return HashMap::new(); + } + match runner.look_up_attempts(attempts.clone()).await { + Ok(found) => found + .into_iter() + .map(|lookup| (lookup.process_id.0, Attempt::from(lookup.found))) + .collect(), + Err(err) => { + tracing::debug!(error = %err, "the Runner could not look up attempts"); + attempts + .into_iter() + .map(|attempt| (attempt.process_id.0, Attempt::Failed)) + .collect() + } + } +} + +#[cfg(not(feature = "devguard"))] +async fn look_up(_runner: &UdsRunner, _records: &[ProcessRecord], _asked: &[usize]) -> Attempts { + Attempts +} + +/// What became of a governed execution, as its Runner recalled it and the resource authority +/// states its attempt. +#[cfg(feature = "devguard")] +fn decide_required(record: &ProcessRecord, recalled: &Recalled, attempts: &Attempts) -> Decision { + let (Some(attempt_id), Some(attempt)) = ( + record.attempt_id.as_deref(), + attempts.get(&record.process_id), + ) else { + return Decision::Blocked(format!( + "process `{}` takes part in the resource authority, and its record names no attempt \ + to look up", + record.process_id + )); + }; + if let Recalled::Held { workspace_id, .. } = recalled { + if *workspace_id != record.workspace_id { + return decide(record, recalled); + } + } + let mismatch = || { + Decision::Blocked(format!( + "the Runner and the resource authority do not agree on process `{}`: the Runner {}, \ + and attempt `{attempt_id}` is {}", + record.process_id, + recalled_as(recalled), + attempt.as_str() + )) + }; + let unsettled = || { + Decision::Pending(format!( + "the resource authority has not settled attempt `{attempt_id}` of process `{}` yet", + record.process_id + )) + }; + match (recalled, attempt) { + (_, Attempt::Failed) => Decision::Pending(format!( + "the resource authority could not be asked about attempt `{attempt_id}` of process \ + `{}`", + record.process_id + )), + (Recalled::Held { status, .. }, attempt) if status.state == ProcessState::Running => { + match attempt { + Attempt::Launched => Decision::Running, + Attempt::Unsettled => unsettled(), + _ => mismatch(), + } + } + (Recalled::Held { .. } | Recalled::Dropped, attempt) => match attempt { + Attempt::ScopeEnded | Attempt::SettledBeforeStart => decide(record, recalled), + Attempt::Launched | Attempt::Unsettled => unsettled(), + _ => mismatch(), + }, + (Recalled::NotReceived | Recalled::Refused, _) + if record.phase != ProcessPhase::Dispatching => + { + decide(record, recalled) + } + (Recalled::NotReceived | Recalled::Refused, attempt) => match attempt { + Attempt::Absent | Attempt::SettledBeforeStart => Decision::NotStarted, + Attempt::Unsettled => unsettled(), + _ => mismatch(), + }, + } +} + +/// A gateway without DevGuard cannot ask about a governed execution. +#[cfg(not(feature = "devguard"))] +fn decide_required(record: &ProcessRecord, _recalled: &Recalled, _attempts: &Attempts) -> Decision { + Decision::Blocked(format!( + "process `{}` takes part in the resource authority, which this gateway cannot ask about", + record.process_id + )) +} + +/// A governed execution's attempt, as the resource authority states it. +#[cfg(feature = "devguard")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Attempt { + /// A launch helper claimed it: the authority holds its reservation. + Launched, + /// The authority observed its scope end, and released it. + ScopeEnded, + /// It ended before it started: denied, cancelled, expired, or released unlaunched. + SettledBeforeStart, + /// No attempt exists: the Runner never asked the authority to admit it, or the authority + /// has no record of it. + Absent, + /// The authority holds it in a phase that may still change. + Unsettled, + /// The authority could not be asked. + Failed, +} + +#[cfg(feature = "devguard")] +impl Attempt { + fn as_str(self) -> &'static str { + match self { + Self::Launched => "launched", + Self::ScopeEnded => "released after its scope ended", + Self::SettledBeforeStart => "settled before it started", + Self::Absent => "absent", + Self::Unsettled => "not settled", + Self::Failed => "not known", + } + } +} + +#[cfg(feature = "devguard")] +impl From for Attempt { + fn from(found: codespace_runner::admission::LookedUp) -> Self { + use codespace_runner::admission::{AttemptState, LookedUp}; + match found { + LookedUp::NotAdmitted | LookedUp::NotFound => Self::Absent, + LookedUp::Failed => Self::Failed, + LookedUp::Found(state) => match state { + AttemptState::Launched => Self::Launched, + AttemptState::ScopeEnded => Self::ScopeEnded, + AttemptState::Denied + | AttemptState::Cancelled + | AttemptState::Expired + | AttemptState::Released => Self::SettledBeforeStart, + AttemptState::NotAsked | AttemptState::NotRecorded => Self::Absent, + AttemptState::Other | AttemptState::Unknown => Self::Unsettled, + }, + } + } +} + +/// An approval consumed by the execution `process_id`, which the Runner created for it. +pub(crate) fn consumed_by( + approval_id: &ApprovalId, + process_id: &ProcessId, +) -> OperationResumeResult { + OperationResumeResult { + approval_id: approval_id.clone(), + state: ApprovalState::Consumed, + apply_patch: None, + exec_command: Some(ExecCommandResult { + process_id: process_id.clone(), + dispatch_status: ExecDispatchStatus::Confirmed, + coordination: None, + }), + } +} + +/// Whether a record proves that its execution started: the Runner answered that it started +/// it, or reported how it ended. +pub(crate) fn proves_started(record: &ProcessRecord) -> bool { + match record.phase { + ProcessPhase::Running => true, + ProcessPhase::Terminal => record.outcome != Some(ProcessOutcome::NotStarted), + ProcessPhase::Dispatching | ProcessPhase::Unknown => false, + } +} + +/// Log what changed in the workspaces where nothing new starts. +fn log_changes(before: &BTreeMap, after: &BTreeMap) { + for (workspace_id, bar) in after { + if before.get(workspace_id) == Some(bar) { + continue; + } + match bar { + Bar::Pending(reason) => tracing::info!( + workspace_id, + reason, + "this workspace waits for its reconciliation with the independent Runner; it is \ + tried again" + ), + Bar::Blocked(reason) => tracing::warn!( + workspace_id, + reason, + "this workspace cannot be reconciled with the independent Runner, so nothing new \ + starts there until an operator resolves it" + ), + } + } + for workspace_id in before.keys().filter(|ws| !after.contains_key(*ws)) { + tracing::info!( + workspace_id, + "reconciled with the independent Runner: new work starts there again" + ); + } +} diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index b4474d7..71284c8 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -218,14 +218,17 @@ impl Drop for RuntimeProcess { } /// After a Claim that found work an earlier controller started, this one included before it -/// lost its connection: nothing new starts (P1R-C01, P1R-C03). +/// lost its connection: nothing new starts in those workspaces until the gateway reconciled +/// them (P1R-C01, P1R-C03, P1R-C04). fn warn_recovery_pending(claim: &RunnerClaim) { if claim.previous_controller { tracing::warn!( epoch = claim.epoch, + barrier = ?claim.barrier, "an earlier controller started work on this independent Runner: its processes can be \ - queried, read, written to, resized and terminated, but new executions, patches and \ - approval resumes are refused with RECOVERY_PENDING" + queried, read, written to, resized and terminated, and new executions, patches and \ + approval resumes in those workspaces are refused with RECOVERY_PENDING until this \ + gateway reconciled them" ); } } diff --git a/crates/server/tests/independent_runner.rs b/crates/server/tests/independent_runner.rs index 40a3447..6d18d7f 100644 --- a/crates/server/tests/independent_runner.rs +++ b/crates/server/tests/independent_runner.rs @@ -1,10 +1,11 @@ //! The gateway with an independent Runner (P1R-C01), each started as an operator starts it: the -//! Runner outlives a gateway, the next gateway inherits its processes but starts nothing, and a -//! gateway that ends leaves or stops the Runner as its exit intent says. Each gateway records -//! its executions in the operations database, which it is the only writer of (P1R-C02), and -//! takes control of the Runner with a Claim for that database (P1R-C03). The tests with a -//! Runner need the worker binary (`CODESPACE_RUNTIME_BIN`) and return early without it, as the -//! gateway's other worker tests do. +//! Runner outlives a gateway, the next gateway inherits its processes, and a gateway that ends +//! leaves or stops the Runner as its exit intent says. Each gateway records its executions in +//! the operations database, which it is the only writer of (P1R-C02), takes control of the +//! Runner with a Claim for that database (P1R-C03), and reconciles what earlier controllers +//! left before it starts work where they did (P1R-C04). The tests with a Runner need the +//! worker binary (`CODESPACE_RUNTIME_BIN`) and return early without it, as the gateway's other +//! worker tests do. use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Stdio}; @@ -245,6 +246,20 @@ impl Gateway { self.started_in("demo", command).await } + /// The refusal of new work in `demo` once the gateway reconciled it (P1R-C04): the + /// refusals with RECOVERY_PENDING while it reconciles are passed over. + async fn refused_once_reconciled(&self) -> serde_json::Value { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let refused = error_body(&self.exec(&["/bin/echo", "new"]).await); + if refused["code"] != "RECOVERY_PENDING" { + return refused; + } + assert!(Instant::now() < deadline, "not reconciled: {refused}"); + sleep(Duration::from_millis(40)).await; + } + } + async fn started_in(&self, workspace_id: &str, command: &[&str]) -> String { let started = self.exec_in(workspace_id, command).await.expect("exec"); let body = payload(&started); @@ -369,8 +384,9 @@ fn parent_of(pid: u32) -> Option { /// A gateway killed while its process runs leaves the process to the Runner: the next gateway /// reads it and sees it end at its original timeout, not at one restarted when that gateway -/// connected, but starts nothing itself, and its normal shutdown with the intent `stop` then -/// stops the Runner (P1R-C01). +/// connected; it reconciles, and holds the workspace for the process again, so new work there +/// is busy (P1R-C04); and its normal shutdown with the intent `stop` then stops the Runner +/// (P1R-C01). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn an_independent_runner_keeps_a_killed_gateways_process_to_its_original_timeout() { let Some(bin) = runtime_bin() else { @@ -391,8 +407,8 @@ async fn an_independent_runner_keeps_a_killed_gateways_process_to_its_original_t let next = setup.gateway("stop").await; assert_eq!(next.status(&process_id).await["state"], "running"); next.output(&process_id, "started").await; - let refused = next.exec(&["/bin/echo", "new"]).await; - assert_eq!(error_body(&refused)["code"], "RECOVERY_PENDING"); + let refused = next.refused_once_reconciled().await; + assert_eq!(refused["code"], "WORKSPACE_BUSY", "{refused}"); let status = loop { let status = next.status(&process_id).await; if status["state"] == "exited" { @@ -740,8 +756,8 @@ async fn a_claim_is_refused_while_a_stopped_gateway_holds_the_runner_and_granted let next = setup.gateway("stop").await; assert_eq!(next.status(&process_id).await["state"], "running"); - let refused = next.exec(&["/bin/echo", "new"]).await; - assert_eq!(error_body(&refused)["code"], "RECOVERY_PENDING"); + let refused = next.refused_once_reconciled().await; + assert_eq!(refused["code"], "WORKSPACE_BUSY", "{refused}"); let ended = next.shut_down().await; assert!(ended.success(), "{ended:?}"); let status = exited(&mut runner).await; diff --git a/crates/store/src/approvals.rs b/crates/store/src/approvals.rs index 973bd3f..0084f7c 100644 --- a/crates/store/src/approvals.rs +++ b/crates/store/src/approvals.rs @@ -348,6 +348,42 @@ impl Store { mark_resuming_on(&conn, approval_id) } + /// The approval with this ID, if there is one. + pub fn approval(&self, approval_id: &ApprovalId) -> Result, ErrorBody> { + let conn = self.conn.lock().expect("sqlite mutex"); + load(&conn, &approval_id.0)? + .map(Row::into_record) + .transpose() + } + + /// Settle an approval whose resume a gateway left `resuming` (P1R-C04), once its + /// reconciliation learned what became of the execution: `Some` result consumes it with + /// the process that the Runner created, `None` gives it back unconsumed because the + /// Runner proved that it created none. Returns whether the approval was settled: not when + /// it is no longer being resumed. Refused while a resume of it is in progress, which + /// settles it itself. + pub fn settle_interrupted_resume( + &self, + approval_id: &ApprovalId, + started: Option<&OperationResumeResult>, + ) -> Result { + let _guard = self.mark_inflight(&approval_id.0)?; + let state = { + let conn = self.conn.lock().expect("sqlite mutex"); + load(&conn, &approval_id.0)? + .ok_or_else(|| not_found(approval_id))? + .state + }; + if state != ApprovalState::Resuming.as_str() { + return Ok(false); + } + match started { + Some(result) => self.finish_resume(approval_id, Ok(result))?, + None => self.return_resume(approval_id)?, + } + Ok(true) + } + pub fn fail_next_finish(&self) { *self.fail_next_finish.lock().expect("fail flag") = true; } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 5f61904..26726e8 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -354,6 +354,15 @@ impl Store { .confirm_process(process_id); } + /// Hold the workspace for a live process found again after a reconnect (P1R-C04): see + /// the resource serializer's `hold_process`. + pub fn hold_process(&self, workspace_id: &str, process_id: &str) -> Result<(), ErrorBody> { + self.locks + .lock() + .expect("lock mutex") + .hold_process(workspace_id, process_id) + } + pub fn clear_shell(&self, workspace_id: &str) { self.locks .lock() @@ -1114,6 +1123,43 @@ mod tests { assert_eq!(status.events[1].name, OperationEventName::Finished); } + /// A live process found again after a reconnect holds its workspace as a confirmed + /// process does: new work is busy, holding it again for the same process changes nothing, + /// a spawn reservation of that process becomes its hold, anything else holding the + /// workspace refuses it, and the process's release frees it (P1R-C04). + #[tokio::test] + async fn a_process_found_again_holds_its_workspace() { + let store = Store::memory().unwrap(); + let demo = Resource::Workspace("demo".into()); + store.hold_process("demo", "proc-1").unwrap(); + store.hold_process("demo", "proc-1").unwrap(); + assert_eq!(store.exclusive_kind(&demo), Some("process")); + assert_eq!( + store.hold_process("demo", "proc-2").unwrap_err().code, + ErrorCode::WorkspaceBusy + ); + assert_eq!( + store.acquire_write("demo").await.err().map(|err| err.code), + Some(ErrorCode::WorkspaceBusy) + ); + store.release_process("proc-1"); + assert_eq!(store.exclusive_kind(&demo), None); + + let mut reservation = store.acquire_shell_busy("demo", "proc-3").await.unwrap(); + reservation.arm_dispatch(); + store.hold_process("demo", "proc-3").unwrap(); + assert_eq!(store.exclusive_kind(&demo), Some("process")); + drop(reservation); + assert_eq!(store.exclusive_kind(&demo), Some("process")); + store.release_process("proc-3"); + + let _write = store.acquire_write("demo").await.unwrap(); + assert_eq!( + store.hold_process("demo", "proc-4").unwrap_err().code, + ErrorCode::WorkspaceBusy + ); + } + #[tokio::test] async fn waiter_closed_is_not_workspace_busy() { let store = Store::memory().unwrap(); diff --git a/crates/store/src/processes.rs b/crates/store/src/processes.rs index ce4859b..46e4a49 100644 --- a/crates/store/src/processes.rs +++ b/crates/store/src/processes.rs @@ -357,6 +357,69 @@ impl Store { load(&conn, process_id) } + /// The records that a gateway reconciles after its Claim of `runner` under `epoch` + /// (P1R-C04): every execution sent to another Runner, or to `runner` under an earlier Claim + /// or before Claims were recorded, that has not ended as far as its record says, and every + /// one whose approval is still being resumed. What the gateway sends under its own Claim is + /// not. Oldest first. + pub fn records_to_reconcile( + &self, + runner: &RunnerRef, + epoch: u64, + ) -> Result, ErrorBody> { + let conn = self.conn.lock().expect("sqlite mutex"); + let mut statement = conn + .prepare(&format!( + "{RECORD_SELECT} + WHERE (phase IN ('dispatching', 'running', 'unknown') + OR approval_id IN + (SELECT approval_id FROM approvals WHERE state = 'resuming')) + AND NOT (runner_boot_id IS ?1 AND runner_pid = ?2 + AND runner_started_at_ms = ?3 AND runner_nonce = ?4 + AND controller_epoch IS NOT NULL AND controller_epoch >= ?5) + ORDER BY created_at_ms, process_id" + )) + .map_err(record_err)?; + let rows = statement + .query_map( + params![ + runner.boot_id, + runner.pid, + to_sql(runner.started_at_ms), + runner.nonce, + to_sql(epoch), + ], + map_record, + ) + .map_err(record_err)?; + let mut records = Vec::new(); + for row in rows { + records.push(row.map_err(record_err)??); + } + Ok(records) + } + + /// The records of the executions that resumed this approval, oldest first (P1R-C04). + pub fn records_of_approval( + &self, + approval_id: &ApprovalId, + ) -> Result, ErrorBody> { + let conn = self.conn.lock().expect("sqlite mutex"); + let mut statement = conn + .prepare(&format!( + "{RECORD_SELECT} WHERE approval_id = ?1 ORDER BY created_at_ms, process_id" + )) + .map_err(record_err)?; + let rows = statement + .query_map(params![approval_id.0], map_record) + .map_err(record_err)?; + let mut records = Vec::new(); + for row in rows { + records.push(row.map_err(record_err)??); + } + Ok(records) + } + /// Change the record of an execution that `runner` was sent, and write it back when it /// changed. Records of executions sent to another Runner are never changed. fn change_record( @@ -955,4 +1018,171 @@ mod tests { ); } } + + /// An execution sent under the controller epoch `epoch`, which may resume `approval_id`. + fn dispatch_under( + store: &Store, + process_id: &str, + epoch: Option, + approval_id: Option<&ApprovalId>, + ) { + dispatch_to(store, &runner(), process_id, epoch, approval_id); + } + + /// An execution sent to `runner` under the controller epoch `epoch`, which may resume + /// `approval_id`. + fn dispatch_to( + store: &Store, + runner: &RunnerRef, + process_id: &str, + epoch: Option, + approval_id: Option<&ApprovalId>, + ) { + store + .record_dispatch(&NewProcessRecord { + process_id, + runner, + controller_epoch: epoch, + workspace_id: "demo", + approval_id, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &argv(), + }) + .unwrap(); + } + + fn ids(records: &[ProcessRecord]) -> Vec<&str> { + let mut ids: Vec<_> = records + .iter() + .map(|record| record.process_id.as_str()) + .collect(); + ids.sort_unstable(); + ids + } + + /// After its Claim under epoch 4 a gateway reconciles what earlier Claims, gateways before + /// Claims were recorded, and other Runners left unsettled: executions whose records have + /// not ended, and those whose approval is still being resumed. What it sends under its own + /// Claim, and what ended, are not (P1R-C04). + #[test] + fn a_gateway_reconciles_what_earlier_claims_left_unsettled() { + let store = Store::memory().unwrap(); + let runner = runner(); + dispatch_under(&store, "proc-dispatching", Some(3), None); + dispatch_under(&store, "proc-running", Some(2), None); + store + .record_running(&runner, "proc-running", Some(&root()), None) + .unwrap(); + dispatch_under(&store, "proc-unrecorded-epoch", None, None); + dispatch_under(&store, "proc-ended", Some(3), None); + store + .record_end(&runner, "proc-ended", &exited(0, 4)) + .unwrap(); + let resumed = queued_approval(&store, "resumed"); + dispatch_under(&store, "proc-resumed", Some(3), Some(&resumed)); + store + .record_end(&runner, "proc-resumed", &ProcessEnd::NotStarted) + .unwrap(); + dispatch_under(&store, "proc-current", Some(4), None); + // Another Runner's epochs are its own; a Runner after a reboot is another Runner. + let other = RunnerRef { + nonce: "nonce-b".into(), + ..runner.clone() + }; + dispatch_to(&store, &other, "proc-other-runner", Some(9), None); + let rebooted = RunnerRef { + boot_id: None, + ..runner.clone() + }; + dispatch_to(&store, &rebooted, "proc-other-boot", Some(4), None); + dispatch_to(&store, &other, "proc-other-ended", Some(1), None); + store + .record_end(&other, "proc-other-ended", &exited(0, 1)) + .unwrap(); + assert_eq!( + ids(&store.records_to_reconcile(&runner, 4).unwrap()), + [ + "proc-dispatching", + "proc-other-boot", + "proc-other-runner", + "proc-resumed", + "proc-running", + "proc-unrecorded-epoch" + ] + ); + assert_eq!( + ids(&store.records_to_reconcile(&runner, 3).unwrap()), + [ + "proc-other-boot", + "proc-other-runner", + "proc-running", + "proc-unrecorded-epoch" + ] + ); + // Once its approval is settled, an ended execution needs nothing more. + assert!(store.settle_interrupted_resume(&resumed, None).unwrap()); + assert_eq!( + ids(&store.records_to_reconcile(&runner, 4).unwrap()), + [ + "proc-dispatching", + "proc-other-boot", + "proc-other-runner", + "proc-running", + "proc-unrecorded-epoch" + ] + ); + assert_eq!( + ids(&store.records_of_approval(&resumed).unwrap()), + ["proc-resumed"] + ); + } + + /// An approval that a gateway left `resuming` is consumed with the process the Runner + /// created, or given back when it created none, once; never while a resume of it is in + /// progress (P1R-C04). + #[test] + fn an_interrupted_resume_is_settled_once() { + let store = Store::memory().unwrap(); + let started = queued_approval(&store, "started"); + dispatch_under(&store, "proc-started", Some(1), Some(&started)); + let result = codespace_domain::OperationResumeResult { + approval_id: started.clone(), + state: ApprovalState::Consumed, + apply_patch: None, + exec_command: Some(codespace_domain::ExecCommandResult { + process_id: codespace_domain::ProcessId("proc-started".into()), + dispatch_status: codespace_domain::ExecDispatchStatus::Confirmed, + coordination: None, + }), + }; + assert!(store + .settle_interrupted_resume(&started, Some(&result)) + .unwrap()); + assert_eq!(approval_state(&store, &started), ApprovalState::Consumed); + assert!(!store.settle_interrupted_resume(&started, None).unwrap()); + assert_eq!(approval_state(&store, &started), ApprovalState::Consumed); + match store.claim_resume(&started).unwrap() { + ResumeClaim::ReplaySuccess(replayed) => assert_eq!(replayed, result), + other => panic!("unexpected {other:?}"), + } + + let unstarted = queued_approval(&store, "unstarted"); + dispatch_under(&store, "proc-unstarted", Some(1), Some(&unstarted)); + { + // A resume in progress settles it itself. + let ResumeClaim::Reconcile(_, _guard) = store.claim_resume(&unstarted).unwrap() else { + panic!("expected reconcile"); + }; + let err = store + .settle_interrupted_resume(&unstarted, None) + .unwrap_err(); + assert_eq!(err.code, ErrorCode::ApprovalConflict); + assert_eq!(approval_state(&store, &unstarted), ApprovalState::Resuming); + } + assert!(store.settle_interrupted_resume(&unstarted, None).unwrap()); + assert_eq!(approval_state(&store, &unstarted), ApprovalState::Granted); + assert!(!store.settle_interrupted_resume(&unstarted, None).unwrap()); + } } diff --git a/crates/store/src/resource.rs b/crates/store/src/resource.rs index 50d455b..bf6e401 100644 --- a/crates/store/src/resource.rs +++ b/crates/store/src/resource.rs @@ -155,6 +155,32 @@ impl ResourceSerializer { } } + /// Hold `workspace_id` for `process_id`, a live process that a gateway found again when it + /// reconciled with its independent Runner (P1R-C04): a confirmed process, as after its + /// spawn, so waiters already queued fail with `WORKSPACE_BUSY`. Holding it again for the + /// same process changes nothing; anything else that holds the workspace refuses it. + pub(crate) fn hold_process( + &mut self, + workspace_id: &str, + process_id: &str, + ) -> Result<(), ErrorBody> { + let resource = Resource::Workspace(workspace_id.to_string()); + match self.exclusive.get(&resource) { + Some(ExclusiveHolder::Process(id)) if id == process_id => return Ok(()), + Some(ExclusiveHolder::Spawning(id)) if id == process_id => {} + Some(_) => return Err(busy(&resource)), + None if self.shared_count(&resource) > 0 => return Err(busy(&resource)), + None => {} + } + self.grant( + resource.clone(), + LockMode::Exclusive, + ExclusiveHolder::Process(process_id.to_string()), + ); + self.fail_waiters(&resource, shell_busy()); + Ok(()) + } + pub(crate) fn clear_shell(&mut self, workspace_id: &str) { let resource = Resource::Workspace(workspace_id.to_string()); if matches!( diff --git a/docs/agent-integration.md b/docs/agent-integration.md index 94c402c..eecc022 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -150,7 +150,7 @@ Interactive input and cancellation use the same handle: A live command occupies the workspace. Wait for it to end or terminate it before applying a patch or starting another command. Reads and searches remain available. Long-lived development servers therefore require a workflow that stops them before edits. -`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. While that gateway claims its Runner again after a lost connection, it refuses requests with `RUNNER_RECONNECTING`, which sent nothing and consumed no approval: retry after a pause. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. +`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. While that gateway claims its Runner again after a lost connection, it refuses requests with `RUNNER_RECONNECTING`, which sent nothing and consumed no approval: retry after a pause. Then, in a workspace where work was started before, it refuses new commands, patches and approval resumes with `RECOVERY_PENDING` until it has [reconciled](runner-isolation.md#reconciliation) that workspace with the Runner; its message says whether retrying after a pause helps. A process it finds running there holds the workspace as before. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. ## Confirm a held mutation @@ -175,7 +175,7 @@ Default workspaces run allowed patches and commands immediately. If the operator } ``` -`approval_resolve` does not change the permission profile. `operation_resume` claims `granted` into `queued`, re-checks policy, waits for the resource, then moves to `resuming` only when dispatch can start. Restart from `queued` reacquires. Restart from `resuming` returns a stored result, recovers a recorded patch, or returns `APPROVAL_AMBIGUOUS`. `consumed` is stored only with a terminal result. An interrupted exec resume is not respawned. Deny is terminal. The same three tools exist when approvals are `off`; only an explicit `approval_create` opens a hold in that mode. v1 does not distinguish host from model: the same MCP caller can grant. +`approval_resolve` does not change the permission profile. `operation_resume` claims `granted` into `queued`, re-checks policy, waits for the resource, then moves to `resuming` only when dispatch can start. Restart from `queued` reacquires. Restart from `resuming` returns a stored result, recovers a recorded patch, or returns `APPROVAL_AMBIGUOUS`. With an [independent Runner](runner-isolation.md#reconciliation), an interrupted exec resume returns the `process_id` that the Runner created for it, or is granted again when the Runner proves that it created none. `consumed` is stored only with a terminal result. An interrupted exec resume is not respawned. Deny is terminal. The same three tools exist when approvals are `off`; only an explicit `approval_create` opens a hold in that mode. v1 does not distinguish host from model: the same MCP caller can grant. ## Retry and recover deliberately diff --git a/docs/architecture.md b/docs/architecture.md index 4eea997..3d5f386 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,7 +51,7 @@ If helper application fails, the Runner attempts per-file snapshot restoration. ## Process lifetime -MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead, and a gateway that lost its connection to it claims it again. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. +MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead, and a gateway that lost its connection to it claims it again and reconciles what earlier controllers left before it starts new work where they did. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. ## Planned execution coordination diff --git a/docs/codex-reuse.md b/docs/codex-reuse.md index 29d5e4c..95b60bd 100644 --- a/docs/codex-reuse.md +++ b/docs/codex-reuse.md @@ -37,7 +37,7 @@ Agent → CodeSpace MCP/policy/store → Runner contract Core crates have no direct Codex dependencies. Adapter crates are separate Cargo workspaces to accommodate the pinned upstream workspace dependencies. The filesystem and PTY adapters are library dependencies of the Runner; patch and Linux sandbox operations use helper processes. An isolated Cargo workspace alone does not create a process or security boundary. -The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 10 and sandbox-helper protocol version 1 are separate contracts. +The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 11 and sandbox-helper protocol version 1 are separate contracts. diff --git a/docs/devguard-integration.md b/docs/devguard-integration.md index a3bf312..df13e4b 100644 --- a/docs/devguard-integration.md +++ b/docs/devguard-integration.md @@ -338,7 +338,7 @@ Default participation remains `off`; `required` is an explicit operator choice i ## Initial Gateway recovery scope -**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. The third unit, P1R-C03, adds the [Claim](runner-isolation.md#claim): one connection at a time controls the Runner, under a controller epoch that each request changing a process or the workspace carries, and a gateway that lost its connection claims the Runner again instead of being restarted. A gateway that controls the Runner after one that started work, the same gateway after a lost connection included, refuses new executions, patches and approval resumes with `RECOVERY_PENDING` until the Runner is stopped and a new one is started. While no gateway controls the Runner, any process of the same user that can reach its socket can claim it, an exposure the [Runner's documentation](runner-isolation.md#claim-exposure) states. Reconciliation and Runner loss come in the later units, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. +**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. The third unit, P1R-C03, adds the [Claim](runner-isolation.md#claim): one connection at a time controls the Runner, under a controller epoch that each request changing a process or the workspace carries, and a gateway that lost its connection claims the Runner again instead of being restarted. While no gateway controls the Runner, any process of the same user that can reach its socket can claim it, an exposure the [Runner's documentation](runner-isolation.md#claim-exposure) states. The fourth unit, P1R-C04, adds [reconciliation](runner-isolation.md#reconciliation): after each Claim, the gateway compares its records, what the Runner holds and remembers, and for a `required` execution its DevGuard attempt, which the Runner looks up, and refuses new executions, patches and approval resumes with `RECOVERY_PENDING` only in the workspaces where an earlier controller left work, until it has reconciled them. A process found running holds its workspace again, an approval left resuming is consumed with the process the Runner created for it or granted again when the Runner proves that it created none, and a workspace whose accounts do not agree, or whose record names another Runner, stays refused for an operator. Runner loss comes in the next unit, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. P1-RECOVERY introduces an operator-selected independent Runner mode and explicit capability. Existing modes keep their current shutdown/disconnect contracts. InProcess shares the Gateway PID and is excluded from live Gateway restart recovery. diff --git a/docs/error-codes.md b/docs/error-codes.md index cf91408..0b759ce 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -54,7 +54,7 @@ Errors use uppercase identifiers and a message. An `operation_id` may be present | `APPROVAL_NOT_FOUND` | Unknown confirmation-hold id | | `APPROVAL_CONFLICT` | Hold is still pending, already decided, or a resume is already in progress | | `APPROVAL_AMBIGUOUS` | Resume was interrupted and the terminal result is not on disk. Includes `approval_id`; patch cases may also include `operation_id` | -| `RECOVERY_PENDING` | A controller before this gateway's Claim started work on its [independent Runner](runner-isolation.md#claim), this gateway's own connection before a lost connection included. The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts until the Runner is stopped and a new one is started. The approval stays as it was. Not occupancy; retrying does not help | +| `RECOVERY_PENDING` | A controller before this gateway's Claim started work in this workspace on its [independent Runner](runner-isolation.md#reconciliation), this gateway's own connection before a lost connection included, and the gateway has not reconciled what it left there yet. The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts there until then. The approval stays as it was. The message says why: when the gateway waits for an answer it tries again by itself, so retry after a pause; when the records, the Runner and DevGuard do not agree, or a record names another Runner, retrying does not help until an operator resolves it. Not occupancy | | `RUNNER_RECONNECTING` | The gateway's connection to its [independent Runner](runner-isolation.md#independent-runner) was lost, and the gateway is claiming the Runner again. The request was not sent: nothing was started, reserved or recorded for it, and an approval it would resume is not consumed. The Runner's processes keep running; retry after a pause | | `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and cannot claim it again: nothing accepts connections at its socket, another Runner answers there, or the Runner refused the Claim for a reason that waiting cannot end. The request was not sent, and the Runner's processes keep running; restart the gateway to reach them | | `STALE_CONTROLLER` | The [independent Runner](runner-isolation.md#claim) refused a request sent under a Claim that another Claim replaced, or from a connection that does not control it. Nothing the request asked for was done, and an approval it resumed is granted again. Retrying sends the request under the gateway's current Claim | diff --git a/docs/execution-substrate.md b/docs/execution-substrate.md index b48952f..6f4617d 100644 --- a/docs/execution-substrate.md +++ b/docs/execution-substrate.md @@ -70,7 +70,7 @@ The Runner may start a recursive filesystem watcher for a workspace on the first `approval_create`, `approval_resolve`, and `operation_resume` are callable. They hold a mutation the workspace profile already allows until the hold is granted. This is not a security boundary: they do not escalate permissions, apply `{ "network": true }` or `ClientClaims.approved`, or write V4A snapshots into the patch operations ledger. The same MCP caller can grant. -When the operator sets workspace `approvals` to `confirm`, a policy-allowed `apply_patch` or `exec_command` returns `APPROVAL_REQUIRED` with an `approval_id` before `begin()` or spawn. Retrying the same logical request reuses that active hold. `off` (the default) still runs those tools immediately; the three tools remain listed so an explicit `approval_create` can open a hold. Grant does not change the profile. Resume claims `granted` into `queued` (restart-safe; no side effect yet), re-checks `allow()`, waits for the resource, then moves to `resuming` only when dispatch can start. `consumed` is recorded only together with the terminal result. A resumed `exec_command` whose governed execution the resource authority refuses with proof that it never started goes back from `resuming` to `granted` instead and can be resumed again ([CSRG-U5](devguard-integration.md#devguard-lifecycle)). A later resume from `queued` reacquires. A later resume from `resuming` returns a stored result, recovers a patch from the operations ledger, or returns `APPROVAL_AMBIGUOUS`. Interrupted exec is not respawned. A denied policy stays `UNAUTHORIZED`. Exec remains on `process_id`. v1 does not authenticate host versus model. The server guarantees the policy re-check on resume plus that durability contract. +When the operator sets workspace `approvals` to `confirm`, a policy-allowed `apply_patch` or `exec_command` returns `APPROVAL_REQUIRED` with an `approval_id` before `begin()` or spawn. Retrying the same logical request reuses that active hold. `off` (the default) still runs those tools immediately; the three tools remain listed so an explicit `approval_create` can open a hold. Grant does not change the profile. Resume claims `granted` into `queued` (restart-safe; no side effect yet), re-checks `allow()`, waits for the resource, then moves to `resuming` only when dispatch can start. `consumed` is recorded only together with the terminal result. A resumed `exec_command` whose governed execution the resource authority refuses with proof that it never started goes back from `resuming` to `granted` instead and can be resumed again ([CSRG-U5](devguard-integration.md#devguard-lifecycle)). A later resume from `queued` reacquires. A later resume from `resuming` returns a stored result, recovers a patch from the operations ledger, or returns `APPROVAL_AMBIGUOUS`. Interrupted exec is not respawned. With an [independent Runner](runner-isolation.md#reconciliation), the gateway settles an interrupted exec from its record and the Runner: consumed with the process that the Runner created for it, or granted again when the Runner proves that it created none. A denied policy stays `UNAUTHORIZED`. Exec remains on `process_id`. v1 does not authenticate host versus model. The server guarantees the policy re-check on resume plus that durability contract. ## Planned managed-execution contracts diff --git a/docs/ko/agent-integration.md b/docs/ko/agent-integration.md index c48f371..c0b7c2d 100644 --- a/docs/ko/agent-integration.md +++ b/docs/ko/agent-integration.md @@ -150,7 +150,7 @@ MCP 클라이언트 SDK로 stdio 또는 Streamable HTTP를 초기화하고, 초 실행 중인 명령은 작업 공간을 점유합니다. 패치를 적용하거나 다른 명령을 시작하려면 기존 명령이 끝날 때까지 기다리거나 종료하세요. 읽기와 검색은 계속 가능합니다. 개발 서버를 오래 실행하는 작업 흐름이라면 수정 전에 서버를 멈추는 절차가 필요합니다. -`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)에서는 `runner_disconnect`가 `keep_running`이고, `gateway_shutdown`은 게이트웨이가 detach하면 `keep_running`, Runner를 중지하면 `terminate`입니다. 그 게이트웨이가 하는 일은 `initialize.instructions`에서 확인하세요. 그 게이트웨이는 연결을 잃은 뒤 Runner를 다시 Claim하는 동안 요청을 `RUNNER_RECONNECTING`으로 거부합니다. 이 거부는 아무것도 보내지 않았고 승인도 소모하지 않았으므로, 잠시 뒤 재시도하세요. spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. +`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)에서는 `runner_disconnect`가 `keep_running`이고, `gateway_shutdown`은 게이트웨이가 detach하면 `keep_running`, Runner를 중지하면 `terminate`입니다. 그 게이트웨이가 하는 일은 `initialize.instructions`에서 확인하세요. 그 게이트웨이는 연결을 잃은 뒤 Runner를 다시 Claim하는 동안 요청을 `RUNNER_RECONNECTING`으로 거부합니다. 이 거부는 아무것도 보내지 않았고 승인도 소모하지 않았으므로, 잠시 뒤 재시도하세요. 그 뒤 이전에 작업을 시작한 workspace에서는 Runner와 그 workspace를 [대조](runner-isolation.md#reconciliation)할 때까지 새 명령, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부합니다. 잠시 뒤 재시도하면 되는지는 그 메시지가 알려 줍니다. 그곳에서 실행 중으로 확인한 프로세스는 지금처럼 workspace를 점유합니다. spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. ## 보류된 변경 확인 @@ -175,7 +175,7 @@ MCP 클라이언트 SDK로 stdio 또는 Streamable HTTP를 초기화하고, 초 } ``` -`approval_resolve`는 권한 프로필을 바꾸지 않습니다. `operation_resume`은 `granted`를 `queued`로 옮긴 뒤 정책을 다시 검사하고, 자원을 받은 다음에야 `resuming`으로 올립니다. `queued`에서 재시작하면 다시 acquire합니다. `resuming`에서 재시작하면 저장한 결과, 패치 원장 복구, 또는 `APPROVAL_AMBIGUOUS`가 반환됩니다. `consumed`는 단말 결과가 저장된 뒤에만 기록됩니다. 중단된 exec 재개는 다시 spawn하지 않습니다. 거절은 단말입니다. `approvals`가 `off`여도 세 도구는 목록에 있으며, 그때는 명시적 `approval_create`만 홀드를 만듭니다. v1은 호스트와 모델을 구분하지 않으므로 같은 MCP 호출자가 grant할 수 있습니다. +`approval_resolve`는 권한 프로필을 바꾸지 않습니다. `operation_resume`은 `granted`를 `queued`로 옮긴 뒤 정책을 다시 검사하고, 자원을 받은 다음에야 `resuming`으로 올립니다. `queued`에서 재시작하면 다시 acquire합니다. `resuming`에서 재시작하면 저장한 결과, 패치 원장 복구, 또는 `APPROVAL_AMBIGUOUS`가 반환됩니다. [독립 Runner](runner-isolation.md#reconciliation)에서는 중단된 exec 재개가 Runner가 그 재개를 위해 만든 `process_id`를 반환하거나, Runner가 아무것도 만들지 않았음이 증명되면 다시 granted가 됩니다. `consumed`는 단말 결과가 저장된 뒤에만 기록됩니다. 중단된 exec 재개는 다시 spawn하지 않습니다. 거절은 단말입니다. `approvals`가 `off`여도 세 도구는 목록에 있으며, 그때는 명시적 `approval_create`만 홀드를 만듭니다. v1은 호스트와 모델을 구분하지 않으므로 같은 MCP 호출자가 grant할 수 있습니다. ## 재시도와 복구 diff --git a/docs/ko/architecture.md b/docs/ko/architecture.md index 7e65a31..83f79c9 100644 --- a/docs/ko/architecture.md +++ b/docs/ko/architecture.md @@ -58,7 +58,7 @@ UDS worker와 기본 러너는 모두 같은 호스트에서 실행합니다. ## 프로세스 수명 -MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)는 이와 달리 게이트웨이보다 오래 살고, Runner와의 연결을 잃은 게이트웨이는 Runner를 다시 Claim합니다. [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. +MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)는 이와 달리 게이트웨이보다 오래 살고, Runner와의 연결을 잃은 게이트웨이는 Runner를 다시 Claim한 뒤 앞선 controller들이 작업을 시작한 곳에서는 그들이 남긴 것을 대조하고 나서 새 작업을 시작합니다. [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. ## 계획된 실행 조정 구조 diff --git a/docs/ko/codex-reuse.md b/docs/ko/codex-reuse.md index 35a2ad3..c6e7528 100644 --- a/docs/ko/codex-reuse.md +++ b/docs/ko/codex-reuse.md @@ -42,7 +42,7 @@ CodeSpace는 특정 버전에 고정한 Codex 소스의 실행 라이브러리 핵심 crate에는 직접적인 Codex 의존성이 없습니다. 어댑터는 고정된 업스트림의 workspace 의존성을 수용하기 위해 별도의 Cargo workspace로 구성합니다. 파일 시스템·PTY 어댑터는 Runner의 라이브러리 의존성이며, 패치와 Linux 샌드박스는 도우미 프로세스를 사용합니다. Cargo workspace를 분리하는 것만으로 프로세스나 보안 경계가 생기지는 않습니다. -Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 10과 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. +Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 11과 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. diff --git a/docs/ko/devguard-integration.md b/docs/ko/devguard-integration.md index 1222cd9..8570780 100644 --- a/docs/ko/devguard-integration.md +++ b/docs/ko/devguard-integration.md @@ -338,7 +338,7 @@ contract/client pin, daemon/helper artifact와 CodeSpace Runner wire는 각각 ## 최초 Gateway 복구 범위 -**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. 두 번째 단위 P1R-C02는 [실행 기록](runner-isolation.md#process-records)을 더한다. gateway는 작업 데이터베이스에 쓰는 유일한 gateway로서 각 실행을 Runner에 보내기 전에 기록하고 끝날 때까지 따라간다. 세 번째 단위 P1R-C03은 [Claim](runner-isolation.md#claim)을 더한다. 한 번에 한 연결만 controller epoch 아래에서 Runner를 제어하고, 프로세스나 workspace를 바꾸는 요청은 모두 그 epoch를 함께 보낸다. 연결을 잃은 gateway는 재시작하지 않고 Runner를 다시 Claim한다. 작업을 시작한 gateway 뒤에 Runner를 제어하는 gateway는, 연결을 잃었다가 다시 Claim한 같은 gateway를 포함해, Runner를 중지하고 새로 시작할 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. Runner를 제어하는 gateway가 없는 동안에는 Runner의 소켓에 닿을 수 있는 같은 사용자의 프로세스라면 무엇이든 Runner를 Claim할 수 있으며, 이 노출은 [Runner 문서](runner-isolation.md#claim-exposure)에 적었다. 대조와 Runner 손실 처리는 이후 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. +**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. 두 번째 단위 P1R-C02는 [실행 기록](runner-isolation.md#process-records)을 더한다. gateway는 작업 데이터베이스에 쓰는 유일한 gateway로서 각 실행을 Runner에 보내기 전에 기록하고 끝날 때까지 따라간다. 세 번째 단위 P1R-C03은 [Claim](runner-isolation.md#claim)을 더한다. 한 번에 한 연결만 controller epoch 아래에서 Runner를 제어하고, 프로세스나 workspace를 바꾸는 요청은 모두 그 epoch를 함께 보낸다. 연결을 잃은 gateway는 재시작하지 않고 Runner를 다시 Claim한다. Runner를 제어하는 gateway가 없는 동안에는 Runner의 소켓에 닿을 수 있는 같은 사용자의 프로세스라면 무엇이든 Runner를 Claim할 수 있으며, 이 노출은 [Runner 문서](runner-isolation.md#claim-exposure)에 적었다. 네 번째 단위 P1R-C04는 [대조](runner-isolation.md#reconciliation)를 더한다. gateway는 Claim할 때마다 자기 기록, Runner가 가진 것과 기억하는 것, `required` 실행이면 Runner가 조회한 DevGuard attempt를 맞춰 보고, 앞선 controller가 작업을 남긴 workspace에서만 대조를 마칠 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. 실행 중으로 확인한 프로세스는 다시 workspace를 점유하고, resuming으로 남은 승인은 Runner가 그 승인을 위해 만든 프로세스로 consumed가 되거나, Runner가 아무것도 만들지 않았음이 증명되면 다시 granted가 된다. 세 가지가 맞지 않거나 기록이 다른 Runner를 가리키는 workspace는 운영자가 판단할 때까지 계속 거부된다. Runner 손실 처리는 다음 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. P1-RECOVERY는 운영자가 선택하는 독립 Runner 모드와 명시적인 capability를 도입한다. 기존 모드의 종료·연결 소실 계약은 유지한다. InProcess는 Gateway와 PID를 공유하므로 Gateway 재시작 중 live 복구 대상에서 제외한다. diff --git a/docs/ko/error-codes.md b/docs/ko/error-codes.md index 20482dc..f3ff0fe 100644 --- a/docs/ko/error-codes.md +++ b/docs/ko/error-codes.md @@ -57,7 +57,7 @@ | `APPROVAL_NOT_FOUND` | 알 수 없는 확인 홀드 ID | | `APPROVAL_CONFLICT` | 홀드가 아직 대기 중이거나 이미 결정되었거나, 재개가 이미 진행 중임 | | `APPROVAL_AMBIGUOUS` | 재개가 중단되어 단말 결과가 디스크에 없음. `approval_id` 포함. 패치는 `operation_id`도 있을 수 있음 | -| `RECOVERY_PENDING` | 이 게이트웨이가 Claim하기 전의 controller가 [독립 Runner](runner-isolation.md#claim)에서 작업을 시작함. 연결을 잃기 전의 이 게이트웨이 자신도 포함함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, Runner를 중지하고 새로 시작하기 전에는 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 점유가 아니므로 재시도해도 소용없음 | +| `RECOVERY_PENDING` | 이 게이트웨이가 Claim하기 전의 controller가 [독립 Runner](runner-isolation.md#reconciliation)에서 이 workspace의 작업을 시작했고, 게이트웨이가 그 controller가 남긴 것을 아직 대조하지 않음. 연결을 잃기 전의 이 게이트웨이 자신도 포함함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, 대조를 마치기 전에는 그 workspace에서 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 메시지가 이유를 알림. 게이트웨이가 답을 기다리는 중이면 스스로 다시 시도하므로 잠시 뒤에 재시도함. 기록, Runner, DevGuard가 서로 맞지 않거나 기록이 다른 Runner를 가리키면 운영자가 해결하기 전에는 재시도해도 소용없음. 점유가 아님 | | `RUNNER_RECONNECTING` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 Runner를 다시 Claim하는 중임. 요청은 보내지 않았음. 그 요청을 위해 시작하거나 예약하거나 기록한 것은 없고, 재개하려던 승인은 소비되지 않음. Runner의 프로세스는 계속 실행됨. 잠시 뒤 재시도할 것 | | `RUNNER_DISCONNECTED` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 다시 Claim할 수 없음. 소켓에서 연결을 받지 않거나, 다른 Runner가 응답하거나, 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부함. 요청은 보내지 않았고 Runner의 프로세스는 계속 실행됨. 그 프로세스에 닿으려면 게이트웨이를 재시작할 것 | | `STALE_CONTROLLER` | [독립 Runner](runner-isolation.md#claim)가 다른 Claim으로 바뀐 Claim 아래에서 보낸 요청, 또는 Runner를 제어하지 않는 연결의 요청을 거부함. 요청이 요구한 일은 하나도 하지 않았고, 요청이 재개한 승인은 다시 승인된 상태가 됨. 재시도하면 게이트웨이의 현재 Claim 아래에서 요청을 보냄 | diff --git a/docs/ko/execution-substrate.md b/docs/ko/execution-substrate.md index f933426..78e6a69 100644 --- a/docs/ko/execution-substrate.md +++ b/docs/ko/execution-substrate.md @@ -79,7 +79,7 @@ Runner는 해당 작업 공간에서 처음 `read`·`find`·`version`·`apply_pa `approval_create`, `approval_resolve`, `operation_resume`을 호출할 수 있습니다. 작업 공간 프로필이 이미 허용한 변경을 홀드가 승인될 때까지 멈춥니다. 보안 경계가 아닙니다. 권한을 높이거나 `{ "network": true }`·`ClientClaims.approved`를 적용하거나 패치 원장에 V4A 스냅샷을 넣지 않습니다. 같은 MCP 호출자가 grant할 수 있습니다. -운영자가 작업 공간 `approvals`를 `confirm`으로 두면, 정책이 허용한 `apply_patch`와 `exec_command`는 `begin()`이나 프로세스 시작 전에 `APPROVAL_REQUIRED`와 `approval_id`를 반환합니다. 같은 논리 요청을 다시 보내면 그 활성 홀드를 재사용합니다. 기본값 `off`에서는 해당 도구가 바로 실행됩니다. 세 도구는 목록에 남아 있으므로 명시적 `approval_create`로 홀드를 만들 수 있습니다. grant는 프로필을 바꾸지 않습니다. 재개는 `granted`를 `queued`로 옮긴 뒤 `allow()`를 다시 검사하고, 자원을 받은 다음에야 `resuming`으로 올립니다. `consumed`는 단말 결과와 함께만 기록됩니다. 재개한 `exec_command`의 관리 대상 실행을 자원 authority가 시작하지 않았다는 증거와 함께 거부하면, 대신 `resuming`에서 `granted`로 돌아가 다시 재개할 수 있습니다([CSRG-U5](devguard-integration.md#devguard-lifecycle)). `queued`에서 재시작하면 다시 acquire합니다. `resuming`에서 이후 재개는 그 결과, 패치 원장 복구, 또는 `APPROVAL_AMBIGUOUS`를 반환합니다. 중단된 exec는 다시 spawn하지 않습니다. 정책 거절은 그대로 `UNAUTHORIZED`입니다. 명령은 `process_id`로 다룹니다. v1은 호스트와 모델을 구분하지 않습니다. 서버가 보장하는 것은 재개 시 정책 재검사와 이 내구성 계약입니다. +운영자가 작업 공간 `approvals`를 `confirm`으로 두면, 정책이 허용한 `apply_patch`와 `exec_command`는 `begin()`이나 프로세스 시작 전에 `APPROVAL_REQUIRED`와 `approval_id`를 반환합니다. 같은 논리 요청을 다시 보내면 그 활성 홀드를 재사용합니다. 기본값 `off`에서는 해당 도구가 바로 실행됩니다. 세 도구는 목록에 남아 있으므로 명시적 `approval_create`로 홀드를 만들 수 있습니다. grant는 프로필을 바꾸지 않습니다. 재개는 `granted`를 `queued`로 옮긴 뒤 `allow()`를 다시 검사하고, 자원을 받은 다음에야 `resuming`으로 올립니다. `consumed`는 단말 결과와 함께만 기록됩니다. 재개한 `exec_command`의 관리 대상 실행을 자원 authority가 시작하지 않았다는 증거와 함께 거부하면, 대신 `resuming`에서 `granted`로 돌아가 다시 재개할 수 있습니다([CSRG-U5](devguard-integration.md#devguard-lifecycle)). `queued`에서 재시작하면 다시 acquire합니다. `resuming`에서 이후 재개는 그 결과, 패치 원장 복구, 또는 `APPROVAL_AMBIGUOUS`를 반환합니다. 중단된 exec는 다시 spawn하지 않습니다. [독립 Runner](runner-isolation.md#reconciliation)에서는 게이트웨이가 중단된 exec를 기록과 Runner로 정리합니다. Runner가 그 exec를 위해 만든 프로세스로 consumed가 되거나, Runner가 아무것도 만들지 않았음이 증명되면 다시 granted가 됩니다. 정책 거절은 그대로 `UNAUTHORIZED`입니다. 명령은 `process_id`로 다룹니다. v1은 호스트와 모델을 구분하지 않습니다. 서버가 보장하는 것은 재개 시 정책 재검사와 이 내구성 계약입니다. ## 계획된 관리 실행 계약 diff --git a/docs/ko/operations.md b/docs/ko/operations.md index 9f8d25a..18951ae 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -115,7 +115,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 살고, 게이트웨이가 그 프로세스를 작업 데이터베이스에 기록하며, 연결을 잃은 게이트웨이가 다시 Claim하는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. +worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 살고, 게이트웨이가 그 프로세스를 작업 데이터베이스에 기록하며, 연결을 잃은 게이트웨이가 다시 Claim한 뒤 자기 기록과 대조하는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. @@ -131,7 +131,7 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 | `CODESPACE_PROCESS_TIMEOUT_SECS` | 양의 정수. 기본 30초. 러너 환경에 설정 | | `CODESPACE_MAX_PROCESSES` | 러너 전체의 실행 중 프로세스 기본 상한 8개. 넘는 요청은 프로세스를 만들기 전에 거부. 작업 공간별 점유 규칙도 적용 | | 프로세스 출력 | 마지막 256 KiB 보관. stdout/stderr를 합침. `read_process`는 `output_lost`와 `retained_from`을 보고하고, 종료는 `process_status`로 조회. 실행 중인 PTY 크기는 `process_resize`로 변경 | -| 종료된 핸들 | 기본 최대 15분, 최대 64개 보관. 영구 저장하지 않음 | +| 종료된 핸들 | 기본 최대 15분, 최대 64개 보관. 영구 저장하지 않음. 독립 Runner는 끝난 프로세스의 상태와 출력을 게이트웨이가 확인할 때까지 보관하며, 핸들을 놓은 뒤에는 최대 64개까지 보관함([끝난 프로세스](runner-isolation.md#reconciliation)) | 로그와 데이터베이스는 토큰·게이트웨이 설정과 같이 관리 대상 작업 공간 밖에 두세요. stderr 로그의 보관·순환은 운영자가 관리합니다. Bearer 토큰을 로그나 커밋에 넣지 마세요. 데이터베이스를 삭제하면 패치 중복 실행 방지 기록과 확인 홀드 행도 사라집니다. @@ -139,7 +139,7 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 **스키마 버전과 이전 버전.** 작업 데이터베이스에는 스키마 버전이 적혀 있습니다. CodeSpace는 자기보다 새 스키마 버전의 데이터베이스를 아무것도 쓰기 전에 거부하고 그대로 둡니다. 그런 데이터베이스는 그것을 쓴 CodeSpace 버전으로 여세요. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 이 데이터베이스에 기록하고, [Runner를 Claim](runner-isolation.md#claim)할 때 밝히는 데이터베이스의 임의 식별자를 이 데이터베이스에 두며, 실행 중에는 그 옆의 lock 파일 `.gateway.lock`을 잡고 있습니다. 실행 기록이 생기기 전의 CodeSpace 버전도 이런 데이터베이스를 열 수 있고 기록은 건드리지 않지만, 스키마 버전을 확인하지도 lock을 걸지도 않습니다. 그러니 독립 Runner의 게이트웨이가 쓰는 데이터베이스에서는 이전 버전을 실행하지 마세요. 독립 모드에서 되돌아갈 때는 그 모드를 쓰지 않고 데이터베이스는 그대로 둡니다. 이전 사본을 복원하면 그 뒤에 쓴 패치 중복 실행 방지 기록, 확인 홀드, 실행 기록이 사라집니다. -패치 응답을 받지 못했다면 `operation_id` 또는 `operation_key` 중 하나만 지정해 `operation_status`를 조회합니다. 재시작 후 미완료 기록은 `unknown`이 되므로 파일을 확인한 뒤 다음 행동을 결정하세요. 프로세스는 `process_id`로 관리하며 `operation_status`로 복구할 수 없습니다. 프로세스가 죽을 때 `queued`이던 확인 홀드는 다시 acquire합니다. `resuming`이던 홀드는 패치 원장에서 복구하거나, 저장된 단말 결과를 재현하거나, `APPROVAL_AMBIGUOUS`를 반환할 수 있습니다. exec는 다시 spawn하지 않습니다. [재시도와 복구 규칙](agent-integration.md)을 참고하세요. +패치 응답을 받지 못했다면 `operation_id` 또는 `operation_key` 중 하나만 지정해 `operation_status`를 조회합니다. 재시작 후 미완료 기록은 `unknown`이 되므로 파일을 확인한 뒤 다음 행동을 결정하세요. 프로세스는 `process_id`로 관리하며 `operation_status`로 복구할 수 없습니다. 프로세스가 죽을 때 `queued`이던 확인 홀드는 다시 acquire합니다. `resuming`이던 홀드는 패치 원장에서 복구하거나, 저장된 단말 결과를 재현하거나, `APPROVAL_AMBIGUOUS`를 반환할 수 있습니다. exec는 다시 spawn하지 않습니다. 독립 Runner에 연결한 게이트웨이는 `resuming`으로 남은 exec 홀드를 [대조](runner-isolation.md#reconciliation)할 때 정리합니다. Runner가 그 홀드를 위해 만든 프로세스로 consumed가 되거나, Runner가 아무것도 만들지 않았음이 증명되면 다시 granted가 됩니다. [재시도와 복구 규칙](agent-integration.md)을 참고하세요. diff --git a/docs/ko/runner-isolation.md b/docs/ko/runner-isolation.md index 74d7741..cda18a3 100644 --- a/docs/ko/runner-isolation.md +++ b/docs/ko/runner-isolation.md @@ -13,7 +13,7 @@ `in-process`는 `codespace-mcp` 안에서 프로세스를 관리합니다. `uds`(Unix domain socket, Unix 도메인 소켓)는 같은 호스트의 `codespace-codex-runtime` 안에서 같은 관리 코드를 실행합니다. worker는 시작 시 Codex의 프로세스 보호 설정을 적용하고 전용 Unix 소켓을 엽니다. worker 자체를 보호하는 것과 실행할 명령에 샌드박스를 적용하는 것은 별개입니다. -게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 10 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. +게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 11 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. 관리 프로세스의 수명은 MCP 연결이 아니라 러너 인스턴스가 소유합니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지합니다. 게이트웨이와 worker의 UDS 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 관리 중인 worker와 자식 프로세스가 종료되고 핸들이 사라집니다. 종료할 때 게이트웨이는 worker와의 연결을 닫으므로 worker는 종료하면서 자기 프로세스를 끝내고, 5초 뒤에도 실행 중인 worker는 강제로 종료됩니다. 재시작은 `process_id`를 복구하지 않습니다. [독립 Runner](#independent-runner)는 예외로, 그 프로세스는 게이트웨이보다 오래 삽니다. @@ -23,7 +23,7 @@ ## 독립 Runner -> **아직 지원하는 모드가 아닙니다.** 독립 Runner와 그 실행의 기록, 그리고 한 게이트웨이에서 다음 게이트웨이로 제어권을 넘기는 Claim은 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 연결을 잃으면 Runner를 다시 Claim합니다. 그러나 작업을 시작한 게이트웨이 뒤에 Runner를 제어하는 게이트웨이는, 연결을 잃었다가 다시 Claim한 같은 게이트웨이를 포함해 새 작업을 시작할 수 없습니다. 상태 대조는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. +> **아직 지원하는 모드가 아닙니다.** 독립 Runner와 그 실행의 기록, 한 게이트웨이에서 다음 게이트웨이로 제어권을 넘기는 Claim, 그리고 Claim마다 뒤따르는 대조는 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 연결을 잃으면 Runner를 다시 Claim하고, 이전 controller가 작업을 시작한 곳에서는 그 controller가 남긴 것을 대조한 뒤에야 새 작업을 시작합니다. Runner 자체를 잃은 뒤의 처리는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. 독립 Runner는 자신에게 연결하는 게이트웨이보다 오래 삽니다. 게이트웨이가 종료되거나 재시작되거나 강제로 종료되어도 프로세스와 그 출력, timeout은 계속됩니다. 운영자나 하네스가 독립 Runner를 시작하고, 게이트웨이는 연결만 합니다. @@ -59,7 +59,25 @@ Claim을 받아들일 때마다 Runner의 controller epoch가 하나 올라갑 **연결을 잃었을 때.** Runner와의 연결을 잃었거나, 호출이 deadline을 넘겨 연결을 닫은 게이트웨이는 다시 연결해 같은 Runner를 Claim합니다. 먼저 소켓에서 응답하는 Runner가 시작할 때 찾은 Runner와 같은 식별 정보를 가졌는지 확인합니다. 바로 한 번 시도하고, 실패하면 100밀리초를 기다린 뒤 다시 시도하며, 실패할 때마다 기다리는 시간을 두 배씩 최대 5초까지 늘립니다. 그동안에는 Runner가 필요한 요청을 모두 `RUNNER_RECONNECTING`으로 거부합니다. 그 요청을 위해서는 아무것도 보내거나 예약하거나 기록하지 않으며, 재개하려던 승인은 소비되지 않습니다. 소켓에서 더 이상 연결을 받지 않거나, 다른 Runner가 응답하거나, 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부하면 게이트웨이는 시도를 멈추고 그런 요청을 모두 `RUNNER_DISCONNECTED`로 거부합니다. 그러면 Runner의 프로세스에 다시 닿으려면 게이트웨이를 재시작해야 합니다. 연결을 잃을 때 이미 보낸 요청은 지금처럼 불확실한 결과를 받습니다. 실행이라면 `dispatch_status: unknown`입니다. Runner가 전하지 못한 종료 이벤트는 다음 controller에게 가고, 게이트웨이는 각 종료를 한 번만 전달합니다. -**이전 controller가 있었을 때.** 새로 시작한 Runner를 처음 Claim한 게이트웨이는 관리 worker와 같이 동작합니다. 한 controller가 그 Runner에서 작업(실행, 패치, DevGuard를 쓰면 준비나 launch)을 시작한 뒤에는, 그 뒤의 controller들이 Runner의 프로세스를 넘겨받습니다. 연결을 잃었다가 Runner를 다시 Claim한 같은 게이트웨이도 마찬가지입니다. 그 게이트웨이는 `process_id`로 프로세스를 조회하고, 출력을 읽고, stdin에 쓰고, 크기를 바꾸고, 종료할 수 있지만, 새 실행, 패치, 승인 재개는 `RECOVERY_PENDING`으로 거부하고 승인은 그대로 둡니다. 넘겨받은 프로세스가 모두 끝난 뒤에도 마찬가지이므로, 작업을 다시 시작하려면 Runner를 중지하고 새로 시작합니다. 게이트웨이는 Runner의 식별 정보(프로세스 ID, 부팅과 시작 시각), Claim의 epoch, 이전 controller가 작업을 시작했는지, 넘겨받은 프로세스 수, 그 목록에 반영된 마지막 종료 이벤트의 번호를 로그에 남깁니다. +**이전 controller가 있었을 때.** 새로 시작한 Runner를 처음 Claim한 게이트웨이는 관리 worker와 같이 동작합니다. 한 controller가 어떤 workspace에서 작업(실행, 패치, DevGuard를 쓰면 준비나 launch)을 시작한 뒤에는, 그 뒤의 controller들이 Runner의 프로세스를 넘겨받습니다. 연결을 잃었다가 Runner를 다시 Claim한 같은 게이트웨이도 마찬가지이며, Claim의 응답은 앞선 controller가 작업을 시작한 workspace를 모두 알립니다. 게이트웨이는 넘겨받은 프로세스를 `process_id`로 바로 조회하고, 출력을 읽고, stdin에 쓰고, 크기를 바꾸고, 종료할 수 있으며, DevGuard가 응답하지 못할 때도 마찬가지입니다. Claim이 알린 workspace, Runner가 실행 중인 프로세스를 가진 workspace, [대조](#reconciliation)할 기록이 남은 workspace에서는 먼저 앞선 controller들이 남긴 것을 대조합니다. 그때까지는 그 workspace의 새 실행, 패치, 승인 재개를 승인을 가져가기 전에 `RECOVERY_PENDING`으로 거부하므로 승인은 그대로 남습니다. Runner도 Claim이 알린 workspace에서는 게이트웨이가 그 workspace를 대조했다고 알릴 때까지 이런 요청을 거부합니다. 다른 workspace는 바로 새 작업을 받습니다. 게이트웨이는 Runner의 식별 정보(프로세스 ID, 부팅과 시작 시각), Claim의 epoch, 앞선 controller가 작업을 시작한 workspace, 넘겨받은 프로세스 수, 그 목록에 반영된 마지막 종료 이벤트의 번호를 로그에 남깁니다. + + + +**대조.** 게이트웨이는 Claim할 때마다, `terminal`이 아니면서 이 Runner에 이번 Claim이나 그 뒤의 Claim으로 보낸 것이 아닌 기록과, 아직 `resuming`인 승인의 기록을 대조합니다. 각 실행에 대해 세 가지를 맞춰 봅니다. 그 실행의 [기록](#process-records), 기록이 가리키는 Runner가 그 프로세스에 대해 가진 것이나 기억하는 것, 그리고 DevGuard를 쓰면 `required` workspace의 실행에 대해 Runner가 DevGuard로 조회한 attempt입니다. Runner는 시작하라고 받은 `process_id`마다, 프로세스를 만들었는지 아니면 요청을 받고도 아무것도 만들지 않았는지를 ID만으로 수명 내내 기억합니다. 요청을 받은 적이 없다는 증명은 Runner 자신에 대해서만 할 수 있으므로, 중지되었거나 잃은 Runner를 포함해 다른 Runner를 가리키는 기록은 이렇게 정리하지 않습니다. + +| 기록 | 기록이 가리키는 Runner | DevGuard attempt(`required`만) | 결과 | +| --- | --- | --- | --- | +| `dispatching` 또는 `running` | 프로세스를 실행 중으로 가짐 | launch됨 | 기록은 `running`이 되고, 프로세스가 다시 workspace를 점유합니다. 그 workspace의 새 작업은 프로세스가 끝나거나 종료될 때까지 `WORKSPACE_BUSY`를 받습니다 | +| `dispatching` 또는 `running` | 끝난 프로세스를 가짐, 또는 만들었지만 더는 갖지 않음 | scope가 끝나 해제됨, 또는 시작 전에 끝남 | 기록은 프로세스가 어떻게 끝났는지 남기며, Runner가 더는 갖지 않으면 종료 방식은 `unknown`입니다. workspace 점유는 해제됩니다 | +| `dispatching` | 요청을 받은 적 없음, 또는 받고 아무것도 만들지 않음 | 없음, 또는 시작 전에 끝남 | 기록은 실행이 시작되지 않았다고 남기고, workspace 점유는 해제됩니다 | +| 없음 | 프로세스를 실행 중으로 가짐 | 무엇이든 | workspace는 계속 거부됩니다 | +| 그 밖의 모든 경우 | | | workspace는 계속 거부됩니다 | + +Runner나 DevGuard에 묻지 못했거나 DevGuard가 attempt를 아직 정리하지 않아 답을 기다리는 workspace는 500밀리초 뒤에 다시 대조하며, 다시 시도할 때마다 기다리는 시간을 두 배씩 최대 30초까지 늘립니다. 세 가지가 서로 맞지 않거나 기록이 다른 Runner를 가리키는 workspace는 운영자가 해결할 때까지 계속 거부되며, 그 절차는 다음 단계에서 더합니다. Runner가 실행 중인 프로세스를 가졌지만 그 프로세스를 설명하는 기록이 없는 workspace는 그 프로세스가 끝나거나 종료될 때까지 계속 거부됩니다. 게이트웨이는 종료가 있을 때마다 다시 대조합니다. 계속 거부하는 workspace는 그 이유와 함께 로그에 남기고, 그 이유는 `RECOVERY_PENDING`에도 들어갑니다. 대조를 마친 workspace도 로그에 남깁니다. 대조할 수 없는 workspace의 점유는 풀지 않습니다. + +앞선 controller가 `resuming`으로 남긴 승인은 Runner가 그 승인을 위해 만든 실행으로 consumed가 되며, 그 뒤의 `operation_resume`은 그 실행의 `process_id`를 `dispatch_status: confirmed`와 함께 돌려줍니다. Runner가 아무것도 만들지 않았음이 증명되면 승인은 다시 granted가 되어 한 번 더 재개할 수 있습니다. 그 밖의 경우에는 `resuming`으로 남고, `operation_resume`은 지금처럼 `APPROVAL_AMBIGUOUS`를 돌려줍니다. 실행이 시작되었을 수도 있는 승인은 다시 granted가 되지 않습니다. + +**끝난 프로세스.** 독립 Runner는 끝난 프로세스와 그 상태, 출력을 controller가 그 기록에 프로세스가 어떻게 끝났는지 남겼다고 확인할 때까지 보관합니다. 그래서 어떤 게이트웨이도 받지 못한 종료도 다음 controller에게 갑니다. 프로세스의 핸들은 관리 worker와 같이 15분이 지나거나 끝난 프로세스가 64개를 넘으면 놓고, 핸들을 놓은 끝난 프로세스는 64개까지 보관하며 가장 오래된 것부터 버립니다. 버린 프로세스에 대해서는 Runner가 그 프로세스를 만들었다는 것만 기억합니다. @@ -69,11 +87,11 @@ Claim을 받아들일 때마다 Runner의 controller epoch가 하나 올라갑 **실행 기록.** 게이트웨이는 각 실행을 Runner에 보내기 전에 작업 데이터베이스에 기록하고, 실행이 끝날 때까지 따라갑니다. 그래서 메모리를 잃은 게이트웨이도 같은 실행을 다시 알아볼 수 있습니다. 기록에는 `process_id`, 실행을 보낸 Runner의 식별 정보(프로세스 ID, 부팅, 시작 시각, 임의의 nonce), 실행을 보낸 [controller epoch](#claim), workspace, 실행이 재개하는 승인, DevGuard attempt, DevGuard 참여 여부, tty 여부, argv의 SHA-256 digest가 들어가며, argv 자체와 그 환경, secret은 들어가지 않습니다. 기록의 단계는 Runner가 실행을 시작했다고 답할 때까지 `dispatching`이고, 그 뒤로는 `running`입니다. Runner가 실행이 끝났다고 알리면 Runner가 보고한 종료 방식, 종료 코드, 신호와 함께 `terminal`이 되고, Runner가 실행을 시작하지 않았다고 답할 때도 `terminal`이 됩니다. 기록에는 Runner가 정한 deadline, Runner가 만든 루트 프로세스, 게이트웨이가 출력을 어디까지 읽었는지도 남습니다. 루트 프로세스는 프로세스 ID와, 나중에 같은 ID를 받은 프로세스와 구별해 주는 시작 시각으로 남기며, `tty: true` 실행의 루트는 남기지 않습니다. -승인을 재개하는 실행은 그 승인을 resuming으로 바꾸는 transaction 안에서 기록합니다. 기록을 쓸 수 없으면 실행을 `INTERNAL`로 거부하고 아무것도 보내지 않습니다. Runner를 제어하는 게이트웨이가 없는 동안 끝난 실행은 Runner가 보관했다가 다음 controller에 알리고, 그 게이트웨이가 기록합니다. `process_status`와 `read_process`도 Runner가 보고한 내용을 기록합니다. 기록이 핸들을 되살리는 일은 없으며, 기록만으로 실행이 시작되었는지나 어떻게 끝났는지가 증명되지도 않습니다. 기준은 Runner이며, 응답을 잃은 실행의 기록은 `dispatching`으로 남습니다. 기록으로 작업을 복구하는 일은 다음 단계에서 더합니다. +승인을 재개하는 실행은 그 승인을 resuming으로 바꾸는 transaction 안에서 기록합니다. 기록을 쓸 수 없으면 실행을 `INTERNAL`로 거부하고 아무것도 보내지 않습니다. 기록에 실행이 어떻게 끝났는지 남기면 게이트웨이는 그것을 Runner에 확인해 줍니다. Runner를 제어하는 게이트웨이가 없는 동안 끝난 실행은 Runner가 보관했다가 다음 controller에 알리고, 그 게이트웨이가 늦어도 대조할 때 기록합니다. Runner를 중지하는 게이트웨이는 Runner의 프로세스가 각각 어떻게 끝났는지 기록합니다. `process_status`와 `read_process`도 Runner가 보고한 내용을 기록합니다. 기록이 핸들을 되살리는 일은 없으며, 기록만으로 실행이 시작되었는지나 어떻게 끝났는지가 증명되지도 않습니다. 기준은 Runner이며, 응답을 잃은 실행의 기록은 [대조](#reconciliation)가 정리할 때까지 `dispatching`으로 남습니다. **writer는 하나.** 독립 Runner에 연결한 게이트웨이는 자기 작업 데이터베이스에 쓰는 유일한 게이트웨이입니다. 데이터베이스를 열기 전에 그 옆의 파일 `.gateway.lock`에 배타 lock을 걸고, 어떤 식으로 종료하든 종료할 때까지 lock을 유지합니다. 이 파일에는 lock을 가진 게이트웨이의 프로세스 ID가 적힙니다. 앞 게이트웨이가 아직 detach하는 동안 새 게이트웨이를 시작했을 때처럼 lock이 이미 잡혀 있으면, 새 게이트웨이는 기다린다는 로그를 남기고 최대 15초 동안 기다린 뒤 시작을 거부합니다. 이 lock은 이 모드에서만 겁니다. 데이터베이스의 스키마 버전과 이전 CodeSpace 버전은 [운영](operations.md#operations-database)을 참고하세요. -**DevGuard.** `devguard` feature로 빌드한 독립 Runner는 worker의 DevGuard 설정으로 스스로 등록하며, 소비자 secret은 `--devguard-credential-file`로 지정한 비공개 파일에서 직접 읽습니다. `--devguard register`로 시작한 게이트웨이는 Runner에 secret을 넘기지 않습니다. 자원 참여를 요구하는 workspace는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 실행하며, 그렇지 않으면 `RESOURCE_POLICY_UNSUPPORTED`로 거부합니다. [DevGuard 결합](devguard-integration.md#resource-gateway-recovery)을 참고하세요. +**DevGuard.** `devguard` feature로 빌드한 독립 Runner는 worker의 DevGuard 설정으로 스스로 등록하며, 소비자 secret은 `--devguard-credential-file`로 지정한 비공개 파일에서 직접 읽습니다. `--devguard register`로 시작한 게이트웨이는 Runner에 secret을 넘기지 않습니다. 자원 참여를 요구하는 workspace는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 실행하며, 그렇지 않으면 `RESOURCE_POLICY_UNSUPPORTED`로 거부합니다. `devguard` feature 없이 빌드한 게이트웨이는 DevGuard에 참여한 실행을 대조할 수 없으므로 그 workspace는 계속 거부됩니다. [DevGuard 결합](devguard-integration.md#resource-gateway-recovery)을 참고하세요. diff --git a/docs/ko/security-model.md b/docs/ko/security-model.md index 9ea5918..6654d88 100644 --- a/docs/ko/security-model.md +++ b/docs/ko/security-model.md @@ -44,7 +44,7 @@ Linux 도우미가 없으면 restricted 정책의 작업 공간에서도 비격 ## 작업과 복구의 안전성 -버전 검사는 예상하지 못한 파일 버전에 패치를 적용하는 것을 막습니다. 작업 키는 패치 중복 요청을 구분하며 인증 토큰이 아닙니다. 작업 공간 점유는 파일을 변경할 수 있는 패치·명령 실행을 직렬화합니다. 요청 소유 충돌은 `acquire()`에서 시작하는 인메모리 FIFO에서 기다리고, 라이브 프로세스는 뒤 대기자와 새 요청에 `WORKSPACE_BUSY`를 반환합니다. 점유 상태는 재시작 후에도 남지 않습니다. 영속적인 프로세스 복구는 없습니다. +버전 검사는 예상하지 못한 파일 버전에 패치를 적용하는 것을 막습니다. 작업 키는 패치 중복 요청을 구분하며 인증 토큰이 아닙니다. 작업 공간 점유는 파일을 변경할 수 있는 패치·명령 실행을 직렬화합니다. 요청 소유 충돌은 `acquire()`에서 시작하는 인메모리 FIFO에서 기다리고, 라이브 프로세스는 뒤 대기자와 새 요청에 `WORKSPACE_BUSY`를 반환합니다. 점유 상태는 재시작 후에도 남지 않습니다. 영속적인 프로세스 복구는 없습니다. 다만 [독립 Runner](runner-isolation.md#reconciliation)에 연결한 게이트웨이는 대조할 때 실행 중으로 확인한 프로세스마다 점유를 복원하며, 그 전에는 그 workspace에서 새 작업을 시작하지 않습니다. `CODESPACE_OPERATIONS_DB`의 `approvals` 확인 홀드는 행이 `pending`·`granted`·`queued`·`resuming`인 동안 V4A 패치나 exec argv를 보관합니다. 패치 원장은 패치 본문이 아니라 해시를 보관합니다. `denied` 또는 `consumed` 뒤에는 홀드 본문을 digest 메타(도구, 작업 공간, fingerprint)로 바꿉니다. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 여기에 기록하는데(`processes`), argv는 SHA-256 digest로만 남기고 argv 자체와 그 환경, secret은 남기지 않습니다. 또 Runner를 Claim할 때 밝히는 데이터베이스의 임의 식별자(`store_identity`)를 여기에 둡니다. 이 식별자는 데이터베이스를 서로 구별할 뿐 secret이 아닙니다. 이 데이터베이스는 토큰·게이트웨이 설정과 같이 작업 공간 루트 밖에 두세요. 홀드는 워크플로 일시정지이며 같은 MCP 호출자가 grant할 수 있습니다. 격리 경계가 아닙니다. diff --git a/docs/operations.md b/docs/operations.md index 7eb489a..bdc646d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -109,7 +109,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, whose gateway records them in the operations database, and which a gateway claims again after a lost connection, is not a supported mode yet. +The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, whose gateway records them in the operations database, and which a gateway claims again after a lost connection and then reconciles with its records, is not a supported mode yet. @@ -123,7 +123,7 @@ The worker runs on the same host and is not a container. The gateway creates a p | `CODESPACE_PROCESS_TIMEOUT_SECS` | Positive integer; default 30 seconds; set in the runner environment | | `CODESPACE_MAX_PROCESSES` | Default 8 live processes across the runner; a request over it is refused before any process is created; workspace occupancy still applies | | Process output | Last 256 KiB retained; stdout/stderr combined; `read_process` reports `output_lost` and `retained_from`; `process_status` reports termination; `process_resize` resizes a running PTY | -| Completed handles | Default retention up to 15 minutes and 64 completed entries; not durable | +| Completed handles | Default retention up to 15 minutes and 64 completed entries; not durable. An independent Runner keeps an ended process's status and output until its gateway acknowledges them, at most 64 once their handles are released ([ended processes](runner-isolation.md#reconciliation)) | Store logs and the database outside the managed workspace, with tokens and gateway configuration. Rotate stderr capture yourself. Do not log Bearer tokens or commit real credentials. Deleting the database also deletes patch idempotency records and confirmation-hold rows. @@ -131,7 +131,7 @@ Store logs and the database outside the managed workspace, with tokens and gatew **Schema version and earlier versions.** The operations database states its schema version. CodeSpace refuses a database with a newer schema version than its own before writing anything, and leaves it as it is: open it with the CodeSpace version that wrote it. A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions in the database, keeps there the random identity of the database that it names when it [claims the Runner](runner-isolation.md#claim), and holds the lock file `.gateway.lock` beside it while it runs. A CodeSpace version from before these records still opens such a database and leaves the records alone, but it neither checks the schema version nor takes the lock, so do not run one on a database that a gateway with an independent Runner uses. To go back from the independent mode, stop using it and keep the database: restoring an earlier copy would drop the patch idempotency records, confirmation holds and process records written since. -After losing a patch response, query `operation_status` with exactly one of `operation_id` or `operation_key`. An unfinished record becomes `unknown` after restart; inspect files before deciding what to do. Processes use `process_id` and cannot be recovered through `operation_status`. A confirmation hold that was `queued` when the process died can be resumed and will reacquire. A hold that was `resuming` may recover a patch from that ledger, replay a stored terminal result, or return `APPROVAL_AMBIGUOUS`; exec is not respawned. See [retry and recovery rules](agent-integration.md). +After losing a patch response, query `operation_status` with exactly one of `operation_id` or `operation_key`. An unfinished record becomes `unknown` after restart; inspect files before deciding what to do. Processes use `process_id` and cannot be recovered through `operation_status`. A confirmation hold that was `queued` when the process died can be resumed and will reacquire. A hold that was `resuming` may recover a patch from that ledger, replay a stored terminal result, or return `APPROVAL_AMBIGUOUS`; exec is not respawned. A gateway with an independent Runner settles an exec hold left `resuming` when it [reconciles](runner-isolation.md#reconciliation): consumed with the process that the Runner created for it, or granted again when the Runner proves that it created none. See [retry and recovery rules](agent-integration.md). diff --git a/docs/runner-isolation.md b/docs/runner-isolation.md index 6541726..6cec195 100644 --- a/docs/runner-isolation.md +++ b/docs/runner-isolation.md @@ -10,7 +10,7 @@ There are three separate questions: which process runs the work, whether command `in-process` runs the supervisor inside `codespace-mcp`. `uds` (Unix domain socket) runs that supervisor inside `codespace-codex-runtime` on the same host. The worker starts with Codex process hardening and binds a private Unix socket. Hardening the worker does not sandbox its commands. -The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 10, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. +The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 11, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. Managed process lifetime is owned by the runner instance, not the MCP connection. Streamable HTTP or MCP client disconnect keeps processes running. Gateway/worker UDS disconnect or gateway shutdown (including stdio EOF) ends the owned worker and its children; process handles are lost. At shutdown the gateway closes the worker's connection, so the worker ends its processes as it exits; a worker still running 5 seconds later is killed. Restart does not restore `process_id`. An [independent Runner](#independent-runner) is the exception: its processes outlive the gateway. @@ -20,7 +20,7 @@ Managed process lifetime is owned by the runner instance, not the MCP connection ## Independent Runner -> **Not a supported mode yet.** The independent Runner, the records of its executions and the Claim that hands its control from one gateway to the next are the first steps of Gateway recovery. A gateway claims the Runner again when its connection is lost, but a gateway that controls it after one that started work, the same gateway after a lost connection included, cannot start new work. A later step adds reconciliation, and the mode is supported only once it is qualified. +> **Not a supported mode yet.** The independent Runner, the records of its executions, the Claim that hands its control from one gateway to the next and the reconciliation after each Claim are the first steps of Gateway recovery. A gateway claims the Runner again when its connection is lost, and starts new work where an earlier controller started work only once it has reconciled what that controller left. A later step adds what follows the loss of the Runner itself, and the mode is supported only once it is qualified. An independent Runner outlives the gateways that connect to it: its processes, their output and their timeouts carry on when a gateway exits, restarts or is killed. An operator or a harness starts it, and a gateway only connects to it: @@ -56,7 +56,25 @@ A gateway that starts while another connection controls the Runner, or while tha **A lost connection.** A gateway whose connection to the Runner is lost, or that closed it after a call passed its deadline, connects and claims the same Runner again. It first checks that the Runner at the socket has the identity it found at its start. It tries at once, waits 100 milliseconds after the first failed try, and twice as long after each further one, up to 5 seconds. Meanwhile it refuses each request that needs the Runner with `RUNNER_RECONNECTING`: nothing is sent, reserved or recorded for the request, and an approval it would resume is not consumed. When nothing accepts connections at the socket any more, another Runner answers there, or the Runner refuses the Claim for a reason that waiting cannot end, the gateway stops trying and refuses every such request with `RUNNER_DISCONNECTED`; it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection was lost keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. An exit event that the Runner could not deliver goes to its next controller, and the gateway hands each exit on once. -**After an earlier controller.** The first gateway that claims a newly started Runner works as it does with a managed worker. Once a controller has started work there (an execution, a patch, or with DevGuard a preparation or a launch), every later controller inherits the Runner's processes, the same gateway included once it claimed the Runner again after a lost connection. It can query, read, write to, resize and terminate them with their `process_id`, but it refuses new executions, patches and approval resumes with `RECOVERY_PENDING`, and leaves the approvals as they are. This holds after the inherited processes have ended too; to start work again, stop the Runner and start a new one. The gateway logs the Runner's identity (its process ID, boot and start time), the epoch of its Claim, whether a controller before it started work, how many processes it inherited, and the number of the last exit event that list reflects. +**After an earlier controller.** The first gateway that claims a newly started Runner works as it does with a managed worker. Once a controller has started work in a workspace (an execution, a patch, or with DevGuard a preparation or a launch), every later controller inherits the Runner's processes, the same gateway included once it claimed the Runner again after a lost connection, and the Claim's answer names each workspace where a controller before it started work. The gateway can query, read, write to, resize and terminate the inherited processes with their `process_id` at once, also while DevGuard cannot answer. In each workspace that the Claim names, where the Runner holds a running process, or that a record left to [reconcile](#reconciliation) names, it first reconciles what the earlier controllers left. Until then it refuses new executions, patches and approval resumes there with `RECOVERY_PENDING`, before it takes up an approval, so the approval stays as it was. The Runner also refuses them in each workspace that the Claim names, until the gateway tells it that the workspace is reconciled. Other workspaces take new work at once. The gateway logs the Runner's identity (its process ID, boot and start time), the epoch of its Claim, the workspaces where a controller before it started work, how many processes it inherited, and the number of the last exit event that list reflects. + + + +**Reconciliation.** After each Claim, the gateway reconciles each record that is not `terminal` and was not sent to this Runner under this Claim or a later one, and each record of an approval that is still `resuming`. It compares three accounts of each execution: its [record](#process-records), what the Runner that the record names holds or remembers of the process, and with DevGuard, for an execution in a `required` workspace, its attempt, which the Runner looks up with DevGuard. For its whole life, the Runner remembers each `process_id` that it was asked to start, by its ID alone: whether it created the process, or received the request and created nothing. It proves that it never received a request only for itself: a record that names another Runner, one that was stopped or lost included, is not settled this way. + +| Record | The Runner that the record names | DevGuard attempt (`required` only) | Outcome | +| --- | --- | --- | --- | +| `dispatching` or `running` | holds the process running | launched | the record says `running`, and the process holds its workspace again: new work there returns `WORKSPACE_BUSY` until it exits or is terminated | +| `dispatching` or `running` | holds the process ended, or created it and no longer holds it | released after its scope ended, or ended before it started | the record says how the process ended, with the termination `unknown` when the Runner no longer holds it, and the workspace is released | +| `dispatching` | never received the request, or received it and created nothing | none, or ended before it started | the record says that the execution did not start, and the workspace is released | +| none | holds a process running | any | the workspace stays refused | +| anything else | | | the workspace stays refused | + +A workspace that waits for an answer, because the Runner or DevGuard could not be asked or DevGuard has not settled an attempt yet, is reconciled again after 500 milliseconds, and twice as long after each further try, up to 30 seconds. A workspace where the accounts do not agree, or whose record names another Runner, stays refused until an operator resolves it, with a procedure that a later step adds; a workspace where the Runner holds a running process that no record accounts for stays refused until that process exits or is terminated. The gateway reconciles again after each exit too. It logs each workspace that stays refused with the reason, which `RECOVERY_PENDING` also states, and each workspace once it is reconciled. It does not release a workspace that it cannot reconcile. + +An approval that an earlier controller left `resuming` is consumed with the execution that the Runner created for it, and `operation_resume` then returns that execution's `process_id` with `dispatch_status: confirmed`. When the Runner proved that it created none, the approval is granted again and can be resumed once more. Otherwise it stays `resuming`, and `operation_resume` returns `APPROVAL_AMBIGUOUS` as before: an approval whose execution may have started is never granted again. + +**Ended processes.** An independent Runner keeps each ended process, its status and its output, until a controller acknowledges that its record says how the process ended, so an exit that no gateway received reaches the next controller. It still releases a process's handle after 15 minutes, or beyond 64 ended processes, as a managed worker does, and keeps at most 64 ended processes whose handles it released, dropping the oldest first. Of a dropped process, the Runner remembers only that it created it. @@ -66,11 +84,11 @@ A gateway that starts while another connection controls the Runner, or while tha **Process records.** The gateway records each execution in its operations database before it sends the execution to the Runner, and follows it to its end, so that a gateway that lost its memory can identify the same execution again. A record holds the `process_id`, the identity of the Runner it was sent to (its process ID, boot, start time and a random nonce), the [controller epoch](#claim) it was sent under, the workspace, the approval it resumes, its DevGuard attempt, whether it takes part in DevGuard, whether it has a tty, and the SHA-256 digest of its argv, never the argv, its environment or a secret. Its phase is `dispatching` until the Runner answers that it started the execution, and `running` after that. It becomes `terminal` once the Runner reports that the execution ended, with the termination, exit code and signal the Runner reported, or once the Runner answers that it did not start it. The record also keeps the deadline the Runner set, the root process the Runner created, by its process ID and the start time that tells it apart from a later process with the same ID (a `tty: true` root is not named), and how far a gateway read the output. -An execution that resumes an approval is recorded in the transaction that marks the approval resuming. When the record cannot be written, the execution is refused with `INTERNAL` and nothing is sent. The next controller records the exits that happened while no gateway controlled the Runner, which the Runner keeps for it, and `process_status` and `read_process` record what the Runner reports. A record never revives a handle, and never by itself proves that an execution started or how it ended: the Runner is the authority, and the record of an execution whose answer was lost stays `dispatching`. Recovering work from the records comes in later steps. +An execution that resumes an approval is recorded in the transaction that marks the approval resuming. When the record cannot be written, the execution is refused with `INTERNAL` and nothing is sent. Once a record says how an execution ended, the gateway acknowledges it to the Runner. The next controller records the exits that happened while no gateway controlled the Runner, which the Runner keeps for it, at the latest when it reconciles; a gateway that stops the Runner records how each of its processes ended; and `process_status` and `read_process` record what the Runner reports. A record never revives a handle, and never by itself proves that an execution started or how it ended: the Runner is the authority, and the record of an execution whose answer was lost stays `dispatching` until a [reconciliation](#reconciliation) settles it. **One writer.** A gateway with an independent Runner is its operations database's only writer. Before it opens the database it takes an exclusive lock on a file beside it, `.gateway.lock`, which names the process ID of the gateway that holds it, and it keeps the lock until it exits, however it exits. A gateway that finds the lock held, such as one started while the gateway before it still detaches, logs that it waits, waits up to 15 seconds, and then refuses to start. Only this mode takes the lock. See [Operations](operations.md#operations-database) for the database's schema version and earlier CodeSpace versions. -**DevGuard.** With the `devguard` feature, the independent Runner registers itself with the worker's DevGuard settings and reads the consumer secret from the private file given with `--devguard-credential-file`; its gateway, started with `--devguard register`, hands it no secret. Workspaces that require resource participation run only when the Runner states that it registers and launches through a launch helper, and are refused with `RESOURCE_POLICY_UNSUPPORTED` otherwise. See [DevGuard integration](devguard-integration.md#resource-gateway-recovery). +**DevGuard.** With the `devguard` feature, the independent Runner registers itself with the worker's DevGuard settings and reads the consumer secret from the private file given with `--devguard-credential-file`; its gateway, started with `--devguard register`, hands it no secret. Workspaces that require resource participation run only when the Runner states that it registers and launches through a launch helper, and are refused with `RESOURCE_POLICY_UNSUPPORTED` otherwise. A gateway without the `devguard` feature cannot reconcile a governed execution, so its workspace stays refused. See [DevGuard integration](devguard-integration.md#resource-gateway-recovery). diff --git a/docs/security-model.md b/docs/security-model.md index f94b509..8abc7de 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -34,7 +34,7 @@ The operator-registered root is a trust anchor. Keep tokens, the operations data ## Operation and recovery safety -Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery. +Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery, except that a gateway with an [independent Runner](runner-isolation.md#reconciliation) restores the occupancy of each process that it finds running when it reconciles, and starts no new work in a workspace before that. Confirmation holds in `CODESPACE_OPERATIONS_DB` (`approvals`) store the V4A patch or exec argv while the row is `pending`, `granted`, `queued`, or `resuming`. The patch operations ledger stores hashes, not patch text. After `denied` or `consumed`, the hold body is replaced with digest metadata (tool, workspace, fingerprint). A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions there (`processes`), each with the SHA-256 digest of its argv, never the argv, its environment or a secret, and keeps there the database's random identity (`store_identity`), which it names when it claims the Runner: it tells one database from another and is not a secret. Keep this database outside the workspace root, with tokens and gateway configuration. The hold is a workflow pause: the same MCP caller can grant it. It is not an isolation boundary. diff --git a/docs/translations.json b/docs/translations.json index d5f6bfa..dbf73e1 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -78,8 +78,8 @@ "최소-도입-조건", "최초-gateway-복구-범위" ], - "source_sha256": "ef6902babea74a960118cef8b12204606c8707a3c13b5f28bb0cec113f633222", - "translation_sha256": "b7562abebafea29eb2da52ea4f421c41a8f2ed5af475a3be49a0937e215a9ae0" + "source_sha256": "73d4de8918976b98bd92f2b1811d0872d9603edf23ac847b59eb9e74a9f36514", + "translation_sha256": "9616be59db270581e1035460a8bef34bf06815f46c2db4d79eb87f2bd66d5a6d" }, { "id": "cs-rg-qualification", @@ -257,8 +257,8 @@ "재시도와-복구", "파일-읽기와-패치" ], - "source_sha256": "0e413aee000ab6f986dd428388789defbf866d86f9cf1376bf9c7bee2b21eab3", - "translation_sha256": "5acba3bbfa22171eb019ede270d3c4f04587454759a1e7af728e5205b567014a" + "source_sha256": "c0257f9844525cb9a2dbbca6fa6ee4dbfa376de877cb8a647d55e4a476031a2d", + "translation_sha256": "095fe129c42f71fab2a7ec47bdbffdd81f0c6141e616e454c46e4f2f3134833b" }, { "id": "operations", @@ -310,8 +310,8 @@ "작업-공간-등록", "첫-연결-확인" ], - "source_sha256": "dc741a1830193cedbf1a5a49327bed25d7d4ba59782e00e24fc3e55b34f872df", - "translation_sha256": "debd98914cb9bdf0f84e1b4ea409695ec5a049a19ad785708ee0ba6f94fbfcb8" + "source_sha256": "3b42dfbca4a44361af4476134e59d02af7313f5093590057b0e81ca35b3f42b5", + "translation_sha256": "b44f15c0a1c415e6015e4d76ca6421a648ef61572b2b959c9e6faa21d8b6b35c" }, { "id": "chatgpt-connector", @@ -400,8 +400,8 @@ "현재-실행-구조", "확장-시-유지할-경계" ], - "source_sha256": "be5e155f374697eb4aedd3cb5eb08db8b00a67ec0c91b91f7b4503f3cf143220", - "translation_sha256": "9bd1f5d137a1cbb5d1ed97be05129f276420927664c337c80e53b9e4283b6477" + "source_sha256": "9d03ae6de59d281f2f992bd39e4bf32516946ea0b490de3e0325d2f78cb0574d", + "translation_sha256": "10d6fffe680c807b9b7dd22148032e2529ea452d1d527ac2ec6f1cade693aa0b" }, { "id": "execution-substrate", @@ -465,8 +465,8 @@ "확인-홀드", "훅과-스킬" ], - "source_sha256": "be934715cd35e1d90c81bbe881681a889592de3a28ff56455b9e448b997bf2e5", - "translation_sha256": "b307eb918b5be50b4a7ec79f78d1ce1dd7d1a05a0ff81fc6f2c21fe382b571f1" + "source_sha256": "93e4ef27d771f9bec226af62ae66a785962aea6030050164195d9d9bcadc82ce", + "translation_sha256": "512a93517c56dd59aebae0271025658fd3c17a9f318bf3e6217b061245eb2b40" }, { "id": "protocol-compatibility", @@ -576,8 +576,8 @@ "프로세스-정직성", "프로필-mvp" ], - "source_sha256": "e3e7395c07e2660aae172e256231cea660d464c1e6f464d7e27f96b56e9c720e", - "translation_sha256": "2e812460939e5716d4ca20d5a097346cab81c0c39062f0e48e9b29118e5ad313" + "source_sha256": "cdb542f72d794730735b7a71cf5c45bfb3aa012d3d6f637196dc28dc41249516", + "translation_sha256": "5bd6ab3149c3c72b7d94a96910ff412be67163ad73b0df1b0c8d12b2b9d109cf" }, { "id": "runner-isolation", @@ -602,6 +602,7 @@ "macos-no-docker", "network-and-filesystem-scope", "process-records", + "reconciliation", "runner-isolation", "what-the-compose-fixture-does", "이후-프로세스-분리", @@ -612,8 +613,8 @@ "컨테이너-실험-구성과-검증", "호스트와-worker-실행" ], - "source_sha256": "b8ca7eaf43a128d33f41ae39999349c275dd677177589f8873d6997dbfb71b76", - "translation_sha256": "004b9bb59acd29d188e81b46d7f1fc9b32f3ca145faeb20e55d579a0bce2bc69" + "source_sha256": "e76149aba184f9de7f6e11441db5f9f519db95f20b62eb367de67e49381f98d2", + "translation_sha256": "950575f22d943f0904b7a805b4cd81f154f218a14e219bc16e92e8449d6ff1f7" }, { "id": "error-codes", @@ -641,8 +642,8 @@ "전송-실패", "전송-실패-작업-없음" ], - "source_sha256": "9dec383f2ff5d3c470603d03f6bac2cee684b87ac775c13ab229d99a73c97936", - "translation_sha256": "70569634b14f9b6419c3eb1c90228eb9b3ef364c6b33a6b61ec8c5db75c3bc26" + "source_sha256": "5892534235706ee8d0d3a9bf18a1d36b18578441567f3eba92f69d91fbce2010", + "translation_sha256": "45046341089415fcc7d64e850629e6605ece6d1ab48b3a05e9579e32f8f56bbd" }, { "id": "codex-reuse", @@ -723,8 +724,8 @@ "핀-6b9826e의-후보", "핵심-대-어댑터" ], - "source_sha256": "aae68b25a95333d12733ae14054ed304a24ce90931cf29c8cafd0200c4238a93", - "translation_sha256": "c9df1c720035bf7138a79d0f748972cf226d8c8728242bad526ab1fa0bd6df19" + "source_sha256": "8012d6d5acbcd330b65c62f1ac75c0fa83b8d0cdf93f3e07bb4816ce6fe5b8de", + "translation_sha256": "539ff72fecf5ae9411a4f4d7941a5b61c12a4aa7aab3ec473ff4fd49190c9a7f" }, { "id": "upstream-lock",