diff --git a/crates/domain/src/lib.rs b/crates/domain/src/lib.rs index dbd9361..49a3a1f 100644 --- a/crates/domain/src/lib.rs +++ b/crates/domain/src/lib.rs @@ -43,7 +43,7 @@ pub use patch::{ OperationEventName, OperationKind, OperationStatusParams, OperationStatusResult, PatchStatus, }; pub use process::{ - ExecCommandParams, ExecCommandResult, ExecDispatchStatus, ProcessResizeParams, + ExecCommandParams, ExecCommandResult, ExecDispatchStatus, LostReason, ProcessResizeParams, ProcessResizeResult, ProcessSignal, ProcessState, ProcessStatusParams, ProcessStatusResult, ProcessTermination, ReadProcessParams, ReadProcessResult, TerminateProcessParams, WriteStdinParams, diff --git a/crates/domain/src/process.rs b/crates/domain/src/process.rs index f9ffd47..a65d193 100644 --- a/crates/domain/src/process.rs +++ b/crates/domain/src/process.rs @@ -58,15 +58,38 @@ pub struct ReadProcessResult { pub output_lost: bool, /// First retained process-output offset (`dropped` in the runner). pub retained_from: u64, + /// Why output is missing or can no longer be read: present whenever output_lost is true, + /// and whenever the Runner no longer holds the process. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lost_reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub coordination: Option, } +/// Why a process's output is missing, or why what became of the process cannot be learned +/// (P1R-C05). None of them is a successful end. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LostReason { + // The output ring kept only the most recent output; the process itself is unaffected. + RingOverflow, + // The Runner no longer keeps the ended process; the gateway answers from its records. + RetentionExpired, + // The Runner that ran the process was lost; the process may still run. + RunnerLost, + // The host restarted since the process started, which ended it. + HostRebooted, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ProcessState { Running, Exited, + // The Runner that ran the process was lost (P1R-C05): it may still run, and CodeSpace + // cannot learn what becomes of it. Not an end. A plain comment keeps the schema a single + // string enum. + Unknown, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -110,6 +133,11 @@ pub struct ProcessStatusResult { pub output_total: u64, pub output_retained_from: u64, pub eof: bool, + /// Why output is missing or the state is uncertain: present whenever + /// output_retained_from is above 0, state is unknown, or the Runner no longer holds the + /// process. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lost_reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub coordination: Option, } @@ -118,7 +146,16 @@ impl ProcessStatusResult { pub fn invariants_hold(&self) -> bool { match self.state { ProcessState::Running => { - self.exit_code.is_none() && self.termination.is_none() && self.signal.is_none() + self.exit_code.is_none() + && self.termination.is_none() + && self.signal.is_none() + && matches!(self.lost_reason, None | Some(LostReason::RingOverflow)) + } + ProcessState::Unknown => { + self.exit_code.is_none() + && self.termination.is_none() + && self.signal.is_none() + && self.lost_reason == Some(LostReason::RunnerLost) } ProcessState::Exited => match self.termination { Some(ProcessTermination::Exited) => self.signal.is_none(), @@ -243,6 +280,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: false, + lost_reason: None, coordination: None, }) .unwrap(); @@ -258,6 +296,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: false, + lost_reason: None, coordination: None, } .invariants_hold()); @@ -270,6 +309,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: false, + lost_reason: None, coordination: None, } .invariants_hold()); @@ -290,6 +330,7 @@ mod tests { output_total: 12, output_retained_from: 0, eof: true, + lost_reason: None, coordination: None, }) .unwrap(); @@ -309,6 +350,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: true, + lost_reason: None, coordination: None, }) .unwrap(); @@ -328,6 +370,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: true, + lost_reason: None, coordination: None, }; let kill = ProcessSignal { @@ -376,15 +419,111 @@ mod tests { eof: false, output_lost: true, retained_from: 8, + lost_reason: Some(LostReason::RingOverflow), coordination: None, }) .unwrap(); assert_eq!(json["output_lost"], true); assert_eq!(json["retained_from"], 8); + assert_eq!(json["lost_reason"], "ring_overflow"); let schema = serde_json::to_value(schemars::schema_for!(ReadProcessResult)).unwrap(); let dumped = schema.to_string(); assert!(dumped.contains("output_lost"), "{dumped}"); assert!(dumped.contains("retained_from"), "{dumped}"); + assert!(dumped.contains("lost_reason"), "{dumped}"); + let complete = serde_json::to_value(ReadProcessResult { + process_id: ProcessId("proc-1".into()), + cursor: 3, + chunk: "abc".into(), + eof: true, + output_lost: false, + retained_from: 0, + lost_reason: None, + coordination: None, + }) + .unwrap(); + assert!(complete.get("lost_reason").is_none(), "{complete}"); + } + + /// Each reason for missing output or an uncertain state is a name of a single string enum + /// (P1R-C05), and so is the `unknown` state. + #[test] + fn reasons_and_the_unknown_state_cross_as_names() { + let reasons = serde_json::to_value(schemars::schema_for!(LostReason)).unwrap(); + assert_eq!( + reasons["enum"], + serde_json::json!([ + "ring_overflow", + "retention_expired", + "runner_lost", + "host_rebooted" + ]), + "{reasons}" + ); + let states = serde_json::to_value(schemars::schema_for!(ProcessState)).unwrap(); + assert_eq!( + states["enum"], + serde_json::json!(["running", "exited", "unknown"]), + "{states}" + ); + let schema = serde_json::to_value(schemars::schema_for!(ProcessStatusResult)).unwrap(); + assert!(schema.to_string().contains("lost_reason"), "{schema}"); + } + + /// A process whose Runner was lost is in no state that reads as an end: `unknown` carries + /// no exit code, termination or signal, and always says why (P1R-C05). A running process + /// can only have lost output to the ring. + #[test] + fn an_unknown_state_says_why_and_is_no_end() { + let status = |state, termination, lost_reason| ProcessStatusResult { + process_id: ProcessId("proc-1".into()), + state, + exit_code: None, + termination, + signal: None, + output_total: 9, + output_retained_from: 9, + eof: true, + lost_reason, + coordination: None, + }; + let lost = status(ProcessState::Unknown, None, Some(LostReason::RunnerLost)); + assert!(lost.invariants_hold()); + let json = serde_json::to_value(&lost).unwrap(); + assert_eq!(json["state"], "unknown"); + assert_eq!(json["lost_reason"], "runner_lost"); + assert!(json.get("exit_code").is_none()); + assert!(json.get("termination").is_none()); + + assert!(!status(ProcessState::Unknown, None, None).invariants_hold()); + assert!(!status( + ProcessState::Unknown, + None, + Some(LostReason::RetentionExpired) + ) + .invariants_hold()); + assert!(!status( + ProcessState::Unknown, + Some(ProcessTermination::Unknown), + Some(LostReason::RunnerLost) + ) + .invariants_hold()); + let mut ended = status(ProcessState::Unknown, None, Some(LostReason::RunnerLost)); + ended.exit_code = Some(0); + assert!(!ended.invariants_hold()); + + assert!( + status(ProcessState::Running, None, Some(LostReason::RingOverflow)).invariants_hold() + ); + assert!( + !status(ProcessState::Running, None, Some(LostReason::RunnerLost)).invariants_hold() + ); + assert!(status( + ProcessState::Exited, + Some(ProcessTermination::Unknown), + Some(LostReason::HostRebooted) + ) + .invariants_hold()); } #[test] diff --git a/crates/runner/src/lib.rs b/crates/runner/src/lib.rs index 37b833a..41c54bc 100644 --- a/crates/runner/src/lib.rs +++ b/crates/runner/src/lib.rs @@ -133,8 +133,9 @@ pub use files::VERSION_ABSENT; pub use fork_handlers::prepare_fork_spawns; pub use patch_helper::ensure_helper_for_tests; pub use process::{ - InProcessRunner, RetentionPolicy, ShellRelease, SlotReservation, DEFAULT_COMPLETED_TTL, - DEFAULT_MAX_COMPLETED, DEFAULT_MAX_PROCESSES, DEFAULT_MAX_UNACKNOWLEDGED, DEFAULT_TIMEOUT, + process_signal, InProcessRunner, RetentionPolicy, ShellRelease, SlotReservation, + DEFAULT_COMPLETED_TTL, DEFAULT_MAX_COMPLETED, DEFAULT_MAX_PROCESSES, + DEFAULT_MAX_UNACKNOWLEDGED, DEFAULT_TIMEOUT, }; /// True when this process probed a working Linux sandbox helper. @@ -143,7 +144,9 @@ pub fn linux_sandbox_available() -> bool { linux_sandbox::probe() } -pub use lifecycle::{RunnerIdentity, RunnerLifecycle}; +pub use lifecycle::{ + host_boot_id, pid_runs, process_runs, process_start_identity, RunnerIdentity, RunnerLifecycle, +}; #[cfg(feature = "devguard")] pub use registration::OwnerRegistration; pub use socket::{ @@ -151,18 +154,18 @@ pub use socket::{ reclaim_leftover_socket, runner_socket_path, RUNNER_SOCKET_NAME, }; pub use uds::{ - runner_disconnected, runner_reconnecting, ClaimError, DisconnectHook, LinkEvent, LinkObserver, - RunnerHello, RunnerStopped, UdsRunner, CLAIM_START_BOUND, RECLAIM_BACKOFF_MAX, + runner_disconnected, runner_lost, runner_reconnecting, ClaimError, DisconnectHook, LinkEvent, + LinkObserver, RunnerHello, RunnerStopped, UdsRunner, CLAIM_START_BOUND, RECLAIM_BACKOFF_MAX, RUNNER_CALL_DEADLINE, }; pub use watch::{FsWatchEvent, FsWatchKind, WatchClosed, WatchSubscription}; #[cfg(feature = "devguard")] pub use wire::serve_runner_connection_with_registration; pub use wire::{ - host_worker, read_frame, recovery_pending, serve_runner_connection, write_frame, ClaimNote, - ClaimObserver, ClaimRefusal, IndependentRunner, RecalledProcess, RunnerClaim, RunnerEvent, - RunnerEvents, RunnerInventoryEntry, RunnerOp, RunnerOpResult, Served, StopHandle, WireEnvelope, - WireKind, CLAIM_WAIT_BOUND, STOP_DRAIN_BOUND, WIRE_PROTOCOL, + host_worker, host_worker_with_retention, read_frame, recovery_pending, serve_runner_connection, + write_frame, ClaimNote, ClaimObserver, ClaimRefusal, IndependentRunner, RecalledProcess, + RunnerClaim, RunnerEvent, RunnerEvents, RunnerInventoryEntry, RunnerOp, RunnerOpResult, Served, + StopHandle, WireEnvelope, WireKind, CLAIM_WAIT_BOUND, STOP_DRAIN_BOUND, WIRE_PROTOCOL, }; /// Execution-plane API. Control-plane fields (`work_id`, coordination, diff --git a/crates/runner/src/lifecycle.rs b/crates/runner/src/lifecycle.rs index 891430b..2709ac1 100644 --- a/crates/runner/src/lifecycle.rs +++ b/crates/runner/src/lifecycle.rs @@ -67,6 +67,12 @@ impl RunnerIdentity { } } +/// The host's current boot session, when it reports one (P1R-C05): Linux's `boot_id`, macOS's +/// `kern.bootsessionuuid`. A process recorded under another boot ended with it. +pub fn host_boot_id() -> Option { + boot_id() +} + #[cfg(target_os = "linux")] fn boot_id() -> Option { std::fs::read_to_string("/proc/sys/kernel/random/boot_id") @@ -108,7 +114,7 @@ fn boot_id() -> Option { /// been reaped yet, as a child that only its own reaper waits for, it is that process's. `None` /// where the host does not report it. #[cfg(target_os = "linux")] -pub(crate) fn process_start_identity(pid: u32) -> Option { +pub fn process_start_identity(pid: u32) -> Option { // `stat`'s 22nd field: the start time in clock ticks since boot. The command name before it // is in parentheses and may itself hold spaces and parentheses. let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; @@ -122,7 +128,83 @@ pub(crate) fn process_start_identity(pid: u32) -> Option { /// been reaped yet, as a child that only its own reaper waits for, it is that process's. `None` /// where the host does not report it. #[cfg(target_os = "macos")] -pub(crate) fn process_start_identity(pid: u32) -> Option { +pub fn process_start_identity(pid: u32) -> Option { + bsd_info(pid).map(|info| macos_start(&info)) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn process_start_identity(_pid: u32) -> Option { + None +} + +/// Whether process `pid` still runs as the process whose start identity is `started` +/// (P1R-C05): a later process with the same PID is another one, and one that exited does not +/// run, even while its parent has not reaped it yet. +#[cfg(target_os = "linux")] +pub fn process_runs(pid: u32, started: &str) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + let Some(close) = stat.rfind(')') else { + return false; + }; + let fields: Vec<&str> = stat[close + 1..].split_whitespace().collect(); + // `stat`'s 3rd field is the state, Z or X once the process exited; its 22nd the start time. + match ( + fields.first(), + fields.get(19).and_then(|ticks| ticks.parse::().ok()), + ) { + (Some(state), Some(ticks)) => { + !matches!(*state, "Z" | "X" | "x") && format!("linux-start-ticks:{ticks}") == started + } + _ => false, + } +} + +/// Whether process `pid` still runs as the process whose start identity is `started` +/// (P1R-C05): a later process with the same PID is another one, and one that exited does not +/// run, even while its parent has not reaped it yet. +#[cfg(target_os = "macos")] +pub fn process_runs(pid: u32, started: &str) -> bool { + bsd_info(pid) + .is_some_and(|info| info.pbi_status != libc::SZOMB && macos_start(&info) == started) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn process_runs(_pid: u32, _started: &str) -> bool { + false +} + +/// Whether some process that has not exited holds the PID `pid` (P1R-C05). A process that +/// exited and that its parent has not reaped yet does not count. Where the host cannot tell, +/// every PID is taken as held. +#[cfg(target_os = "linux")] +pub fn pid_runs(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + let state = stat + .rfind(')') + .and_then(|close| stat[close + 1..].split_whitespace().next()); + !matches!(state, None | Some("Z" | "X" | "x")) +} + +/// Whether some process that has not exited holds the PID `pid` (P1R-C05). A process that +/// exited and that its parent has not reaped yet does not count. Where the host cannot tell, +/// every PID is taken as held. +#[cfg(target_os = "macos")] +pub fn pid_runs(pid: u32) -> bool { + bsd_info(pid).is_some_and(|info| info.pbi_status != libc::SZOMB) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn pid_runs(_pid: u32) -> bool { + true +} + +/// What macOS reports of process `pid`, an exited one that is not reaped yet included. +#[cfg(target_os = "macos")] +fn bsd_info(pid: u32) -> Option { // SAFETY: an all-zero `proc_bsdinfo` is a valid value of this plain C structure. let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; let size = std::mem::size_of::() as libc::c_int; @@ -137,18 +219,15 @@ pub(crate) fn process_start_identity(pid: u32) -> Option { size, ) }; - if written != size || info.pbi_pid != pid { - return None; - } - Some(format!( - "macos-start:{}.{:06}", - info.pbi_start_tvsec, info.pbi_start_tvusec - )) + (written == size && info.pbi_pid == pid).then_some(info) } -#[cfg(not(any(target_os = "linux", target_os = "macos")))] -pub(crate) fn process_start_identity(_pid: u32) -> Option { - None +#[cfg(target_os = "macos")] +fn macos_start(info: &libc::proc_bsdinfo) -> String { + format!( + "macos-start:{}.{:06}", + info.pbi_start_tvsec, info.pbi_start_tvusec + ) } /// 128 random bits in hex. A host without `/dev/urandom` gets the time and the process ID @@ -212,14 +291,15 @@ mod tests { assert_eq!(own, None); return; } - let own = own.expect("this process's start identity"); + assert!(own.is_some(), "this process's start identity"); let mut child = std::process::Command::new("sleep") .arg("30") .spawn() .unwrap(); let pid = child.id(); + // Only with its PID does an identity name one process: on Linux, two processes that + // start within one clock tick, as this one and its child may, have the same start time. let started = process_start_identity(pid).expect("a running child's start identity"); - assert_ne!(started, own); assert_eq!(process_start_identity(pid).as_ref(), Some(&started)); child.kill().unwrap(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); @@ -243,6 +323,48 @@ mod tests { ); } + /// A process runs, for an operator who looks for what a lost Runner left (P1R-C05), only + /// by its PID and its start identity together, and not once it exited, even before it is + /// reaped: a later process with its PID is never taken for it. + #[test] + fn a_process_runs_as_itself_until_it_exits() { + if cfg!(not(any(target_os = "linux", target_os = "macos"))) { + assert!(!process_runs(std::process::id(), "any")); + return; + } + let mut child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let pid = child.id(); + let started = process_start_identity(pid).expect("a running child's start identity"); + assert!(process_runs(pid, &started)); + assert!(pid_runs(pid)); + assert!( + !process_runs(pid, &format!("{started}0")), + "another start identity with the same PID" + ); + child.kill().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !exited_unreaped(pid) { + assert!( + std::time::Instant::now() < deadline, + "the child did not exit" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + !process_runs(pid, &started), + "an exited child that is not reaped yet" + ); + assert!(!pid_runs(pid), "an exited child that is not reaped yet"); + child.wait().unwrap(); + assert!(!process_runs(pid, &started), "a reaped child"); + assert!(pid_runs(std::process::id())); + // Beyond the largest PID that Linux and macOS give a process. + assert!(!pid_runs(4_194_999)); + } + #[test] fn lifecycles_cross_the_wire_by_name() { assert_eq!( diff --git a/crates/runner/src/process.rs b/crates/runner/src/process.rs index 4c0e4ac..c1405ab 100644 --- a/crates/runner/src/process.rs +++ b/crates/runner/src/process.rs @@ -1500,7 +1500,7 @@ fn live_count(map: &HashMap) -> usize { } /// A signal number of this host, named when it is one of the standard signals. -fn process_signal(number: i32) -> ProcessSignal { +pub fn process_signal(number: i32) -> ProcessSignal { const NAMES: [(i32, &str); 21] = [ (libc::SIGHUP, "SIGHUP"), (libc::SIGINT, "SIGINT"), diff --git a/crates/runner/src/uds.rs b/crates/runner/src/uds.rs index bf80232..5899d5c 100644 --- a/crates/runner/src/uds.rs +++ b/crates/runner/src/uds.rs @@ -78,6 +78,21 @@ pub fn runner_disconnected(reason: &str) -> ErrorBody { ) } +/// The refusal of a request to an independent Runner that this gateway lost (P1R-C05): nothing +/// accepts connections at its socket any more, or another worker answers there. The request was +/// not sent. +pub fn runner_lost(reason: &str) -> ErrorBody { + ErrorBody::new( + ErrorCode::RunnerDisconnected, + format!( + "this gateway lost its independent Runner ({reason}), so the request was not sent. \ + What became of the Runner's processes cannot be learned, and some may still run: \ + process_status and read_process answer from this gateway's records, and an \ + operator finds the processes and releases their workspaces" + ), + ) +} + /// The refusal of a request while this gateway claims its independent Runner again /// (P1R-C03): the request was not sent. pub fn runner_reconnecting() -> ErrorBody { @@ -101,6 +116,10 @@ pub enum LinkEvent { /// The Runner cannot be claimed again, and why: every later request is refused with /// `RUNNER_DISCONNECTED`. Gone(String), + /// The Runner is lost, and why (P1R-C05): nothing accepts connections at its socket any + /// more, or another worker answers there. What becomes of its processes cannot be learned; + /// every later request is refused with `RUNNER_DISCONNECTED`. + RunnerLost(String), } pub type LinkObserver = Arc; @@ -160,6 +179,8 @@ enum State { Reclaiming, /// The Runner cannot be claimed again, and why (P1R-C03). Gone(String), + /// The Runner is lost, and why (P1R-C05). + Lost(String), } /// Where and for what a client claims its independent Runner again (P1R-C03). @@ -173,8 +194,10 @@ struct Reclaim { enum Attempt { /// Trying again may succeed. Retry(String), - /// Trying again cannot succeed. + /// Trying again cannot succeed: the Runner refused. Gone(String), + /// Trying again cannot succeed: the Runner is lost (P1R-C05). + Lost(String), } struct Client { @@ -203,6 +226,8 @@ struct Client { last_event: AtomicU64, /// The Claim of the client's current connection, once it has one (P1R-C04). claims: watch::Sender>, + /// Why the client lost its Runner, once it did (P1R-C05). + lost: watch::Sender>, } pub struct UdsRunner { @@ -346,6 +371,12 @@ impl UdsRunner { self.client.claims.subscribe() } + /// Why this client lost its independent Runner, once it did (P1R-C05): nothing accepts + /// connections at the Runner's socket any more, or another worker answers there. + pub fn lost(&self) -> watch::Receiver> { + self.client.lost.subscribe() + } + /// Connect to the independent Runner at `socket` and claim it for the operations database /// `store_id` (P1R-C03). While another connection controls the Runner, or that connection's /// requests run, and when the Claim's answer was lost, the Claim is asked again over a new @@ -451,7 +482,9 @@ impl UdsRunner { Err(Attempt::Retry(message)) => { Err(ClaimError::Transport(RunnerError::before_dispatch(message))) } - Err(Attempt::Gone(message)) => Err(ClaimError::NotIndependent(message)), + Err(Attempt::Gone(message) | Attempt::Lost(message)) => { + Err(ClaimError::NotIndependent(message)) + } } } @@ -791,6 +824,7 @@ impl Client { leaving: AtomicBool::new(false), last_event: AtomicU64::new(0), claims: watch::Sender::new(None), + lost: watch::Sender::new(None), } } @@ -839,6 +873,7 @@ impl Client { ))), State::Reclaiming => Err(RunnerError::execution(runner_reconnecting())), State::Gone(reason) => Err(RunnerError::execution(runner_disconnected(reason))), + State::Lost(reason) => Err(RunnerError::execution(runner_lost(reason))), } } @@ -985,6 +1020,12 @@ impl Client { (reclaim.observer)(LinkEvent::Gone(reason)); return; } + Err(Attempt::Lost(reason)) => { + *self.state() = State::Lost(reason.clone()); + self.lost.send_replace(Some(reason.clone())); + (reclaim.observer)(LinkEvent::RunnerLost(reason)); + return; + } Err(Attempt::Retry(reason)) => { (reclaim.observer)(LinkEvent::Retrying { reason, @@ -998,8 +1039,9 @@ impl Client { } /// One try to claim the Runner at `socket` again over a new connection, which is not the - /// client's until the caller makes it so. Nothing answering at the socket, another Runner - /// there, or a refusal that waiting cannot end, means the Runner cannot be claimed again. + /// client's until the caller makes it so. Nothing accepting connections at the socket, or + /// another worker there, means the Runner is lost (P1R-C05); a refusal that waiting cannot + /// end means it cannot be claimed again. async fn link_and_claim( self: &Arc, socket: &Path, @@ -1013,7 +1055,7 @@ impl Client { io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused ) => { - return Err(Attempt::Gone(format!( + return Err(Attempt::Lost(format!( "nothing accepts connections at {}: {err}", socket.display() ))) @@ -1059,16 +1101,16 @@ impl Client { if identity.is_some() && identity == expected { Ok(()) } else { - Err(Attempt::Gone(format!( + Err(Attempt::Lost(format!( "another Runner answers at its socket (pid {})", identity.map_or_else(|| "unknown".into(), |found| found.pid.to_string()) ))) } } - Ok(other) => Err(Attempt::Gone(format!( + Ok(other) => Err(Attempt::Lost(format!( "the worker at its socket answered Hello with {other:?}" ))), - Err(RunnerError::Execution(body)) => Err(Attempt::Gone(format!( + Err(RunnerError::Execution(body)) => Err(Attempt::Lost(format!( "the worker at its socket refused Hello: {}", body.message ))), diff --git a/crates/runner/src/wire.rs b/crates/runner/src/wire.rs index 96c41b9..3a4c0b0 100644 --- a/crates/runner/src/wire.rs +++ b/crates/runner/src/wire.rs @@ -28,9 +28,9 @@ use tokio::task::JoinSet; use crate::lifecycle::{RunnerIdentity, RunnerLifecycle}; use crate::{ - InProcessRunner, Recalled, Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, - RunnerError, RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, RunnerReadProcess, - RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, ShellRelease, + InProcessRunner, Recalled, RetentionPolicy, Runner, RunnerApplyPatchRequest, + RunnerApplyPatchResult, RunnerError, RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, + RunnerReadProcess, RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, ShellRelease, DEFAULT_MAX_UNACKNOWLEDGED, }; @@ -488,6 +488,11 @@ pub struct RunnerEvents { /// Host worker: `InProcessRunner` plus its numbered `ProcessExited` events. pub fn host_worker() -> (InProcessRunner, RunnerEvents) { + host_worker_with_retention(RetentionPolicy::default()) +} + +/// A host worker that keeps its ended processes as `retention` says. +pub fn host_worker_with_retention(retention: RetentionPolicy) -> (InProcessRunner, RunnerEvents) { let (tx, receiver) = mpsc::unbounded_channel(); let numbered = Arc::new(std::sync::Mutex::new(0u64)); let numbering = numbered.clone(); @@ -503,7 +508,7 @@ pub fn host_worker() -> (InProcessRunner, RunnerEvents) { )); }); ( - InProcessRunner::new(on_release), + InProcessRunner::with_retention(on_release, retention), RunnerEvents { receiver, numbered }, ) } @@ -4221,8 +4226,8 @@ mod tests { } /// A client stops claiming its Runner again once another Runner answers at the socket - /// or nothing accepts connections there, and refuses every request with - /// RUNNER_DISCONNECTED from then on, saying why (P1R-C03). + /// or nothing accepts connections there: it lost the Runner, says so, and refuses + /// every request with RUNNER_DISCONNECTED from then on, saying why (P1R-C03, P1R-C05). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_client_that_cannot_claim_its_runner_again_is_disconnected() { let host = Host::start(); @@ -4244,18 +4249,28 @@ mod tests { ) .await .unwrap(); + let lost = gateway.lost(); + assert_eq!(*lost.borrow(), None); relay.retarget(&other.socket); relay.cut(); assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); match next_link_event(&mut events).await { - LinkEvent::Gone(reason) => { + LinkEvent::RunnerLost(reason) => { assert!(reason.contains("another Runner answers"), "{reason}") } other => panic!("unexpected {other:?}"), } + assert!( + lost.borrow() + .as_deref() + .is_some_and(|reason| reason.contains("another Runner answers")), + "{:?}", + *lost.borrow() + ); let refusal = disconnected(&gateway, &ProcessId("proc-1".into())).await; assert!( - refusal.message.contains("another Runner answers"), + refusal.message.contains("another Runner answers") + && refusal.message.contains("lost its independent Runner"), "{refusal:?}" ); // The other Runner was asked nothing but its `Hello`. @@ -4275,12 +4290,17 @@ mod tests { assert_eq!(stopping.served().await, Served::Stopped { terminated: 0 }); assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); match next_link_event(&mut events).await { - LinkEvent::Gone(reason) => { + LinkEvent::RunnerLost(reason) => { assert!(reason.contains("nothing accepts connections"), "{reason}") } other => panic!("unexpected {other:?}"), } - disconnected(&gateway, &ProcessId("proc-1".into())).await; + let refusal = disconnected(&gateway, &ProcessId("proc-1".into())).await; + assert!( + refusal.message.contains("lost its independent Runner"), + "{refusal:?}" + ); + assert!(gateway.lost().borrow().is_some()); } /// An explicit stop ends the processes, a tty's process group included, and the Runner. diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index cbde08c..fbb0d9b 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -1,4 +1,4 @@ -use clap::{Parser, ValueEnum}; +use clap::{Parser, Subcommand, ValueEnum}; pub const MCP_PATH: &str = "/mcp"; pub const INBOX_PATH: &str = "/inbox"; @@ -48,6 +48,32 @@ pub enum GatewayLifecycle { Independent { exit_intent: RunnerExitIntent }, } +/// An operator's command instead of serving MCP (P1R-C05). +#[derive(Debug, Clone, PartialEq, Eq, Subcommand)] +pub enum Command { + /// The process records of a gateway with an independent Runner, in the database that + /// --operations-db names. + #[command(subcommand)] + Records(RecordsCommand), +} + +/// What an operator does with process records (P1R-C05, D8). +#[derive(Debug, Clone, PartialEq, Eq, Subcommand)] +pub enum RecordsCommand { + /// Print, as JSON, every record that may still hold a workspace: those that have not ended + /// and were not released, with whether their root process still runs. Reads only, so a + /// gateway may be running. + List, + /// Release the workspaces that these records hold after their Runner was lost, once nothing + /// of their processes runs. Refused while a gateway records into the database, for a record + /// whose root process still runs, and for one whose Runner may still run. + Release { + /// The process IDs of the records to release. + #[arg(required = true)] + process_ids: Vec, + }, +} + #[derive(Debug, Clone, Parser)] #[command( name = "codespace-mcp", @@ -55,6 +81,10 @@ pub enum GatewayLifecycle { about = "CodeSpace execution-tools MCP" )] pub struct Cli { + /// An operator's command; without one, the gateway serves MCP. + #[command(subcommand)] + pub command: Option, + /// Use Streamable HTTP instead of stdio. #[arg(long)] pub http: bool, @@ -79,8 +109,8 @@ pub struct Cli { /// SQLite file for operations. Unset uses an in-memory database (no replay across restarts). /// Required with `--runner-lifecycle independent`, whose gateway records its executions there - /// and holds the lock file beside it (P1R-C02). - #[arg(long, env = "CODESPACE_OPERATIONS_DB")] + /// and holds the lock file beside it (P1R-C02), and with `records`. + #[arg(long, env = "CODESPACE_OPERATIONS_DB", global = true)] pub operations_db: Option, /// Runner backend. Allowed values: `in-process` (default host supervisor) or `uds`. @@ -302,6 +332,41 @@ mod tests { assert!(err.to_string().contains("detached"), "{err}"); } + /// The operator's records commands name the operations database, before or after the + /// command, and release at least one record (P1R-C05). + #[test] + fn the_records_commands_take_the_operations_database() { + let parse = |args: &[&str]| { + let mut argv = vec!["codespace-mcp"]; + argv.extend_from_slice(args); + Cli::try_parse_from(argv) + }; + let cli = parse(&["records", "list", "--operations-db", "/var/cs/ops.sqlite"]).unwrap(); + assert_eq!(cli.command, Some(Command::Records(RecordsCommand::List))); + assert_eq!( + cli.operations_db.as_deref(), + Some(std::path::Path::new("/var/cs/ops.sqlite")) + ); + let cli = parse(&[ + "--operations-db", + "/var/cs/ops.sqlite", + "records", + "release", + "proc-1", + "proc-2", + ]) + .unwrap(); + assert_eq!( + cli.command, + Some(Command::Records(RecordsCommand::Release { + process_ids: vec!["proc-1".into(), "proc-2".into()] + })) + ); + assert!(parse(&["records", "release"]).is_err()); + assert!(parse(&["records"]).is_err()); + assert_eq!(parse(&[]).unwrap().command, None); + } + /// Without the `devguard` feature the gateway has no DevGuard setting at all. #[cfg(not(feature = "devguard"))] #[test] diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index a0a52be..c1e32e1 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -8,6 +8,7 @@ pub mod http; pub mod inbox; pub mod logging; pub mod mcp; +pub mod operator; pub mod patch_helper; pub mod protocol; pub mod records; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 6c74924..9808ea5 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -10,6 +10,7 @@ use codespace_server::config::{ }; use codespace_server::http::serve_http_handler; use codespace_server::logging; +use codespace_server::operator; use codespace_server::records::{Leases, ProcessRecords}; use codespace_server::recovery::Recovery; use codespace_server::runtime::RuntimeProcess; @@ -32,6 +33,12 @@ fn main() -> Result<()> { .expect("Failed building the Runtime"); logging::init(); let cli = Cli::parse(); + // An operator's command on the process records, instead of a gateway (P1R-C05). + if let Some(command) = &cli.command { + let done = + operator::run(command, cli.operations_db.as_deref()).map_err(anyhow::Error::msg)?; + std::process::exit(if done { 0 } else { 1 }); + } let lifecycle = cli.lifecycle().map_err(anyhow::Error::msg)?; // Released as the gateway exits, once its runtime is shut down. let _writer = lock_operations_db(&cli, lifecycle)?; @@ -251,6 +258,9 @@ async fn start_runner( let backend = RuntimeBackend::Uds(runner.clone()); let records = ProcessRecords::start(store.clone(), &identity, backend.clone(), exits); + // Once the Runner is lost, what was sent to it and had not ended becomes + // unknown, and keeps its workspace (P1R-C05). + records.follow_loss(runner.lost()); // What earlier claims left is reconciled before new work starts where they // started work, and again after each later Claim (P1R-C04). let recovery = Recovery::start(store, runner, records.clone(), leases, &claim); diff --git a/crates/server/src/mcp.rs b/crates/server/src/mcp.rs index c7b35a7..2380b54 100644 --- a/crates/server/src/mcp.rs +++ b/crates/server/src/mcp.rs @@ -6,7 +6,7 @@ use codespace_domain::{ ApprovalId, ApprovalResolveParams, ApprovalResolveResult, ApprovalState, ApprovalTargetTool, ClientEnvironmentKind, CoordinationHint, EffectivePermissionInfo, EnvironmentExecutionInfo, ErrorBody, ErrorCode, ExecCommandParams, ExecCommandResult, ExecDispatchStatus, FindParams, - FindResult, NetworkPolicyState, OperationResumeParams, OperationResumeResult, + FindResult, LostReason, NetworkPolicyState, OperationResumeParams, OperationResumeResult, OperationStatusParams, OperationStatusResult, PatchStatus, ProcessId, ProcessResizeParams, ProcessResizeResult, ProcessState, ProcessStatusParams, ProcessStatusResult, ReadParams, ReadProcessParams, ReadProcessResult, ReadResult, SteerClaimNextResult, SteerCompleteParams, @@ -133,7 +133,9 @@ a successful exit. signaled, timeout, terminated, and unknown are not success \ even when eof is true. read_process results include output_lost and retained_from. If output_lost is \ -true, the retained window is not the complete log. +true, the retained window is not the complete log, and lost_reason says why; \ +ring_overflow means that only the most recent output was kept. process_status \ +gives the same lost_reason when output_retained_from is above 0. Managed process lifetime is owned by the runner instance, not the MCP \ session. Client or HTTP disconnect keeps the process running. Losing the \ @@ -193,7 +195,17 @@ without a sandbox, can take control of it. Terminating a pipe process \ signals that process only, not its process group, so processes it started \ may outlive it. If you lose the spawn response before receiving \ process_id, do not search for the lost handle and do not start a duplicate \ -command."; +command. Once the Runner no longer keeps an ended process, process_status and \ +read_process answer from the gateway's records with lost_reason \ +retention_expired: how it ended, with no output retained. If the gateway \ +loses the Runner itself, because nothing accepts connections at its socket \ +any more or another Runner answers there, what becomes of its processes \ +cannot be learned, and some may still run: process_status reports state \ +unknown with lost_reason runner_lost for each process whose end was not \ +recorded, which is not an end, or exited with termination unknown and \ +lost_reason host_rebooted when the host restarted since. Do not start a \ +duplicate of such a process. New work in its workspace is refused until an \ +operator confirms that nothing of it runs and releases it."; const DETACH_SHUTDOWN: &str = "\ When this gateway shuts down it detaches: the Runner keeps the processes, \ @@ -500,20 +512,32 @@ impl CodeSpace { #[tool( name = "read_process", - description = "Read output from a managed process starting at cursor. Output is bounded; output_lost means the retained window is not the complete log. process_id cannot be invented." + description = "Read output from a managed process starting at cursor. Output is bounded; output_lost means the retained window is not the complete log, and lost_reason says why: ring_overflow, retention_expired, runner_lost, or host_rebooted. process_id cannot be invented." )] async fn read_process( &self, Parameters(params): Parameters, ) -> Result, String> { - let result = self + let result = match self .runner .read_process(RunnerReadProcess { process_id: params.process_id.clone(), cursor: params.cursor, }) .await - .map_err(runner_err_json)?; + { + Ok(result) => result, + // The Runner no longer holds it, or was lost: the records answer (P1R-C05). + Err(err) => match self.recorded(¶ms.process_id, &err) { + Some(recorded) => { + return Ok(Json(ReadProcessResult { + coordination: self.process_hint(¶ms.process_id.0).await, + ..recorded.read(params.cursor) + })) + } + None => return Err(runner_err_json(err)), + }, + }; if let Some(records) = &self.records { records.observed_output(&result.process_id, result.cursor); } @@ -524,23 +548,32 @@ impl CodeSpace { eof: result.eof, output_lost: result.output_lost, retained_from: result.retained_from, + lost_reason: result.output_lost.then_some(LostReason::RingOverflow), coordination: self.process_hint(¶ms.process_id.0).await, })) } #[tool( name = "process_status", - description = "Observe a managed process lifecycle. state is running or exited. termination is present only after exit (exited, signaled, timeout, terminated, or unknown). exit_code is present only when termination is exited; signal only when it is signaled, a signal CodeSpace did not send. A tty process reports a signal death as exited with exit_code 1, or 128 plus the signal number when it was launched through DevGuard. EOF is not success. Unknown process_id is rejected." + description = "Observe a managed process lifecycle. state is running, exited, or unknown: unknown means the independent Runner that ran it was lost, so the process may still run and what becomes of it cannot be learned; it is not an end. termination is present only after exit (exited, signaled, timeout, terminated, or unknown). exit_code is present only when termination is exited; signal only when it is signaled, a signal CodeSpace did not send. A tty process reports a signal death as exited with exit_code 1, or 128 plus the signal number when it was launched through DevGuard. lost_reason says why output is missing or the state is uncertain: ring_overflow, retention_expired, runner_lost, or host_rebooted. EOF is not success. Unknown process_id is rejected." )] async fn process_status( &self, Parameters(params): Parameters, ) -> Result, String> { - let result = self - .runner - .process_status(¶ms.process_id) - .await - .map_err(runner_err_json)?; + let result = match self.runner.process_status(¶ms.process_id).await { + Ok(result) => result, + // The Runner no longer holds it, or was lost: the records answer (P1R-C05). + Err(err) => match self.recorded(¶ms.process_id, &err) { + Some(recorded) => { + return Ok(Json(ProcessStatusResult { + coordination: self.process_hint(¶ms.process_id.0).await, + ..recorded.status() + })) + } + None => return Err(runner_err_json(err)), + }, + }; if let Some(records) = &self.records { match result.state { ProcessState::Exited => { @@ -549,6 +582,8 @@ impl CodeSpace { ProcessState::Running => { records.observed_output(&result.process_id, result.output_total) } + // A Runner reports no process it lost. + ProcessState::Unknown => {} } } Ok(Json(ProcessStatusResult { @@ -560,10 +595,21 @@ impl CodeSpace { output_total: result.output_total, output_retained_from: result.output_retained_from, eof: result.eof, + lost_reason: (result.output_retained_from > 0).then_some(LostReason::RingOverflow), coordination: self.process_hint(¶ms.process_id.0).await, })) } + /// What the process records say of `process_id`, which the Runner did not answer about but + /// with `err` (P1R-C05). + fn recorded( + &self, + process_id: &ProcessId, + err: &RunnerError, + ) -> Option { + self.records.as_ref()?.recorded(process_id, err) + } + #[tool( name = "process_resize", description = "Change the PTY size of a running managed process. Requires tty=true spawn. rows and cols must be at least 1. Pipe-backed processes return PROCESS_NOT_TTY. An exited handle returns PROCESS_NOT_RUNNING. Unknown process_id is rejected." @@ -4527,12 +4573,13 @@ mod independent_tests { ProcessTermination, TerminateProcessParams, WorkspaceId, }; use codespace_runner::{ - host_worker, IndependentRunner, LinkEvent, RunnerIdentity, RunnerOp, RunnerOpResult, + host_boot_id, host_worker, host_worker_with_retention, InProcessRunner, IndependentRunner, + LinkEvent, RetentionPolicy, RunnerEvents, RunnerIdentity, RunnerOp, RunnerOpResult, StopHandle, UdsRunner, }; use codespace_store::{ - NewProcessRecord, ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, - RunnerRef, + NewProcessRecord, ProcessEnd, ProcessLoss, ProcessOutcome, ProcessPhase, ProcessRecord, + RecordedParticipation, RunnerRef, }; use std::collections::BTreeMap; use std::time::{Duration, Instant}; @@ -4552,7 +4599,16 @@ mod independent_tests { impl Runner { fn start() -> Self { - let (runner, events) = host_worker(); + Self::serving(host_worker()) + } + + /// One that keeps an ended process, once a controller acknowledged it, as `retention` + /// says. + fn keeping(retention: RetentionPolicy) -> Self { + Self::serving(host_worker_with_retention(retention)) + } + + fn serving((runner, events): (InProcessRunner, RunnerEvents)) -> Self { let independent = IndependentRunner::new(runner, events); let stop = independent.stop_handle(); let (streams, incoming) = mpsc::unbounded_channel::(); @@ -4618,6 +4674,7 @@ mod independent_tests { .expect("an independent Runner's identity"); let backend = RuntimeBackend::Uds(runner.clone()); let records = ProcessRecords::start(store.clone(), &identity, backend.clone(), exits); + records.follow_loss(runner.lost()); let recovery = Recovery::start(store.clone(), runner, records.clone(), leases, claim); CodeSpace::with_store_and_runner(participation_registry(root, "off"), store, backend) .with_lifecycle(DETACHING) @@ -6182,10 +6239,10 @@ mod independent_tests { } /// A workspace where another Runner was sent an execution whose record does not say it - /// ended stays blocked (P1R-C04): only that Runner could prove what became of it. New work - /// there is refused with RECOVERY_PENDING, an approval resume before its approval is - /// claimed, so a granted approval stays granted; the record stays as it was, and the other - /// workspaces work. + /// ended stays blocked (P1R-C04): only that Runner could prove what became of it, and it is + /// lost, here to a restart of the host, so the record says so until an operator releases + /// it (P1R-C05). New work there is refused with RECOVERY_PENDING, an approval resume before + /// its approval is claimed, so a granted approval stays granted; the other workspaces work. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn an_independent_gateway_blocks_a_workspace_another_runner_left_work_in() { let dir = tempfile::tempdir().unwrap(); @@ -6213,8 +6270,8 @@ mod independent_tests { assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); assert!( refusal.message.contains("proc-elsewhere") - && refusal.message.contains("another Runner") - && refusal.message.contains("operator"), + && refusal.message.contains("the host's restart") + && refusal.message.contains("codespace-mcp records"), "{refusal:?}" ); assert_eq!( @@ -6267,13 +6324,390 @@ mod independent_tests { .unwrap() .unwrap(); assert_eq!( - (record.phase, record.runner), - (ProcessPhase::Running, elsewhere) + (record.phase, record.runner, record.lost), + ( + ProcessPhase::Unknown, + elsewhere, + Some(ProcessLoss::HostRebooted) + ) ); assert!(!root.join("runs").exists()); assert!(!root.join("added.txt").exists()); } + /// The status of `process_id`, as the gateway answers it. + async fn status_of(gateway: &CodeSpace, process_id: &str) -> ProcessStatusResult { + gateway + .process_status(Parameters(ProcessStatusParams { + process_id: ProcessId(process_id.into()), + })) + .await + .unwrap() + .0 + } + + /// A read of `process_id` from `cursor`, as the gateway answers it. + async fn read_from(gateway: &CodeSpace, process_id: &str, cursor: u64) -> ReadProcessResult { + gateway + .read_process(Parameters(ReadProcessParams { + process_id: ProcessId(process_id.into()), + cursor, + })) + .await + .unwrap() + .0 + } + + /// An ended process is re-queried as the Runner reported it for as long as the Runner keeps + /// it, and once the Runner let it go after its retention time, from its record (P1R-C05): + /// how it ended, with nothing of its output retained, and why: `retention_expired`. The + /// next gateway answers the same. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_ended_process_is_answered_from_its_record_once_the_runner_let_it_go() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::keeping(RetentionPolicy { + ttl: Duration::from_millis(300), + max_completed: 64, + }); + let gateway = runner.gateway(&root, &database).await; + let started = gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/sh", "-c", "echo done; exit 3"], + false, + ))) + .await + .unwrap() + .0; + let process_id = started.process_id.0.as_str(); + let read = output_of(&gateway, &started.process_id).await; + assert_eq!( + (read.chunk.as_str(), read.output_lost, read.lost_reason), + ("done\n", false, None) + ); + let status = status_of(&gateway, process_id).await; + assert_eq!( + (status.state, status.exit_code, status.lost_reason), + (ProcessState::Exited, Some(3), None) + ); + + // Its end recorded and acknowledged, the Runner lets it go after its retention time. + record_once(&database, &started.process_id, terminal).await; + let deadline = Instant::now() + Duration::from_secs(10); + let status = loop { + let status = status_of(&gateway, process_id).await; + if status.lost_reason.is_some() { + break status; + } + assert!(Instant::now() < deadline, "{status:?}"); + tokio::time::sleep(Duration::from_millis(50)).await; + }; + let expected = ProcessStatusResult { + process_id: started.process_id.clone(), + state: ProcessState::Exited, + exit_code: Some(3), + termination: Some(ProcessTermination::Exited), + signal: None, + output_total: 5, + output_retained_from: 5, + eof: true, + lost_reason: Some(LostReason::RetentionExpired), + coordination: None, + }; + assert_eq!( + ProcessStatusResult { + coordination: None, + ..status + }, + expected + ); + let read = read_from(&gateway, process_id, 0).await; + assert_eq!( + ReadProcessResult { + coordination: None, + ..read + }, + ReadProcessResult { + process_id: started.process_id.clone(), + cursor: 5, + chunk: String::new(), + eof: true, + output_lost: true, + retained_from: 5, + lost_reason: Some(LostReason::RetentionExpired), + coordination: None, + } + ); + detach(&gateway).await; + + let next = runner.gateway(&root, &database).await; + let status = status_of(&next, process_id).await; + assert_eq!( + ProcessStatusResult { + coordination: None, + ..status + }, + expected + ); + } + + /// Executions sent to Runners that are lost are answered from their records (P1R-C05): one + /// whose Runner was lost on this boot of the host is `unknown`, since it may still run, and + /// one whose Runner the host's restart ended exited, how is not known. Each says why, keeps + /// its deadline and the output offset a gateway saw, with nothing of its output retained. + /// Their workspace stays blocked, and nothing starts for them, until an operator releases + /// their records; the next gateway then starts new work there. An execution that ended on + /// another Runner is answered as it ended. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_answers_for_a_lost_runners_processes_until_released() { + const DEADLINE: u64 = 1_900_000_000_000; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let lost = runner_ref(RunnerIdentity { + boot_id: host_boot_id(), + ..fake_identity() + }); + let rebooted = runner_ref(fake_identity()); + { + let store = Store::open(&database).unwrap(); + for (runner, process_id) in [ + (&lost, "proc-lost"), + (&rebooted, "proc-rebooted"), + (&lost, "proc-ended"), + ] { + store + .record_dispatch(&NewProcessRecord { + process_id, + runner, + controller_epoch: Some(1), + workspace_id: "demo", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["/bin/sleep".to_string(), "30".to_string()], + }) + .unwrap(); + store + .record_running(runner, process_id, None, Some(DEADLINE)) + .unwrap(); + store.record_output(runner, process_id, 12).unwrap(); + } + store + .record_end( + &lost, + "proc-ended", + &ProcessEnd::Ended { + termination: Some(ProcessTermination::Exited), + exit_code: Some(3), + signal: None, + output_total: Some(15), + }, + ) + .unwrap(); + } + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + + let status = status_of(&gateway, "proc-lost").await; + assert_eq!( + ( + status.state, + status.exit_code, + status.termination, + status.signal.clone() + ), + (ProcessState::Unknown, None, None, None), + "{status:?}" + ); + assert_eq!( + ( + status.output_total, + status.output_retained_from, + status.eof, + status.lost_reason + ), + (12, 12, true, Some(LostReason::RunnerLost)) + ); + for (cursor, after) in [(0, 12), (20, 20)] { + let read = read_from(&gateway, "proc-lost", cursor).await; + assert_eq!( + ( + read.chunk.as_str(), + read.cursor, + read.eof, + read.output_lost, + read.retained_from, + read.lost_reason + ), + ("", after, true, true, 12, Some(LostReason::RunnerLost)) + ); + } + let status = status_of(&gateway, "proc-rebooted").await; + assert_eq!( + ( + status.state, + status.termination, + status.exit_code, + status.lost_reason + ), + ( + ProcessState::Exited, + Some(ProcessTermination::Unknown), + None, + Some(LostReason::HostRebooted) + ) + ); + let read = read_from(&gateway, "proc-rebooted", 0).await; + assert_eq!(read.lost_reason, Some(LostReason::HostRebooted)); + let status = status_of(&gateway, "proc-ended").await; + assert_eq!( + ( + status.state, + status.exit_code, + status.output_retained_from, + status.lost_reason + ), + ( + ProcessState::Exited, + Some(3), + 15, + Some(LostReason::RetentionExpired) + ) + ); + let missing = refused( + gateway + .process_status(Parameters(ProcessStatusParams { + process_id: ProcessId("proc-unrecorded".into()), + })) + .await, + ); + assert_eq!(missing.code, ErrorCode::ProcessNotFound); + + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert!( + refusal.message.contains("proc-lost") + && refusal.message.contains("may still run") + && refusal.message.contains("codespace-mcp records"), + "{refusal:?}" + ); + for (process_id, loss) in [ + ("proc-lost", ProcessLoss::RunnerLost), + ("proc-rebooted", ProcessLoss::HostRebooted), + ] { + let record = gateway.store.process_record(process_id).unwrap().unwrap(); + assert_eq!( + (record.phase, record.lost, record.deadline_at_ms), + (ProcessPhase::Unknown, Some(loss), Some(DEADLINE)), + "{process_id}" + ); + assert_eq!(record.released_at_ms, None); + } + detach(&gateway).await; + assert_eq!(recorded_executions(&database), 3); + + // An operator confirmed that nothing of them runs, and released them. + let store = Store::open(&database).unwrap(); + for process_id in ["proc-lost", "proc-rebooted"] { + assert!(store.release_record(process_id).unwrap(), "{process_id}"); + } + let next = runner.gateway(&root, &database).await; + admitted(&next, "demo").await; + let status = status_of(&next, "proc-lost").await; + assert_eq!( + (status.state, status.lost_reason), + (ProcessState::Unknown, Some(LostReason::RunnerLost)) + ); + let started = next + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await + .unwrap() + .0; + record_once(&database, &started.process_id, terminal).await; + assert_eq!(recorded_executions(&database), 4); + } + + /// An approval whose execution was sent to a Runner that is lost is never run again + /// (P1R-C05): it stays resuming while its workspace is blocked, and once an operator + /// released the record, resuming it answers APPROVAL_AMBIGUOUS, every time. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_approval_whose_execution_a_lost_runner_held_is_never_run_again() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let first = runner.gateway(&root, &database).await; + let command = ["/bin/sh", "-c", "echo run >> runs"]; + let approval_id = granted(&first, "gov-confirm", &command).await; + assert!(matches!( + first.store.claim_resume(&approval_id).unwrap(), + ResumeClaim::Execute(..) + )); + let lost = runner_ref(RunnerIdentity { + boot_id: host_boot_id(), + ..fake_identity() + }); + first + .store + .record_dispatch(&NewProcessRecord { + process_id: "proc-lost", + runner: &lost, + controller_epoch: Some(1), + workspace_id: "gov-confirm", + approval_id: Some(&approval_id), + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &command.map(String::from), + }) + .unwrap(); + first + .store + .record_running(&lost, "proc-lost", None, None) + .unwrap(); + detach(&first).await; + + let next = runner.gateway(&root, &database).await; + let refusal = refused(resume(&next, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert_eq!( + next.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Resuming + ); + detach(&next).await; + assert!(Store::open(&database) + .unwrap() + .release_record("proc-lost") + .unwrap()); + + let last = runner.gateway(&root, &database).await; + for _ in 0..2 { + let refusal = refused(resume(&last, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::ApprovalAmbiguous, "{refusal:?}"); + } + assert!(!root.join("runs").exists()); + assert_eq!(recorded_executions(&database), 1); + } + /// A workspace whose record and Runner disagree stays blocked, and so does one where the /// Runner runs a process that no record accounts for, until that process ends (P1R-C04): /// the refusal names the process, which can be queried and terminated meanwhile. diff --git a/crates/server/src/operator.rs b/crates/server/src/operator.rs new file mode 100644 index 0000000..e231005 --- /dev/null +++ b/crates/server/src/operator.rs @@ -0,0 +1,530 @@ +//! The operator's commands on the process records of a gateway with an independent Runner +//! (P1R-C05, D8). A record whose Runner was lost before its execution ended holds its workspace, +//! since a process of a killed Runner can outlive it: `records list` shows the records that may +//! still hold one, with whether each root process still runs, and `records release` releases +//! the ones an operator confirmed nothing runs of. Neither ends a process, and neither asks the +//! resource authority anything. + +use std::path::Path; +use std::time::Duration; + +use codespace_runner::{host_boot_id, pid_runs, process_runs}; +use codespace_store::{ + unsettled_records_at, ProcessLoss, ProcessPhase, ProcessRecord, RootProcess, Store, WriterLock, +}; +use serde_json::{json, Value}; + +use crate::config::{Command, RecordsCommand}; + +/// Run `command` on the operations database `database`, print its report as JSON, and return +/// whether it did all it was asked. +pub fn run(command: &Command, database: Option<&Path>) -> Result { + let database = + database.ok_or("records needs --operations-db, the gateway's operations database")?; + let boot_id = host_boot_id(); + let (report, done) = match command { + Command::Records(RecordsCommand::List) => (list(database, boot_id.as_deref())?, true), + Command::Records(RecordsCommand::Release { process_ids }) => { + release(database, boot_id.as_deref(), process_ids)? + } + }; + let text = serde_json::to_string_pretty(&report).map_err(|err| err.to_string())?; + println!("{text}"); + Ok(done) +} + +/// Every record that may still hold a workspace, read without writing to the database. +fn list(database: &Path, boot_id: Option<&str>) -> Result { + let records = unsettled_records_at(database)?; + Ok(json!({ + "operations_db": database.display().to_string(), + "boot_id": boot_id, + "records": records + .iter() + .map(|record| describe(record, boot_id)) + .collect::>(), + })) +} + +/// Release the records `process_ids` while no gateway records into `database`. Returns the +/// report, and whether every record was released. +fn release( + database: &Path, + boot_id: Option<&str>, + process_ids: &[String], +) -> Result<(Value, bool), String> { + if !database.exists() { + return Err(format!( + "the operations database {} does not exist", + database.display() + )); + } + let _writer = WriterLock::acquire(database, Duration::ZERO, |_| {}).map_err(|err| { + format!("records release changes records only while no gateway runs on the database: {err}") + })?; + let store = Store::open(database)?; + let mut released = Vec::new(); + let mut already = Vec::new(); + let mut refused = Vec::new(); + for process_id in process_ids { + match release_one(&store, boot_id, process_id)? { + Released::Now => released.push(process_id.clone()), + Released::Before => already.push(process_id.clone()), + Released::Refused(reason) => refused.push(json!({ + "process_id": process_id, + "reason": reason, + })), + } + } + let done = refused.is_empty(); + Ok(( + json!({ + "operations_db": database.display().to_string(), + "released": released, + "already_released": already, + "refused": refused, + }), + done, + )) +} + +/// What became of one release. +enum Released { + Now, + Before, + Refused(String), +} + +/// Release the record of `process_id`, unless it does not wait for a release or its root +/// process still runs. A record whose Runner ran on an earlier boot of the host is lost to that +/// restart, and one whose Runner's PID runs no process is lost with that Runner, even when no +/// gateway said so yet: a Runner that died while no gateway ran leaves its records running. +/// While the Runner's PID runs a process, that may be the Runner, which still holds the record. +fn release_one(store: &Store, boot_id: Option<&str>, process_id: &str) -> Result { + let Some(mut record) = store + .process_record(process_id) + .map_err(|err| err.message)? + else { + return Ok(Released::Refused("no record has this process ID".into())); + }; + if record.released_at_ms.is_some() { + return Ok(Released::Before); + } + match record.phase { + ProcessPhase::Terminal => { + return Ok(Released::Refused( + "its execution ended, so its record holds no workspace".into(), + )) + } + ProcessPhase::Dispatching | ProcessPhase::Running => { + let loss = ProcessLoss::of(&record.runner, boot_id); + if loss == ProcessLoss::RunnerLost && pid_runs(record.runner.pid) { + return Ok(Released::Refused(format!( + "a process with its Runner's PID {} runs, and may be that Runner, which still \ + holds the record: stop that Runner first, or start the gateway, which marks \ + the record lost when no Runner answers for it", + record.runner.pid + ))); + } + store + .record_runner_lost(&record.runner, loss) + .map_err(|err| err.message)?; + record = store + .process_record(process_id) + .map_err(|err| err.message)? + .ok_or("the record disappeared")?; + } + ProcessPhase::Unknown => {} + } + if let Some(root) = record.root.as_ref() { + if root_runs(&record, root, boot_id) == Some(true) { + return Ok(Released::Refused(format!( + "its root process {} still runs: end it, and what it started, first", + root.pid + ))); + } + } + Ok( + if store + .release_record(process_id) + .map_err(|err| err.message)? + { + Released::Now + } else { + Released::Refused("its record does not wait for a release".into()) + }, + ) +} + +/// Whether the root process of `record` still runs on this host, by its PID and its start +/// identity, so that a later process with the same PID is not taken for it. None when the +/// record cannot tell: it names no start identity. +fn root_runs(record: &ProcessRecord, root: &RootProcess, boot_id: Option<&str>) -> Option { + if ProcessLoss::of(&record.runner, boot_id) == ProcessLoss::HostRebooted { + return Some(false); + } + Some(process_runs(root.pid, root.started.as_deref()?)) +} + +/// One record, as `records list` prints it. +fn describe(record: &ProcessRecord, boot_id: Option<&str>) -> Value { + let this_boot = match (record.runner.boot_id.as_deref(), boot_id) { + (Some(then), Some(now)) => Some(then == now), + _ => None, + }; + json!({ + "process_id": record.process_id, + "workspace_id": record.workspace_id, + "phase": record.phase.as_str(), + "lost": record.lost.map(ProcessLoss::as_str), + "participation": record.participation.as_str(), + "attempt_id": record.attempt_id, + "approval_id": record.approval_id.as_ref().map(|id| id.0.as_str()), + "tty": record.tty, + "runner": { + "pid": record.runner.pid, + "boot_id": record.runner.boot_id, + "started_at_ms": record.runner.started_at_ms, + "this_boot": this_boot, + "pid_running": ProcessLoss::of(&record.runner, boot_id) != ProcessLoss::HostRebooted + && pid_runs(record.runner.pid), + }, + "root": record.root.as_ref().map(|root| json!({ + "pid": root.pid, + "started": root.started, + "running": root_runs(record, root, boot_id), + })), + "deadline_at_ms": record.deadline_at_ms, + "output_cursor": record.output_cursor, + "created_at_ms": record.created_at_ms, + "updated_at_ms": record.updated_at_ms, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use codespace_domain::ProcessTermination; + use codespace_runner::process_start_identity; + use codespace_store::{NewProcessRecord, ProcessEnd, RecordedParticipation, RunnerRef}; + + /// A Runner that ran on `boot_id`, whose PID runs no process: it is beyond the largest PID + /// that Linux and macOS give a process. + fn runner_on(boot_id: Option) -> RunnerRef { + RunnerRef { + boot_id, + pid: 4_194_999, + started_at_ms: 1_000, + nonce: "lost-runner".into(), + } + } + + /// A Runner that ran on `boot_id`, whose PID this test's own process runs now. + fn runner_running_on(boot_id: Option) -> RunnerRef { + RunnerRef { + pid: std::process::id(), + ..runner_on(boot_id) + } + } + + /// Record an execution sent to `runner` in `workspace_id`, started with the root `root`. + fn sent(store: &Store, runner: &RunnerRef, process_id: &str, root: Option<&RootProcess>) { + store + .record_dispatch(&NewProcessRecord { + process_id, + runner, + controller_epoch: Some(1), + workspace_id: "demo", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["/bin/sleep".to_string(), "30".to_string()], + }) + .unwrap(); + store + .record_running(runner, process_id, root, None) + .unwrap(); + } + + /// A child that runs until it is killed, and the root that names it. + fn live_root() -> (std::process::Child, RootProcess) { + let child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let root = RootProcess { + pid: child.id(), + started: process_start_identity(child.id()), + }; + (child, root) + } + + fn ids(report: &Value, key: &str) -> Vec { + report[key] + .as_array() + .unwrap() + .iter() + .map(|id| id.as_str().unwrap().to_string()) + .collect() + } + + /// `records list` shows each record that may still hold a workspace, with whether its root + /// process runs, and reads the database while a gateway holds it (P1R-C05). + #[test] + fn records_list_shows_what_may_still_hold_a_workspace() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + let runner = runner_running_on(Some(boot.clone())); + let (mut child, root) = live_root(); + sent(&store, &runner, "proc-running", Some(&root)); + sent(&store, &runner, "proc-ended", None); + store + .record_end( + &runner, + "proc-ended", + &ProcessEnd::Ended { + termination: Some(ProcessTermination::Exited), + exit_code: Some(0), + signal: None, + output_total: Some(3), + }, + ) + .unwrap(); + let earlier = runner_on(Some("an-earlier-boot".into())); + sent(&store, &earlier, "proc-rebooted", Some(&root)); + let _gateway = WriterLock::acquire(&database, Duration::ZERO, |_| {}).unwrap(); + + let report = list(&database, Some(&boot)).unwrap(); + assert_eq!(report["boot_id"], boot.as_str()); + let records = report["records"].as_array().unwrap(); + let listed: Vec<&str> = records + .iter() + .map(|record| record["process_id"].as_str().unwrap()) + .collect(); + assert_eq!(listed, ["proc-running", "proc-rebooted"], "{report}"); + let running = &records[0]; + assert_eq!(running["phase"], "running"); + assert_eq!(running["lost"], Value::Null); + assert_eq!(running["workspace_id"], "demo"); + assert_eq!(running["participation"], "off"); + assert_eq!(running["runner"]["this_boot"], true); + assert_eq!(running["runner"]["pid_running"], true); + assert_eq!(running["root"]["pid"], root.pid); + assert_eq!(running["root"]["running"], true, "{running}"); + // The same PIDs on a later boot are other processes. + let rebooted = &records[1]; + assert_eq!(rebooted["runner"]["this_boot"], false); + assert_eq!(rebooted["runner"]["pid_running"], false); + assert_eq!(rebooted["root"]["running"], false, "{rebooted}"); + + child.kill().unwrap(); + child.wait().unwrap(); + let report = list(&database, Some(&boot)).unwrap(); + assert_eq!(report["records"][0]["root"]["running"], false, "{report}"); + assert!(list(&dir.path().join("missing.sqlite"), Some(&boot)).is_err()); + } + + /// `records release` changes nothing while a gateway holds the database, and releases only + /// a record whose Runner was lost once its root process no longer runs: never one that + /// ended or that its Runner may still hold, and a later process with the root's PID is not + /// the root (P1R-C05, D8). + #[test] + fn records_release_waits_for_the_gateway_and_the_root() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + let runner = runner_running_on(Some(boot.clone())); + let (mut child, root) = live_root(); + sent(&store, &runner, "proc-lost", Some(&root)); + let (mut other, reused) = live_root(); + // A record whose root PID a later process took. + let gone = RootProcess { + pid: reused.pid, + started: Some("linux-start-ticks:1".into()), + }; + sent(&store, &runner, "proc-reused", Some(&gone)); + sent(&store, &runner, "proc-ended", None); + store + .record_end( + &runner, + "proc-ended", + &ProcessEnd::Ended { + termination: Some(ProcessTermination::Exited), + exit_code: Some(0), + signal: None, + output_total: None, + }, + ) + .unwrap(); + let release_all = || { + release( + &database, + Some(&boot), + &[ + "proc-lost".to_string(), + "proc-reused".to_string(), + "proc-ended".to_string(), + "proc-none".to_string(), + ], + ) + }; + + let gateway = WriterLock::acquire(&database, Duration::ZERO, |_| {}).unwrap(); + let refusal = release_all().unwrap_err(); + assert!(refusal.contains("while no gateway runs"), "{refusal}"); + drop(gateway); + + // Nothing said yet that the Runner was lost, and its PID runs. + let (report, done) = release_all().unwrap(); + assert!(!done); + assert!(ids(&report, "released").is_empty(), "{report}"); + let refused = report["refused"].as_array().unwrap(); + assert_eq!(refused.len(), 4, "{report}"); + for held in &refused[..2] { + let reason = held["reason"].as_str().unwrap(); + assert!( + reason.contains("still holds the record") + && reason.contains(&std::process::id().to_string()), + "{reason}" + ); + } + assert!(refused[2]["reason"].as_str().unwrap().contains("ended")); + assert!(refused[3]["reason"].as_str().unwrap().contains("no record")); + let record = store.process_record("proc-lost").unwrap().unwrap(); + assert_eq!((record.phase, record.lost), (ProcessPhase::Running, None)); + + store + .record_runner_lost(&runner, ProcessLoss::RunnerLost) + .unwrap(); + let (report, done) = release_all().unwrap(); + assert!(!done); + assert_eq!(ids(&report, "released"), ["proc-reused"], "{report}"); + let refused = report["refused"].as_array().unwrap(); + assert_eq!(refused[0]["process_id"], "proc-lost"); + let reason = refused[0]["reason"].as_str().unwrap(); + assert!( + reason.contains("still runs") && reason.contains(&root.pid.to_string()), + "{reason}" + ); + + child.kill().unwrap(); + child.wait().unwrap(); + let (report, _) = release_all().unwrap(); + assert_eq!(ids(&report, "released"), ["proc-lost"], "{report}"); + assert_eq!( + ids(&report, "already_released"), + ["proc-reused"], + "{report}" + ); + let record = store.process_record("proc-lost").unwrap().unwrap(); + assert_eq!( + (record.phase, record.lost), + (ProcessPhase::Unknown, Some(ProcessLoss::RunnerLost)) + ); + assert!(record.released_at_ms.is_some()); + assert!(unsettled_records_at(&database).unwrap().is_empty()); + other.kill().unwrap(); + other.wait().unwrap(); + } + + /// A Runner that died while no gateway ran left its records running: once its PID runs no + /// process, `records release` marks every record of that Runner lost, as a gateway would, + /// and releases the ones asked for whose root processes no longer run (P1R-C05). + #[test] + fn records_release_takes_a_runner_gone_unseen_as_lost() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + let gone = runner_on(Some(boot.clone())); + let (mut child, root) = live_root(); + sent(&store, &gone, "proc-survivor", Some(&root)); + store + .record_dispatch(&NewProcessRecord { + process_id: "proc-sent", + runner: &gone, + controller_epoch: Some(1), + workspace_id: "other", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["/bin/true".to_string()], + }) + .unwrap(); + let listed = list(&database, Some(&boot)).unwrap(); + assert_eq!( + listed["records"][0]["runner"]["pid_running"], false, + "{listed}" + ); + + let (report, done) = release(&database, Some(&boot), &["proc-sent".into()]).unwrap(); + assert!(done, "{report}"); + assert_eq!(ids(&report, "released"), ["proc-sent"]); + let survivor = store.process_record("proc-survivor").unwrap().unwrap(); + assert_eq!( + (survivor.phase, survivor.lost, survivor.released_at_ms), + (ProcessPhase::Unknown, Some(ProcessLoss::RunnerLost), None) + ); + let (report, done) = release(&database, Some(&boot), &["proc-survivor".into()]).unwrap(); + assert!(!done); + let reason = report["refused"][0]["reason"].as_str().unwrap(); + assert!(reason.contains("still runs"), "{reason}"); + + child.kill().unwrap(); + child.wait().unwrap(); + let (report, done) = release(&database, Some(&boot), &["proc-survivor".into()]).unwrap(); + assert!(done, "{report}"); + assert!(unsettled_records_at(&database).unwrap().is_empty()); + } + + /// A record whose Runner ran on an earlier boot of the host is released as one that the + /// restart ended, though no gateway said so yet, and whatever runs with its root's PID now + /// (P1R-C05). + #[test] + fn records_release_takes_a_restart_as_the_end() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + // Its Runner's PID and its root's PID run other processes now. + let earlier = runner_running_on(Some("an-earlier-boot".into())); + let (mut child, root) = live_root(); + sent(&store, &earlier, "proc-rebooted", Some(&root)); + let (report, done) = release(&database, Some(&boot), &["proc-rebooted".into()]).unwrap(); + assert!(done, "{report}"); + assert_eq!(ids(&report, "released"), ["proc-rebooted"]); + let record = store.process_record("proc-rebooted").unwrap().unwrap(); + assert_eq!( + (record.phase, record.lost), + (ProcessPhase::Unknown, Some(ProcessLoss::HostRebooted)) + ); + assert!(record.released_at_ms.is_some()); + child.kill().unwrap(); + child.wait().unwrap(); + } + + /// The records commands need the operations database, and release needs one that exists. + #[test] + fn records_commands_need_their_database() { + let refusal = run(&Command::Records(RecordsCommand::List), None).unwrap_err(); + assert!(refusal.contains("--operations-db"), "{refusal}"); + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("missing.sqlite"); + let refusal = release(&missing, None, &["proc-1".into()]).unwrap_err(); + assert!(refusal.contains("does not exist"), "{refusal}"); + assert!(!missing.exists()); + } +} diff --git a/crates/server/src/records.rs b/crates/server/src/records.rs index 6387472..c110a87 100644 --- a/crates/server/src/records.rs +++ b/crates/server/src/records.rs @@ -5,20 +5,26 @@ //! ended: the Runner is the authority on live processes. Once a record says how an execution //! ended, the gateway acknowledges it to the Runner, which until then keeps the ended process //! for a controller (P1R-C04). +//! +//! When the Runner no longer holds a process, or the gateway lost the Runner, the records answer +//! what the Runner cannot, with why (P1R-C05): an execution whose end was recorded is reported +//! as it ended, and one whose Runner was lost before it ended as `unknown`, never as an end. use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; -use codespace_domain::ProcessTermination; use codespace_domain::{ - ApprovalId, ErrorBody, ErrorCode, ExecCommandParams, ProcessId, ProcessState, + ApprovalId, ErrorBody, ErrorCode, ExecCommandParams, LostReason, ProcessId, ProcessState, + ProcessStatusResult, ProcessTermination, ReadProcessResult, }; use codespace_runner::{ - Runner, RunnerIdentity, RunnerInventoryEntry, RunnerProcessRoot, RunnerProcessStatus, - RuntimeBackend, ShellRelease, + process_signal, Runner, RunnerError, RunnerIdentity, RunnerInventoryEntry, RunnerProcessRoot, + RunnerProcessStatus, RuntimeBackend, ShellRelease, }; use codespace_store::{ - NewProcessRecord, ProcessEnd, RecordedParticipation, RootProcess, RunnerRef, Store, + NewProcessRecord, ProcessEnd, ProcessLoss, ProcessOutcome, ProcessPhase, ProcessRecord, + RecordedParticipation, RootProcess, RunnerRef, Store, }; use tokio::sync::{mpsc, watch}; @@ -87,6 +93,8 @@ pub struct ProcessRecords { runner: RunnerRef, /// Bumped after each end that the records follow from an exit event (P1R-C04). ends: watch::Sender, + /// Set once the gateway lost the Runner (P1R-C05). + lost: AtomicBool, } impl ProcessRecords { @@ -120,6 +128,7 @@ impl ProcessRecords { nonce: identity.nonce.clone(), }, ends: watch::Sender::new(0), + lost: AtomicBool::new(false), }); let following = records.clone(); let Exits(mut exited) = exits; @@ -155,6 +164,92 @@ impl ProcessRecords { records } + /// Follow the loss of the Runner these records are of (P1R-C05): once `lost` says why the + /// gateway lost it, see [`Self::runner_lost`]. + pub fn follow_loss(self: &Arc, mut lost: watch::Receiver>) { + let records = self.clone(); + tokio::spawn(async move { + loop { + if let Some(reason) = lost.borrow_and_update().clone() { + records.runner_lost(&reason); + return; + } + if lost.changed().await.is_err() { + return; + } + } + }); + } + + /// The gateway lost the Runner these records are of, for `reason` (P1R-C05): what became of + /// each execution sent to it that had not ended cannot be learned any more, and some may + /// still run, so each of their records becomes `unknown` and keeps its workspace held until + /// an operator releases it. An end the Runner reported before still ends its record. + pub(crate) fn runner_lost(&self, reason: &str) { + self.lost.store(true, Ordering::SeqCst); + match self + .store + .record_runner_lost(&self.runner, ProcessLoss::RunnerLost) + { + Ok(processes) => tracing::warn!( + ?processes, + reason, + "the independent Runner was lost before these processes ended: what became of \ + them cannot be learned, so their records say unknown and their workspaces stay \ + held until an operator finds them and releases them" + ), + Err(err) => tracing::error!( + reason, + error = %err.message, + "the independent Runner was lost, and the records of its processes could not say \ + so: their workspaces stay held" + ), + } + } + + /// The record to answer from about `process_id`, which the Runner did not answer about but + /// with `err` (P1R-C05): when it no longer holds the process, or the gateway lost it, and + /// the record says that the execution was sent. None when the Runner remains the one to + /// ask, as for an execution of its own that has not ended as far as its record says. + pub(crate) fn recorded(&self, process_id: &ProcessId, err: &RunnerError) -> Option { + let lost_runner = match err.as_execution().map(|body| body.code) { + Some(ErrorCode::ProcessNotFound) => false, + Some(ErrorCode::RunnerDisconnected) if self.lost.load(Ordering::SeqCst) => true, + _ => return None, + }; + let record = match self.store.process_record(&process_id.0) { + Ok(Some(record)) => record, + Ok(None) => return None, + Err(err) => { + tracing::warn!( + process_id = %process_id.0, + error = %err.message, + "the process record could not be read" + ); + return None; + } + }; + if record.outcome == Some(ProcessOutcome::NotStarted) { + return None; + } + let ours = record.runner == self.runner; + let loss_since = || ProcessLoss::of(&record.runner, self.runner.boot_id.as_deref()); + let reason = match (record.lost, record.phase) { + (Some(loss), _) => lost_reason(loss), + (None, ProcessPhase::Terminal) if ours && lost_runner => LostReason::RunnerLost, + // Its end is recorded: only what the Runner retained is gone. + (None, ProcessPhase::Terminal) => match loss_since() { + ProcessLoss::HostRebooted => LostReason::HostRebooted, + ProcessLoss::RunnerLost => LostReason::RetentionExpired, + }, + (None, _) if ours && lost_runner => LostReason::RunnerLost, + (None, _) if ours => return None, + // Another Runner answers at its Runner's socket. + (None, _) => lost_reason(loss_since()), + }; + Some(Recorded { record, reason }) + } + /// The Runner these records are of. pub(crate) fn runner(&self) -> &RunnerRef { &self.runner @@ -310,3 +405,86 @@ impl ProcessRecords { } } } + +/// An execution as its record tells it, when its Runner cannot (P1R-C05). +pub(crate) struct Recorded { + record: ProcessRecord, + reason: LostReason, +} + +impl Recorded { + /// The furthest output offset the gateway knows of. + fn output_total(&self) -> u64 { + self.record.output_cursor.unwrap_or(0) + } + + /// The execution's status: as its record says it ended, with nothing of its output + /// retained. One whose Runner was lost before it ended is `unknown`, never an end; after + /// a host restart, it ended, and how is not known. + pub(crate) fn status(&self) -> ProcessStatusResult { + let record = &self.record; + let unknown_end = ( + ProcessState::Exited, + Some(ProcessTermination::Unknown), + None, + None, + ); + let (state, termination, exit_code, signal) = match (record.phase, record.outcome) { + (ProcessPhase::Terminal, Some(ProcessOutcome::Ended(termination))) => match termination + { + ProcessTermination::Exited => ( + ProcessState::Exited, + Some(ProcessTermination::Exited), + record.exit_code, + None, + ), + ProcessTermination::Signaled => match record.signal { + Some(number) => ( + ProcessState::Exited, + Some(ProcessTermination::Signaled), + None, + Some(process_signal(number)), + ), + None => unknown_end, + }, + termination => (ProcessState::Exited, Some(termination), None, None), + }, + (ProcessPhase::Terminal, _) => unknown_end, + _ if self.reason == LostReason::HostRebooted => unknown_end, + _ => (ProcessState::Unknown, None, None, None), + }; + ProcessStatusResult { + process_id: ProcessId(record.process_id.clone()), + state, + exit_code, + termination, + signal, + output_total: self.output_total(), + output_retained_from: self.output_total(), + eof: true, + lost_reason: Some(self.reason), + coordination: None, + } + } + + /// A read from `cursor`: nothing of the output is retained, and nothing more will be. + pub(crate) fn read(&self, cursor: u64) -> ReadProcessResult { + ReadProcessResult { + process_id: ProcessId(self.record.process_id.clone()), + cursor: cursor.max(self.output_total()), + chunk: String::new(), + eof: true, + output_lost: true, + retained_from: self.output_total(), + lost_reason: Some(self.reason), + coordination: None, + } + } +} + +fn lost_reason(loss: ProcessLoss) -> LostReason { + match loss { + ProcessLoss::RunnerLost => LostReason::RunnerLost, + ProcessLoss::HostRebooted => LostReason::HostRebooted, + } +} diff --git a/crates/server/src/recovery.rs b/crates/server/src/recovery.rs index 43304dc..ee8999d 100644 --- a/crates/server/src/recovery.rs +++ b/crates/server/src/recovery.rs @@ -16,11 +16,14 @@ //! | none | holds it running | any | the workspace stays blocked | //! | anything else | | | the workspace stays blocked: a failed lookup is asked again, a mismatch waits for an operator | //! -//! A Runner proves that it never received a request only for itself: a record sent to another -//! Runner keeps its workspace blocked. An approval left `resuming` is consumed with the process -//! the Runner created for it, given back when the Runner proved that it created none, and -//! otherwise left as it is. A workspace that waits is reconciled again after each exit, and -//! one that waits for an answer also after a growing pause. +//! A Runner proves that it never received a request only for itself. A record of an execution +//! sent to another Runner that has not ended means that Runner was lost, since another one +//! answers at its socket (P1R-C05): the record becomes `unknown`, with why, and like every +//! record whose Runner was lost it keeps its workspace blocked until an operator releases it +//! (D8). An approval left `resuming` is consumed with the process the Runner created for it, +//! given back when the Runner proved that it created none, and otherwise left as it is. A +//! workspace that waits is reconciled again after each exit, and one that waits for an answer +//! also after a growing pause. use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak}; @@ -31,7 +34,10 @@ use codespace_domain::{ OperationResumeResult, ProcessId, ProcessState, }; use codespace_runner::{Recalled, RunnerClaim, RunnerProcessStatus, UdsRunner}; -use codespace_store::{ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, Store}; +use codespace_store::{ + ProcessLoss, ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, RunnerRef, + Store, +}; use tokio::sync::watch; use crate::records::{Leases, ProcessRecords}; @@ -278,6 +284,7 @@ impl Recovery { ) -> BTreeMap { let mut bars = Bars::default(); let ours = self.records.runner(); + let lost = self.lose_other_runners(records); let mut decisions: Vec> = Vec::with_capacity(records.len()); let mut asked = Vec::new(); for (index, record) in records.iter().enumerate() { @@ -285,16 +292,24 @@ impl Recovery { ProcessPhase::Terminal => Some(Decision::Settled { started: record.outcome != Some(ProcessOutcome::NotStarted), }), - // Only the Runner that was sent it can prove what became of it (P1R-C05). - _ if record.runner != *ours => Some(Decision::Blocked(format!( - "process `{}` was sent to another Runner, which this gateway cannot ask \ - what became of it", - record.process_id - ))), - ProcessPhase::Unknown => Some(Decision::Blocked(format!( - "the record of process `{}` says that what became of it cannot be learned", - record.process_id + ProcessPhase::Unknown => Some(Decision::Blocked(awaits_release( + record, + record.lost.unwrap_or(ProcessLoss::RunnerLost), ))), + // Only the Runner that was sent it can prove what became of it, and that Runner + // is lost (P1R-C05). + _ if record.runner != *ours => Some(match lost.get(&record.process_id) { + Some(Ok(loss)) => Decision::Blocked(awaits_release(record, *loss)), + Some(Err(err)) => Decision::Pending(format!( + "the Runner that process `{}` was sent to is lost, and its record could \ + not say so: {err}", + record.process_id + )), + None => Decision::Pending(format!( + "the record of process `{}` was not marked lost", + record.process_id + )), + }), ProcessPhase::Dispatching | ProcessPhase::Running => { asked.push(index); None @@ -462,6 +477,53 @@ impl Recovery { bars.0 } + /// The Runners other than this gateway's that `records` sent executions to which have not + /// ended are lost: another Runner answers at their socket (P1R-C05). Each of their records + /// that had not ended becomes `unknown`, for a restart of the host when the lost Runner ran + /// on an earlier boot. Returns, for each such record, its loss, or why it could not be + /// recorded. + fn lose_other_runners( + &self, + records: &[ProcessRecord], + ) -> HashMap> { + let ours = self.records.runner(); + let mut runners: Vec<&RunnerRef> = Vec::new(); + for record in records { + if matches!( + record.phase, + ProcessPhase::Dispatching | ProcessPhase::Running + ) && record.runner != *ours + && !runners.contains(&&record.runner) + { + runners.push(&record.runner); + } + } + let mut lost = HashMap::new(); + for runner in runners { + let loss = ProcessLoss::of(runner, ours.boot_id.as_deref()); + let marked = self + .store + .record_runner_lost(runner, loss) + .map_err(|err| err.message); + if let Ok(processes) = &marked { + if !processes.is_empty() { + tracing::warn!( + runner_pid = runner.pid, + loss = loss.as_str(), + ?processes, + "an earlier independent Runner was lost before these processes ended: \ + another Runner answers at its socket, so what became of them cannot be \ + learned; their workspaces stay blocked until an operator releases them" + ); + } + } + for record in records.iter().filter(|record| record.runner == *runner) { + lost.insert(record.process_id.clone(), marked.clone().map(|_| loss)); + } + } + lost + } + /// Act on `decision` for `record`, and return what it says of the execution's start. fn apply( &self, @@ -669,6 +731,23 @@ impl Bars { } } +/// Why `record`, whose Runner was lost for `loss`, keeps its workspace blocked until an +/// operator releases it (P1R-C05, D8). +fn awaits_release(record: &ProcessRecord, loss: ProcessLoss) -> String { + let lost = match loss { + ProcessLoss::RunnerLost => { + "was sent to a Runner that was lost, so what became of it cannot be learned and it \ + may still run" + } + ProcessLoss::HostRebooted => "was sent to a Runner that the host's restart ended", + }; + format!( + "process `{}` {lost}; an operator confirms that nothing of it runs and releases its \ + record (`codespace-mcp records`)", + record.process_id + ) +} + /// What became of an execution that takes no part in the resource authority, as its Runner /// recalled it. fn decide(record: &ProcessRecord, recalled: &Recalled) -> Decision { diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index 71284c8..9af5fd7 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -277,6 +277,13 @@ fn link_observer() -> LinkObserver { stop, and this gateway refuses every request that needs the Runner with \ RUNNER_DISCONNECTED until it is restarted" ), + LinkEvent::RunnerLost(reason) => tracing::error!( + %reason, + "lost the independent Runner: what became of its processes cannot be learned and some \ + may still run, so their records become unknown and keep their workspaces until an \ + operator finds the processes and releases them (`codespace-mcp records list`); this \ + gateway refuses every request that needs the Runner with RUNNER_DISCONNECTED" + ), }) } diff --git a/crates/server/tests/independent_runner.rs b/crates/server/tests/independent_runner.rs index 6d18d7f..b70e19b 100644 --- a/crates/server/tests/independent_runner.rs +++ b/crates/server/tests/independent_runner.rs @@ -3,9 +3,11 @@ //! leaves or stops the Runner as its exit intent says. Each gateway records its executions in //! the operations database, which it is the only writer of (P1R-C02), takes control of the //! Runner with a Claim for that database (P1R-C03), and reconciles what earlier controllers -//! left before it starts work where they did (P1R-C04). The tests with a Runner need the -//! worker binary (`CODESPACE_RUNTIME_BIN`) and return early without it, as the gateway's other -//! worker tests do. +//! left before it starts work where they did (P1R-C04). A gateway whose Runner is lost answers +//! for its executions from their records, and an operator releases what they hold once nothing +//! of them runs (P1R-C05). The tests with a Runner need the worker binary +//! (`CODESPACE_RUNTIME_BIN`) and return early without it, as the gateway's other worker tests +//! do. use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Stdio}; @@ -190,6 +192,21 @@ async fn refused(mut command: Command) -> String { String::from_utf8_lossy(&output.stderr).into_owned() } +/// What `codespace-mcp records` prints with `args` for the operations database, and whether +/// it did all it was asked (P1R-C05). +async fn records(setup: &Setup, args: &[&str]) -> (serde_json::Value, bool) { + let args = [&["records"], args].concat(); + let output = timeout( + Duration::from_secs(30), + setup.gateway_command(&args).stdin(Stdio::null()).output(), + ) + .await + .expect("the records command ends") + .expect("run the records command"); + let report = serde_json::from_slice(&output.stdout).unwrap_or_else(|_| panic!("{output:?}")); + (report, output.status.success()) +} + /// The lines a process writes to `stderr`, read for as long as it writes them. fn log_lines(stderr: tokio::process::ChildStderr) -> mpsc::UnboundedReceiver { let (lines, logged) = mpsc::unbounded_channel(); @@ -267,6 +284,34 @@ impl Gateway { body["process_id"].as_str().unwrap().to_string() } + async fn read(&self, process_id: &str, cursor: u64) -> serde_json::Value { + let read = self + .client + .call_tool( + CallToolRequestParams::new(TOOL_READ_PROCESS) + .with_arguments(object!({ "process_id": process_id, "cursor": cursor })), + ) + .await + .expect("read_process"); + payload(&read) + } + + /// Start `command` once new work in `demo` is no longer refused while the gateway + /// reconciles (P1R-C04), and return its process id. + async fn started_once_reconciled(&self, command: &[&str]) -> String { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let started = self.exec(command).await; + let body = payload(started.as_ref().expect("exec")); + if body["code"] != "RECOVERY_PENDING" { + assert_eq!(body["dispatch_status"], "confirmed", "{body}"); + return body["process_id"].as_str().unwrap().to_string(); + } + assert!(Instant::now() < deadline, "not reconciled: {body}"); + sleep(Duration::from_millis(40)).await; + } + } + async fn status(&self, process_id: &str) -> serde_json::Value { let status = self .client @@ -863,3 +908,168 @@ async fn a_sandboxed_command_cannot_reach_the_runner_socket() { let status = exited(&mut runner).await; assert!(status.success(), "{status:?}"); } + +/// The record of `process_id` once `done` holds of it. +async fn record_once( + setup: &Setup, + process_id: &str, + done: impl Fn(&codespace_store::ProcessRecord) -> bool, +) -> codespace_store::ProcessRecord { + let store = Store::open(&setup.database()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match store.process_record(process_id).unwrap() { + Some(record) if done(&record) => return record, + record => assert!(Instant::now() < deadline, "{record:?}"), + } + sleep(Duration::from_millis(20)).await; + } +} + +/// An independent Runner killed with SIGKILL is lost (P1R-C05). Its gateway answers for the +/// executions it was sent from their records, one that had not ended as `unknown` and one that +/// ended as it ended, each with `runner_lost`, and refuses new work. Where no command sandbox +/// ties it to the Runner, a pipe process of the killed Runner keeps running: nothing ended it, +/// and `records list` shows it. `records release` waits for the gateway to end and for that +/// process, then releases the record, and a new Runner's gateway starts new work there. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_killed_runners_processes_stay_held_until_an_operator_releases_them() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let gateway = setup.gateway("detach").await; + let ended = gateway + .started(&["/bin/sh", "-c", "echo ended; exit 5"]) + .await; + gateway.ended(&ended).await; + gateway.output(&ended, "ended").await; + record_once(&setup, &ended, |record| { + record.phase == ProcessPhase::Terminal + }) + .await; + let beat = setup.ws.join("beat"); + let running = gateway + .started(&[ + "/bin/sh", + "-c", + "i=0; while :; do i=$((i+1)); echo $i > beat; sleep 0.1; done", + ]) + .await; + assert!(still_beating(&beat).await); + + runner.kill().await.expect("SIGKILL the Runner"); + let deadline = Instant::now() + Duration::from_secs(15); + let status = loop { + let status = gateway.status(&running).await; + if status["state"] == "unknown" { + break status; + } + assert!( + Instant::now() < deadline, + "the Runner is not lost: {status}" + ); + sleep(Duration::from_millis(40)).await; + }; + assert_eq!(status["lost_reason"], "runner_lost", "{status}"); + assert_eq!(status["eof"], true, "{status}"); + assert!(status.get("exit_code").is_none(), "{status}"); + let read = gateway.read(&running, 0).await; + assert_eq!( + (&read["chunk"], &read["eof"], &read["output_lost"]), + (&"".into(), &true.into(), &true.into()), + "{read}" + ); + assert_eq!(read["lost_reason"], "runner_lost", "{read}"); + let status = gateway.status(&ended).await; + assert_eq!( + ( + &status["state"], + &status["exit_code"], + &status["lost_reason"] + ), + (&"exited".into(), &5.into(), &"runner_lost".into()), + "{status}" + ); + let new_work = error_body(&gateway.exec(&["/bin/echo", "new"]).await); + assert_eq!(new_work["code"], "RUNNER_DISCONNECTED", "{new_work}"); + assert!( + new_work["message"] + .as_str() + .unwrap() + .contains("lost its independent Runner"), + "{new_work}" + ); + // The command sandbox ends its processes with the Runner; without it, nothing does. + let survives = !codespace_runner::linux_sandbox_available(); + assert_eq!(still_beating(&beat).await, survives); + + // The list is read while the gateway runs. + let (listed, done) = records(&setup, &["list"]).await; + assert!(done, "{listed}"); + let held = listed["records"].as_array().unwrap(); + assert_eq!(held.len(), 1, "{listed}"); + assert_eq!( + (&held[0]["process_id"], &held[0]["phase"], &held[0]["lost"]), + ( + &running.as_str().into(), + &"unknown".into(), + &"runner_lost".into() + ), + "{listed}" + ); + assert_eq!(held[0]["runner"]["this_boot"], true, "{listed}"); + assert_eq!(held[0]["root"]["running"], survives, "{listed}"); + let root_pid = held[0]["root"]["pid"].as_u64().unwrap(); + let refusal = refused(setup.gateway_command(&["records", "release", &running])).await; + assert!(refusal.contains("while no gateway runs"), "{refusal}"); + let shut = gateway.shut_down().await; + assert!(shut.success(), "{shut:?}"); + + if survives { + let (report, done) = records(&setup, &["release", &running]).await; + assert!(!done, "{report}"); + assert!( + report["refused"][0]["reason"] + .as_str() + .unwrap() + .contains("still runs"), + "{report}" + ); + let killed = std::process::Command::new("kill") + .args(["-KILL", &root_pid.to_string()]) + .status() + .unwrap(); + assert!(killed.success()); + assert!(!still_beating(&beat).await); + } + let deadline = Instant::now() + Duration::from_secs(15); + let report = loop { + // An exited root that its new parent has not reaped yet runs no more. + let (report, done) = records(&setup, &["release", &running]).await; + if done { + break report; + } + assert!(Instant::now() < deadline, "{report}"); + sleep(Duration::from_millis(100)).await; + }; + assert_eq!(report["released"], serde_json::json!([running]), "{report}"); + let (listed, _) = records(&setup, &["list"]).await; + assert_eq!(listed["records"], serde_json::json!([]), "{listed}"); + + let mut runner = setup.start_runner(&bin).await; + let next = setup.gateway("stop").await; + let process_id = next.started_once_reconciled(&["/bin/echo", "after"]).await; + next.output(&process_id, "after").await; + let status = next.status(&running).await; + assert_eq!( + (&status["state"], &status["lost_reason"]), + (&"unknown".into(), &"runner_lost".into()), + "{status}" + ); + let shut = next.shut_down().await; + assert!(shut.success(), "{shut:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 26726e8..2ac6cae 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -29,8 +29,8 @@ use sha2::{Digest, Sha256}; pub use approvals::{ApprovalRecord, ResumeClaim}; pub use coord::CreateIntent; pub use processes::{ - NewProcessRecord, ProcessEnd, ProcessOutcome, ProcessPhase, ProcessRecord, - RecordedParticipation, RootProcess, RunnerRef, + unsettled_records_at, NewProcessRecord, ProcessEnd, ProcessLoss, ProcessOutcome, ProcessPhase, + ProcessRecord, RecordedParticipation, RootProcess, RunnerRef, }; use resource::AcquireOutcome; pub use resource::{LockMode, Resource, ResourceGuard}; @@ -251,6 +251,7 @@ impl Store { .map_err(|e| e.to_string())?; migrate_operations(&conn)?; migrate_approvals(&conn)?; + processes::migrate(&conn)?; // Last, so a database whose upgrade was cut short is upgraded again. if version < SCHEMA_VERSION { conn.execute_batch(&format!("PRAGMA user_version = {SCHEMA_VERSION}")) @@ -1464,6 +1465,102 @@ mod tests { ); } + /// The `processes` table of a CodeSpace from before P1R-C05, which records no loss. + const PROCESSES_BEFORE_LOSS: &str = "CREATE TABLE processes ( + process_id TEXT PRIMARY KEY, + runner_boot_id TEXT, + runner_pid INTEGER NOT NULL, + runner_started_at_ms INTEGER NOT NULL, + runner_nonce TEXT NOT NULL, + controller_epoch INTEGER, + workspace_id TEXT NOT NULL, + approval_id TEXT, + attempt_id TEXT, + participation TEXT NOT NULL, + tty INTEGER NOT NULL, + argv_digest TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ('dispatching', 'running', 'terminal', 'unknown')), + deadline_at_ms INTEGER, + root_pid INTEGER, + root_started TEXT, + termination TEXT, + exit_code INTEGER, + signal INTEGER, + output_cursor INTEGER, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + INSERT INTO processes (process_id, runner_boot_id, runner_pid, runner_started_at_ms, + runner_nonce, controller_epoch, workspace_id, participation, tty, argv_digest, phase, + root_pid, root_started, created_at_ms, updated_at_ms) + VALUES ('proc-1', 'boot-a', 4242, 1760000000000, 'nonce-a', 2, 'demo', 'off', 0, + 'sha256:00', 'running', 777, 'linux:100', 1, 1); + PRAGMA user_version = 1;"; + + /// Process records from before losses were recorded (P1R-C05) can be listed without + /// anything being written to the database, and gain the loss columns, empty, once a gateway + /// opens it: each record stays as it was, and so does the schema version. + #[test] + fn records_from_before_losses_are_listed_as_they_are_and_gain_the_columns() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ops.sqlite"); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch(PROCESSES_BEFORE_LOSS) + .unwrap(); + let before = std::fs::read(&path).unwrap(); + let listed = unsettled_records_at(&path).unwrap(); + assert_eq!(listed.len(), 1, "{listed:?}"); + let earlier = &listed[0]; + assert_eq!(earlier.process_id, "proc-1"); + assert_eq!(earlier.phase, ProcessPhase::Running); + assert_eq!(earlier.runner, runner()); + assert_eq!((earlier.lost, earlier.released_at_ms), (None, None)); + assert_eq!(std::fs::read(&path).unwrap(), before, "listing wrote to it"); + + let store = Store::open(&path).unwrap(); + let conn = rusqlite::Connection::open(&path).unwrap(); + let columns = table_columns(&conn, "processes").unwrap(); + assert!(columns.ends_with(&["lost".to_string(), "released_at_ms".to_string()])); + assert_eq!(user_version(&path), SCHEMA_VERSION); + assert_eq!( + store.process_record("proc-1").unwrap().as_ref(), + Some(earlier) + ); + assert_eq!(unsettled_records_at(&path).unwrap(), listed); + // A database brought up to date is opened again as it is. + drop(store); + let store = Store::open(&path).unwrap(); + assert_eq!(store.unsettled_records().unwrap(), listed); + } + + /// Listing records reads only a database it knows: one that does not exist, or that a newer + /// CodeSpace wrote, is refused, and one without process records has none (P1R-C05). + #[test] + fn listing_records_reads_only_a_database_it_knows() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("missing.sqlite"); + let err = unsettled_records_at(&missing).unwrap_err(); + assert!(err.contains("does not exist"), "{err}"); + assert!(!missing.exists()); + + let newer = dir.path().join("newer.sqlite"); + dispatch(&Store::open(&newer).unwrap(), "proc-1"); + rusqlite::Connection::open(&newer) + .unwrap() + .execute_batch(&format!("PRAGMA user_version = {}", SCHEMA_VERSION + 1)) + .unwrap(); + let err = unsettled_records_at(&newer).unwrap_err(); + assert!(err.contains("newer than this CodeSpace's"), "{err}"); + + let plain = dir.path().join("plain.sqlite"); + rusqlite::Connection::open(&plain) + .unwrap() + .execute_batch("CREATE TABLE operations (operation_id TEXT PRIMARY KEY);") + .unwrap(); + assert!(unsettled_records_at(&plain).unwrap().is_empty()); + } + /// A database keeps its identity across reopenings, and another database has its own /// (P1R-C03). The schema version stays. #[test] diff --git a/crates/store/src/processes.rs b/crates/store/src/processes.rs index 46e4a49..77c700c 100644 --- a/crates/store/src/processes.rs +++ b/crates/store/src/processes.rs @@ -5,6 +5,10 @@ //! the authority on live processes. //! //! A record keeps the argv's digest, never the argv, its environment or a secret. +//! +//! When the Runner an execution was sent to is lost, what became of the execution cannot be +//! learned any more (P1R-C05): a record that had not ended becomes `unknown`, with why, and +//! holds its workspace until an operator confirms that nothing of it runs and releases it. use codespace_domain::{ApprovalId, ErrorBody, ErrorCode, ProcessTermination}; use rusqlite::{params, Connection, OptionalExtension}; @@ -34,10 +38,31 @@ pub(crate) const SCHEMA: &str = "CREATE TABLE IF NOT EXISTS processes ( signal INTEGER, output_cursor INTEGER, created_at_ms INTEGER NOT NULL, - updated_at_ms INTEGER NOT NULL + updated_at_ms INTEGER NOT NULL, + lost TEXT, + released_at_ms INTEGER ); CREATE INDEX IF NOT EXISTS idx_processes_workspace_phase ON processes(workspace_id, phase);"; +/// The columns that a database from before P1R-C05 lacks: added, empty, when it is opened. A +/// CodeSpace from before them leaves them alone, so the schema version stays. +const LOSS_COLUMNS: [(&str, &str); 2] = [("lost", "TEXT"), ("released_at_ms", "INTEGER")]; + +/// Add the columns that a `processes` table from before P1R-C05 lacks. +pub(crate) fn migrate(conn: &Connection) -> Result<(), String> { + let columns = crate::table_columns(conn, "processes")?; + for (name, kind) in LOSS_COLUMNS { + if !columns.iter().any(|column| column == name) { + conn.execute( + &format!("ALTER TABLE processes ADD COLUMN {name} {kind}"), + [], + ) + .map_err(|err| err.to_string())?; + } + } + Ok(()) +} + /// The Runner an execution was sent to, by the identity its `Hello` states (P1R-C01). A Runner /// with an earlier Runner's PID, after a restart or a reboot, is another Runner. #[derive(Debug, Clone, PartialEq, Eq)] @@ -57,12 +82,13 @@ pub enum ProcessPhase { Running, /// The Runner reported that it ended, or answered that it did not start it. Terminal, - /// What became of it cannot be learned any more (P1R-C05). Nothing records it yet. + /// What became of it cannot be learned any more: the Runner it was sent to was lost before + /// it ended (P1R-C05). The record says why. Unknown, } impl ProcessPhase { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Self::Dispatching => "dispatching", Self::Running => "running", @@ -82,6 +108,44 @@ impl ProcessPhase { } } +/// Why what became of an execution cannot be learned any more (P1R-C05). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessLoss { + /// The Runner it was sent to was lost: nothing accepts connections at the Runner's socket, + /// or another Runner answers there. A process of a killed Runner can outlive it, so the + /// execution may still run. + RunnerLost, + /// The host restarted since the Runner it was sent to started, which ended it. + HostRebooted, +} + +impl ProcessLoss { + pub fn as_str(self) -> &'static str { + match self { + Self::RunnerLost => "runner_lost", + Self::HostRebooted => "host_rebooted", + } + } + + fn parse(raw: &str) -> Option { + Some(match raw { + "runner_lost" => Self::RunnerLost, + "host_rebooted" => Self::HostRebooted, + _ => return None, + }) + } + + /// Why an execution sent to the Runner `lost` cannot be followed any more, on the host + /// whose current boot is `boot_id`: a Runner of another boot ended with that boot. Without + /// both boots, nothing proves a restart. + pub fn of(lost: &RunnerRef, boot_id: Option<&str>) -> Self { + match (lost.boot_id.as_deref(), boot_id) { + (Some(then), Some(now)) if then != now => Self::HostRebooted, + _ => Self::RunnerLost, + } + } +} + /// How an execution takes part in the resource authority. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RecordedParticipation { @@ -90,7 +154,7 @@ pub enum RecordedParticipation { } impl RecordedParticipation { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Self::Off => "off", Self::Required => "required", @@ -217,6 +281,12 @@ pub struct ProcessRecord { pub output_cursor: Option, pub created_at_ms: u64, pub updated_at_ms: u64, + /// Why what became of the execution cannot be learned, once its Runner was lost before it + /// ended (P1R-C05). + pub lost: Option, + /// When an operator released the workspace that the record held after its Runner was lost, + /// in milliseconds since the Unix epoch (P1R-C05, D8). + pub released_at_ms: Option, } impl ProcessRecord { @@ -271,6 +341,12 @@ impl ProcessRecord { self.output_cursor = Some(cursor); } } + + /// Whether the record still holds its workspace although what became of its execution + /// cannot be learned: it waits for an operator (P1R-C05). + pub fn awaits_release(&self) -> bool { + self.phase == ProcessPhase::Unknown && self.released_at_ms.is_none() + } } impl Store { @@ -361,7 +437,7 @@ impl Store { /// (P1R-C04): every execution sent to another Runner, or to `runner` under an earlier Claim /// or before Claims were recorded, that has not ended as far as its record says, and every /// one whose approval is still being resumed. What the gateway sends under its own Claim is - /// not. Oldest first. + /// not, nor what an operator released (P1R-C05). Oldest first. pub fn records_to_reconcile( &self, runner: &RunnerRef, @@ -374,6 +450,7 @@ impl Store { WHERE (phase IN ('dispatching', 'running', 'unknown') OR approval_id IN (SELECT approval_id FROM approvals WHERE state = 'resuming')) + AND released_at_ms IS NULL AND NOT (runner_boot_id IS ?1 AND runner_pid = ?2 AND runner_started_at_ms = ?3 AND runner_nonce = ?4 AND controller_epoch IS NOT NULL AND controller_epoch >= ?5) @@ -420,6 +497,80 @@ impl Store { Ok(records) } + /// The Runner `lost` was lost (P1R-C05): what became of each execution sent to it that had + /// not ended, as far as its record says, cannot be learned any more. Each such record + /// becomes `unknown` for `loss`, and holds its workspace until an operator releases it. + /// Returns their IDs, oldest first; a record that ended stays as it is. + pub fn record_runner_lost( + &self, + lost: &RunnerRef, + loss: ProcessLoss, + ) -> Result, ErrorBody> { + let mut conn = self.conn.lock().expect("sqlite mutex"); + let tx = conn.transaction().map_err(record_err)?; + let identity = params![ + lost.boot_id, + lost.pid, + to_sql(lost.started_at_ms), + lost.nonce + ]; + let unended = "runner_boot_id IS ?1 AND runner_pid = ?2 AND runner_started_at_ms = ?3 + AND runner_nonce = ?4 AND phase IN ('dispatching', 'running')"; + let ids = { + let mut statement = tx + .prepare(&format!( + "SELECT process_id FROM processes WHERE {unended} + ORDER BY created_at_ms, process_id" + )) + .map_err(record_err)?; + let rows = statement + .query_map(identity, |row| row.get::<_, String>(0)) + .map_err(record_err)?; + rows.collect::, _>>().map_err(record_err)? + }; + tx.execute( + &format!( + "UPDATE processes SET phase = 'unknown', lost = ?5, updated_at_ms = ?6 + WHERE {unended}" + ), + params![ + lost.boot_id, + lost.pid, + to_sql(lost.started_at_ms), + lost.nonce, + loss.as_str(), + to_sql(now_ms()), + ], + ) + .map_err(record_err)?; + tx.commit().map_err(record_err)?; + Ok(ids) + } + + /// An operator confirmed that nothing of the execution `process_id` runs any more, whose + /// Runner was lost, and releases the workspace its record holds (P1R-C05, D8). The record + /// stays `unknown`: how the execution ended was never learned. Returns whether it was + /// released now: not when it is not waiting for a release, or already was released. + pub fn release_record(&self, process_id: &str) -> Result { + let conn = self.conn.lock().expect("sqlite mutex"); + let now = to_sql(now_ms()); + let released = conn + .execute( + "UPDATE processes SET released_at_ms = ?2, updated_at_ms = ?2 + WHERE process_id = ?1 AND phase = 'unknown' AND released_at_ms IS NULL", + params![process_id, now], + ) + .map_err(record_err)?; + Ok(released == 1) + } + + /// The records that may still hold a workspace (P1R-C05): every execution whose record has + /// not ended and was not released, whichever Runner it was sent to, oldest first. + pub fn unsettled_records(&self) -> Result, ErrorBody> { + let conn = self.conn.lock().expect("sqlite mutex"); + unsettled(&conn, RECORD_SELECT) + } + /// Change the record of an execution that `runner` was sent, and write it back when it /// changed. Records of executions sent to another Runner are never changed. fn change_record( @@ -481,7 +632,73 @@ pub(crate) static CRASH_BEFORE_COMMIT: std::sync::atomic::AtomicBool = const RECORD_SELECT: &str = "SELECT process_id, runner_boot_id, runner_pid, runner_started_at_ms, runner_nonce, controller_epoch, workspace_id, approval_id, attempt_id, participation, tty, argv_digest, phase, deadline_at_ms, root_pid, root_started, termination, exit_code, signal, - output_cursor, created_at_ms, updated_at_ms FROM processes"; + output_cursor, created_at_ms, updated_at_ms, lost, released_at_ms FROM processes"; + +/// [`RECORD_SELECT`] for a table from before P1R-C05, which records no loss. +const RECORD_SELECT_BEFORE_LOSS: &str = "SELECT * FROM (SELECT process_id, runner_boot_id, + runner_pid, runner_started_at_ms, runner_nonce, controller_epoch, workspace_id, approval_id, + attempt_id, participation, tty, argv_digest, phase, deadline_at_ms, root_pid, root_started, + termination, exit_code, signal, output_cursor, created_at_ms, updated_at_ms, + NULL AS lost, NULL AS released_at_ms FROM processes)"; + +/// The records that may still hold a workspace, read with `select`. +fn unsettled(conn: &Connection, select: &str) -> Result, ErrorBody> { + let mut statement = conn + .prepare(&format!( + "{select} + WHERE phase IN ('dispatching', 'running', 'unknown') AND released_at_ms IS NULL + ORDER BY created_at_ms, process_id" + )) + .map_err(record_err)?; + let rows = statement.query_map([], map_record).map_err(record_err)?; + let mut records = Vec::new(); + for row in rows { + records.push(row.map_err(record_err)??); + } + Ok(records) +} + +/// The records that may still hold a workspace in the operations database at `database`, read +/// without writing anything to it, so that an operator can look while a gateway records into +/// it (P1R-C05). A database without process records has none; one that a newer CodeSpace wrote +/// is refused. +pub fn unsettled_records_at(database: &std::path::Path) -> Result, String> { + use rusqlite::OpenFlags; + if !database.exists() { + return Err(format!( + "the operations database {} does not exist", + database.display() + )); + } + let conn = Connection::open_with_flags( + database, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(|err| format!("cannot open {}: {err}", database.display()))?; + conn.busy_timeout(std::time::Duration::from_secs(5)) + .map_err(|err| err.to_string())?; + let version = crate::schema_version(&conn)?; + if version > crate::SCHEMA_VERSION { + return Err(format!( + "the operations database has schema version {version}, newer than this \ + CodeSpace's {}: read it with the CodeSpace that wrote it", + crate::SCHEMA_VERSION + )); + } + let columns = crate::table_columns(&conn, "processes")?; + if columns.is_empty() { + return Ok(Vec::new()); + } + let select = if LOSS_COLUMNS + .iter() + .all(|(name, _)| columns.iter().any(|column| column == name)) + { + RECORD_SELECT + } else { + RECORD_SELECT_BEFORE_LOSS + }; + unsettled(&conn, select).map_err(|err| err.message) +} fn load(conn: &Connection, process_id: &str) -> Result, ErrorBody> { conn.query_row( @@ -501,6 +718,7 @@ fn map_record(row: &rusqlite::Row<'_>) -> rusqlite::Result = row.get(14)?; let root_started: Option = row.get(15)?; let termination: Option = row.get(16)?; + let lost: Option = row.get(22)?; let unreadable = |what: &str, value: &str| { Err(ErrorBody::new( ErrorCode::Internal, @@ -520,6 +738,13 @@ fn map_record(row: &rusqlite::Row<'_>) -> rusqlite::Result return Ok(unreadable("termination", &raw)), }, }; + let lost = match lost { + None => None, + Some(raw) => match ProcessLoss::parse(&raw) { + Some(loss) => Some(loss), + None => return Ok(unreadable("loss", &raw)), + }, + }; Ok(Ok(ProcessRecord { runner: RunnerRef { boot_id: row.get(1)?, @@ -546,6 +771,8 @@ fn map_record(row: &rusqlite::Row<'_>) -> rusqlite::Result>(19)?.map(from_sql), created_at_ms: from_sql(row.get(20)?), updated_at_ms: from_sql(row.get(21)?), + lost, + released_at_ms: row.get::<_, Option>(23)?.map(from_sql), process_id, })) } @@ -1139,6 +1366,132 @@ mod tests { ); } + /// Once a Runner is lost, each record of what was sent to it that had not ended becomes + /// `unknown`, with why, and holds its workspace: a gateway reconciles it, and it is listed, + /// until an operator releases it (P1R-C05). A record that ended, and one of another Runner, + /// stay as they are. An end the lost Runner reported still ends its record. + #[test] + fn a_lost_runners_unended_records_wait_for_a_release() { + let store = Store::memory().unwrap(); + let runner = runner(); + let other = RunnerRef { + nonce: "nonce-b".into(), + ..runner.clone() + }; + dispatch_under(&store, "proc-dispatching", Some(1), None); + dispatch_under(&store, "proc-running", Some(1), None); + store + .record_running(&runner, "proc-running", Some(&root()), None) + .unwrap(); + dispatch_under(&store, "proc-ended", Some(1), None); + store + .record_end(&runner, "proc-ended", &exited(0, 3)) + .unwrap(); + dispatch_to(&store, &other, "proc-other", Some(1), None); + let ended = record(&store, "proc-ended"); + + assert_eq!( + store + .record_runner_lost(&runner, ProcessLoss::RunnerLost) + .unwrap(), + ["proc-dispatching", "proc-running"] + ); + for process_id in ["proc-dispatching", "proc-running"] { + let lost = record(&store, process_id); + assert_eq!(lost.phase, ProcessPhase::Unknown, "{process_id}"); + assert_eq!(lost.lost, Some(ProcessLoss::RunnerLost), "{process_id}"); + assert_eq!(lost.released_at_ms, None); + assert!(lost.awaits_release()); + } + assert_eq!(record(&store, "proc-running").root, Some(root())); + assert_eq!(record(&store, "proc-ended"), ended); + assert_eq!( + record(&store, "proc-other").phase, + ProcessPhase::Dispatching + ); + // Nothing is left to lose. + assert!(store + .record_runner_lost(&runner, ProcessLoss::HostRebooted) + .unwrap() + .is_empty()); + assert_eq!( + record(&store, "proc-running").lost, + Some(ProcessLoss::RunnerLost) + ); + + let current = RunnerRef { + nonce: "nonce-c".into(), + ..runner.clone() + }; + assert_eq!( + ids(&store.unsettled_records().unwrap()), + ["proc-dispatching", "proc-other", "proc-running"] + ); + assert_eq!( + ids(&store.records_to_reconcile(¤t, 1).unwrap()), + ["proc-dispatching", "proc-other", "proc-running"] + ); + + // Only a record that waits for it is released, once. + assert!(store.release_record("proc-running").unwrap()); + assert!(!store.release_record("proc-running").unwrap()); + for process_id in ["proc-ended", "proc-other", "proc-unrecorded"] { + assert!(!store.release_record(process_id).unwrap(), "{process_id}"); + } + let released = record(&store, "proc-running"); + assert_eq!(released.phase, ProcessPhase::Unknown); + assert!(released.released_at_ms.is_some()); + assert!(!released.awaits_release()); + assert_eq!( + ids(&store.unsettled_records().unwrap()), + ["proc-dispatching", "proc-other"] + ); + assert_eq!( + ids(&store.records_to_reconcile(¤t, 1).unwrap()), + ["proc-dispatching", "proc-other"] + ); + + // The lost Runner reported this end before it was lost: the record ends. + assert!(store + .record_end(&runner, "proc-dispatching", &exited(2, 1)) + .unwrap()); + let ended = record(&store, "proc-dispatching"); + assert_eq!(ended.phase, ProcessPhase::Terminal); + assert_eq!( + (ended.exit_code, ended.lost), + (Some(2), Some(ProcessLoss::RunnerLost)) + ); + assert!(!store.release_record("proc-dispatching").unwrap()); + assert_eq!(ids(&store.unsettled_records().unwrap()), ["proc-other"]); + } + + /// A lost Runner of an earlier boot of the host ended with that boot; without both boots, + /// nothing proves a restart (P1R-C05). + #[test] + fn a_loss_names_a_restart_only_across_two_known_boots() { + let then = runner(); + assert_eq!( + ProcessLoss::of(&then, Some("boot-b")), + ProcessLoss::HostRebooted + ); + assert_eq!( + ProcessLoss::of(&then, Some("boot-a")), + ProcessLoss::RunnerLost + ); + assert_eq!(ProcessLoss::of(&then, None), ProcessLoss::RunnerLost); + let unbooted = RunnerRef { + boot_id: None, + ..then + }; + assert_eq!( + ProcessLoss::of(&unbooted, Some("boot-b")), + ProcessLoss::RunnerLost + ); + for loss in [ProcessLoss::RunnerLost, ProcessLoss::HostRebooted] { + assert_eq!(ProcessLoss::parse(loss.as_str()), Some(loss)); + } + } + /// An approval that a gateway left `resuming` is consumed with the process the Runner /// created, or given back when it created none, once; never while a resume of it is in /// progress (P1R-C04). diff --git a/crates/store/src/writer.rs b/crates/store/src/writer.rs index 3b41754..9e9f80d 100644 --- a/crates/store/src/writer.rs +++ b/crates/store/src/writer.rs @@ -18,7 +18,7 @@ const RETRY: Duration = Duration::from_millis(100); /// ends, however it ends. #[derive(Debug)] pub struct WriterLock { - _file: File, + file: File, path: PathBuf, } @@ -88,7 +88,7 @@ impl WriterLock { .and_then(|()| file.seek(SeekFrom::Start(0))) .and_then(|_| file.write_all(format!("{}\n", std::process::id()).as_bytes())) .map_err(|err| format!("cannot write {}: {err}", path.display()))?; - Ok(Self { _file: file, path }) + Ok(Self { file, path }) } pub fn path(&self) -> &Path { @@ -96,6 +96,15 @@ impl WriterLock { } } +impl Drop for WriterLock { + fn drop(&mut self) { + // Let go at once: a child that another thread is forking holds a copy of this + // descriptor, and with it the lock, until it runs its program. + // SAFETY: flock unlocks the open file description behind a descriptor this lock owns. + unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; + } +} + /// Take the lock if no other open file holds it. fn try_lock(file: &File) -> std::io::Result { loop { @@ -194,6 +203,20 @@ mod tests { ); } + /// A writer lets go when it is dropped, though a copy of its descriptor lives on, as in a + /// child that another thread forked and that has not run its program yet. + #[test] + fn a_writer_lets_go_though_a_copy_of_its_descriptor_lives_on() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let first = WriterLock::acquire(&database, Duration::ZERO, not_waiting).unwrap(); + let copy = first.file.try_clone().unwrap(); + drop(first); + let second = WriterLock::acquire(&database, Duration::ZERO, not_waiting).unwrap(); + drop(second); + drop(copy); + } + /// A gateway that is killed while it holds the lock lets go of it with its process: the /// next one never has to remove a stale lock file. #[test] diff --git a/docs/agent-integration.md b/docs/agent-integration.md index eecc022..a85c924 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -85,7 +85,7 @@ The response contains a server-issued `process_id` and `dispatch_status`. `confi } ``` -Each result has `chunk`, `cursor`, `eof`, `output_lost`, and `retained_from`. Output combines stdout/stderr without preserving their identity. EOF means output collection is complete; it is not a successful exit status. Judge termination with `process_status`: `state` is `running` or `exited`. After exit, `termination` is `exited`, `signaled`, `timeout`, `terminated`, or `unknown`. `exit_code` is present only when `termination` is `exited`. `signaled` means that a signal CodeSpace did not send ended the process, for example a crash or a kill from another process. `signal` then gives its `number` and, when known, its `name`, such as `SIGKILL`. Signal numbers differ between operating systems. A kill that CodeSpace requested is reported as `timeout` or `terminated` instead. `exited` does not rule out a signal: a `tty: true` process reports a signal death as `exited`, with `exit_code` 1, the same as a command that failed with status 1, or, when CodeSpace launches it through DevGuard (participation `required`), with 128 plus the signal number, such as 143 for SIGTERM. Under the Linux command sandbox the status is that of the sandbox helper ([runner isolation](runner-isolation.md)). `signaled`, `timeout`, `terminated`, and `unknown` are not success, even when `eof` is true. The last 256 KiB are retained. If `output_lost` is true, the retained window is not the complete log. Do not claim a build or test passed solely from EOF or an incomplete log. If success cannot be established from a reliable task-specific result, report it as unverified. +Each result has `chunk`, `cursor`, `eof`, `output_lost`, and `retained_from`. Output combines stdout/stderr without preserving their identity. EOF means output collection is complete; it is not a successful exit status. Judge termination with `process_status`: `state` is `running` or `exited`, or, after an [independent Runner](runner-isolation.md#runner-loss) was lost, `unknown`. After exit, `termination` is `exited`, `signaled`, `timeout`, `terminated`, or `unknown`. `exit_code` is present only when `termination` is `exited`. `signaled` means that a signal CodeSpace did not send ended the process, for example a crash or a kill from another process. `signal` then gives its `number` and, when known, its `name`, such as `SIGKILL`. Signal numbers differ between operating systems. A kill that CodeSpace requested is reported as `timeout` or `terminated` instead. `exited` does not rule out a signal: a `tty: true` process reports a signal death as `exited`, with `exit_code` 1, the same as a command that failed with status 1, or, when CodeSpace launches it through DevGuard (participation `required`), with 128 plus the signal number, such as 143 for SIGTERM. Under the Linux command sandbox the status is that of the sandbox helper ([runner isolation](runner-isolation.md)). `signaled`, `timeout`, `terminated`, and `unknown` are not success, even when `eof` is true. The last 256 KiB are retained. If `output_lost` is true, the retained window is not the complete log. `lost_reason`, when present, says why output, or what became of the process, is missing: `ring_overflow` when only the most recent output is kept, and with an independent Runner `retention_expired` when the Runner no longer keeps an ended process, `runner_lost` when the Runner that ran it was lost, and `host_rebooted` when the host restarted since it started. Such an answer comes from the gateway's record and has no output. With `runner_lost`, `state` may be `unknown`: the process may still run, and that is not an end. Report the command as unverified and do not start it again: its workspace stays refused until an operator settles it. Do not claim a build or test passed solely from EOF or an incomplete log. If success cannot be established from a reliable task-specific result, report it as unverified. ```json { @@ -150,7 +150,7 @@ Interactive input and cancellation use the same handle: A live command occupies the workspace. Wait for it to end or terminate it before applying a patch or starting another command. Reads and searches remain available. Long-lived development servers therefore require a workflow that stops them before edits. -`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. While that gateway claims its Runner again after a lost connection, it refuses requests with `RUNNER_RECONNECTING`, which sent nothing and consumed no approval: retry after a pause. Then, in a workspace where work was started before, it refuses new commands, patches and approval resumes with `RECOVERY_PENDING` until it has [reconciled](runner-isolation.md#reconciliation) that workspace with the Runner; its message says whether retrying after a pause helps. A process it finds running there holds the workspace as before. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. +`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. While that gateway claims its Runner again after a lost connection, it refuses requests with `RUNNER_RECONNECTING`, which sent nothing and consumed no approval: retry after a pause. Then, in a workspace where work was started before, it refuses new commands, patches and approval resumes with `RECOVERY_PENDING` until it has [reconciled](runner-isolation.md#reconciliation) that workspace with the Runner; its message says whether retrying after a pause helps. A process it finds running there holds the workspace as before. If the gateway loses the Runner itself, it answers `process_status` and `read_process` for that Runner's processes from its records, with `lost_reason`, refuses other requests that need the Runner with `RUNNER_DISCONNECTED`, and keeps their workspaces refused, with `RECOVERY_PENDING` in the next gateway, until an operator releases them ([Runner loss](runner-isolation.md#runner-loss)). If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. ## Confirm a held mutation diff --git a/docs/architecture.md b/docs/architecture.md index 3d5f386..3032687 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,7 +51,7 @@ If helper application fails, the Runner attempts per-file snapshot restoration. ## Process lifetime -MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead, and a gateway that lost its connection to it claims it again and reconciles what earlier controllers left before it starts new work where they did. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. +MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead, and a gateway that lost its connection to it claims it again and reconciles what earlier controllers left before it starts new work where they did. When the Runner itself is lost, the gateway answers for its executions from their records, and their workspaces wait for an operator ([Runner loss](runner-isolation.md#runner-loss)). See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. ## Planned execution coordination diff --git a/docs/devguard-integration.md b/docs/devguard-integration.md index df13e4b..6c2b2c7 100644 --- a/docs/devguard-integration.md +++ b/docs/devguard-integration.md @@ -338,7 +338,7 @@ Default participation remains `off`; `required` is an explicit operator choice i ## Initial Gateway recovery scope -**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. The third unit, P1R-C03, adds the [Claim](runner-isolation.md#claim): one connection at a time controls the Runner, under a controller epoch that each request changing a process or the workspace carries, and a gateway that lost its connection claims the Runner again instead of being restarted. While no gateway controls the Runner, any process of the same user that can reach its socket can claim it, an exposure the [Runner's documentation](runner-isolation.md#claim-exposure) states. The fourth unit, P1R-C04, adds [reconciliation](runner-isolation.md#reconciliation): after each Claim, the gateway compares its records, what the Runner holds and remembers, and for a `required` execution its DevGuard attempt, which the Runner looks up, and refuses new executions, patches and approval resumes with `RECOVERY_PENDING` only in the workspaces where an earlier controller left work, until it has reconciled them. A process found running holds its workspace again, an approval left resuming is consumed with the process the Runner created for it or granted again when the Runner proves that it created none, and a workspace whose accounts do not agree, or whose record names another Runner, stays refused for an operator. Runner loss comes in the next unit, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. +**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. The third unit, P1R-C03, adds the [Claim](runner-isolation.md#claim): one connection at a time controls the Runner, under a controller epoch that each request changing a process or the workspace carries, and a gateway that lost its connection claims the Runner again instead of being restarted. While no gateway controls the Runner, any process of the same user that can reach its socket can claim it, an exposure the [Runner's documentation](runner-isolation.md#claim-exposure) states. The fourth unit, P1R-C04, adds [reconciliation](runner-isolation.md#reconciliation): after each Claim, the gateway compares its records, what the Runner holds and remembers, and for a `required` execution its DevGuard attempt, which the Runner looks up, and refuses new executions, patches and approval resumes with `RECOVERY_PENDING` only in the workspaces where an earlier controller left work, until it has reconciled them. A process found running holds its workspace again, an approval left resuming is consumed with the process the Runner created for it or granted again when the Runner proves that it created none, and a workspace whose accounts do not agree, or whose record names another Runner, stays refused for an operator. The fifth unit, P1R-C05, adds [Runner loss](runner-isolation.md#runner-loss): when the Runner is killed, crashes or stops, or the host restarts, the gateway answers `process_status` and `read_process` for its executions from their records, with a `lost_reason`, and each execution that had not ended becomes `unknown` and holds its workspace until an operator finds its processes and releases its record with `codespace-mcp records` ([Operations](operations.md#runner-loss)). The operator releases a `required` execution's record only once DevGuard no longer charges its attempt, which only DevGuard's administrator quiescence shows, since DevGuard answers a lookup only to the attempt's own instance; until then DevGuard keeps the lost Runner's instance, so a new Runner cannot register as the `codespace` consumer's one instance. The mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. P1-RECOVERY introduces an operator-selected independent Runner mode and explicit capability. Existing modes keep their current shutdown/disconnect contracts. InProcess shares the Gateway PID and is excluded from live Gateway restart recovery. diff --git a/docs/error-codes.md b/docs/error-codes.md index 0b759ce..e80646a 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -54,9 +54,9 @@ Errors use uppercase identifiers and a message. An `operation_id` may be present | `APPROVAL_NOT_FOUND` | Unknown confirmation-hold id | | `APPROVAL_CONFLICT` | Hold is still pending, already decided, or a resume is already in progress | | `APPROVAL_AMBIGUOUS` | Resume was interrupted and the terminal result is not on disk. Includes `approval_id`; patch cases may also include `operation_id` | -| `RECOVERY_PENDING` | A controller before this gateway's Claim started work in this workspace on its [independent Runner](runner-isolation.md#reconciliation), this gateway's own connection before a lost connection included, and the gateway has not reconciled what it left there yet. The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts there until then. The approval stays as it was. The message says why: when the gateway waits for an answer it tries again by itself, so retry after a pause; when the records, the Runner and DevGuard do not agree, or a record names another Runner, retrying does not help until an operator resolves it. Not occupancy | +| `RECOVERY_PENDING` | A controller before this gateway's Claim started work in this workspace on its [independent Runner](runner-isolation.md#reconciliation), this gateway's own connection before a lost connection included, and the gateway has not reconciled what it left there yet. The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts there until then. The approval stays as it was. The message says why: when the gateway waits for an answer it tries again by itself, so retry after a pause; when the records, the Runner and DevGuard do not agree, retrying does not help until an operator resolves it, and when a record is of a lost Runner, until an operator confirms that nothing of it runs and [releases it](operations.md#runner-loss). Not occupancy | | `RUNNER_RECONNECTING` | The gateway's connection to its [independent Runner](runner-isolation.md#independent-runner) was lost, and the gateway is claiming the Runner again. The request was not sent: nothing was started, reserved or recorded for it, and an approval it would resume is not consumed. The Runner's processes keep running; retry after a pause | -| `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and cannot claim it again: nothing accepts connections at its socket, another Runner answers there, or the Runner refused the Claim for a reason that waiting cannot end. The request was not sent, and the Runner's processes keep running; restart the gateway to reach them | +| `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and cannot claim it again: nothing accepts connections at its socket, another Runner answers there, or the Runner refused the Claim for a reason that waiting cannot end. The request was not sent. When the Runner refused the Claim, its processes keep running; restart the gateway to reach them. Otherwise the gateway lost the Runner: what became of its processes cannot be learned and some may still run, `process_status` and `read_process` answer for them from the gateway's records with `lost_reason`, and an operator finds them and releases their workspaces ([Runner loss](operations.md#runner-loss)) | | `STALE_CONTROLLER` | The [independent Runner](runner-isolation.md#claim) refused a request sent under a Claim that another Claim replaced, or from a connection that does not control it. Nothing the request asked for was done, and an approval it resumed is granted again. Retrying sends the request under the gateway's current Claim | | `INTERNAL` | Scheduler or store invariant failed, or a gateway with an [independent Runner](runner-isolation.md#process-records) could not record an execution, which it then did not start. Not occupancy; do not retry as `WORKSPACE_BUSY` | diff --git a/docs/execution-substrate.md b/docs/execution-substrate.md index 6f4617d..3b62dbb 100644 --- a/docs/execution-substrate.md +++ b/docs/execution-substrate.md @@ -41,7 +41,7 @@ The operator registry maps `read-only` and `workspace-write` to effective permis Gateway fills workspace-root cwd, runner-local environment defaults, time/output limits, PTY choice, and policy into the internal Runner request. Only `tty` is exposed as a spawn-time terminal option. `tty_size` is not a spawn argument. A running PTY is resized with `process_resize`. Public calls do not accept arbitrary cwd/env/timeout overrides. See [operations](operations.md) for defaults and [Agent Loop integration](agent-integration.md) for result handling. -`exec_command` returns dispatch identity (`process_id`, `dispatch_status`). `process_status` reports `running` or `exited` plus termination metadata. `process_resize` changes the size of a running PTY. `read_process` reports output including `output_lost` and `retained_from`. EOF is not success. After handle eviction the next lookup is `PROCESS_NOT_FOUND`, not a new state. Pipe-backed resize is `PROCESS_NOT_TTY`; an exited handle is `PROCESS_NOT_RUNNING`. On Linux sandbox, the wait status is that of the managed child (the helper argv); it is not documented as identical to the user argv. +`exec_command` returns dispatch identity (`process_id`, `dispatch_status`). `process_status` reports `running` or `exited` plus termination metadata, or `unknown` after an [independent Runner](runner-isolation.md#runner-loss) was lost, with `lost_reason`. `process_resize` changes the size of a running PTY. `read_process` reports output including `output_lost`, `retained_from` and, when output is missing, `lost_reason`. EOF is not success. After handle eviction the next lookup is `PROCESS_NOT_FOUND`, not a new state, except that a gateway with an independent Runner answers from the execution's record, with `lost_reason` `retention_expired`. Pipe-backed resize is `PROCESS_NOT_TTY`; an exited handle is `PROCESS_NOT_RUNNING`. On Linux sandbox, the wait status is that of the managed child (the helper argv); it is not documented as identical to the user argv. A workspace mutation lease prevents simultaneous patch/exec mutations. FIFO order starts when an eligible request attempts to acquire the resource, not when the MCP message arrives. Request-owned patch and exec work on the same workspace waits in an in-memory per-resource FIFO until the current request releases the lease. When an exec becomes a live process, trailing waiters fail with `WORKSPACE_BUSY` and later arrivals are rejected immediately. Queue saturation is `RESOURCE_QUEUE_FULL`. That wait is not a durable or thread-keyed queue. Read and find remain available while a command runs, so filesystem I/O must reject symlink races at open time rather than rely on a prior path check. Runner file operations use `codespace-fs`; patch execution uses the separate patch helper. `operation_status` exposes the recorded patch ledger (`kind` is `patch`); live commands stay on `process_id` and are not recovered through that lookup. diff --git a/docs/ko/agent-integration.md b/docs/ko/agent-integration.md index c0b7c2d..d572904 100644 --- a/docs/ko/agent-integration.md +++ b/docs/ko/agent-integration.md @@ -85,7 +85,7 @@ MCP 클라이언트 SDK로 stdio 또는 Streamable HTTP를 초기화하고, 초 } ``` -결과에는 `chunk`, `cursor`, `eof`, `output_lost`, `retained_from`이 있습니다. stdout/stderr는 구분 없이 합쳐집니다. EOF는 출력 수집이 끝났다는 뜻이며 성공 종료를 나타내지 않습니다. 종료는 `process_status`로 판정합니다. `state`는 `running` 또는 `exited`입니다. 종료 후 `termination`은 `exited`·`signaled`·`timeout`·`terminated`·`unknown` 중 하나입니다. `exit_code`는 `termination`이 `exited`일 때만 있을 수 있습니다. `signaled`는 CodeSpace가 보내지 않은 신호로 프로세스가 끝났다는 뜻입니다. 크래시나 다른 프로세스가 보낸 kill이 여기에 해당합니다. 이때 `signal`에는 신호 번호 `number`와, 알 수 있는 경우 `SIGKILL` 같은 이름 `name`이 들어갑니다. 신호 번호는 운영체제마다 다릅니다. CodeSpace가 요청한 kill은 `signaled`가 아니라 `timeout`이나 `terminated`로 보고됩니다. `exited`라고 해서 신호로 끝나지 않았다는 뜻은 아닙니다. `tty: true` 프로세스는 신호로 끝나도 `exited`로 보고됩니다. `exit_code`는 1이어서 상태 1로 실패한 명령과 구별되지 않고, CodeSpace가 DevGuard를 거쳐 띄운 프로세스(participation `required`)라면 128에 신호 번호를 더한 값(SIGTERM이면 143)입니다. Linux 명령 샌드박스에서는 샌드박스 도우미의 종료 상태가 보고됩니다([러너 격리](runner-isolation.md)). `signaled`·`timeout`·`terminated`·`unknown`은 `eof`가 참이어도 성공이 아닙니다. 마지막 256 KiB만 보관합니다. `output_lost`가 참이면 보관 창이 전체 로그가 아닙니다. EOF나 불완전한 로그만으로 빌드·테스트 성공을 선언하지 마세요. 신뢰할 수 있는 작업별 결과로 성공을 확인할 수 없다면 미검증으로 보고해야 합니다. +결과에는 `chunk`, `cursor`, `eof`, `output_lost`, `retained_from`이 있습니다. stdout/stderr는 구분 없이 합쳐집니다. EOF는 출력 수집이 끝났다는 뜻이며 성공 종료를 나타내지 않습니다. 종료는 `process_status`로 판정합니다. `state`는 `running` 또는 `exited`이고, [독립 Runner](runner-isolation.md#runner-loss)를 잃은 뒤에는 `unknown`일 수 있습니다. 종료 후 `termination`은 `exited`·`signaled`·`timeout`·`terminated`·`unknown` 중 하나입니다. `exit_code`는 `termination`이 `exited`일 때만 있을 수 있습니다. `signaled`는 CodeSpace가 보내지 않은 신호로 프로세스가 끝났다는 뜻입니다. 크래시나 다른 프로세스가 보낸 kill이 여기에 해당합니다. 이때 `signal`에는 신호 번호 `number`와, 알 수 있는 경우 `SIGKILL` 같은 이름 `name`이 들어갑니다. 신호 번호는 운영체제마다 다릅니다. CodeSpace가 요청한 kill은 `signaled`가 아니라 `timeout`이나 `terminated`로 보고됩니다. `exited`라고 해서 신호로 끝나지 않았다는 뜻은 아닙니다. `tty: true` 프로세스는 신호로 끝나도 `exited`로 보고됩니다. `exit_code`는 1이어서 상태 1로 실패한 명령과 구별되지 않고, CodeSpace가 DevGuard를 거쳐 띄운 프로세스(participation `required`)라면 128에 신호 번호를 더한 값(SIGTERM이면 143)입니다. Linux 명령 샌드박스에서는 샌드박스 도우미의 종료 상태가 보고됩니다([러너 격리](runner-isolation.md)). `signaled`·`timeout`·`terminated`·`unknown`은 `eof`가 참이어도 성공이 아닙니다. 마지막 256 KiB만 보관합니다. `output_lost`가 참이면 보관 창이 전체 로그가 아닙니다. `lost_reason`이 있으면 출력이나 프로세스의 결과가 왜 빠졌는지 알려 줍니다. `ring_overflow`는 가장 최근 출력만 보관했다는 뜻입니다. 독립 Runner에서 `retention_expired`는 Runner가 끝난 프로세스를 더는 보관하지 않는다는 뜻이고, `runner_lost`는 그 프로세스를 실행한 Runner를 잃었다는 뜻, `host_rebooted`는 프로세스가 시작된 뒤 호스트가 재시작했다는 뜻입니다. 이런 답은 게이트웨이의 기록에서 오며 출력이 없습니다. `runner_lost`이면 `state`가 `unknown`일 수 있습니다. 프로세스가 아직 실행 중일 수 있으며, 이는 종료가 아닙니다. 명령을 미검증으로 보고하고 다시 시작하지 마세요. 운영자가 정리할 때까지 그 workspace는 계속 거부됩니다. EOF나 불완전한 로그만으로 빌드·테스트 성공을 선언하지 마세요. 신뢰할 수 있는 작업별 결과로 성공을 확인할 수 없다면 미검증으로 보고해야 합니다. ```json { @@ -150,7 +150,7 @@ MCP 클라이언트 SDK로 stdio 또는 Streamable HTTP를 초기화하고, 초 실행 중인 명령은 작업 공간을 점유합니다. 패치를 적용하거나 다른 명령을 시작하려면 기존 명령이 끝날 때까지 기다리거나 종료하세요. 읽기와 검색은 계속 가능합니다. 개발 서버를 오래 실행하는 작업 흐름이라면 수정 전에 서버를 멈추는 절차가 필요합니다. -`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)에서는 `runner_disconnect`가 `keep_running`이고, `gateway_shutdown`은 게이트웨이가 detach하면 `keep_running`, Runner를 중지하면 `terminate`입니다. 그 게이트웨이가 하는 일은 `initialize.instructions`에서 확인하세요. 그 게이트웨이는 연결을 잃은 뒤 Runner를 다시 Claim하는 동안 요청을 `RUNNER_RECONNECTING`으로 거부합니다. 이 거부는 아무것도 보내지 않았고 승인도 소모하지 않았으므로, 잠시 뒤 재시도하세요. 그 뒤 이전에 작업을 시작한 workspace에서는 Runner와 그 workspace를 [대조](runner-isolation.md#reconciliation)할 때까지 새 명령, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부합니다. 잠시 뒤 재시도하면 되는지는 그 메시지가 알려 줍니다. 그곳에서 실행 중으로 확인한 프로세스는 지금처럼 workspace를 점유합니다. spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. +`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)에서는 `runner_disconnect`가 `keep_running`이고, `gateway_shutdown`은 게이트웨이가 detach하면 `keep_running`, Runner를 중지하면 `terminate`입니다. 그 게이트웨이가 하는 일은 `initialize.instructions`에서 확인하세요. 그 게이트웨이는 연결을 잃은 뒤 Runner를 다시 Claim하는 동안 요청을 `RUNNER_RECONNECTING`으로 거부합니다. 이 거부는 아무것도 보내지 않았고 승인도 소모하지 않았으므로, 잠시 뒤 재시도하세요. 그 뒤 이전에 작업을 시작한 workspace에서는 Runner와 그 workspace를 [대조](runner-isolation.md#reconciliation)할 때까지 새 명령, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부합니다. 잠시 뒤 재시도하면 되는지는 그 메시지가 알려 줍니다. 그곳에서 실행 중으로 확인한 프로세스는 지금처럼 workspace를 점유합니다. 게이트웨이가 Runner 자체를 잃으면 그 Runner의 프로세스에 대한 `process_status`와 `read_process`에는 기록으로 `lost_reason`과 함께 답하고, Runner가 필요한 다른 요청은 `RUNNER_DISCONNECTED`로 거부하며, 운영자가 그 기록을 풀 때까지 그 workspace를 계속 거부합니다. 다음 게이트웨이에서는 `RECOVERY_PENDING`으로 거부합니다([Runner 손실](runner-isolation.md#runner-loss)). spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. ## 보류된 변경 확인 diff --git a/docs/ko/architecture.md b/docs/ko/architecture.md index 83f79c9..82dae0e 100644 --- a/docs/ko/architecture.md +++ b/docs/ko/architecture.md @@ -58,7 +58,7 @@ UDS worker와 기본 러너는 모두 같은 호스트에서 실행합니다. ## 프로세스 수명 -MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)는 이와 달리 게이트웨이보다 오래 살고, Runner와의 연결을 잃은 게이트웨이는 Runner를 다시 Claim한 뒤 앞선 controller들이 작업을 시작한 곳에서는 그들이 남긴 것을 대조하고 나서 새 작업을 시작합니다. [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. +MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)는 이와 달리 게이트웨이보다 오래 살고, Runner와의 연결을 잃은 게이트웨이는 Runner를 다시 Claim한 뒤 앞선 controller들이 작업을 시작한 곳에서는 그들이 남긴 것을 대조하고 나서 새 작업을 시작합니다. Runner 자체를 잃으면 게이트웨이는 그 실행들에 대해 기록으로 답하고, 그 workspace는 운영자를 기다립니다([Runner 손실](runner-isolation.md#runner-loss)). [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. ## 계획된 실행 조정 구조 diff --git a/docs/ko/devguard-integration.md b/docs/ko/devguard-integration.md index 8570780..12d7543 100644 --- a/docs/ko/devguard-integration.md +++ b/docs/ko/devguard-integration.md @@ -338,7 +338,7 @@ contract/client pin, daemon/helper artifact와 CodeSpace Runner wire는 각각 ## 최초 Gateway 복구 범위 -**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. 두 번째 단위 P1R-C02는 [실행 기록](runner-isolation.md#process-records)을 더한다. gateway는 작업 데이터베이스에 쓰는 유일한 gateway로서 각 실행을 Runner에 보내기 전에 기록하고 끝날 때까지 따라간다. 세 번째 단위 P1R-C03은 [Claim](runner-isolation.md#claim)을 더한다. 한 번에 한 연결만 controller epoch 아래에서 Runner를 제어하고, 프로세스나 workspace를 바꾸는 요청은 모두 그 epoch를 함께 보낸다. 연결을 잃은 gateway는 재시작하지 않고 Runner를 다시 Claim한다. Runner를 제어하는 gateway가 없는 동안에는 Runner의 소켓에 닿을 수 있는 같은 사용자의 프로세스라면 무엇이든 Runner를 Claim할 수 있으며, 이 노출은 [Runner 문서](runner-isolation.md#claim-exposure)에 적었다. 네 번째 단위 P1R-C04는 [대조](runner-isolation.md#reconciliation)를 더한다. gateway는 Claim할 때마다 자기 기록, Runner가 가진 것과 기억하는 것, `required` 실행이면 Runner가 조회한 DevGuard attempt를 맞춰 보고, 앞선 controller가 작업을 남긴 workspace에서만 대조를 마칠 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. 실행 중으로 확인한 프로세스는 다시 workspace를 점유하고, resuming으로 남은 승인은 Runner가 그 승인을 위해 만든 프로세스로 consumed가 되거나, Runner가 아무것도 만들지 않았음이 증명되면 다시 granted가 된다. 세 가지가 맞지 않거나 기록이 다른 Runner를 가리키는 workspace는 운영자가 판단할 때까지 계속 거부된다. Runner 손실 처리는 다음 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. +**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. 두 번째 단위 P1R-C02는 [실행 기록](runner-isolation.md#process-records)을 더한다. gateway는 작업 데이터베이스에 쓰는 유일한 gateway로서 각 실행을 Runner에 보내기 전에 기록하고 끝날 때까지 따라간다. 세 번째 단위 P1R-C03은 [Claim](runner-isolation.md#claim)을 더한다. 한 번에 한 연결만 controller epoch 아래에서 Runner를 제어하고, 프로세스나 workspace를 바꾸는 요청은 모두 그 epoch를 함께 보낸다. 연결을 잃은 gateway는 재시작하지 않고 Runner를 다시 Claim한다. Runner를 제어하는 gateway가 없는 동안에는 Runner의 소켓에 닿을 수 있는 같은 사용자의 프로세스라면 무엇이든 Runner를 Claim할 수 있으며, 이 노출은 [Runner 문서](runner-isolation.md#claim-exposure)에 적었다. 네 번째 단위 P1R-C04는 [대조](runner-isolation.md#reconciliation)를 더한다. gateway는 Claim할 때마다 자기 기록, Runner가 가진 것과 기억하는 것, `required` 실행이면 Runner가 조회한 DevGuard attempt를 맞춰 보고, 앞선 controller가 작업을 남긴 workspace에서만 대조를 마칠 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. 실행 중으로 확인한 프로세스는 다시 workspace를 점유하고, resuming으로 남은 승인은 Runner가 그 승인을 위해 만든 프로세스로 consumed가 되거나, Runner가 아무것도 만들지 않았음이 증명되면 다시 granted가 된다. 세 가지가 맞지 않거나 기록이 다른 Runner를 가리키는 workspace는 운영자가 판단할 때까지 계속 거부된다. 다섯 번째 단위 P1R-C05는 [Runner 손실](runner-isolation.md#runner-loss) 처리를 더한다. Runner가 kill되거나, 크래시하거나, 멈추거나, 호스트가 재시작하면 gateway는 그 실행들에 대한 `process_status`와 `read_process`에 기록으로 `lost_reason`과 함께 답하고, 끝나지 않은 실행은 `unknown`이 되어 운영자가 그 프로세스를 찾고 `codespace-mcp records`로 기록을 풀 때까지 workspace를 점유한다([운영](operations.md#runner-loss)). `required` 실행의 기록은 DevGuard가 그 attempt를 더는 과금하지 않을 때에만 운영자가 푼다. DevGuard는 attempt를 소유한 인스턴스의 조회에만 답하므로, 이는 DevGuard 관리자의 quiescence로만 확인할 수 있다. 그때까지 DevGuard는 잃은 Runner의 인스턴스를 유지하므로, 새 Runner는 `codespace` 소비자의 유일한 인스턴스로 등록할 수 없다. 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. P1-RECOVERY는 운영자가 선택하는 독립 Runner 모드와 명시적인 capability를 도입한다. 기존 모드의 종료·연결 소실 계약은 유지한다. InProcess는 Gateway와 PID를 공유하므로 Gateway 재시작 중 live 복구 대상에서 제외한다. diff --git a/docs/ko/error-codes.md b/docs/ko/error-codes.md index f3ff0fe..30a1716 100644 --- a/docs/ko/error-codes.md +++ b/docs/ko/error-codes.md @@ -57,9 +57,9 @@ | `APPROVAL_NOT_FOUND` | 알 수 없는 확인 홀드 ID | | `APPROVAL_CONFLICT` | 홀드가 아직 대기 중이거나 이미 결정되었거나, 재개가 이미 진행 중임 | | `APPROVAL_AMBIGUOUS` | 재개가 중단되어 단말 결과가 디스크에 없음. `approval_id` 포함. 패치는 `operation_id`도 있을 수 있음 | -| `RECOVERY_PENDING` | 이 게이트웨이가 Claim하기 전의 controller가 [독립 Runner](runner-isolation.md#reconciliation)에서 이 workspace의 작업을 시작했고, 게이트웨이가 그 controller가 남긴 것을 아직 대조하지 않음. 연결을 잃기 전의 이 게이트웨이 자신도 포함함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, 대조를 마치기 전에는 그 workspace에서 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 메시지가 이유를 알림. 게이트웨이가 답을 기다리는 중이면 스스로 다시 시도하므로 잠시 뒤에 재시도함. 기록, Runner, DevGuard가 서로 맞지 않거나 기록이 다른 Runner를 가리키면 운영자가 해결하기 전에는 재시도해도 소용없음. 점유가 아님 | +| `RECOVERY_PENDING` | 이 게이트웨이가 Claim하기 전의 controller가 [독립 Runner](runner-isolation.md#reconciliation)에서 이 workspace의 작업을 시작했고, 게이트웨이가 그 controller가 남긴 것을 아직 대조하지 않음. 연결을 잃기 전의 이 게이트웨이 자신도 포함함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, 대조를 마치기 전에는 그 workspace에서 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 메시지가 이유를 알림. 게이트웨이가 답을 기다리는 중이면 스스로 다시 시도하므로 잠시 뒤에 재시도함. 기록, Runner, DevGuard가 서로 맞지 않으면 운영자가 해결하기 전에는, 기록이 잃은 Runner의 것이면 운영자가 그중 아무것도 실행 중이지 않음을 확인하고 [기록을 풀기](operations.md#runner-loss) 전에는 재시도해도 소용없음. 점유가 아님 | | `RUNNER_RECONNECTING` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 Runner를 다시 Claim하는 중임. 요청은 보내지 않았음. 그 요청을 위해 시작하거나 예약하거나 기록한 것은 없고, 재개하려던 승인은 소비되지 않음. Runner의 프로세스는 계속 실행됨. 잠시 뒤 재시도할 것 | -| `RUNNER_DISCONNECTED` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 다시 Claim할 수 없음. 소켓에서 연결을 받지 않거나, 다른 Runner가 응답하거나, 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부함. 요청은 보내지 않았고 Runner의 프로세스는 계속 실행됨. 그 프로세스에 닿으려면 게이트웨이를 재시작할 것 | +| `RUNNER_DISCONNECTED` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 다시 Claim할 수 없음. 소켓에서 연결을 받지 않거나, 다른 Runner가 응답하거나, 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부함. 요청은 보내지 않았음. Runner가 Claim을 거부했다면 그 프로세스는 계속 실행되며, 그 프로세스에 닿으려면 게이트웨이를 재시작할 것. 그 밖의 경우는 게이트웨이가 Runner를 잃은 것임. 그 프로세스가 어떻게 되었는지 알 수 없고 일부는 아직 실행 중일 수 있음. `process_status`와 `read_process`는 게이트웨이의 기록으로 `lost_reason`과 함께 답하며, 운영자가 그 프로세스를 찾고 workspace를 풂([Runner 손실](operations.md#runner-loss)) | | `STALE_CONTROLLER` | [독립 Runner](runner-isolation.md#claim)가 다른 Claim으로 바뀐 Claim 아래에서 보낸 요청, 또는 Runner를 제어하지 않는 연결의 요청을 거부함. 요청이 요구한 일은 하나도 하지 않았고, 요청이 재개한 승인은 다시 승인된 상태가 됨. 재시도하면 게이트웨이의 현재 Claim 아래에서 요청을 보냄 | | `INTERNAL` | 스케줄러·스토어 불변조건 실패, 또는 [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이가 실행을 기록하지 못해 시작하지 않음. 점유가 아니므로 `WORKSPACE_BUSY`로 재시도하지 말 것 | diff --git a/docs/ko/execution-substrate.md b/docs/ko/execution-substrate.md index 78e6a69..02c0428 100644 --- a/docs/ko/execution-substrate.md +++ b/docs/ko/execution-substrate.md @@ -46,7 +46,7 @@ 게이트웨이는 내부 Runner 요청에 작업 공간 루트 cwd, 러너 환경 기본값, 시간·출력 제한, PTY 선택, 정책을 채웁니다. spawn 시 공개된 터미널 옵션은 `tty`뿐입니다. `tty_size`는 spawn 인자가 아닙니다. 실행 중인 PTY는 `process_resize`로 크기를 바꿉니다. 공개 호출은 임의의 cwd·환경변수·제한 시간 변경을 받지 않습니다. 기본값은 [운영](operations.md), 결과 처리는 [Agent Loop 연동](agent-integration.md)을 참고하세요. -`exec_command`는 디스패치 식별(`process_id`, `dispatch_status`)만 반환합니다. 종료 판정은 `process_status`의 `running`/`exited`와 termination 메타데이터를 사용합니다. 실행 중인 PTY 크기는 `process_resize`로 바꿉니다. `read_process`는 `output_lost`와 `retained_from`을 포함한 출력을 반환합니다. EOF는 성공이 아닙니다. 핸들이 만료된 뒤의 조회는 새 상태가 아니라 `PROCESS_NOT_FOUND`입니다. 파이프 프로세스의 크기 변경은 `PROCESS_NOT_TTY`, 종료된 핸들은 `PROCESS_NOT_RUNNING`입니다. Linux 샌드박스에서 wait 상태는 관리 자식(헬퍼 argv)의 코드이며, 사용자 argv와 동일하다고 문서화하지 않습니다. +`exec_command`는 디스패치 식별(`process_id`, `dispatch_status`)만 반환합니다. 종료 판정은 `process_status`의 `running`/`exited`와 termination 메타데이터를 사용하며, [독립 Runner](runner-isolation.md#runner-loss)를 잃은 뒤에는 `lost_reason`과 함께 `unknown`이 올 수 있습니다. 실행 중인 PTY 크기는 `process_resize`로 바꿉니다. `read_process`는 `output_lost`와 `retained_from`, 출력이 빠졌으면 `lost_reason`을 포함한 출력을 반환합니다. EOF는 성공이 아닙니다. 핸들이 만료된 뒤의 조회는 새 상태가 아니라 `PROCESS_NOT_FOUND`입니다. 다만 독립 Runner에 연결한 게이트웨이는 그 실행의 기록으로 `lost_reason` `retention_expired`와 함께 답합니다. 파이프 프로세스의 크기 변경은 `PROCESS_NOT_TTY`, 종료된 핸들은 `PROCESS_NOT_RUNNING`입니다. Linux 샌드박스에서 wait 상태는 관리 자식(헬퍼 argv)의 코드이며, 사용자 argv와 동일하다고 문서화하지 않습니다. 작업 공간 잠금은 패치와 명령이 동시에 파일을 변경하지 못하게 합니다. FIFO 순서는 MCP 메시지 도착이 아니라, 적격 요청이 자원을 `acquire()`할 때 시작됩니다. 같은 작업 공간에서 요청이 소유한 패치·exec는 현재 요청이 임대를 놓을 때까지 메모리 FIFO로 기다립니다. exec가 라이브 프로세스가 되면 이미 줄 서 있던 대기자와 이후 도착은 `WORKSPACE_BUSY`로 거절됩니다. 큐가 가득 차면 `RESOURCE_QUEUE_FULL`입니다. 이 대기는 SQLite나 스레드 키 큐가 아닙니다. 명령 실행 중에도 읽기와 검색은 가능하므로 파일 I/O는 사전 경로 검사에만 의존하지 않고 파일을 여는 시점의 심볼릭 링크 변경도 거부해야 합니다. Runner 파일 작업은 `codespace-fs`, 패치 적용은 별도 패치 도우미를 사용합니다. `operation_status`는 기록된 패치 원장(`kind`는 `patch`)을 조회하며, 실행 중인 명령은 `process_id`로만 다루고 이 조회로 복구하지 않습니다. diff --git a/docs/ko/operations.md b/docs/ko/operations.md index 18951ae..4c596f6 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -115,7 +115,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 살고, 게이트웨이가 그 프로세스를 작업 데이터베이스에 기록하며, 연결을 잃은 게이트웨이가 다시 Claim한 뒤 자기 기록과 대조하는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. +worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 살고, 게이트웨이가 그 프로세스를 작업 데이터베이스에 기록하며, 연결을 잃은 게이트웨이가 다시 Claim한 뒤 자기 기록과 대조하는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. 이런 Runner 자체를 잃으면 운영자가 `codespace-mcp records`로 그 Runner가 실행하던 것을 정리합니다([Runner 손실](#runner-loss)). @@ -130,8 +130,8 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 | `CODESPACE_OPERATIONS_DB` 미설정 | 패치 작업과 지시 큐를 메모리에 보관하며 재시작 시 사라짐. 독립 Runner에 연결하는 게이트웨이는 이 설정 없이 시작하지 않음 | | `CODESPACE_PROCESS_TIMEOUT_SECS` | 양의 정수. 기본 30초. 러너 환경에 설정 | | `CODESPACE_MAX_PROCESSES` | 러너 전체의 실행 중 프로세스 기본 상한 8개. 넘는 요청은 프로세스를 만들기 전에 거부. 작업 공간별 점유 규칙도 적용 | -| 프로세스 출력 | 마지막 256 KiB 보관. stdout/stderr를 합침. `read_process`는 `output_lost`와 `retained_from`을 보고하고, 종료는 `process_status`로 조회. 실행 중인 PTY 크기는 `process_resize`로 변경 | -| 종료된 핸들 | 기본 최대 15분, 최대 64개 보관. 영구 저장하지 않음. 독립 Runner는 끝난 프로세스의 상태와 출력을 게이트웨이가 확인할 때까지 보관하며, 핸들을 놓은 뒤에는 최대 64개까지 보관함([끝난 프로세스](runner-isolation.md#reconciliation)) | +| 프로세스 출력 | 마지막 256 KiB 보관. stdout/stderr를 합침. `read_process`는 `output_lost`와 `retained_from`을 보고하고, 출력이 빠진 이유는 `lost_reason`이 알려 줌. 종료는 `process_status`로 조회. 실행 중인 PTY 크기는 `process_resize`로 변경 | +| 종료된 핸들 | 기본 최대 15분, 최대 64개 보관. 영구 저장하지 않음. 독립 Runner는 끝난 프로세스의 상태와 출력을 게이트웨이가 확인할 때까지 보관하며, 핸들을 놓은 뒤에는 최대 64개까지 보관함([끝난 프로세스](runner-isolation.md#reconciliation)). 그 뒤에는 게이트웨이가 출력 없이 실행 기록으로 답하며 `lost_reason`은 `retention_expired` | 로그와 데이터베이스는 토큰·게이트웨이 설정과 같이 관리 대상 작업 공간 밖에 두세요. stderr 로그의 보관·순환은 운영자가 관리합니다. Bearer 토큰을 로그나 커밋에 넣지 마세요. 데이터베이스를 삭제하면 패치 중복 실행 방지 기록과 확인 홀드 행도 사라집니다. @@ -141,6 +141,43 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 패치 응답을 받지 못했다면 `operation_id` 또는 `operation_key` 중 하나만 지정해 `operation_status`를 조회합니다. 재시작 후 미완료 기록은 `unknown`이 되므로 파일을 확인한 뒤 다음 행동을 결정하세요. 프로세스는 `process_id`로 관리하며 `operation_status`로 복구할 수 없습니다. 프로세스가 죽을 때 `queued`이던 확인 홀드는 다시 acquire합니다. `resuming`이던 홀드는 패치 원장에서 복구하거나, 저장된 단말 결과를 재현하거나, `APPROVAL_AMBIGUOUS`를 반환할 수 있습니다. exec는 다시 spawn하지 않습니다. 독립 Runner에 연결한 게이트웨이는 `resuming`으로 남은 exec 홀드를 [대조](runner-isolation.md#reconciliation)할 때 정리합니다. Runner가 그 홀드를 위해 만든 프로세스로 consumed가 되거나, Runner가 아무것도 만들지 않았음이 증명되면 다시 granted가 됩니다. [재시도와 복구 규칙](agent-integration.md)을 참고하세요. + + +## Runner 손실 + +[독립 Runner](runner-isolation.md#independent-runner)에 연결한 게이트웨이에만 해당하며, 이 모드는 아직 지원하지 않습니다. 그 Runner가 kill되거나, 크래시하거나, 멈추거나, 호스트가 재시작하면 Runner가 실행하던 것을 Runner에서 더는 알아낼 수 없고, 그 프로세스는 Runner보다 오래 살 수 있습니다([Runner 손실](runner-isolation.md#runner-loss)). 게이트웨이는 그 프로세스에 대해 기록으로 `lost_reason`과 함께 `unknown`이라고 답합니다. 그 프로세스가 점유하던 workspace는 운영자가 그중 아무것도 실행 중이지 않음을 확인하고 기록을 풀 때까지 계속 거부되며, 다음 게이트웨이에서는 `RECOVERY_PENDING`으로 거부됩니다. CodeSpace는 이 프로세스를 하나도 종료하지 않고, 아무것도 다시 시작하지 않으며, 이 프로세스에 대해 DevGuard에 묻지도 않습니다. + +1. **기록을 나열합니다.** 아직 workspace를 점유할 수 있는 기록을 보여 줍니다. 데이터베이스를 읽기만 하므로 게이트웨이가 실행 중이어도 됩니다. + + ```bash + ./dist/codespace-mcp records list --operations-db "$CODESPACE_OPERATIONS_DB" + ``` + + 결과는 JSON입니다. 각 기록에는 `process_id`와 `workspace_id`, `dispatching`·`running`·`unknown` 중 하나인 `phase`, 게이트웨이나 release가 손실로 표시했으면 `runner_lost` 또는 `host_rebooted`이고 그 전에는 null인 `lost`, `participation`과 DevGuard `attempt_id`, 그 실행이 재개한 `approval_id`, `tty`가 있습니다. `runner`에는 Runner의 PID와 부팅, 호스트의 이번 부팅에서 실행되었는지를 뜻하는 `this_boot`, 그 PID를 가진 프로세스가 지금 실행 중인지를 뜻하는 `pid_running`이 있습니다. 이 프로세스는 그 PID를 이어받은 다른 프로세스일 수 있습니다. `root`에는 root 프로세스와, 바로 그 프로세스가 아직 실행 중인지를 뜻하는 `running`이 있으며, 기록으로 알 수 없으면 `running`은 null입니다. + +2. **아직 실행 중인 것을 찾습니다.** 호스트가 재시작했다면(`host_rebooted`이거나 `this_boot`가 false) 아무것도 실행 중이지 않습니다. 그렇지 않다면 다음을 확인합니다. + - `running`이 true이면 root 프로세스가 실행 중입니다. `ps -o pid,pgid,sid,lstart,command -p `는 프로세스를 그 프로세스 그룹, 세션과 함께 보여 줍니다. + - pipe 프로세스와 그 프로세스가 시작한 것은, 스스로 벗어나지 않는 한 자신이 시작된 프로세스 그룹에 남습니다. 그 그룹은 Runner의 프로세스 그룹이거나, 네트워크를 허용한 샌드박스 도우미와 DevGuard의 launch helper처럼 root가 이끄는 그룹입니다. `pgrep -g `는 그룹을, `pgrep -P `는 프로세스의 자식을 나열하고, `lsof -d cwd`는 어떤 프로세스가 workspace 디렉터리에서 작업하는지 보여 줍니다. Runner가 끝난 뒤 출력에 쓴 프로세스는, SIGPIPE를 무시하지 않았다면 그 신호로 끝났을 가능성이 큽니다. + - `tty: true` 프로세스는 자기 세션을 이끌며, `pgrep -s `가 그 세션을 나열합니다. 이 프로세스는 Runner가 끝날 때 받았을 가능성이 큰 SIGHUP을 무시한 경우에만 Runner보다 오래 삽니다. + - Runner를 잃었을 때 아직 `dispatching`이어서 root가 없는 기록은 프로세스를 시작했을 수도, 시작하지 않았을 수도 있습니다. 위와 같이 찾아봅니다. + - Linux 명령 샌드박스에서 네트워크를 허용하지 않은 workspace의 명령은 Runner와 함께 끝나지만, 네트워크를 허용한 workspace의 명령은 그렇지 않습니다. 나머지는 Runner의 서비스가 멈출 때 서비스 관리자가 이미 종료했을 수 있습니다. systemd는 기본값 `KillMode=control-group`에 따라 서비스의 control group을, launchd는 job이 `AbandonProcessGroup`을 설정하지 않았다면 job의 프로세스 그룹을 종료합니다. 가정하지 말고 확인하세요. + +3. **아직 실행 중인 것을 종료합니다.** 끝까지 실행하지 않기로 정했다면 `kill -TERM `를 보내고, 끝나지 않으면 `kill -KILL `를 보냅니다. deadline을 지키게 하던 Runner가 없으므로 이제 deadline이 이 프로세스를 끝내지 않습니다. + +4. **`required` 기록은 DevGuard가 그 attempt를 더는 과금하지 않을 때까지 기다립니다.** 잃은 Runner는 DevGuard의 실행 소유자였고, DevGuard는 attempt의 프로세스가 끝났음을 관찰하거나 호스트가 재시작할 때까지 그 attempt를 과금합니다. macOS에서 `devguard admission --open`은 DevGuard의 `quiescence`를 출력합니다. `charged_attempts`는 과금 중인 attempt 수이고, `attempts`는 그 key를 최대 16개 보여 줍니다. 그 key 중 하나가 기록의 `attempt_id`를 가졌거나 `charged_attempts`가 나열된 key 수보다 크면 DevGuard가 아직 그 attempt를 과금하고 있을 수 있습니다. 이 명령은 admission이 닫혀 있으면 다시 엽니다. 중단된 DevGuard upgrade가 admission을 닫힌 채로 남기므로, 그런 upgrade는 [DevGuard 운영 문서](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/operations.md#제공-명령)대로 먼저 마치세요. DevGuard에는 quiescence를 읽기만 하는 명령이 없습니다. `quiescence`가 null로 출력되면 실행 중인 DevGuard release가 이를 보고하지 않는 것이므로 DevGuard가 그 attempt를 아직 과금하는지 알 수 없습니다. 이때는 호스트가 재시작할 때까지 기록을 그대로 두세요. 호스트가 재시작하면 DevGuard에서도 그 attempt가 끝납니다. [DevGuard 대조 계약](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/contracts.md#대조)이 설명하듯 DevGuard가 attempt의 프로세스를 추적하지 못하게 되면, 프로세스가 모두 끝난 뒤에도 호스트가 재시작할 때까지 그 attempt를 과금합니다. DevGuard가 attempt를 과금하는 동안에는 `required` 기록을 풀지 마세요. + +5. **게이트웨이를 멈추고 기록을 풉니다.** + + ```bash + ./dist/codespace-mcp records release --operations-db "$CODESPACE_OPERATIONS_DB" ... + ``` + + 실행 중인 게이트웨이는 lock 파일을 잡고 있으므로, 이 명령은 그 데이터베이스에서 게이트웨이가 실행 중이지 않을 때만 기록을 바꿉니다. Runner를 잃은 기록을 그 root 프로세스가 더는 실행 중이지 않을 때 풀고, `released`, `already_released`, 그리고 각각 이유가 붙은 `refused`를 JSON으로 출력합니다. 하나라도 거부하면 종료 상태 1로 끝납니다. 끝난 기록은 workspace를 점유하지 않으므로 거부하고, root 프로세스가 아직 실행 중인 기록과 Runner가 아직 실행 중일 수 있는 기록도 거부합니다. 게이트웨이가 실행되지 않는 동안 멈춘 Runner는 자기 기록을 `running`으로 남깁니다. 이 명령은 그 Runner의 PID를 가진 프로세스가 하나도 없을 때만 그 기록을 그 Runner의 끝나지 않은 다른 기록과 함께 손실로 표시합니다. 이 명령은 프로세스를 종료하지 않습니다. 푼 기록은 `unknown`으로 남고, 그 실행이 재개한 승인에 대해 `operation_resume`은 `APPROVAL_AMBIGUOUS`로 답합니다. 그 명령은 다시 실행되지 않습니다. + +6. **Runner와 게이트웨이를 다시 시작합니다.** 게이트웨이는 각 workspace를 대조하며, 기록이 모두 끝났거나 풀린 workspace는 다시 새 작업을 받습니다. DevGuard를 쓰면 새 Runner는 `codespace` 소비자의 인스턴스 하나로 등록하는데, DevGuard는 잃은 Runner의 인스턴스를 retire한 뒤에만, 즉 그 Runner의 attempt가 더는 과금되지 않을 때만 이를 허용합니다. 그때까지 `workspace_info`는 `resource_authority.registration.state`를 `refused`로 보고하고 `required` workspace는 실행되지 않습니다. `off` workspace는 영향을 받지 않습니다. DevGuard는 소비자의 인스턴스가 하나라도 retire되지 않은 동안 그 소비자의 인스턴스 한도를 바꾸지 못하게 합니다. + +`RECOVERY_PENDING`이 기록, Runner, DevGuard가 서로 맞지 않는다고 알리는 workspace는 그 Runner가 실행 중인 동안 `records release`로 정리되지 않습니다. 그 Runner를 멈추면 그 Runner가 점유하던 모든 workspace에서 끝나지 않은 기록이 잃은 Runner의 기록이 되고, 그 뒤에는 위 단계를 따릅니다. + @@ -158,5 +195,6 @@ worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테 | Linux에서 샌드박스가 없다고 표시 | 도우미 위치, bubblewrap, 네임스페이스 지원 확인 | | `enabled` 명령 시작 실패 | 사용 가능한 Linux 샌드박스 도우미 필요 | | HTTP 요청 거부 | `/mcp` 경로, Bearer 헤더, Host 검증 | +| 게이트웨이가 Runner를 잃었다고 알리는 `RUNNER_DISCONNECTED`, 또는 `codespace-mcp records`를 알려 주는 `RECOVERY_PENDING` | [Runner 손실](#runner-loss) 절차를 따르기 | 로컬 전송 테스트는 실제 ChatGPT 연결, 모든 패키지 관리자의 프록시 호환성, 커널·컨테이너 탈출 방어까지 검증하지 않습니다. diff --git a/docs/ko/runner-isolation.md b/docs/ko/runner-isolation.md index cda18a3..54f4e3c 100644 --- a/docs/ko/runner-isolation.md +++ b/docs/ko/runner-isolation.md @@ -23,7 +23,7 @@ ## 독립 Runner -> **아직 지원하는 모드가 아닙니다.** 독립 Runner와 그 실행의 기록, 한 게이트웨이에서 다음 게이트웨이로 제어권을 넘기는 Claim, 그리고 Claim마다 뒤따르는 대조는 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 연결을 잃으면 Runner를 다시 Claim하고, 이전 controller가 작업을 시작한 곳에서는 그 controller가 남긴 것을 대조한 뒤에야 새 작업을 시작합니다. Runner 자체를 잃은 뒤의 처리는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. +> **아직 지원하는 모드가 아닙니다.** 독립 Runner와 그 실행의 기록, 한 게이트웨이에서 다음 게이트웨이로 제어권을 넘기는 Claim, 그리고 Claim마다 뒤따르는 대조는 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 연결을 잃으면 Runner를 다시 Claim하고, 이전 controller가 작업을 시작한 곳에서는 그 controller가 남긴 것을 대조한 뒤에야 새 작업을 시작합니다. Runner 자체를 잃으면 게이트웨이는 그 실행들에 대해 기록으로 답하고, 운영자가 풀 때까지 그 workspace를 점유한 채로 둡니다([Runner 손실](#runner-loss)). 이 모드는 qualification을 마친 뒤에야 지원합니다. 독립 Runner는 자신에게 연결하는 게이트웨이보다 오래 삽니다. 게이트웨이가 종료되거나 재시작되거나 강제로 종료되어도 프로세스와 그 출력, timeout은 계속됩니다. 운영자나 하네스가 독립 Runner를 시작하고, 게이트웨이는 연결만 합니다. @@ -57,27 +57,46 @@ Claim을 받아들일 때마다 Runner의 controller epoch가 하나 올라갑 다른 연결이 Runner를 제어하는 동안이나 그 연결의 요청이 처리되는 동안 시작한 게이트웨이는 간격을 늘려 가며 최대 30초 동안 다시 요청하고, 그래도 안 되면 시작을 거부합니다. 다른 작업 데이터베이스를 쓰는 경우처럼 다른 이유로 거부된 게이트웨이는 바로 시작을 거부하며, Runner는 그대로 남습니다. -**연결을 잃었을 때.** Runner와의 연결을 잃었거나, 호출이 deadline을 넘겨 연결을 닫은 게이트웨이는 다시 연결해 같은 Runner를 Claim합니다. 먼저 소켓에서 응답하는 Runner가 시작할 때 찾은 Runner와 같은 식별 정보를 가졌는지 확인합니다. 바로 한 번 시도하고, 실패하면 100밀리초를 기다린 뒤 다시 시도하며, 실패할 때마다 기다리는 시간을 두 배씩 최대 5초까지 늘립니다. 그동안에는 Runner가 필요한 요청을 모두 `RUNNER_RECONNECTING`으로 거부합니다. 그 요청을 위해서는 아무것도 보내거나 예약하거나 기록하지 않으며, 재개하려던 승인은 소비되지 않습니다. 소켓에서 더 이상 연결을 받지 않거나, 다른 Runner가 응답하거나, 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부하면 게이트웨이는 시도를 멈추고 그런 요청을 모두 `RUNNER_DISCONNECTED`로 거부합니다. 그러면 Runner의 프로세스에 다시 닿으려면 게이트웨이를 재시작해야 합니다. 연결을 잃을 때 이미 보낸 요청은 지금처럼 불확실한 결과를 받습니다. 실행이라면 `dispatch_status: unknown`입니다. Runner가 전하지 못한 종료 이벤트는 다음 controller에게 가고, 게이트웨이는 각 종료를 한 번만 전달합니다. +**연결을 잃었을 때.** Runner와의 연결을 잃었거나, 호출이 deadline을 넘겨 연결을 닫은 게이트웨이는 다시 연결해 같은 Runner를 Claim합니다. 먼저 소켓에서 응답하는 Runner가 시작할 때 찾은 Runner와 같은 식별 정보를 가졌는지 확인합니다. 바로 한 번 시도하고, 실패하면 100밀리초를 기다린 뒤 다시 시도하며, 실패할 때마다 기다리는 시간을 두 배씩 최대 5초까지 늘립니다. 그동안에는 Runner가 필요한 요청을 모두 `RUNNER_RECONNECTING`으로 거부합니다. 그 요청을 위해서는 아무것도 보내거나 예약하거나 기록하지 않으며, 재개하려던 승인은 소비되지 않습니다. 소켓에서 더 이상 연결을 받지 않거나 다른 Runner가 응답하면 Runner를 [잃은](#runner-loss) 것입니다. 기다려도 풀리지 않는 이유로 Runner가 Claim을 거부할 때도 게이트웨이는 시도를 멈춥니다. 어느 쪽이든 그런 요청을 모두 `RUNNER_DISCONNECTED`로 거부하며, Runner의 프로세스에 다시 닿으려면 게이트웨이를 재시작해야 합니다. 연결을 잃을 때 이미 보낸 요청은 지금처럼 불확실한 결과를 받습니다. 실행이라면 `dispatch_status: unknown`입니다. Runner가 전하지 못한 종료 이벤트는 다음 controller에게 가고, 게이트웨이는 각 종료를 한 번만 전달합니다. **이전 controller가 있었을 때.** 새로 시작한 Runner를 처음 Claim한 게이트웨이는 관리 worker와 같이 동작합니다. 한 controller가 어떤 workspace에서 작업(실행, 패치, DevGuard를 쓰면 준비나 launch)을 시작한 뒤에는, 그 뒤의 controller들이 Runner의 프로세스를 넘겨받습니다. 연결을 잃었다가 Runner를 다시 Claim한 같은 게이트웨이도 마찬가지이며, Claim의 응답은 앞선 controller가 작업을 시작한 workspace를 모두 알립니다. 게이트웨이는 넘겨받은 프로세스를 `process_id`로 바로 조회하고, 출력을 읽고, stdin에 쓰고, 크기를 바꾸고, 종료할 수 있으며, DevGuard가 응답하지 못할 때도 마찬가지입니다. Claim이 알린 workspace, Runner가 실행 중인 프로세스를 가진 workspace, [대조](#reconciliation)할 기록이 남은 workspace에서는 먼저 앞선 controller들이 남긴 것을 대조합니다. 그때까지는 그 workspace의 새 실행, 패치, 승인 재개를 승인을 가져가기 전에 `RECOVERY_PENDING`으로 거부하므로 승인은 그대로 남습니다. Runner도 Claim이 알린 workspace에서는 게이트웨이가 그 workspace를 대조했다고 알릴 때까지 이런 요청을 거부합니다. 다른 workspace는 바로 새 작업을 받습니다. 게이트웨이는 Runner의 식별 정보(프로세스 ID, 부팅과 시작 시각), Claim의 epoch, 앞선 controller가 작업을 시작한 workspace, 넘겨받은 프로세스 수, 그 목록에 반영된 마지막 종료 이벤트의 번호를 로그에 남깁니다. -**대조.** 게이트웨이는 Claim할 때마다, `terminal`이 아니면서 이 Runner에 이번 Claim이나 그 뒤의 Claim으로 보낸 것이 아닌 기록과, 아직 `resuming`인 승인의 기록을 대조합니다. 각 실행에 대해 세 가지를 맞춰 봅니다. 그 실행의 [기록](#process-records), 기록이 가리키는 Runner가 그 프로세스에 대해 가진 것이나 기억하는 것, 그리고 DevGuard를 쓰면 `required` workspace의 실행에 대해 Runner가 DevGuard로 조회한 attempt입니다. Runner는 시작하라고 받은 `process_id`마다, 프로세스를 만들었는지 아니면 요청을 받고도 아무것도 만들지 않았는지를 ID만으로 수명 내내 기억합니다. 요청을 받은 적이 없다는 증명은 Runner 자신에 대해서만 할 수 있으므로, 중지되었거나 잃은 Runner를 포함해 다른 Runner를 가리키는 기록은 이렇게 정리하지 않습니다. +**대조.** 게이트웨이는 Claim할 때마다, `terminal`이 아니면서 이 Runner에 이번 Claim이나 그 뒤의 Claim으로 보낸 것이 아닌 기록과, 아직 `resuming`인 승인의 기록을 대조합니다. 각 실행에 대해 세 가지를 맞춰 봅니다. 그 실행의 [기록](#process-records), 기록이 가리키는 Runner가 그 프로세스에 대해 가진 것이나 기억하는 것, 그리고 DevGuard를 쓰면 `required` workspace의 실행에 대해 Runner가 DevGuard로 조회한 attempt입니다. Runner는 시작하라고 받은 `process_id`마다, 프로세스를 만들었는지 아니면 요청을 받고도 아무것도 만들지 않았는지를 ID만으로 수명 내내 기억합니다. 요청을 받은 적이 없다는 증명은 Runner 자신에 대해서만 할 수 있습니다. 다른 Runner를 가리키면서 끝나지 않은 기록은 그 소켓에서 다른 Runner가 응답하므로 그 Runner를 [잃었다](#runner-loss)는 뜻이고, 운영자만 정리할 수 있습니다. | 기록 | 기록이 가리키는 Runner | DevGuard attempt(`required`만) | 결과 | | --- | --- | --- | --- | | `dispatching` 또는 `running` | 프로세스를 실행 중으로 가짐 | launch됨 | 기록은 `running`이 되고, 프로세스가 다시 workspace를 점유합니다. 그 workspace의 새 작업은 프로세스가 끝나거나 종료될 때까지 `WORKSPACE_BUSY`를 받습니다 | | `dispatching` 또는 `running` | 끝난 프로세스를 가짐, 또는 만들었지만 더는 갖지 않음 | scope가 끝나 해제됨, 또는 시작 전에 끝남 | 기록은 프로세스가 어떻게 끝났는지 남기며, Runner가 더는 갖지 않으면 종료 방식은 `unknown`입니다. workspace 점유는 해제됩니다 | | `dispatching` | 요청을 받은 적 없음, 또는 받고 아무것도 만들지 않음 | 없음, 또는 시작 전에 끝남 | 기록은 실행이 시작되지 않았다고 남기고, workspace 점유는 해제됩니다 | +| `dispatching`, `running` 또는 `unknown` | 다른 Runner이며, 잃었음 | 무엇이든 | 기록은 이유와 함께 `unknown`이 되고, 운영자가 기록을 풀 때까지 workspace는 계속 거부됩니다 | | 없음 | 프로세스를 실행 중으로 가짐 | 무엇이든 | workspace는 계속 거부됩니다 | | 그 밖의 모든 경우 | | | workspace는 계속 거부됩니다 | -Runner나 DevGuard에 묻지 못했거나 DevGuard가 attempt를 아직 정리하지 않아 답을 기다리는 workspace는 500밀리초 뒤에 다시 대조하며, 다시 시도할 때마다 기다리는 시간을 두 배씩 최대 30초까지 늘립니다. 세 가지가 서로 맞지 않거나 기록이 다른 Runner를 가리키는 workspace는 운영자가 해결할 때까지 계속 거부되며, 그 절차는 다음 단계에서 더합니다. Runner가 실행 중인 프로세스를 가졌지만 그 프로세스를 설명하는 기록이 없는 workspace는 그 프로세스가 끝나거나 종료될 때까지 계속 거부됩니다. 게이트웨이는 종료가 있을 때마다 다시 대조합니다. 계속 거부하는 workspace는 그 이유와 함께 로그에 남기고, 그 이유는 `RECOVERY_PENDING`에도 들어갑니다. 대조를 마친 workspace도 로그에 남깁니다. 대조할 수 없는 workspace의 점유는 풀지 않습니다. +Runner나 DevGuard에 묻지 못했거나 DevGuard가 attempt를 아직 정리하지 않아 답을 기다리는 workspace는 500밀리초 뒤에 다시 대조하며, 다시 시도할 때마다 기다리는 시간을 두 배씩 최대 30초까지 늘립니다. 세 가지가 서로 맞지 않는 workspace는 운영자가 해결할 때까지, 잃은 Runner의 기록이 있는 workspace는 운영자가 그 기록을 풀 때까지 계속 거부됩니다([운영](operations.md#runner-loss)). Runner가 실행 중인 프로세스를 가졌지만 그 프로세스를 설명하는 기록이 없는 workspace는 그 프로세스가 끝나거나 종료될 때까지 계속 거부됩니다. 게이트웨이는 종료가 있을 때마다 다시 대조합니다. 계속 거부하는 workspace는 그 이유와 함께 로그에 남기고, 그 이유는 `RECOVERY_PENDING`에도 들어갑니다. 대조를 마친 workspace도 로그에 남깁니다. 대조할 수 없는 workspace의 점유는 풀지 않습니다. 앞선 controller가 `resuming`으로 남긴 승인은 Runner가 그 승인을 위해 만든 실행으로 consumed가 되며, 그 뒤의 `operation_resume`은 그 실행의 `process_id`를 `dispatch_status: confirmed`와 함께 돌려줍니다. Runner가 아무것도 만들지 않았음이 증명되면 승인은 다시 granted가 되어 한 번 더 재개할 수 있습니다. 그 밖의 경우에는 `resuming`으로 남고, `operation_resume`은 지금처럼 `APPROVAL_AMBIGUOUS`를 돌려줍니다. 실행이 시작되었을 수도 있는 승인은 다시 granted가 되지 않습니다. -**끝난 프로세스.** 독립 Runner는 끝난 프로세스와 그 상태, 출력을 controller가 그 기록에 프로세스가 어떻게 끝났는지 남겼다고 확인할 때까지 보관합니다. 그래서 어떤 게이트웨이도 받지 못한 종료도 다음 controller에게 갑니다. 프로세스의 핸들은 관리 worker와 같이 15분이 지나거나 끝난 프로세스가 64개를 넘으면 놓고, 핸들을 놓은 끝난 프로세스는 64개까지 보관하며 가장 오래된 것부터 버립니다. 버린 프로세스에 대해서는 Runner가 그 프로세스를 만들었다는 것만 기억합니다. +**끝난 프로세스.** 독립 Runner는 끝난 프로세스와 그 상태, 출력을 controller가 그 기록에 프로세스가 어떻게 끝났는지 남겼다고 확인할 때까지 보관합니다. 그래서 어떤 게이트웨이도 받지 못한 종료도 다음 controller에게 갑니다. 프로세스의 핸들은 관리 worker와 같이 15분이 지나거나 끝난 프로세스가 64개를 넘으면 놓고, 핸들을 놓은 끝난 프로세스는 64개까지 보관하며 가장 오래된 것부터 버립니다. 버린 프로세스에 대해서는 Runner가 그 프로세스를 만들었다는 것만 기억합니다. Runner가 더는 갖지 않는 끝난 프로세스에 대해서는 `process_status`와 `read_process`가 그 기록으로 답하며, `lost_reason`은 `retention_expired`입니다. + + + +**Runner 손실.** Runner와의 연결을 잃은 게이트웨이는 소켓에서 더 이상 연결을 받지 않거나, 그 소켓의 worker가 다른 식별 정보를 알리거나 아무 식별 정보도 알리지 않으면 Runner를 잃었다고 봅니다. Runner가 강제로 종료되었거나, 비정상 종료했거나, 중지되었고, 다른 Runner로 바뀌었을 수도 있습니다. 그때부터는 그 Runner에 보낸 실행이 어떻게 되었는지 알 수 없습니다. 강제로 종료된 Runner는 아무것도 끝내지 않으므로, 그 파이프 프로세스는 자신이 시작한 프로세스와 함께, 자신이 시작된 프로세스 그룹 안에서 계속 실행됩니다. 그 그룹은 Runner의 프로세스 그룹이거나, 네트워크를 허용한 샌드박스 도우미와 DevGuard의 launch helper처럼 root가 이끄는 그룹입니다. 그 뒤 출력에 쓰는 파이프 프로세스는 SIGPIPE를 받을 가능성이 크고, `tty: true` 프로세스는 Runner가 끝나며 터미널이 닫힐 때 SIGHUP을 받을 가능성이 크지만, 어느 쪽이든 이를 무시할 수 있으므로 끝났다고 보지 않습니다. Linux 명령 샌드박스 안에서 네트워크를 허용하지 않은 workspace의 명령은, 샌드박스가 그 프로세스를 Runner의 수명에 묶으므로 그 명령을 시작한 Runner와 함께 끝납니다. 이 경우의 시험이 이를 확인합니다. 네트워크를 허용하면 명령과 그 네트워크 프록시를 실행하는 샌드박스 도우미가 Runner에 묶이지 않으므로, 명령이 Runner보다 오래 살 수 있습니다. 게이트웨이는 손실과 그에 해당하는 프로세스를 로그에 남기고, Runner가 필요한 요청을 모두 `RUNNER_DISCONNECTED`로 거부합니다. + +잃은 Runner에 보낸 것 중 끝나지 않은 기록은 모두 이유와 함께 `unknown`이 되고 workspace를 계속 점유합니다. Runner를 잃은 게이트웨이에서도, [대조](#reconciliation)하며 다른 Runner의 기록을 찾는 다음 게이트웨이에서도 그렇습니다. Runner가 호스트의 이전 부팅에서 실행된 기록은 호스트가 재시작해 실행이 끝났다고 남깁니다. Runner가 손실 전에 알린 종료는 그대로 그 기록을 끝냅니다. 이런 실행은 다시 시작하지 않으며, 다음 게이트웨이는 운영자가 그 실행이 하나도 남아 있지 않음을 확인하고 기록을 풀 때까지 그 workspace의 새 작업을 `RECOVERY_PENDING`으로 거부합니다([운영](operations.md#runner-loss)). 이 오류는 프로세스와 `codespace-mcp records`를 알려 줍니다. 이런 실행이 재개한 승인은 `resuming`으로 남고, 그 기록을 풀면 `operation_resume`은 그 승인에 대해 `APPROVAL_AMBIGUOUS`를 돌려줍니다. 기록의 deadline과 argv digest는 그대로입니다. + +Runner가 더는 갖지 않는 실행, 또는 게이트웨이가 Runner와 함께 잃은 실행에 대해서는 `process_status`와 `read_process`가 기록으로 답합니다. + +| 기록 | `state` | `lost_reason` | +| --- | --- | --- | +| 끝났고 Runner가 더는 갖지 않음 | 기록된 종료 방식, 종료 코드, 신호와 함께 `exited` | `retention_expired` | +| 끝났고 게이트웨이가 Runner를 잃음 | 기록대로 `exited` | `runner_lost` | +| 호스트 재시작 전에 끝남 | 기록대로 `exited` | `host_rebooted` | +| 끝나지 않았고, 이번 부팅에서 Runner를 잃음 | `unknown`: 프로세스가 아직 실행 중일 수 있으며, 종료가 아닙니다 | `runner_lost` | +| 끝나지 않았고, Runner가 이전 부팅에서 실행됨 | 종료 방식 `unknown`과 함께 `exited` | `host_rebooted` | + +이런 답에는 출력이 없습니다. `read_process`는 빈 `chunk`와 함께 `eof`와 `output_lost`를 true로 돌려주고, `retained_from`과 cursor는 게이트웨이가 본 가장 먼 출력 위치에 둡니다. 요청한 cursor가 더 멀면 그 cursor에 둡니다. Runner가 프로세스는 갖고 있지만 출력의 일부만 갖고 있는 동안에는 `lost_reason`이 `ring_overflow`이고, `read_process`에서는 `output_lost`가, `process_status`에서는 0보다 큰 `output_retained_from`이 함께 옵니다. Runner 손실 뒤에는 출력을 되살리지 않습니다. diff --git a/docs/ko/security-model.md b/docs/ko/security-model.md index 6654d88..81b90ae 100644 --- a/docs/ko/security-model.md +++ b/docs/ko/security-model.md @@ -44,7 +44,7 @@ Linux 도우미가 없으면 restricted 정책의 작업 공간에서도 비격 ## 작업과 복구의 안전성 -버전 검사는 예상하지 못한 파일 버전에 패치를 적용하는 것을 막습니다. 작업 키는 패치 중복 요청을 구분하며 인증 토큰이 아닙니다. 작업 공간 점유는 파일을 변경할 수 있는 패치·명령 실행을 직렬화합니다. 요청 소유 충돌은 `acquire()`에서 시작하는 인메모리 FIFO에서 기다리고, 라이브 프로세스는 뒤 대기자와 새 요청에 `WORKSPACE_BUSY`를 반환합니다. 점유 상태는 재시작 후에도 남지 않습니다. 영속적인 프로세스 복구는 없습니다. 다만 [독립 Runner](runner-isolation.md#reconciliation)에 연결한 게이트웨이는 대조할 때 실행 중으로 확인한 프로세스마다 점유를 복원하며, 그 전에는 그 workspace에서 새 작업을 시작하지 않습니다. +버전 검사는 예상하지 못한 파일 버전에 패치를 적용하는 것을 막습니다. 작업 키는 패치 중복 요청을 구분하며 인증 토큰이 아닙니다. 작업 공간 점유는 파일을 변경할 수 있는 패치·명령 실행을 직렬화합니다. 요청 소유 충돌은 `acquire()`에서 시작하는 인메모리 FIFO에서 기다리고, 라이브 프로세스는 뒤 대기자와 새 요청에 `WORKSPACE_BUSY`를 반환합니다. 점유 상태는 재시작 후에도 남지 않습니다. 영속적인 프로세스 복구는 없습니다. 다만 [독립 Runner](runner-isolation.md#reconciliation)에 연결한 게이트웨이는 대조할 때 실행 중으로 확인한 프로세스마다 점유를 복원하며, 그 전에는 그 workspace에서 새 작업을 시작하지 않습니다. 그 Runner를 잃으면 끝나지 않은 실행마다, 운영자가 그중 아무것도 실행 중이지 않음을 확인하고 프로세스 하나씩 풀 때까지 workspace를 점유합니다. 아무것도 다시 실행하지 않습니다([Runner 손실](operations.md#runner-loss)). `CODESPACE_OPERATIONS_DB`의 `approvals` 확인 홀드는 행이 `pending`·`granted`·`queued`·`resuming`인 동안 V4A 패치나 exec argv를 보관합니다. 패치 원장은 패치 본문이 아니라 해시를 보관합니다. `denied` 또는 `consumed` 뒤에는 홀드 본문을 digest 메타(도구, 작업 공간, fingerprint)로 바꿉니다. [독립 Runner](runner-isolation.md#process-records)에 연결한 게이트웨이는 실행도 여기에 기록하는데(`processes`), argv는 SHA-256 digest로만 남기고 argv 자체와 그 환경, secret은 남기지 않습니다. 또 Runner를 Claim할 때 밝히는 데이터베이스의 임의 식별자(`store_identity`)를 여기에 둡니다. 이 식별자는 데이터베이스를 서로 구별할 뿐 secret이 아닙니다. 이 데이터베이스는 토큰·게이트웨이 설정과 같이 작업 공간 루트 밖에 두세요. 홀드는 워크플로 일시정지이며 같은 MCP 호출자가 grant할 수 있습니다. 격리 경계가 아닙니다. diff --git a/docs/operations.md b/docs/operations.md index bdc646d..f0bb7fa 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -109,7 +109,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, whose gateway records them in the operations database, and which a gateway claims again after a lost connection and then reconciles with its records, is not a supported mode yet. +The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, whose gateway records them in the operations database, and which a gateway claims again after a lost connection and then reconciles with its records, is not a supported mode yet; when such a Runner itself is lost, an operator settles what it ran with `codespace-mcp records` ([Runner loss](#runner-loss)). @@ -122,8 +122,8 @@ The worker runs on the same host and is not a container. The gateway creates a p | `CODESPACE_OPERATIONS_DB` unset | In-memory patch operations and instruction queue; lost on restart. A gateway with an independent Runner does not start without it | | `CODESPACE_PROCESS_TIMEOUT_SECS` | Positive integer; default 30 seconds; set in the runner environment | | `CODESPACE_MAX_PROCESSES` | Default 8 live processes across the runner; a request over it is refused before any process is created; workspace occupancy still applies | -| Process output | Last 256 KiB retained; stdout/stderr combined; `read_process` reports `output_lost` and `retained_from`; `process_status` reports termination; `process_resize` resizes a running PTY | -| Completed handles | Default retention up to 15 minutes and 64 completed entries; not durable. An independent Runner keeps an ended process's status and output until its gateway acknowledges them, at most 64 once their handles are released ([ended processes](runner-isolation.md#reconciliation)) | +| Process output | Last 256 KiB retained; stdout/stderr combined; `read_process` reports `output_lost` and `retained_from`, and `lost_reason` says why output is missing; `process_status` reports termination; `process_resize` resizes a running PTY | +| Completed handles | Default retention up to 15 minutes and 64 completed entries; not durable. An independent Runner keeps an ended process's status and output until its gateway acknowledges them, at most 64 once their handles are released ([ended processes](runner-isolation.md#reconciliation)); after that, its gateway answers from the process record, without output, with `lost_reason` `retention_expired` | Store logs and the database outside the managed workspace, with tokens and gateway configuration. Rotate stderr capture yourself. Do not log Bearer tokens or commit real credentials. Deleting the database also deletes patch idempotency records and confirmation-hold rows. @@ -133,6 +133,43 @@ Store logs and the database outside the managed workspace, with tokens and gatew After losing a patch response, query `operation_status` with exactly one of `operation_id` or `operation_key`. An unfinished record becomes `unknown` after restart; inspect files before deciding what to do. Processes use `process_id` and cannot be recovered through `operation_status`. A confirmation hold that was `queued` when the process died can be resumed and will reacquire. A hold that was `resuming` may recover a patch from that ledger, replay a stored terminal result, or return `APPROVAL_AMBIGUOUS`; exec is not respawned. A gateway with an independent Runner settles an exec hold left `resuming` when it [reconciles](runner-isolation.md#reconciliation): consumed with the process that the Runner created for it, or granted again when the Runner proves that it created none. See [retry and recovery rules](agent-integration.md). + + +## Runner loss + +This applies to a gateway with an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet. When that Runner is killed, crashes or stops, or the host restarts, what it was running can no longer be learned from it, and its processes may outlive it ([Runner loss](runner-isolation.md#runner-loss)). The gateway answers for them from their records, as `unknown` with a `lost_reason`, and each workspace they held stays refused, with `RECOVERY_PENDING` in the next gateway, until you confirm that nothing of them runs and release their records. CodeSpace ends none of these processes, starts nothing again, and asks DevGuard nothing about them. + +1. **List the records** that may still hold a workspace. This only reads the database, so it also works while a gateway runs: + + ```bash + ./dist/codespace-mcp records list --operations-db "$CODESPACE_OPERATIONS_DB" + ``` + + It prints JSON. Each record has its `process_id` and `workspace_id`; its `phase`, `dispatching`, `running` or `unknown`; `lost`, which is `runner_lost` or `host_rebooted` once a gateway or a release marked it lost, and null before; its `participation` and DevGuard `attempt_id`; the `approval_id` it resumed; `tty`; its `runner`, with the Runner's PID and boot, `this_boot`, whether it ran on this boot of the host, and `pid_running`, whether a process with that PID runs now, which may be another process that took the PID; and its `root` process, with `running`, whether that very process still runs, or null when the record cannot tell. + +2. **Find what still runs** of each record. After a restart of the host, `host_rebooted` or `this_boot` false, nothing does. Otherwise: + - The root process runs when `running` is true. `ps -o pid,pgid,sid,lstart,command -p ` shows a process with its process group and session. + - A pipe process, and what it starts, stays in the process group it started in unless it leaves it: the Runner's own, or one that the root leads, as a sandbox helper with network enabled and DevGuard's launch helper do. `pgrep -g ` lists a group, `pgrep -P ` a process's children, and `lsof -d cwd` shows which processes work in the workspace's directories. One that wrote to its output since the Runner's end most likely ended with SIGPIPE, unless it ignores that signal. + - A `tty: true` process leads its own session, which `pgrep -s ` lists, and outlives the Runner only if it ignored the SIGHUP that the Runner's end most likely sent it. + - A record without a root, still `dispatching` when the Runner was lost, may or may not have started its process: look for it as above. + - Under the Linux command sandbox, commands in a workspace without network end with the Runner; with network enabled, they do not. A service manager may have ended the rest when the Runner's service stopped, as systemd does for a service's control group with its default `KillMode=control-group`, and launchd for a job's process group unless the job sets `AbandonProcessGroup`. Check rather than assume. + +3. **End what still runs**, once you decide that it should not finish: `kill -TERM `, then `kill -KILL ` if it does not exit. No deadline ends it any more, since the Runner that enforced the deadline is gone. + +4. **For a `required` record, wait until DevGuard no longer charges its attempt.** The lost Runner was DevGuard's execution owner, and DevGuard keeps the attempt charged until it observes that the attempt's processes ended, or until the host restarts. On macOS, `devguard admission --open` prints DevGuard's `quiescence`: `charged_attempts`, how many attempts it charges, and `attempts`, up to 16 of their keys. While a key there has the record's `attempt_id`, or `charged_attempts` is larger than the number of keys listed, DevGuard may still charge it. That command also reopens admission when it is closed, as an interrupted DevGuard upgrade leaves it, so complete such an upgrade first as [DevGuard's operations guide](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/operations.md#available-commands) says. DevGuard has no command that only reads its quiescence. When it prints `quiescence` as null, the serving DevGuard release does not report it, and nothing tells whether DevGuard still charges the attempt: keep the record until the host restarts, which ends the attempt in DevGuard too. When DevGuard lost track of an attempt's processes, as its [reconciliation contract](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/contracts.md#reconciliation) describes, it keeps the attempt charged until the host restarts, even after they all ended. Do not release a `required` record while DevGuard charges its attempt. + +5. **Stop the gateway and release the records:** + + ```bash + ./dist/codespace-mcp records release --operations-db "$CODESPACE_OPERATIONS_DB" ... + ``` + + It changes records only while no gateway runs on the database, since a running gateway holds its lock file. It releases each record whose Runner was lost once the record's root process no longer runs, prints JSON with `released`, `already_released` and `refused`, each refusal with its reason, and exits with status 1 when it refused any. It refuses a record that ended, which holds no workspace, one whose root process still runs, and one whose Runner may still run: a Runner that stopped while no gateway ran left its records `running`, and they are marked lost, with every other unended record of that Runner, only once no process has that Runner's PID. It ends no process. A released record stays `unknown`, and `operation_resume` answers an approval it resumed with `APPROVAL_AMBIGUOUS`: that command never runs again. + +6. **Start a Runner and the gateway again.** The gateway reconciles each workspace, and one whose records all ended or were released takes new work again. With DevGuard, the new Runner registers as the `codespace` consumer's one instance, which DevGuard allows only once it has retired the lost Runner's instance, when none of that Runner's attempts is charged any more. Until then `workspace_info` reports `resource_authority.registration.state` as `refused`, and `required` workspaces do not run; `off` workspaces are not affected. DevGuard does not let a consumer's instance limit change while one of its instances is not retired. + +A workspace where `RECOVERY_PENDING` says that the records, the Runner and DevGuard do not agree is not settled by `records release` while its Runner runs. Stopping that Runner makes its unended records records of a lost Runner, in every workspace it held, and the steps above then apply. + @@ -148,5 +185,6 @@ After losing a patch response, query `operation_status` with exactly one of `ope | Linux reports no sandbox | Helper location, bubblewrap, namespace support; see isolation guide | | `enabled` command fails before starting | A working Linux sandbox helper is required | | HTTP rejects the request | `/mcp` path, Bearer header, and Host validation | +| `RUNNER_DISCONNECTED` saying that the gateway lost its Runner, or `RECOVERY_PENDING` naming `codespace-mcp records` | Follow [Runner loss](#runner-loss) | Local transport tests do not establish a live ChatGPT connection, compatibility with every package manager through the proxy, or protection against kernel/container escapes. diff --git a/docs/runner-isolation.md b/docs/runner-isolation.md index 6cec195..334ba0c 100644 --- a/docs/runner-isolation.md +++ b/docs/runner-isolation.md @@ -20,7 +20,7 @@ Managed process lifetime is owned by the runner instance, not the MCP connection ## Independent Runner -> **Not a supported mode yet.** The independent Runner, the records of its executions, the Claim that hands its control from one gateway to the next and the reconciliation after each Claim are the first steps of Gateway recovery. A gateway claims the Runner again when its connection is lost, and starts new work where an earlier controller started work only once it has reconciled what that controller left. A later step adds what follows the loss of the Runner itself, and the mode is supported only once it is qualified. +> **Not a supported mode yet.** The independent Runner, the records of its executions, the Claim that hands its control from one gateway to the next and the reconciliation after each Claim are the first steps of Gateway recovery. A gateway claims the Runner again when its connection is lost, and starts new work where an earlier controller started work only once it has reconciled what that controller left. When the Runner itself is lost, the gateway answers for its executions from their records and keeps their workspaces until an operator releases them ([Runner loss](#runner-loss)). The mode is supported only once it is qualified. An independent Runner outlives the gateways that connect to it: its processes, their output and their timeouts carry on when a gateway exits, restarts or is killed. An operator or a harness starts it, and a gateway only connects to it: @@ -54,27 +54,46 @@ Each granted Claim raises the Runner's controller epoch by one. The epoch never A gateway that starts while another connection controls the Runner, or while that connection's requests run, asks again with a growing pause for up to 30 seconds, and then refuses to start. A gateway refused for another reason, such as another operations database, refuses to start at once and leaves the Runner as it was. -**A lost connection.** A gateway whose connection to the Runner is lost, or that closed it after a call passed its deadline, connects and claims the same Runner again. It first checks that the Runner at the socket has the identity it found at its start. It tries at once, waits 100 milliseconds after the first failed try, and twice as long after each further one, up to 5 seconds. Meanwhile it refuses each request that needs the Runner with `RUNNER_RECONNECTING`: nothing is sent, reserved or recorded for the request, and an approval it would resume is not consumed. When nothing accepts connections at the socket any more, another Runner answers there, or the Runner refuses the Claim for a reason that waiting cannot end, the gateway stops trying and refuses every such request with `RUNNER_DISCONNECTED`; it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection was lost keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. An exit event that the Runner could not deliver goes to its next controller, and the gateway hands each exit on once. +**A lost connection.** A gateway whose connection to the Runner is lost, or that closed it after a call passed its deadline, connects and claims the same Runner again. It first checks that the Runner at the socket has the identity it found at its start. It tries at once, waits 100 milliseconds after the first failed try, and twice as long after each further one, up to 5 seconds. Meanwhile it refuses each request that needs the Runner with `RUNNER_RECONNECTING`: nothing is sent, reserved or recorded for the request, and an approval it would resume is not consumed. When nothing accepts connections at the socket any more, or another Runner answers there, the Runner is [lost](#runner-loss). When the Runner refuses the Claim for a reason that waiting cannot end, the gateway stops trying too. Either way it refuses every such request with `RUNNER_DISCONNECTED`, and it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection was lost keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. An exit event that the Runner could not deliver goes to its next controller, and the gateway hands each exit on once. **After an earlier controller.** The first gateway that claims a newly started Runner works as it does with a managed worker. Once a controller has started work in a workspace (an execution, a patch, or with DevGuard a preparation or a launch), every later controller inherits the Runner's processes, the same gateway included once it claimed the Runner again after a lost connection, and the Claim's answer names each workspace where a controller before it started work. The gateway can query, read, write to, resize and terminate the inherited processes with their `process_id` at once, also while DevGuard cannot answer. In each workspace that the Claim names, where the Runner holds a running process, or that a record left to [reconcile](#reconciliation) names, it first reconciles what the earlier controllers left. Until then it refuses new executions, patches and approval resumes there with `RECOVERY_PENDING`, before it takes up an approval, so the approval stays as it was. The Runner also refuses them in each workspace that the Claim names, until the gateway tells it that the workspace is reconciled. Other workspaces take new work at once. The gateway logs the Runner's identity (its process ID, boot and start time), the epoch of its Claim, the workspaces where a controller before it started work, how many processes it inherited, and the number of the last exit event that list reflects. -**Reconciliation.** After each Claim, the gateway reconciles each record that is not `terminal` and was not sent to this Runner under this Claim or a later one, and each record of an approval that is still `resuming`. It compares three accounts of each execution: its [record](#process-records), what the Runner that the record names holds or remembers of the process, and with DevGuard, for an execution in a `required` workspace, its attempt, which the Runner looks up with DevGuard. For its whole life, the Runner remembers each `process_id` that it was asked to start, by its ID alone: whether it created the process, or received the request and created nothing. It proves that it never received a request only for itself: a record that names another Runner, one that was stopped or lost included, is not settled this way. +**Reconciliation.** After each Claim, the gateway reconciles each record that is not `terminal` and was not sent to this Runner under this Claim or a later one, and each record of an approval that is still `resuming`. It compares three accounts of each execution: its [record](#process-records), what the Runner that the record names holds or remembers of the process, and with DevGuard, for an execution in a `required` workspace, its attempt, which the Runner looks up with DevGuard. For its whole life, the Runner remembers each `process_id` that it was asked to start, by its ID alone: whether it created the process, or received the request and created nothing. It proves that it never received a request only for itself: a record that names another Runner and has not ended means that that Runner is [lost](#runner-loss), since another Runner answers at its socket, and only an operator settles it. | Record | The Runner that the record names | DevGuard attempt (`required` only) | Outcome | | --- | --- | --- | --- | | `dispatching` or `running` | holds the process running | launched | the record says `running`, and the process holds its workspace again: new work there returns `WORKSPACE_BUSY` until it exits or is terminated | | `dispatching` or `running` | holds the process ended, or created it and no longer holds it | released after its scope ended, or ended before it started | the record says how the process ended, with the termination `unknown` when the Runner no longer holds it, and the workspace is released | | `dispatching` | never received the request, or received it and created nothing | none, or ended before it started | the record says that the execution did not start, and the workspace is released | +| `dispatching`, `running` or `unknown` | is another Runner, which is lost | any | the record says `unknown`, with why, and the workspace stays refused until an operator releases the record | | none | holds a process running | any | the workspace stays refused | | anything else | | | the workspace stays refused | -A workspace that waits for an answer, because the Runner or DevGuard could not be asked or DevGuard has not settled an attempt yet, is reconciled again after 500 milliseconds, and twice as long after each further try, up to 30 seconds. A workspace where the accounts do not agree, or whose record names another Runner, stays refused until an operator resolves it, with a procedure that a later step adds; a workspace where the Runner holds a running process that no record accounts for stays refused until that process exits or is terminated. The gateway reconciles again after each exit too. It logs each workspace that stays refused with the reason, which `RECOVERY_PENDING` also states, and each workspace once it is reconciled. It does not release a workspace that it cannot reconcile. +A workspace that waits for an answer, because the Runner or DevGuard could not be asked or DevGuard has not settled an attempt yet, is reconciled again after 500 milliseconds, and twice as long after each further try, up to 30 seconds. A workspace where the accounts do not agree stays refused until an operator resolves it, and one with a record of a lost Runner until an operator releases that record ([Operations](operations.md#runner-loss)); a workspace where the Runner holds a running process that no record accounts for stays refused until that process exits or is terminated. The gateway reconciles again after each exit too. It logs each workspace that stays refused with the reason, which `RECOVERY_PENDING` also states, and each workspace once it is reconciled. It does not release a workspace that it cannot reconcile. An approval that an earlier controller left `resuming` is consumed with the execution that the Runner created for it, and `operation_resume` then returns that execution's `process_id` with `dispatch_status: confirmed`. When the Runner proved that it created none, the approval is granted again and can be resumed once more. Otherwise it stays `resuming`, and `operation_resume` returns `APPROVAL_AMBIGUOUS` as before: an approval whose execution may have started is never granted again. -**Ended processes.** An independent Runner keeps each ended process, its status and its output, until a controller acknowledges that its record says how the process ended, so an exit that no gateway received reaches the next controller. It still releases a process's handle after 15 minutes, or beyond 64 ended processes, as a managed worker does, and keeps at most 64 ended processes whose handles it released, dropping the oldest first. Of a dropped process, the Runner remembers only that it created it. +**Ended processes.** An independent Runner keeps each ended process, its status and its output, until a controller acknowledges that its record says how the process ended, so an exit that no gateway received reaches the next controller. It still releases a process's handle after 15 minutes, or beyond 64 ended processes, as a managed worker does, and keeps at most 64 ended processes whose handles it released, dropping the oldest first. Of a dropped process, the Runner remembers only that it created it. Once the Runner no longer keeps an ended process, `process_status` and `read_process` answer from its record, with `lost_reason` `retention_expired`. + + + +**Runner loss.** A gateway whose connection to the Runner is lost finds the Runner lost when nothing accepts connections at its socket any more, or when the worker there states another identity, or none: the Runner was killed, crashed or stopped, and perhaps replaced. What became of the executions it was sent cannot be learned from then on. A killed Runner ends nothing, so its pipe processes keep running, with the processes they started, in the process group they started in: the Runner's own, or one that their root leads, as a sandbox helper with network enabled and DevGuard's launch helper do. One that writes to its output afterwards most likely receives SIGPIPE, and a `tty: true` process most likely receives SIGHUP once the Runner's end closes its terminal, but either may ignore it, so neither is taken as ended. Under the Linux command sandbox, a command in a workspace without network ends with the Runner that started it, since the sandbox ties its processes to the Runner's life, as the tests of this case check. With network enabled, the sandbox helper that runs the command and its network proxy is not tied to the Runner, so the command can outlive it. The gateway logs the loss and the processes it concerns, and refuses every request that needs the Runner with `RUNNER_DISCONNECTED`. + +Each record of what was sent to the lost Runner that had not ended becomes `unknown`, with why, and keeps its workspace: in the gateway that lost the Runner, and in the next gateway, which finds records of another Runner as it [reconciles](#reconciliation). A record whose Runner ran on an earlier boot of the host says that the host restarted, which ended it. An end that the Runner reported before it was lost still ends its record. Nothing starts again for such an execution, and the next gateway refuses new work in its workspace with `RECOVERY_PENDING`, which names the process and `codespace-mcp records`, until an operator confirms that nothing of it runs and releases the record ([Operations](operations.md#runner-loss)). An approval that such an execution resumed stays `resuming`, and once its record is released, `operation_resume` returns `APPROVAL_AMBIGUOUS` for it. The record's deadline and argv digest stay as they were. + +`process_status` and `read_process` answer for an execution that the Runner no longer holds, or that the gateway lost with the Runner, from its record: + +| The record | `state` | `lost_reason` | +| --- | --- | --- | +| ended, and the Runner no longer keeps it | `exited`, with the recorded termination, exit code and signal | `retention_expired` | +| ended, and the gateway lost the Runner | `exited`, as recorded | `runner_lost` | +| ended before a restart of the host | `exited`, as recorded | `host_rebooted` | +| not ended, and the Runner was lost on this boot | `unknown`: the process may still run, and this is not an end | `runner_lost` | +| not ended, and the Runner ran on an earlier boot | `exited`, with the termination `unknown` | `host_rebooted` | + +Such an answer has no output: `read_process` returns an empty `chunk` with `eof` and `output_lost` true, and `retained_from` and the cursor at the furthest output offset a gateway saw, or at the cursor asked for when that is further. While the Runner keeps a process but not all of its output, `lost_reason` is `ring_overflow`, with `output_lost` in `read_process` and an `output_retained_from` above 0 in `process_status`. Output is never restored after a Runner loss. diff --git a/docs/security-model.md b/docs/security-model.md index 8abc7de..6dafbe0 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -34,7 +34,7 @@ The operator-registered root is a trust anchor. Keep tokens, the operations data ## Operation and recovery safety -Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery, except that a gateway with an [independent Runner](runner-isolation.md#reconciliation) restores the occupancy of each process that it finds running when it reconciles, and starts no new work in a workspace before that. +Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery, except that a gateway with an [independent Runner](runner-isolation.md#reconciliation) restores the occupancy of each process that it finds running when it reconciles, and starts no new work in a workspace before that. When its Runner is lost, each execution that had not ended keeps its workspace until an operator confirms that nothing of it runs and releases it, one process at a time; nothing is run again ([Runner loss](operations.md#runner-loss)). Confirmation holds in `CODESPACE_OPERATIONS_DB` (`approvals`) store the V4A patch or exec argv while the row is `pending`, `granted`, `queued`, or `resuming`. The patch operations ledger stores hashes, not patch text. After `denied` or `consumed`, the hold body is replaced with digest metadata (tool, workspace, fingerprint). A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions there (`processes`), each with the SHA-256 digest of its argv, never the argv, its environment or a secret, and keeps there the database's random identity (`store_identity`), which it names when it claims the Runner: it tells one database from another and is not a secret. Keep this database outside the workspace root, with tokens and gateway configuration. The hold is a workflow pause: the same MCP caller can grant it. It is not an isolation boundary. diff --git a/docs/translations.json b/docs/translations.json index 435a08c..2384194 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -78,8 +78,8 @@ "최소-도입-조건", "최초-gateway-복구-범위" ], - "source_sha256": "73d4de8918976b98bd92f2b1811d0872d9603edf23ac847b59eb9e74a9f36514", - "translation_sha256": "9616be59db270581e1035460a8bef34bf06815f46c2db4d79eb87f2bd66d5a6d" + "source_sha256": "77bada859adefafb165febb3273f179c6135fcb5b6f757396dd639eb22c3ca0f", + "translation_sha256": "319176abbe5b1e8029493d5e3a637e712287a40c518068c6cd0e8b5e38f7b40b" }, { "id": "cs-rg-qualification", @@ -257,8 +257,8 @@ "재시도와-복구", "파일-읽기와-패치" ], - "source_sha256": "c0257f9844525cb9a2dbbca6fa6ee4dbfa376de877cb8a647d55e4a476031a2d", - "translation_sha256": "095fe129c42f71fab2a7ec47bdbffdd81f0c6141e616e454c46e4f2f3134833b" + "source_sha256": "a3a4705eb2a425b423aa936b0c3f9b448fbb217f58ec4bbbb8ca7031333ff8d2", + "translation_sha256": "6cb63bc559a6b7e9f6fc9c1fa425113c67a08970faef1f84eb98094ef41e6b10" }, { "id": "operations", @@ -284,6 +284,8 @@ "register-a-workspace", "reproduce-the-mvp-flow", "run-the-gateway", + "runner-loss", + "runner-손실", "start-the-server", "troubleshooting", "verify-the-first-connection", @@ -310,8 +312,8 @@ "작업-공간-등록", "첫-연결-확인" ], - "source_sha256": "3b42dfbca4a44361af4476134e59d02af7313f5093590057b0e81ca35b3f42b5", - "translation_sha256": "b44f15c0a1c415e6015e4d76ca6421a648ef61572b2b959c9e6faa21d8b6b35c" + "source_sha256": "5950758a72f81664dce25afd7c281c9201fab11fa4ae4d78461bdaeb00ce978b", + "translation_sha256": "09ca3ed836f6b34f0528e9f750882af013389b47a50047cae4bd512a64250355" }, { "id": "chatgpt-connector", @@ -400,8 +402,8 @@ "현재-실행-구조", "확장-시-유지할-경계" ], - "source_sha256": "9d03ae6de59d281f2f992bd39e4bf32516946ea0b490de3e0325d2f78cb0574d", - "translation_sha256": "10d6fffe680c807b9b7dd22148032e2529ea452d1d527ac2ec6f1cade693aa0b" + "source_sha256": "686f0cd9906db317058fa6128080362e74da25bbdd8863095ac849a566d99bb8", + "translation_sha256": "02d127df0e182eab1c9c504194143ad9f282162ffed7ca3a8cf53cc3d3d75d6b" }, { "id": "execution-substrate", @@ -465,8 +467,8 @@ "확인-홀드", "훅과-스킬" ], - "source_sha256": "93e4ef27d771f9bec226af62ae66a785962aea6030050164195d9d9bcadc82ce", - "translation_sha256": "512a93517c56dd59aebae0271025658fd3c17a9f318bf3e6217b061245eb2b40" + "source_sha256": "366ffeb1ad5ef8112a9dcddc57486851147faf6f257e1bb95be2a764f4a9579a", + "translation_sha256": "23aa9948c2ca43fb207ec9843e9b4a93dfb1c23e8023e8d3c8afa6be8da133df" }, { "id": "protocol-compatibility", @@ -576,8 +578,8 @@ "프로세스-정직성", "프로필-mvp" ], - "source_sha256": "cdb542f72d794730735b7a71cf5c45bfb3aa012d3d6f637196dc28dc41249516", - "translation_sha256": "5bd6ab3149c3c72b7d94a96910ff412be67163ad73b0df1b0c8d12b2b9d109cf" + "source_sha256": "c9ca06af27ed21c9cce3be7528c9d5aa5cdbe03db171dc02ae24880ffc755859", + "translation_sha256": "80cd04a917bbbaf09f0c9f3191fa16b6ef79c1222cf65428a5d0801c5435e70f" }, { "id": "runner-isolation", @@ -604,6 +606,7 @@ "process-records", "reconciliation", "runner-isolation", + "runner-loss", "what-the-compose-fixture-does", "이후-프로세스-분리", "네트워크와-파일-접근-범위", @@ -613,8 +616,8 @@ "컨테이너-실험-구성과-검증", "호스트와-worker-실행" ], - "source_sha256": "e76149aba184f9de7f6e11441db5f9f519db95f20b62eb367de67e49381f98d2", - "translation_sha256": "950575f22d943f0904b7a805b4cd81f154f218a14e219bc16e92e8449d6ff1f7" + "source_sha256": "ada35d6e3907abe03287e595acf28acfda9ba546ede51d5d65ce4115db67268a", + "translation_sha256": "3336edb2d573475407bb95ff5b8220b3195836e42debb39f1feff0ae61c5a429" }, { "id": "error-codes", @@ -642,8 +645,8 @@ "전송-실패", "전송-실패-작업-없음" ], - "source_sha256": "5892534235706ee8d0d3a9bf18a1d36b18578441567f3eba92f69d91fbce2010", - "translation_sha256": "45046341089415fcc7d64e850629e6605ece6d1ab48b3a05e9579e32f8f56bbd" + "source_sha256": "cc10d7c2915b2a339723dfcf81a02201c45aa6396292b2d24a06631561bd45bd", + "translation_sha256": "0f20fb1831799f4aae566f1dffff42f20030718959918f819cdcea469974dba7" }, { "id": "codex-reuse", diff --git a/scripts/validate-upstream.py b/scripts/validate-upstream.py index 66c8149..a5a59ec 100644 --- a/scripts/validate-upstream.py +++ b/scripts/validate-upstream.py @@ -89,13 +89,15 @@ def stages(): # against the worker, the fixture authority and the helper built here. With them, the # independent Runner (P1R-C01): its identity, socket directory and wire, its Claim with # the peer credentials that macOS reads through `getpeereid` (P1R-C03), the gateway's - # settings, refusals and reconnect, then the worker binary and the gateway binary - # against it, with the default worker built again over the DevGuard one. + # settings, refusals and reconnect, the operator's records commands, which read the + # host's boot session and whether a process still runs as itself (P1R-C05), then the + # worker binary and the gateway binary against it, with the default worker built again + # over the DevGuard one. + devguard_binaries() + [cargo('test', 'devguard'), cargo('test', 'codex-runtime', '--features', 'devguard', '--bin', 'codespace-codex-runtime'), cargo('test', 'root', '-p', 'codespace-runner', '--features', 'devguard', '--lib', '--', 'registration', 'wire', 'admission', 'launch', 'independent', 'lifecycle'), - cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--lib', '--', 'devguard', 'independent'), + cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--lib', '--', 'devguard', 'independent', 'operator'), cargo('build', 'codex-runtime', '--bin', 'codespace-codex-runtime'), cargo('test', 'codex-runtime', '--test', 'runtime_binary', '--', 'independent'), cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--test', 'independent_runner')],