From 272ca060536c9104c0f984f22f47c365ed464218 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 11:25:46 +0000 Subject: [PATCH 1/3] runner, server, store: answer for a lost Runner's processes and let an operator release them (P1R-C05) A gateway whose independent Runner was killed, crashed or stopped refused every request with RUNNER_DISCONNECTED, process_status and read_process included, though processes of that Runner might still run, and the next gateway kept each workspace with an unended record of that Runner refused with RECOVERY_PENDING that nothing could release. An ended process that the Runner no longer kept was PROCESS_NOT_FOUND, and nothing said why output was missing. read_process and process_status now say why output is missing, or why what became of a process cannot be learned, in lost_reason: ring_overflow, retention_expired, runner_lost or host_rebooted. The client tells a lost Runner, when nothing accepts connections at its socket or another worker answers there, from a Claim refused for a reason that waiting cannot end, and reports the loss once. The gateway then marks each unended record of that Runner unknown and answers for it from the record: state unknown with runner_lost, which is never an end, or after a restart of the host exited with termination unknown and host_rebooted. Reconciliation marks the unended records of any other Runner the same way. Each such record keeps its workspace refused, and nothing of it runs again, until an operator releases it. codespace-mcp records list prints, as JSON, every record that may still hold a workspace, with whether its root process still runs as itself and whether a process holds its Runner's PID; it reads the database while a gateway runs. records release releases a lost Runner's record once its root process no longer runs, only while no gateway holds the database. It marks the records of a Runner that died while no gateway ran lost only once no process holds that Runner's PID. It ends no process and asks DevGuard nothing; an approval whose execution the record held is answered APPROVAL_AMBIGUOUS once it is released. The processes table gains lost and released_at_ms in place. A new operations runbook, in English and Korean, says how to find what still runs, when a required record may be released given DevGuard's charges (DevGuard has no command that only reads them), and why a new Runner cannot register while DevGuard keeps the lost Runner's instance. The mode stays unsupported. The P1R-C02 test of start identities compared a child's identity with its parent's; on Linux two processes that start in one clock tick have the same start time, so it now compares identities only with the same PID. macOS CI also runs the gateway's operator tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/domain/src/lib.rs | 2 +- crates/domain/src/process.rs | 141 +++++- crates/runner/src/lib.rs | 21 +- crates/runner/src/lifecycle.rs | 150 +++++- crates/runner/src/process.rs | 2 +- crates/runner/src/uds.rs | 58 ++- crates/runner/src/wire.rs | 40 +- crates/server/src/config.rs | 71 ++- crates/server/src/lib.rs | 1 + crates/server/src/main.rs | 10 + crates/server/src/mcp.rs | 482 +++++++++++++++++++- crates/server/src/operator.rs | 530 ++++++++++++++++++++++ crates/server/src/records.rs | 188 +++++++- crates/server/src/recovery.rs | 109 ++++- crates/server/src/runtime.rs | 7 + crates/server/tests/independent_runner.rs | 216 ++++++++- crates/store/src/lib.rs | 101 ++++- crates/store/src/processes.rs | 365 ++++++++++++++- docs/agent-integration.md | 4 +- docs/architecture.md | 2 +- docs/devguard-integration.md | 2 +- docs/error-codes.md | 4 +- docs/execution-substrate.md | 2 +- docs/ko/agent-integration.md | 4 +- docs/ko/architecture.md | 2 +- docs/ko/devguard-integration.md | 2 +- docs/ko/error-codes.md | 4 +- docs/ko/execution-substrate.md | 2 +- docs/ko/operations.md | 44 +- docs/ko/runner-isolation.md | 29 +- docs/ko/security-model.md | 2 +- docs/operations.md | 44 +- docs/runner-isolation.md | 29 +- docs/security-model.md | 2 +- docs/translations.json | 35 +- scripts/validate-upstream.py | 8 +- 36 files changed, 2562 insertions(+), 153 deletions(-) create mode 100644 crates/server/src/operator.rs diff --git a/crates/domain/src/lib.rs b/crates/domain/src/lib.rs index dbd9361..49a3a1f 100644 --- a/crates/domain/src/lib.rs +++ b/crates/domain/src/lib.rs @@ -43,7 +43,7 @@ pub use patch::{ OperationEventName, OperationKind, OperationStatusParams, OperationStatusResult, PatchStatus, }; pub use process::{ - ExecCommandParams, ExecCommandResult, ExecDispatchStatus, ProcessResizeParams, + ExecCommandParams, ExecCommandResult, ExecDispatchStatus, LostReason, ProcessResizeParams, ProcessResizeResult, ProcessSignal, ProcessState, ProcessStatusParams, ProcessStatusResult, ProcessTermination, ReadProcessParams, ReadProcessResult, TerminateProcessParams, WriteStdinParams, diff --git a/crates/domain/src/process.rs b/crates/domain/src/process.rs index f9ffd47..a65d193 100644 --- a/crates/domain/src/process.rs +++ b/crates/domain/src/process.rs @@ -58,15 +58,38 @@ pub struct ReadProcessResult { pub output_lost: bool, /// First retained process-output offset (`dropped` in the runner). pub retained_from: u64, + /// Why output is missing or can no longer be read: present whenever output_lost is true, + /// and whenever the Runner no longer holds the process. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lost_reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub coordination: Option, } +/// Why a process's output is missing, or why what became of the process cannot be learned +/// (P1R-C05). None of them is a successful end. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LostReason { + // The output ring kept only the most recent output; the process itself is unaffected. + RingOverflow, + // The Runner no longer keeps the ended process; the gateway answers from its records. + RetentionExpired, + // The Runner that ran the process was lost; the process may still run. + RunnerLost, + // The host restarted since the process started, which ended it. + HostRebooted, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ProcessState { Running, Exited, + // The Runner that ran the process was lost (P1R-C05): it may still run, and CodeSpace + // cannot learn what becomes of it. Not an end. A plain comment keeps the schema a single + // string enum. + Unknown, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -110,6 +133,11 @@ pub struct ProcessStatusResult { pub output_total: u64, pub output_retained_from: u64, pub eof: bool, + /// Why output is missing or the state is uncertain: present whenever + /// output_retained_from is above 0, state is unknown, or the Runner no longer holds the + /// process. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lost_reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub coordination: Option, } @@ -118,7 +146,16 @@ impl ProcessStatusResult { pub fn invariants_hold(&self) -> bool { match self.state { ProcessState::Running => { - self.exit_code.is_none() && self.termination.is_none() && self.signal.is_none() + self.exit_code.is_none() + && self.termination.is_none() + && self.signal.is_none() + && matches!(self.lost_reason, None | Some(LostReason::RingOverflow)) + } + ProcessState::Unknown => { + self.exit_code.is_none() + && self.termination.is_none() + && self.signal.is_none() + && self.lost_reason == Some(LostReason::RunnerLost) } ProcessState::Exited => match self.termination { Some(ProcessTermination::Exited) => self.signal.is_none(), @@ -243,6 +280,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: false, + lost_reason: None, coordination: None, }) .unwrap(); @@ -258,6 +296,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: false, + lost_reason: None, coordination: None, } .invariants_hold()); @@ -270,6 +309,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: false, + lost_reason: None, coordination: None, } .invariants_hold()); @@ -290,6 +330,7 @@ mod tests { output_total: 12, output_retained_from: 0, eof: true, + lost_reason: None, coordination: None, }) .unwrap(); @@ -309,6 +350,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: true, + lost_reason: None, coordination: None, }) .unwrap(); @@ -328,6 +370,7 @@ mod tests { output_total: 0, output_retained_from: 0, eof: true, + lost_reason: None, coordination: None, }; let kill = ProcessSignal { @@ -376,15 +419,111 @@ mod tests { eof: false, output_lost: true, retained_from: 8, + lost_reason: Some(LostReason::RingOverflow), coordination: None, }) .unwrap(); assert_eq!(json["output_lost"], true); assert_eq!(json["retained_from"], 8); + assert_eq!(json["lost_reason"], "ring_overflow"); let schema = serde_json::to_value(schemars::schema_for!(ReadProcessResult)).unwrap(); let dumped = schema.to_string(); assert!(dumped.contains("output_lost"), "{dumped}"); assert!(dumped.contains("retained_from"), "{dumped}"); + assert!(dumped.contains("lost_reason"), "{dumped}"); + let complete = serde_json::to_value(ReadProcessResult { + process_id: ProcessId("proc-1".into()), + cursor: 3, + chunk: "abc".into(), + eof: true, + output_lost: false, + retained_from: 0, + lost_reason: None, + coordination: None, + }) + .unwrap(); + assert!(complete.get("lost_reason").is_none(), "{complete}"); + } + + /// Each reason for missing output or an uncertain state is a name of a single string enum + /// (P1R-C05), and so is the `unknown` state. + #[test] + fn reasons_and_the_unknown_state_cross_as_names() { + let reasons = serde_json::to_value(schemars::schema_for!(LostReason)).unwrap(); + assert_eq!( + reasons["enum"], + serde_json::json!([ + "ring_overflow", + "retention_expired", + "runner_lost", + "host_rebooted" + ]), + "{reasons}" + ); + let states = serde_json::to_value(schemars::schema_for!(ProcessState)).unwrap(); + assert_eq!( + states["enum"], + serde_json::json!(["running", "exited", "unknown"]), + "{states}" + ); + let schema = serde_json::to_value(schemars::schema_for!(ProcessStatusResult)).unwrap(); + assert!(schema.to_string().contains("lost_reason"), "{schema}"); + } + + /// A process whose Runner was lost is in no state that reads as an end: `unknown` carries + /// no exit code, termination or signal, and always says why (P1R-C05). A running process + /// can only have lost output to the ring. + #[test] + fn an_unknown_state_says_why_and_is_no_end() { + let status = |state, termination, lost_reason| ProcessStatusResult { + process_id: ProcessId("proc-1".into()), + state, + exit_code: None, + termination, + signal: None, + output_total: 9, + output_retained_from: 9, + eof: true, + lost_reason, + coordination: None, + }; + let lost = status(ProcessState::Unknown, None, Some(LostReason::RunnerLost)); + assert!(lost.invariants_hold()); + let json = serde_json::to_value(&lost).unwrap(); + assert_eq!(json["state"], "unknown"); + assert_eq!(json["lost_reason"], "runner_lost"); + assert!(json.get("exit_code").is_none()); + assert!(json.get("termination").is_none()); + + assert!(!status(ProcessState::Unknown, None, None).invariants_hold()); + assert!(!status( + ProcessState::Unknown, + None, + Some(LostReason::RetentionExpired) + ) + .invariants_hold()); + assert!(!status( + ProcessState::Unknown, + Some(ProcessTermination::Unknown), + Some(LostReason::RunnerLost) + ) + .invariants_hold()); + let mut ended = status(ProcessState::Unknown, None, Some(LostReason::RunnerLost)); + ended.exit_code = Some(0); + assert!(!ended.invariants_hold()); + + assert!( + status(ProcessState::Running, None, Some(LostReason::RingOverflow)).invariants_hold() + ); + assert!( + !status(ProcessState::Running, None, Some(LostReason::RunnerLost)).invariants_hold() + ); + assert!(status( + ProcessState::Exited, + Some(ProcessTermination::Unknown), + Some(LostReason::HostRebooted) + ) + .invariants_hold()); } #[test] diff --git a/crates/runner/src/lib.rs b/crates/runner/src/lib.rs index 37b833a..41c54bc 100644 --- a/crates/runner/src/lib.rs +++ b/crates/runner/src/lib.rs @@ -133,8 +133,9 @@ pub use files::VERSION_ABSENT; pub use fork_handlers::prepare_fork_spawns; pub use patch_helper::ensure_helper_for_tests; pub use process::{ - InProcessRunner, RetentionPolicy, ShellRelease, SlotReservation, DEFAULT_COMPLETED_TTL, - DEFAULT_MAX_COMPLETED, DEFAULT_MAX_PROCESSES, DEFAULT_MAX_UNACKNOWLEDGED, DEFAULT_TIMEOUT, + process_signal, InProcessRunner, RetentionPolicy, ShellRelease, SlotReservation, + DEFAULT_COMPLETED_TTL, DEFAULT_MAX_COMPLETED, DEFAULT_MAX_PROCESSES, + DEFAULT_MAX_UNACKNOWLEDGED, DEFAULT_TIMEOUT, }; /// True when this process probed a working Linux sandbox helper. @@ -143,7 +144,9 @@ pub fn linux_sandbox_available() -> bool { linux_sandbox::probe() } -pub use lifecycle::{RunnerIdentity, RunnerLifecycle}; +pub use lifecycle::{ + host_boot_id, pid_runs, process_runs, process_start_identity, RunnerIdentity, RunnerLifecycle, +}; #[cfg(feature = "devguard")] pub use registration::OwnerRegistration; pub use socket::{ @@ -151,18 +154,18 @@ pub use socket::{ reclaim_leftover_socket, runner_socket_path, RUNNER_SOCKET_NAME, }; pub use uds::{ - runner_disconnected, runner_reconnecting, ClaimError, DisconnectHook, LinkEvent, LinkObserver, - RunnerHello, RunnerStopped, UdsRunner, CLAIM_START_BOUND, RECLAIM_BACKOFF_MAX, + runner_disconnected, runner_lost, runner_reconnecting, ClaimError, DisconnectHook, LinkEvent, + LinkObserver, RunnerHello, RunnerStopped, UdsRunner, CLAIM_START_BOUND, RECLAIM_BACKOFF_MAX, RUNNER_CALL_DEADLINE, }; pub use watch::{FsWatchEvent, FsWatchKind, WatchClosed, WatchSubscription}; #[cfg(feature = "devguard")] pub use wire::serve_runner_connection_with_registration; pub use wire::{ - host_worker, read_frame, recovery_pending, serve_runner_connection, write_frame, ClaimNote, - ClaimObserver, ClaimRefusal, IndependentRunner, RecalledProcess, RunnerClaim, RunnerEvent, - RunnerEvents, RunnerInventoryEntry, RunnerOp, RunnerOpResult, Served, StopHandle, WireEnvelope, - WireKind, CLAIM_WAIT_BOUND, STOP_DRAIN_BOUND, WIRE_PROTOCOL, + host_worker, host_worker_with_retention, read_frame, recovery_pending, serve_runner_connection, + write_frame, ClaimNote, ClaimObserver, ClaimRefusal, IndependentRunner, RecalledProcess, + RunnerClaim, RunnerEvent, RunnerEvents, RunnerInventoryEntry, RunnerOp, RunnerOpResult, Served, + StopHandle, WireEnvelope, WireKind, CLAIM_WAIT_BOUND, STOP_DRAIN_BOUND, WIRE_PROTOCOL, }; /// Execution-plane API. Control-plane fields (`work_id`, coordination, diff --git a/crates/runner/src/lifecycle.rs b/crates/runner/src/lifecycle.rs index 891430b..2709ac1 100644 --- a/crates/runner/src/lifecycle.rs +++ b/crates/runner/src/lifecycle.rs @@ -67,6 +67,12 @@ impl RunnerIdentity { } } +/// The host's current boot session, when it reports one (P1R-C05): Linux's `boot_id`, macOS's +/// `kern.bootsessionuuid`. A process recorded under another boot ended with it. +pub fn host_boot_id() -> Option { + boot_id() +} + #[cfg(target_os = "linux")] fn boot_id() -> Option { std::fs::read_to_string("/proc/sys/kernel/random/boot_id") @@ -108,7 +114,7 @@ fn boot_id() -> Option { /// been reaped yet, as a child that only its own reaper waits for, it is that process's. `None` /// where the host does not report it. #[cfg(target_os = "linux")] -pub(crate) fn process_start_identity(pid: u32) -> Option { +pub fn process_start_identity(pid: u32) -> Option { // `stat`'s 22nd field: the start time in clock ticks since boot. The command name before it // is in parentheses and may itself hold spaces and parentheses. let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; @@ -122,7 +128,83 @@ pub(crate) fn process_start_identity(pid: u32) -> Option { /// been reaped yet, as a child that only its own reaper waits for, it is that process's. `None` /// where the host does not report it. #[cfg(target_os = "macos")] -pub(crate) fn process_start_identity(pid: u32) -> Option { +pub fn process_start_identity(pid: u32) -> Option { + bsd_info(pid).map(|info| macos_start(&info)) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn process_start_identity(_pid: u32) -> Option { + None +} + +/// Whether process `pid` still runs as the process whose start identity is `started` +/// (P1R-C05): a later process with the same PID is another one, and one that exited does not +/// run, even while its parent has not reaped it yet. +#[cfg(target_os = "linux")] +pub fn process_runs(pid: u32, started: &str) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + let Some(close) = stat.rfind(')') else { + return false; + }; + let fields: Vec<&str> = stat[close + 1..].split_whitespace().collect(); + // `stat`'s 3rd field is the state, Z or X once the process exited; its 22nd the start time. + match ( + fields.first(), + fields.get(19).and_then(|ticks| ticks.parse::().ok()), + ) { + (Some(state), Some(ticks)) => { + !matches!(*state, "Z" | "X" | "x") && format!("linux-start-ticks:{ticks}") == started + } + _ => false, + } +} + +/// Whether process `pid` still runs as the process whose start identity is `started` +/// (P1R-C05): a later process with the same PID is another one, and one that exited does not +/// run, even while its parent has not reaped it yet. +#[cfg(target_os = "macos")] +pub fn process_runs(pid: u32, started: &str) -> bool { + bsd_info(pid) + .is_some_and(|info| info.pbi_status != libc::SZOMB && macos_start(&info) == started) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn process_runs(_pid: u32, _started: &str) -> bool { + false +} + +/// Whether some process that has not exited holds the PID `pid` (P1R-C05). A process that +/// exited and that its parent has not reaped yet does not count. Where the host cannot tell, +/// every PID is taken as held. +#[cfg(target_os = "linux")] +pub fn pid_runs(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + let state = stat + .rfind(')') + .and_then(|close| stat[close + 1..].split_whitespace().next()); + !matches!(state, None | Some("Z" | "X" | "x")) +} + +/// Whether some process that has not exited holds the PID `pid` (P1R-C05). A process that +/// exited and that its parent has not reaped yet does not count. Where the host cannot tell, +/// every PID is taken as held. +#[cfg(target_os = "macos")] +pub fn pid_runs(pid: u32) -> bool { + bsd_info(pid).is_some_and(|info| info.pbi_status != libc::SZOMB) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn pid_runs(_pid: u32) -> bool { + true +} + +/// What macOS reports of process `pid`, an exited one that is not reaped yet included. +#[cfg(target_os = "macos")] +fn bsd_info(pid: u32) -> Option { // SAFETY: an all-zero `proc_bsdinfo` is a valid value of this plain C structure. let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; let size = std::mem::size_of::() as libc::c_int; @@ -137,18 +219,15 @@ pub(crate) fn process_start_identity(pid: u32) -> Option { size, ) }; - if written != size || info.pbi_pid != pid { - return None; - } - Some(format!( - "macos-start:{}.{:06}", - info.pbi_start_tvsec, info.pbi_start_tvusec - )) + (written == size && info.pbi_pid == pid).then_some(info) } -#[cfg(not(any(target_os = "linux", target_os = "macos")))] -pub(crate) fn process_start_identity(_pid: u32) -> Option { - None +#[cfg(target_os = "macos")] +fn macos_start(info: &libc::proc_bsdinfo) -> String { + format!( + "macos-start:{}.{:06}", + info.pbi_start_tvsec, info.pbi_start_tvusec + ) } /// 128 random bits in hex. A host without `/dev/urandom` gets the time and the process ID @@ -212,14 +291,15 @@ mod tests { assert_eq!(own, None); return; } - let own = own.expect("this process's start identity"); + assert!(own.is_some(), "this process's start identity"); let mut child = std::process::Command::new("sleep") .arg("30") .spawn() .unwrap(); let pid = child.id(); + // Only with its PID does an identity name one process: on Linux, two processes that + // start within one clock tick, as this one and its child may, have the same start time. let started = process_start_identity(pid).expect("a running child's start identity"); - assert_ne!(started, own); assert_eq!(process_start_identity(pid).as_ref(), Some(&started)); child.kill().unwrap(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); @@ -243,6 +323,48 @@ mod tests { ); } + /// A process runs, for an operator who looks for what a lost Runner left (P1R-C05), only + /// by its PID and its start identity together, and not once it exited, even before it is + /// reaped: a later process with its PID is never taken for it. + #[test] + fn a_process_runs_as_itself_until_it_exits() { + if cfg!(not(any(target_os = "linux", target_os = "macos"))) { + assert!(!process_runs(std::process::id(), "any")); + return; + } + let mut child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let pid = child.id(); + let started = process_start_identity(pid).expect("a running child's start identity"); + assert!(process_runs(pid, &started)); + assert!(pid_runs(pid)); + assert!( + !process_runs(pid, &format!("{started}0")), + "another start identity with the same PID" + ); + child.kill().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !exited_unreaped(pid) { + assert!( + std::time::Instant::now() < deadline, + "the child did not exit" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + !process_runs(pid, &started), + "an exited child that is not reaped yet" + ); + assert!(!pid_runs(pid), "an exited child that is not reaped yet"); + child.wait().unwrap(); + assert!(!process_runs(pid, &started), "a reaped child"); + assert!(pid_runs(std::process::id())); + // Beyond the largest PID that Linux and macOS give a process. + assert!(!pid_runs(4_194_999)); + } + #[test] fn lifecycles_cross_the_wire_by_name() { assert_eq!( diff --git a/crates/runner/src/process.rs b/crates/runner/src/process.rs index 4c0e4ac..c1405ab 100644 --- a/crates/runner/src/process.rs +++ b/crates/runner/src/process.rs @@ -1500,7 +1500,7 @@ fn live_count(map: &HashMap) -> usize { } /// A signal number of this host, named when it is one of the standard signals. -fn process_signal(number: i32) -> ProcessSignal { +pub fn process_signal(number: i32) -> ProcessSignal { const NAMES: [(i32, &str); 21] = [ (libc::SIGHUP, "SIGHUP"), (libc::SIGINT, "SIGINT"), diff --git a/crates/runner/src/uds.rs b/crates/runner/src/uds.rs index bf80232..5899d5c 100644 --- a/crates/runner/src/uds.rs +++ b/crates/runner/src/uds.rs @@ -78,6 +78,21 @@ pub fn runner_disconnected(reason: &str) -> ErrorBody { ) } +/// The refusal of a request to an independent Runner that this gateway lost (P1R-C05): nothing +/// accepts connections at its socket any more, or another worker answers there. The request was +/// not sent. +pub fn runner_lost(reason: &str) -> ErrorBody { + ErrorBody::new( + ErrorCode::RunnerDisconnected, + format!( + "this gateway lost its independent Runner ({reason}), so the request was not sent. \ + What became of the Runner's processes cannot be learned, and some may still run: \ + process_status and read_process answer from this gateway's records, and an \ + operator finds the processes and releases their workspaces" + ), + ) +} + /// The refusal of a request while this gateway claims its independent Runner again /// (P1R-C03): the request was not sent. pub fn runner_reconnecting() -> ErrorBody { @@ -101,6 +116,10 @@ pub enum LinkEvent { /// The Runner cannot be claimed again, and why: every later request is refused with /// `RUNNER_DISCONNECTED`. Gone(String), + /// The Runner is lost, and why (P1R-C05): nothing accepts connections at its socket any + /// more, or another worker answers there. What becomes of its processes cannot be learned; + /// every later request is refused with `RUNNER_DISCONNECTED`. + RunnerLost(String), } pub type LinkObserver = Arc; @@ -160,6 +179,8 @@ enum State { Reclaiming, /// The Runner cannot be claimed again, and why (P1R-C03). Gone(String), + /// The Runner is lost, and why (P1R-C05). + Lost(String), } /// Where and for what a client claims its independent Runner again (P1R-C03). @@ -173,8 +194,10 @@ struct Reclaim { enum Attempt { /// Trying again may succeed. Retry(String), - /// Trying again cannot succeed. + /// Trying again cannot succeed: the Runner refused. Gone(String), + /// Trying again cannot succeed: the Runner is lost (P1R-C05). + Lost(String), } struct Client { @@ -203,6 +226,8 @@ struct Client { last_event: AtomicU64, /// The Claim of the client's current connection, once it has one (P1R-C04). claims: watch::Sender>, + /// Why the client lost its Runner, once it did (P1R-C05). + lost: watch::Sender>, } pub struct UdsRunner { @@ -346,6 +371,12 @@ impl UdsRunner { self.client.claims.subscribe() } + /// Why this client lost its independent Runner, once it did (P1R-C05): nothing accepts + /// connections at the Runner's socket any more, or another worker answers there. + pub fn lost(&self) -> watch::Receiver> { + self.client.lost.subscribe() + } + /// Connect to the independent Runner at `socket` and claim it for the operations database /// `store_id` (P1R-C03). While another connection controls the Runner, or that connection's /// requests run, and when the Claim's answer was lost, the Claim is asked again over a new @@ -451,7 +482,9 @@ impl UdsRunner { Err(Attempt::Retry(message)) => { Err(ClaimError::Transport(RunnerError::before_dispatch(message))) } - Err(Attempt::Gone(message)) => Err(ClaimError::NotIndependent(message)), + Err(Attempt::Gone(message) | Attempt::Lost(message)) => { + Err(ClaimError::NotIndependent(message)) + } } } @@ -791,6 +824,7 @@ impl Client { leaving: AtomicBool::new(false), last_event: AtomicU64::new(0), claims: watch::Sender::new(None), + lost: watch::Sender::new(None), } } @@ -839,6 +873,7 @@ impl Client { ))), State::Reclaiming => Err(RunnerError::execution(runner_reconnecting())), State::Gone(reason) => Err(RunnerError::execution(runner_disconnected(reason))), + State::Lost(reason) => Err(RunnerError::execution(runner_lost(reason))), } } @@ -985,6 +1020,12 @@ impl Client { (reclaim.observer)(LinkEvent::Gone(reason)); return; } + Err(Attempt::Lost(reason)) => { + *self.state() = State::Lost(reason.clone()); + self.lost.send_replace(Some(reason.clone())); + (reclaim.observer)(LinkEvent::RunnerLost(reason)); + return; + } Err(Attempt::Retry(reason)) => { (reclaim.observer)(LinkEvent::Retrying { reason, @@ -998,8 +1039,9 @@ impl Client { } /// One try to claim the Runner at `socket` again over a new connection, which is not the - /// client's until the caller makes it so. Nothing answering at the socket, another Runner - /// there, or a refusal that waiting cannot end, means the Runner cannot be claimed again. + /// client's until the caller makes it so. Nothing accepting connections at the socket, or + /// another worker there, means the Runner is lost (P1R-C05); a refusal that waiting cannot + /// end means it cannot be claimed again. async fn link_and_claim( self: &Arc, socket: &Path, @@ -1013,7 +1055,7 @@ impl Client { io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused ) => { - return Err(Attempt::Gone(format!( + return Err(Attempt::Lost(format!( "nothing accepts connections at {}: {err}", socket.display() ))) @@ -1059,16 +1101,16 @@ impl Client { if identity.is_some() && identity == expected { Ok(()) } else { - Err(Attempt::Gone(format!( + Err(Attempt::Lost(format!( "another Runner answers at its socket (pid {})", identity.map_or_else(|| "unknown".into(), |found| found.pid.to_string()) ))) } } - Ok(other) => Err(Attempt::Gone(format!( + Ok(other) => Err(Attempt::Lost(format!( "the worker at its socket answered Hello with {other:?}" ))), - Err(RunnerError::Execution(body)) => Err(Attempt::Gone(format!( + Err(RunnerError::Execution(body)) => Err(Attempt::Lost(format!( "the worker at its socket refused Hello: {}", body.message ))), diff --git a/crates/runner/src/wire.rs b/crates/runner/src/wire.rs index 96c41b9..3a4c0b0 100644 --- a/crates/runner/src/wire.rs +++ b/crates/runner/src/wire.rs @@ -28,9 +28,9 @@ use tokio::task::JoinSet; use crate::lifecycle::{RunnerIdentity, RunnerLifecycle}; use crate::{ - InProcessRunner, Recalled, Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, - RunnerError, RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, RunnerReadProcess, - RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, ShellRelease, + InProcessRunner, Recalled, RetentionPolicy, Runner, RunnerApplyPatchRequest, + RunnerApplyPatchResult, RunnerError, RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, + RunnerReadProcess, RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, ShellRelease, DEFAULT_MAX_UNACKNOWLEDGED, }; @@ -488,6 +488,11 @@ pub struct RunnerEvents { /// Host worker: `InProcessRunner` plus its numbered `ProcessExited` events. pub fn host_worker() -> (InProcessRunner, RunnerEvents) { + host_worker_with_retention(RetentionPolicy::default()) +} + +/// A host worker that keeps its ended processes as `retention` says. +pub fn host_worker_with_retention(retention: RetentionPolicy) -> (InProcessRunner, RunnerEvents) { let (tx, receiver) = mpsc::unbounded_channel(); let numbered = Arc::new(std::sync::Mutex::new(0u64)); let numbering = numbered.clone(); @@ -503,7 +508,7 @@ pub fn host_worker() -> (InProcessRunner, RunnerEvents) { )); }); ( - InProcessRunner::new(on_release), + InProcessRunner::with_retention(on_release, retention), RunnerEvents { receiver, numbered }, ) } @@ -4221,8 +4226,8 @@ mod tests { } /// A client stops claiming its Runner again once another Runner answers at the socket - /// or nothing accepts connections there, and refuses every request with - /// RUNNER_DISCONNECTED from then on, saying why (P1R-C03). + /// or nothing accepts connections there: it lost the Runner, says so, and refuses + /// every request with RUNNER_DISCONNECTED from then on, saying why (P1R-C03, P1R-C05). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_client_that_cannot_claim_its_runner_again_is_disconnected() { let host = Host::start(); @@ -4244,18 +4249,28 @@ mod tests { ) .await .unwrap(); + let lost = gateway.lost(); + assert_eq!(*lost.borrow(), None); relay.retarget(&other.socket); relay.cut(); assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); match next_link_event(&mut events).await { - LinkEvent::Gone(reason) => { + LinkEvent::RunnerLost(reason) => { assert!(reason.contains("another Runner answers"), "{reason}") } other => panic!("unexpected {other:?}"), } + assert!( + lost.borrow() + .as_deref() + .is_some_and(|reason| reason.contains("another Runner answers")), + "{:?}", + *lost.borrow() + ); let refusal = disconnected(&gateway, &ProcessId("proc-1".into())).await; assert!( - refusal.message.contains("another Runner answers"), + refusal.message.contains("another Runner answers") + && refusal.message.contains("lost its independent Runner"), "{refusal:?}" ); // The other Runner was asked nothing but its `Hello`. @@ -4275,12 +4290,17 @@ mod tests { assert_eq!(stopping.served().await, Served::Stopped { terminated: 0 }); assert_eq!(next_link_event(&mut events).await, LinkEvent::Lost); match next_link_event(&mut events).await { - LinkEvent::Gone(reason) => { + LinkEvent::RunnerLost(reason) => { assert!(reason.contains("nothing accepts connections"), "{reason}") } other => panic!("unexpected {other:?}"), } - disconnected(&gateway, &ProcessId("proc-1".into())).await; + let refusal = disconnected(&gateway, &ProcessId("proc-1".into())).await; + assert!( + refusal.message.contains("lost its independent Runner"), + "{refusal:?}" + ); + assert!(gateway.lost().borrow().is_some()); } /// An explicit stop ends the processes, a tty's process group included, and the Runner. diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index cbde08c..fbb0d9b 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -1,4 +1,4 @@ -use clap::{Parser, ValueEnum}; +use clap::{Parser, Subcommand, ValueEnum}; pub const MCP_PATH: &str = "/mcp"; pub const INBOX_PATH: &str = "/inbox"; @@ -48,6 +48,32 @@ pub enum GatewayLifecycle { Independent { exit_intent: RunnerExitIntent }, } +/// An operator's command instead of serving MCP (P1R-C05). +#[derive(Debug, Clone, PartialEq, Eq, Subcommand)] +pub enum Command { + /// The process records of a gateway with an independent Runner, in the database that + /// --operations-db names. + #[command(subcommand)] + Records(RecordsCommand), +} + +/// What an operator does with process records (P1R-C05, D8). +#[derive(Debug, Clone, PartialEq, Eq, Subcommand)] +pub enum RecordsCommand { + /// Print, as JSON, every record that may still hold a workspace: those that have not ended + /// and were not released, with whether their root process still runs. Reads only, so a + /// gateway may be running. + List, + /// Release the workspaces that these records hold after their Runner was lost, once nothing + /// of their processes runs. Refused while a gateway records into the database, for a record + /// whose root process still runs, and for one whose Runner may still run. + Release { + /// The process IDs of the records to release. + #[arg(required = true)] + process_ids: Vec, + }, +} + #[derive(Debug, Clone, Parser)] #[command( name = "codespace-mcp", @@ -55,6 +81,10 @@ pub enum GatewayLifecycle { about = "CodeSpace execution-tools MCP" )] pub struct Cli { + /// An operator's command; without one, the gateway serves MCP. + #[command(subcommand)] + pub command: Option, + /// Use Streamable HTTP instead of stdio. #[arg(long)] pub http: bool, @@ -79,8 +109,8 @@ pub struct Cli { /// SQLite file for operations. Unset uses an in-memory database (no replay across restarts). /// Required with `--runner-lifecycle independent`, whose gateway records its executions there - /// and holds the lock file beside it (P1R-C02). - #[arg(long, env = "CODESPACE_OPERATIONS_DB")] + /// and holds the lock file beside it (P1R-C02), and with `records`. + #[arg(long, env = "CODESPACE_OPERATIONS_DB", global = true)] pub operations_db: Option, /// Runner backend. Allowed values: `in-process` (default host supervisor) or `uds`. @@ -302,6 +332,41 @@ mod tests { assert!(err.to_string().contains("detached"), "{err}"); } + /// The operator's records commands name the operations database, before or after the + /// command, and release at least one record (P1R-C05). + #[test] + fn the_records_commands_take_the_operations_database() { + let parse = |args: &[&str]| { + let mut argv = vec!["codespace-mcp"]; + argv.extend_from_slice(args); + Cli::try_parse_from(argv) + }; + let cli = parse(&["records", "list", "--operations-db", "/var/cs/ops.sqlite"]).unwrap(); + assert_eq!(cli.command, Some(Command::Records(RecordsCommand::List))); + assert_eq!( + cli.operations_db.as_deref(), + Some(std::path::Path::new("/var/cs/ops.sqlite")) + ); + let cli = parse(&[ + "--operations-db", + "/var/cs/ops.sqlite", + "records", + "release", + "proc-1", + "proc-2", + ]) + .unwrap(); + assert_eq!( + cli.command, + Some(Command::Records(RecordsCommand::Release { + process_ids: vec!["proc-1".into(), "proc-2".into()] + })) + ); + assert!(parse(&["records", "release"]).is_err()); + assert!(parse(&["records"]).is_err()); + assert_eq!(parse(&[]).unwrap().command, None); + } + /// Without the `devguard` feature the gateway has no DevGuard setting at all. #[cfg(not(feature = "devguard"))] #[test] diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index a0a52be..c1e32e1 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -8,6 +8,7 @@ pub mod http; pub mod inbox; pub mod logging; pub mod mcp; +pub mod operator; pub mod patch_helper; pub mod protocol; pub mod records; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 6c74924..9808ea5 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -10,6 +10,7 @@ use codespace_server::config::{ }; use codespace_server::http::serve_http_handler; use codespace_server::logging; +use codespace_server::operator; use codespace_server::records::{Leases, ProcessRecords}; use codespace_server::recovery::Recovery; use codespace_server::runtime::RuntimeProcess; @@ -32,6 +33,12 @@ fn main() -> Result<()> { .expect("Failed building the Runtime"); logging::init(); let cli = Cli::parse(); + // An operator's command on the process records, instead of a gateway (P1R-C05). + if let Some(command) = &cli.command { + let done = + operator::run(command, cli.operations_db.as_deref()).map_err(anyhow::Error::msg)?; + std::process::exit(if done { 0 } else { 1 }); + } let lifecycle = cli.lifecycle().map_err(anyhow::Error::msg)?; // Released as the gateway exits, once its runtime is shut down. let _writer = lock_operations_db(&cli, lifecycle)?; @@ -251,6 +258,9 @@ async fn start_runner( let backend = RuntimeBackend::Uds(runner.clone()); let records = ProcessRecords::start(store.clone(), &identity, backend.clone(), exits); + // Once the Runner is lost, what was sent to it and had not ended becomes + // unknown, and keeps its workspace (P1R-C05). + records.follow_loss(runner.lost()); // What earlier claims left is reconciled before new work starts where they // started work, and again after each later Claim (P1R-C04). let recovery = Recovery::start(store, runner, records.clone(), leases, &claim); diff --git a/crates/server/src/mcp.rs b/crates/server/src/mcp.rs index c7b35a7..2380b54 100644 --- a/crates/server/src/mcp.rs +++ b/crates/server/src/mcp.rs @@ -6,7 +6,7 @@ use codespace_domain::{ ApprovalId, ApprovalResolveParams, ApprovalResolveResult, ApprovalState, ApprovalTargetTool, ClientEnvironmentKind, CoordinationHint, EffectivePermissionInfo, EnvironmentExecutionInfo, ErrorBody, ErrorCode, ExecCommandParams, ExecCommandResult, ExecDispatchStatus, FindParams, - FindResult, NetworkPolicyState, OperationResumeParams, OperationResumeResult, + FindResult, LostReason, NetworkPolicyState, OperationResumeParams, OperationResumeResult, OperationStatusParams, OperationStatusResult, PatchStatus, ProcessId, ProcessResizeParams, ProcessResizeResult, ProcessState, ProcessStatusParams, ProcessStatusResult, ReadParams, ReadProcessParams, ReadProcessResult, ReadResult, SteerClaimNextResult, SteerCompleteParams, @@ -133,7 +133,9 @@ a successful exit. signaled, timeout, terminated, and unknown are not success \ even when eof is true. read_process results include output_lost and retained_from. If output_lost is \ -true, the retained window is not the complete log. +true, the retained window is not the complete log, and lost_reason says why; \ +ring_overflow means that only the most recent output was kept. process_status \ +gives the same lost_reason when output_retained_from is above 0. Managed process lifetime is owned by the runner instance, not the MCP \ session. Client or HTTP disconnect keeps the process running. Losing the \ @@ -193,7 +195,17 @@ without a sandbox, can take control of it. Terminating a pipe process \ signals that process only, not its process group, so processes it started \ may outlive it. If you lose the spawn response before receiving \ process_id, do not search for the lost handle and do not start a duplicate \ -command."; +command. Once the Runner no longer keeps an ended process, process_status and \ +read_process answer from the gateway's records with lost_reason \ +retention_expired: how it ended, with no output retained. If the gateway \ +loses the Runner itself, because nothing accepts connections at its socket \ +any more or another Runner answers there, what becomes of its processes \ +cannot be learned, and some may still run: process_status reports state \ +unknown with lost_reason runner_lost for each process whose end was not \ +recorded, which is not an end, or exited with termination unknown and \ +lost_reason host_rebooted when the host restarted since. Do not start a \ +duplicate of such a process. New work in its workspace is refused until an \ +operator confirms that nothing of it runs and releases it."; const DETACH_SHUTDOWN: &str = "\ When this gateway shuts down it detaches: the Runner keeps the processes, \ @@ -500,20 +512,32 @@ impl CodeSpace { #[tool( name = "read_process", - description = "Read output from a managed process starting at cursor. Output is bounded; output_lost means the retained window is not the complete log. process_id cannot be invented." + description = "Read output from a managed process starting at cursor. Output is bounded; output_lost means the retained window is not the complete log, and lost_reason says why: ring_overflow, retention_expired, runner_lost, or host_rebooted. process_id cannot be invented." )] async fn read_process( &self, Parameters(params): Parameters, ) -> Result, String> { - let result = self + let result = match self .runner .read_process(RunnerReadProcess { process_id: params.process_id.clone(), cursor: params.cursor, }) .await - .map_err(runner_err_json)?; + { + Ok(result) => result, + // The Runner no longer holds it, or was lost: the records answer (P1R-C05). + Err(err) => match self.recorded(¶ms.process_id, &err) { + Some(recorded) => { + return Ok(Json(ReadProcessResult { + coordination: self.process_hint(¶ms.process_id.0).await, + ..recorded.read(params.cursor) + })) + } + None => return Err(runner_err_json(err)), + }, + }; if let Some(records) = &self.records { records.observed_output(&result.process_id, result.cursor); } @@ -524,23 +548,32 @@ impl CodeSpace { eof: result.eof, output_lost: result.output_lost, retained_from: result.retained_from, + lost_reason: result.output_lost.then_some(LostReason::RingOverflow), coordination: self.process_hint(¶ms.process_id.0).await, })) } #[tool( name = "process_status", - description = "Observe a managed process lifecycle. state is running or exited. termination is present only after exit (exited, signaled, timeout, terminated, or unknown). exit_code is present only when termination is exited; signal only when it is signaled, a signal CodeSpace did not send. A tty process reports a signal death as exited with exit_code 1, or 128 plus the signal number when it was launched through DevGuard. EOF is not success. Unknown process_id is rejected." + description = "Observe a managed process lifecycle. state is running, exited, or unknown: unknown means the independent Runner that ran it was lost, so the process may still run and what becomes of it cannot be learned; it is not an end. termination is present only after exit (exited, signaled, timeout, terminated, or unknown). exit_code is present only when termination is exited; signal only when it is signaled, a signal CodeSpace did not send. A tty process reports a signal death as exited with exit_code 1, or 128 plus the signal number when it was launched through DevGuard. lost_reason says why output is missing or the state is uncertain: ring_overflow, retention_expired, runner_lost, or host_rebooted. EOF is not success. Unknown process_id is rejected." )] async fn process_status( &self, Parameters(params): Parameters, ) -> Result, String> { - let result = self - .runner - .process_status(¶ms.process_id) - .await - .map_err(runner_err_json)?; + let result = match self.runner.process_status(¶ms.process_id).await { + Ok(result) => result, + // The Runner no longer holds it, or was lost: the records answer (P1R-C05). + Err(err) => match self.recorded(¶ms.process_id, &err) { + Some(recorded) => { + return Ok(Json(ProcessStatusResult { + coordination: self.process_hint(¶ms.process_id.0).await, + ..recorded.status() + })) + } + None => return Err(runner_err_json(err)), + }, + }; if let Some(records) = &self.records { match result.state { ProcessState::Exited => { @@ -549,6 +582,8 @@ impl CodeSpace { ProcessState::Running => { records.observed_output(&result.process_id, result.output_total) } + // A Runner reports no process it lost. + ProcessState::Unknown => {} } } Ok(Json(ProcessStatusResult { @@ -560,10 +595,21 @@ impl CodeSpace { output_total: result.output_total, output_retained_from: result.output_retained_from, eof: result.eof, + lost_reason: (result.output_retained_from > 0).then_some(LostReason::RingOverflow), coordination: self.process_hint(¶ms.process_id.0).await, })) } + /// What the process records say of `process_id`, which the Runner did not answer about but + /// with `err` (P1R-C05). + fn recorded( + &self, + process_id: &ProcessId, + err: &RunnerError, + ) -> Option { + self.records.as_ref()?.recorded(process_id, err) + } + #[tool( name = "process_resize", description = "Change the PTY size of a running managed process. Requires tty=true spawn. rows and cols must be at least 1. Pipe-backed processes return PROCESS_NOT_TTY. An exited handle returns PROCESS_NOT_RUNNING. Unknown process_id is rejected." @@ -4527,12 +4573,13 @@ mod independent_tests { ProcessTermination, TerminateProcessParams, WorkspaceId, }; use codespace_runner::{ - host_worker, IndependentRunner, LinkEvent, RunnerIdentity, RunnerOp, RunnerOpResult, + host_boot_id, host_worker, host_worker_with_retention, InProcessRunner, IndependentRunner, + LinkEvent, RetentionPolicy, RunnerEvents, RunnerIdentity, RunnerOp, RunnerOpResult, StopHandle, UdsRunner, }; use codespace_store::{ - NewProcessRecord, ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, - RunnerRef, + NewProcessRecord, ProcessEnd, ProcessLoss, ProcessOutcome, ProcessPhase, ProcessRecord, + RecordedParticipation, RunnerRef, }; use std::collections::BTreeMap; use std::time::{Duration, Instant}; @@ -4552,7 +4599,16 @@ mod independent_tests { impl Runner { fn start() -> Self { - let (runner, events) = host_worker(); + Self::serving(host_worker()) + } + + /// One that keeps an ended process, once a controller acknowledged it, as `retention` + /// says. + fn keeping(retention: RetentionPolicy) -> Self { + Self::serving(host_worker_with_retention(retention)) + } + + fn serving((runner, events): (InProcessRunner, RunnerEvents)) -> Self { let independent = IndependentRunner::new(runner, events); let stop = independent.stop_handle(); let (streams, incoming) = mpsc::unbounded_channel::(); @@ -4618,6 +4674,7 @@ mod independent_tests { .expect("an independent Runner's identity"); let backend = RuntimeBackend::Uds(runner.clone()); let records = ProcessRecords::start(store.clone(), &identity, backend.clone(), exits); + records.follow_loss(runner.lost()); let recovery = Recovery::start(store.clone(), runner, records.clone(), leases, claim); CodeSpace::with_store_and_runner(participation_registry(root, "off"), store, backend) .with_lifecycle(DETACHING) @@ -6182,10 +6239,10 @@ mod independent_tests { } /// A workspace where another Runner was sent an execution whose record does not say it - /// ended stays blocked (P1R-C04): only that Runner could prove what became of it. New work - /// there is refused with RECOVERY_PENDING, an approval resume before its approval is - /// claimed, so a granted approval stays granted; the record stays as it was, and the other - /// workspaces work. + /// ended stays blocked (P1R-C04): only that Runner could prove what became of it, and it is + /// lost, here to a restart of the host, so the record says so until an operator releases + /// it (P1R-C05). New work there is refused with RECOVERY_PENDING, an approval resume before + /// its approval is claimed, so a granted approval stays granted; the other workspaces work. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn an_independent_gateway_blocks_a_workspace_another_runner_left_work_in() { let dir = tempfile::tempdir().unwrap(); @@ -6213,8 +6270,8 @@ mod independent_tests { assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); assert!( refusal.message.contains("proc-elsewhere") - && refusal.message.contains("another Runner") - && refusal.message.contains("operator"), + && refusal.message.contains("the host's restart") + && refusal.message.contains("codespace-mcp records"), "{refusal:?}" ); assert_eq!( @@ -6267,13 +6324,390 @@ mod independent_tests { .unwrap() .unwrap(); assert_eq!( - (record.phase, record.runner), - (ProcessPhase::Running, elsewhere) + (record.phase, record.runner, record.lost), + ( + ProcessPhase::Unknown, + elsewhere, + Some(ProcessLoss::HostRebooted) + ) ); assert!(!root.join("runs").exists()); assert!(!root.join("added.txt").exists()); } + /// The status of `process_id`, as the gateway answers it. + async fn status_of(gateway: &CodeSpace, process_id: &str) -> ProcessStatusResult { + gateway + .process_status(Parameters(ProcessStatusParams { + process_id: ProcessId(process_id.into()), + })) + .await + .unwrap() + .0 + } + + /// A read of `process_id` from `cursor`, as the gateway answers it. + async fn read_from(gateway: &CodeSpace, process_id: &str, cursor: u64) -> ReadProcessResult { + gateway + .read_process(Parameters(ReadProcessParams { + process_id: ProcessId(process_id.into()), + cursor, + })) + .await + .unwrap() + .0 + } + + /// An ended process is re-queried as the Runner reported it for as long as the Runner keeps + /// it, and once the Runner let it go after its retention time, from its record (P1R-C05): + /// how it ended, with nothing of its output retained, and why: `retention_expired`. The + /// next gateway answers the same. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_ended_process_is_answered_from_its_record_once_the_runner_let_it_go() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::keeping(RetentionPolicy { + ttl: Duration::from_millis(300), + max_completed: 64, + }); + let gateway = runner.gateway(&root, &database).await; + let started = gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/sh", "-c", "echo done; exit 3"], + false, + ))) + .await + .unwrap() + .0; + let process_id = started.process_id.0.as_str(); + let read = output_of(&gateway, &started.process_id).await; + assert_eq!( + (read.chunk.as_str(), read.output_lost, read.lost_reason), + ("done\n", false, None) + ); + let status = status_of(&gateway, process_id).await; + assert_eq!( + (status.state, status.exit_code, status.lost_reason), + (ProcessState::Exited, Some(3), None) + ); + + // Its end recorded and acknowledged, the Runner lets it go after its retention time. + record_once(&database, &started.process_id, terminal).await; + let deadline = Instant::now() + Duration::from_secs(10); + let status = loop { + let status = status_of(&gateway, process_id).await; + if status.lost_reason.is_some() { + break status; + } + assert!(Instant::now() < deadline, "{status:?}"); + tokio::time::sleep(Duration::from_millis(50)).await; + }; + let expected = ProcessStatusResult { + process_id: started.process_id.clone(), + state: ProcessState::Exited, + exit_code: Some(3), + termination: Some(ProcessTermination::Exited), + signal: None, + output_total: 5, + output_retained_from: 5, + eof: true, + lost_reason: Some(LostReason::RetentionExpired), + coordination: None, + }; + assert_eq!( + ProcessStatusResult { + coordination: None, + ..status + }, + expected + ); + let read = read_from(&gateway, process_id, 0).await; + assert_eq!( + ReadProcessResult { + coordination: None, + ..read + }, + ReadProcessResult { + process_id: started.process_id.clone(), + cursor: 5, + chunk: String::new(), + eof: true, + output_lost: true, + retained_from: 5, + lost_reason: Some(LostReason::RetentionExpired), + coordination: None, + } + ); + detach(&gateway).await; + + let next = runner.gateway(&root, &database).await; + let status = status_of(&next, process_id).await; + assert_eq!( + ProcessStatusResult { + coordination: None, + ..status + }, + expected + ); + } + + /// Executions sent to Runners that are lost are answered from their records (P1R-C05): one + /// whose Runner was lost on this boot of the host is `unknown`, since it may still run, and + /// one whose Runner the host's restart ended exited, how is not known. Each says why, keeps + /// its deadline and the output offset a gateway saw, with nothing of its output retained. + /// Their workspace stays blocked, and nothing starts for them, until an operator releases + /// their records; the next gateway then starts new work there. An execution that ended on + /// another Runner is answered as it ended. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_answers_for_a_lost_runners_processes_until_released() { + const DEADLINE: u64 = 1_900_000_000_000; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let lost = runner_ref(RunnerIdentity { + boot_id: host_boot_id(), + ..fake_identity() + }); + let rebooted = runner_ref(fake_identity()); + { + let store = Store::open(&database).unwrap(); + for (runner, process_id) in [ + (&lost, "proc-lost"), + (&rebooted, "proc-rebooted"), + (&lost, "proc-ended"), + ] { + store + .record_dispatch(&NewProcessRecord { + process_id, + runner, + controller_epoch: Some(1), + workspace_id: "demo", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["/bin/sleep".to_string(), "30".to_string()], + }) + .unwrap(); + store + .record_running(runner, process_id, None, Some(DEADLINE)) + .unwrap(); + store.record_output(runner, process_id, 12).unwrap(); + } + store + .record_end( + &lost, + "proc-ended", + &ProcessEnd::Ended { + termination: Some(ProcessTermination::Exited), + exit_code: Some(3), + signal: None, + output_total: Some(15), + }, + ) + .unwrap(); + } + let runner = Runner::start(); + let gateway = runner.gateway(&root, &database).await; + + let status = status_of(&gateway, "proc-lost").await; + assert_eq!( + ( + status.state, + status.exit_code, + status.termination, + status.signal.clone() + ), + (ProcessState::Unknown, None, None, None), + "{status:?}" + ); + assert_eq!( + ( + status.output_total, + status.output_retained_from, + status.eof, + status.lost_reason + ), + (12, 12, true, Some(LostReason::RunnerLost)) + ); + for (cursor, after) in [(0, 12), (20, 20)] { + let read = read_from(&gateway, "proc-lost", cursor).await; + assert_eq!( + ( + read.chunk.as_str(), + read.cursor, + read.eof, + read.output_lost, + read.retained_from, + read.lost_reason + ), + ("", after, true, true, 12, Some(LostReason::RunnerLost)) + ); + } + let status = status_of(&gateway, "proc-rebooted").await; + assert_eq!( + ( + status.state, + status.termination, + status.exit_code, + status.lost_reason + ), + ( + ProcessState::Exited, + Some(ProcessTermination::Unknown), + None, + Some(LostReason::HostRebooted) + ) + ); + let read = read_from(&gateway, "proc-rebooted", 0).await; + assert_eq!(read.lost_reason, Some(LostReason::HostRebooted)); + let status = status_of(&gateway, "proc-ended").await; + assert_eq!( + ( + status.state, + status.exit_code, + status.output_retained_from, + status.lost_reason + ), + ( + ProcessState::Exited, + Some(3), + 15, + Some(LostReason::RetentionExpired) + ) + ); + let missing = refused( + gateway + .process_status(Parameters(ProcessStatusParams { + process_id: ProcessId("proc-unrecorded".into()), + })) + .await, + ); + assert_eq!(missing.code, ErrorCode::ProcessNotFound); + + let refusal = refused( + gateway + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert!( + refusal.message.contains("proc-lost") + && refusal.message.contains("may still run") + && refusal.message.contains("codespace-mcp records"), + "{refusal:?}" + ); + for (process_id, loss) in [ + ("proc-lost", ProcessLoss::RunnerLost), + ("proc-rebooted", ProcessLoss::HostRebooted), + ] { + let record = gateway.store.process_record(process_id).unwrap().unwrap(); + assert_eq!( + (record.phase, record.lost, record.deadline_at_ms), + (ProcessPhase::Unknown, Some(loss), Some(DEADLINE)), + "{process_id}" + ); + assert_eq!(record.released_at_ms, None); + } + detach(&gateway).await; + assert_eq!(recorded_executions(&database), 3); + + // An operator confirmed that nothing of them runs, and released them. + let store = Store::open(&database).unwrap(); + for process_id in ["proc-lost", "proc-rebooted"] { + assert!(store.release_record(process_id).unwrap(), "{process_id}"); + } + let next = runner.gateway(&root, &database).await; + admitted(&next, "demo").await; + let status = status_of(&next, "proc-lost").await; + assert_eq!( + (status.state, status.lost_reason), + (ProcessState::Unknown, Some(LostReason::RunnerLost)) + ); + let started = next + .exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await + .unwrap() + .0; + record_once(&database, &started.process_id, terminal).await; + assert_eq!(recorded_executions(&database), 4); + } + + /// An approval whose execution was sent to a Runner that is lost is never run again + /// (P1R-C05): it stays resuming while its workspace is blocked, and once an operator + /// released the record, resuming it answers APPROVAL_AMBIGUOUS, every time. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_approval_whose_execution_a_lost_runner_held_is_never_run_again() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let first = runner.gateway(&root, &database).await; + let command = ["/bin/sh", "-c", "echo run >> runs"]; + let approval_id = granted(&first, "gov-confirm", &command).await; + assert!(matches!( + first.store.claim_resume(&approval_id).unwrap(), + ResumeClaim::Execute(..) + )); + let lost = runner_ref(RunnerIdentity { + boot_id: host_boot_id(), + ..fake_identity() + }); + first + .store + .record_dispatch(&NewProcessRecord { + process_id: "proc-lost", + runner: &lost, + controller_epoch: Some(1), + workspace_id: "gov-confirm", + approval_id: Some(&approval_id), + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &command.map(String::from), + }) + .unwrap(); + first + .store + .record_running(&lost, "proc-lost", None, None) + .unwrap(); + detach(&first).await; + + let next = runner.gateway(&root, &database).await; + let refusal = refused(resume(&next, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert_eq!( + next.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Resuming + ); + detach(&next).await; + assert!(Store::open(&database) + .unwrap() + .release_record("proc-lost") + .unwrap()); + + let last = runner.gateway(&root, &database).await; + for _ in 0..2 { + let refusal = refused(resume(&last, &approval_id).await); + assert_eq!(refusal.code, ErrorCode::ApprovalAmbiguous, "{refusal:?}"); + } + assert!(!root.join("runs").exists()); + assert_eq!(recorded_executions(&database), 1); + } + /// A workspace whose record and Runner disagree stays blocked, and so does one where the /// Runner runs a process that no record accounts for, until that process ends (P1R-C04): /// the refusal names the process, which can be queried and terminated meanwhile. diff --git a/crates/server/src/operator.rs b/crates/server/src/operator.rs new file mode 100644 index 0000000..e231005 --- /dev/null +++ b/crates/server/src/operator.rs @@ -0,0 +1,530 @@ +//! The operator's commands on the process records of a gateway with an independent Runner +//! (P1R-C05, D8). A record whose Runner was lost before its execution ended holds its workspace, +//! since a process of a killed Runner can outlive it: `records list` shows the records that may +//! still hold one, with whether each root process still runs, and `records release` releases +//! the ones an operator confirmed nothing runs of. Neither ends a process, and neither asks the +//! resource authority anything. + +use std::path::Path; +use std::time::Duration; + +use codespace_runner::{host_boot_id, pid_runs, process_runs}; +use codespace_store::{ + unsettled_records_at, ProcessLoss, ProcessPhase, ProcessRecord, RootProcess, Store, WriterLock, +}; +use serde_json::{json, Value}; + +use crate::config::{Command, RecordsCommand}; + +/// Run `command` on the operations database `database`, print its report as JSON, and return +/// whether it did all it was asked. +pub fn run(command: &Command, database: Option<&Path>) -> Result { + let database = + database.ok_or("records needs --operations-db, the gateway's operations database")?; + let boot_id = host_boot_id(); + let (report, done) = match command { + Command::Records(RecordsCommand::List) => (list(database, boot_id.as_deref())?, true), + Command::Records(RecordsCommand::Release { process_ids }) => { + release(database, boot_id.as_deref(), process_ids)? + } + }; + let text = serde_json::to_string_pretty(&report).map_err(|err| err.to_string())?; + println!("{text}"); + Ok(done) +} + +/// Every record that may still hold a workspace, read without writing to the database. +fn list(database: &Path, boot_id: Option<&str>) -> Result { + let records = unsettled_records_at(database)?; + Ok(json!({ + "operations_db": database.display().to_string(), + "boot_id": boot_id, + "records": records + .iter() + .map(|record| describe(record, boot_id)) + .collect::>(), + })) +} + +/// Release the records `process_ids` while no gateway records into `database`. Returns the +/// report, and whether every record was released. +fn release( + database: &Path, + boot_id: Option<&str>, + process_ids: &[String], +) -> Result<(Value, bool), String> { + if !database.exists() { + return Err(format!( + "the operations database {} does not exist", + database.display() + )); + } + let _writer = WriterLock::acquire(database, Duration::ZERO, |_| {}).map_err(|err| { + format!("records release changes records only while no gateway runs on the database: {err}") + })?; + let store = Store::open(database)?; + let mut released = Vec::new(); + let mut already = Vec::new(); + let mut refused = Vec::new(); + for process_id in process_ids { + match release_one(&store, boot_id, process_id)? { + Released::Now => released.push(process_id.clone()), + Released::Before => already.push(process_id.clone()), + Released::Refused(reason) => refused.push(json!({ + "process_id": process_id, + "reason": reason, + })), + } + } + let done = refused.is_empty(); + Ok(( + json!({ + "operations_db": database.display().to_string(), + "released": released, + "already_released": already, + "refused": refused, + }), + done, + )) +} + +/// What became of one release. +enum Released { + Now, + Before, + Refused(String), +} + +/// Release the record of `process_id`, unless it does not wait for a release or its root +/// process still runs. A record whose Runner ran on an earlier boot of the host is lost to that +/// restart, and one whose Runner's PID runs no process is lost with that Runner, even when no +/// gateway said so yet: a Runner that died while no gateway ran leaves its records running. +/// While the Runner's PID runs a process, that may be the Runner, which still holds the record. +fn release_one(store: &Store, boot_id: Option<&str>, process_id: &str) -> Result { + let Some(mut record) = store + .process_record(process_id) + .map_err(|err| err.message)? + else { + return Ok(Released::Refused("no record has this process ID".into())); + }; + if record.released_at_ms.is_some() { + return Ok(Released::Before); + } + match record.phase { + ProcessPhase::Terminal => { + return Ok(Released::Refused( + "its execution ended, so its record holds no workspace".into(), + )) + } + ProcessPhase::Dispatching | ProcessPhase::Running => { + let loss = ProcessLoss::of(&record.runner, boot_id); + if loss == ProcessLoss::RunnerLost && pid_runs(record.runner.pid) { + return Ok(Released::Refused(format!( + "a process with its Runner's PID {} runs, and may be that Runner, which still \ + holds the record: stop that Runner first, or start the gateway, which marks \ + the record lost when no Runner answers for it", + record.runner.pid + ))); + } + store + .record_runner_lost(&record.runner, loss) + .map_err(|err| err.message)?; + record = store + .process_record(process_id) + .map_err(|err| err.message)? + .ok_or("the record disappeared")?; + } + ProcessPhase::Unknown => {} + } + if let Some(root) = record.root.as_ref() { + if root_runs(&record, root, boot_id) == Some(true) { + return Ok(Released::Refused(format!( + "its root process {} still runs: end it, and what it started, first", + root.pid + ))); + } + } + Ok( + if store + .release_record(process_id) + .map_err(|err| err.message)? + { + Released::Now + } else { + Released::Refused("its record does not wait for a release".into()) + }, + ) +} + +/// Whether the root process of `record` still runs on this host, by its PID and its start +/// identity, so that a later process with the same PID is not taken for it. None when the +/// record cannot tell: it names no start identity. +fn root_runs(record: &ProcessRecord, root: &RootProcess, boot_id: Option<&str>) -> Option { + if ProcessLoss::of(&record.runner, boot_id) == ProcessLoss::HostRebooted { + return Some(false); + } + Some(process_runs(root.pid, root.started.as_deref()?)) +} + +/// One record, as `records list` prints it. +fn describe(record: &ProcessRecord, boot_id: Option<&str>) -> Value { + let this_boot = match (record.runner.boot_id.as_deref(), boot_id) { + (Some(then), Some(now)) => Some(then == now), + _ => None, + }; + json!({ + "process_id": record.process_id, + "workspace_id": record.workspace_id, + "phase": record.phase.as_str(), + "lost": record.lost.map(ProcessLoss::as_str), + "participation": record.participation.as_str(), + "attempt_id": record.attempt_id, + "approval_id": record.approval_id.as_ref().map(|id| id.0.as_str()), + "tty": record.tty, + "runner": { + "pid": record.runner.pid, + "boot_id": record.runner.boot_id, + "started_at_ms": record.runner.started_at_ms, + "this_boot": this_boot, + "pid_running": ProcessLoss::of(&record.runner, boot_id) != ProcessLoss::HostRebooted + && pid_runs(record.runner.pid), + }, + "root": record.root.as_ref().map(|root| json!({ + "pid": root.pid, + "started": root.started, + "running": root_runs(record, root, boot_id), + })), + "deadline_at_ms": record.deadline_at_ms, + "output_cursor": record.output_cursor, + "created_at_ms": record.created_at_ms, + "updated_at_ms": record.updated_at_ms, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use codespace_domain::ProcessTermination; + use codespace_runner::process_start_identity; + use codespace_store::{NewProcessRecord, ProcessEnd, RecordedParticipation, RunnerRef}; + + /// A Runner that ran on `boot_id`, whose PID runs no process: it is beyond the largest PID + /// that Linux and macOS give a process. + fn runner_on(boot_id: Option) -> RunnerRef { + RunnerRef { + boot_id, + pid: 4_194_999, + started_at_ms: 1_000, + nonce: "lost-runner".into(), + } + } + + /// A Runner that ran on `boot_id`, whose PID this test's own process runs now. + fn runner_running_on(boot_id: Option) -> RunnerRef { + RunnerRef { + pid: std::process::id(), + ..runner_on(boot_id) + } + } + + /// Record an execution sent to `runner` in `workspace_id`, started with the root `root`. + fn sent(store: &Store, runner: &RunnerRef, process_id: &str, root: Option<&RootProcess>) { + store + .record_dispatch(&NewProcessRecord { + process_id, + runner, + controller_epoch: Some(1), + workspace_id: "demo", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["/bin/sleep".to_string(), "30".to_string()], + }) + .unwrap(); + store + .record_running(runner, process_id, root, None) + .unwrap(); + } + + /// A child that runs until it is killed, and the root that names it. + fn live_root() -> (std::process::Child, RootProcess) { + let child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let root = RootProcess { + pid: child.id(), + started: process_start_identity(child.id()), + }; + (child, root) + } + + fn ids(report: &Value, key: &str) -> Vec { + report[key] + .as_array() + .unwrap() + .iter() + .map(|id| id.as_str().unwrap().to_string()) + .collect() + } + + /// `records list` shows each record that may still hold a workspace, with whether its root + /// process runs, and reads the database while a gateway holds it (P1R-C05). + #[test] + fn records_list_shows_what_may_still_hold_a_workspace() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + let runner = runner_running_on(Some(boot.clone())); + let (mut child, root) = live_root(); + sent(&store, &runner, "proc-running", Some(&root)); + sent(&store, &runner, "proc-ended", None); + store + .record_end( + &runner, + "proc-ended", + &ProcessEnd::Ended { + termination: Some(ProcessTermination::Exited), + exit_code: Some(0), + signal: None, + output_total: Some(3), + }, + ) + .unwrap(); + let earlier = runner_on(Some("an-earlier-boot".into())); + sent(&store, &earlier, "proc-rebooted", Some(&root)); + let _gateway = WriterLock::acquire(&database, Duration::ZERO, |_| {}).unwrap(); + + let report = list(&database, Some(&boot)).unwrap(); + assert_eq!(report["boot_id"], boot.as_str()); + let records = report["records"].as_array().unwrap(); + let listed: Vec<&str> = records + .iter() + .map(|record| record["process_id"].as_str().unwrap()) + .collect(); + assert_eq!(listed, ["proc-running", "proc-rebooted"], "{report}"); + let running = &records[0]; + assert_eq!(running["phase"], "running"); + assert_eq!(running["lost"], Value::Null); + assert_eq!(running["workspace_id"], "demo"); + assert_eq!(running["participation"], "off"); + assert_eq!(running["runner"]["this_boot"], true); + assert_eq!(running["runner"]["pid_running"], true); + assert_eq!(running["root"]["pid"], root.pid); + assert_eq!(running["root"]["running"], true, "{running}"); + // The same PIDs on a later boot are other processes. + let rebooted = &records[1]; + assert_eq!(rebooted["runner"]["this_boot"], false); + assert_eq!(rebooted["runner"]["pid_running"], false); + assert_eq!(rebooted["root"]["running"], false, "{rebooted}"); + + child.kill().unwrap(); + child.wait().unwrap(); + let report = list(&database, Some(&boot)).unwrap(); + assert_eq!(report["records"][0]["root"]["running"], false, "{report}"); + assert!(list(&dir.path().join("missing.sqlite"), Some(&boot)).is_err()); + } + + /// `records release` changes nothing while a gateway holds the database, and releases only + /// a record whose Runner was lost once its root process no longer runs: never one that + /// ended or that its Runner may still hold, and a later process with the root's PID is not + /// the root (P1R-C05, D8). + #[test] + fn records_release_waits_for_the_gateway_and_the_root() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + let runner = runner_running_on(Some(boot.clone())); + let (mut child, root) = live_root(); + sent(&store, &runner, "proc-lost", Some(&root)); + let (mut other, reused) = live_root(); + // A record whose root PID a later process took. + let gone = RootProcess { + pid: reused.pid, + started: Some("linux-start-ticks:1".into()), + }; + sent(&store, &runner, "proc-reused", Some(&gone)); + sent(&store, &runner, "proc-ended", None); + store + .record_end( + &runner, + "proc-ended", + &ProcessEnd::Ended { + termination: Some(ProcessTermination::Exited), + exit_code: Some(0), + signal: None, + output_total: None, + }, + ) + .unwrap(); + let release_all = || { + release( + &database, + Some(&boot), + &[ + "proc-lost".to_string(), + "proc-reused".to_string(), + "proc-ended".to_string(), + "proc-none".to_string(), + ], + ) + }; + + let gateway = WriterLock::acquire(&database, Duration::ZERO, |_| {}).unwrap(); + let refusal = release_all().unwrap_err(); + assert!(refusal.contains("while no gateway runs"), "{refusal}"); + drop(gateway); + + // Nothing said yet that the Runner was lost, and its PID runs. + let (report, done) = release_all().unwrap(); + assert!(!done); + assert!(ids(&report, "released").is_empty(), "{report}"); + let refused = report["refused"].as_array().unwrap(); + assert_eq!(refused.len(), 4, "{report}"); + for held in &refused[..2] { + let reason = held["reason"].as_str().unwrap(); + assert!( + reason.contains("still holds the record") + && reason.contains(&std::process::id().to_string()), + "{reason}" + ); + } + assert!(refused[2]["reason"].as_str().unwrap().contains("ended")); + assert!(refused[3]["reason"].as_str().unwrap().contains("no record")); + let record = store.process_record("proc-lost").unwrap().unwrap(); + assert_eq!((record.phase, record.lost), (ProcessPhase::Running, None)); + + store + .record_runner_lost(&runner, ProcessLoss::RunnerLost) + .unwrap(); + let (report, done) = release_all().unwrap(); + assert!(!done); + assert_eq!(ids(&report, "released"), ["proc-reused"], "{report}"); + let refused = report["refused"].as_array().unwrap(); + assert_eq!(refused[0]["process_id"], "proc-lost"); + let reason = refused[0]["reason"].as_str().unwrap(); + assert!( + reason.contains("still runs") && reason.contains(&root.pid.to_string()), + "{reason}" + ); + + child.kill().unwrap(); + child.wait().unwrap(); + let (report, _) = release_all().unwrap(); + assert_eq!(ids(&report, "released"), ["proc-lost"], "{report}"); + assert_eq!( + ids(&report, "already_released"), + ["proc-reused"], + "{report}" + ); + let record = store.process_record("proc-lost").unwrap().unwrap(); + assert_eq!( + (record.phase, record.lost), + (ProcessPhase::Unknown, Some(ProcessLoss::RunnerLost)) + ); + assert!(record.released_at_ms.is_some()); + assert!(unsettled_records_at(&database).unwrap().is_empty()); + other.kill().unwrap(); + other.wait().unwrap(); + } + + /// A Runner that died while no gateway ran left its records running: once its PID runs no + /// process, `records release` marks every record of that Runner lost, as a gateway would, + /// and releases the ones asked for whose root processes no longer run (P1R-C05). + #[test] + fn records_release_takes_a_runner_gone_unseen_as_lost() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + let gone = runner_on(Some(boot.clone())); + let (mut child, root) = live_root(); + sent(&store, &gone, "proc-survivor", Some(&root)); + store + .record_dispatch(&NewProcessRecord { + process_id: "proc-sent", + runner: &gone, + controller_epoch: Some(1), + workspace_id: "other", + approval_id: None, + attempt_id: None, + participation: RecordedParticipation::Off, + tty: false, + argv: &["/bin/true".to_string()], + }) + .unwrap(); + let listed = list(&database, Some(&boot)).unwrap(); + assert_eq!( + listed["records"][0]["runner"]["pid_running"], false, + "{listed}" + ); + + let (report, done) = release(&database, Some(&boot), &["proc-sent".into()]).unwrap(); + assert!(done, "{report}"); + assert_eq!(ids(&report, "released"), ["proc-sent"]); + let survivor = store.process_record("proc-survivor").unwrap().unwrap(); + assert_eq!( + (survivor.phase, survivor.lost, survivor.released_at_ms), + (ProcessPhase::Unknown, Some(ProcessLoss::RunnerLost), None) + ); + let (report, done) = release(&database, Some(&boot), &["proc-survivor".into()]).unwrap(); + assert!(!done); + let reason = report["refused"][0]["reason"].as_str().unwrap(); + assert!(reason.contains("still runs"), "{reason}"); + + child.kill().unwrap(); + child.wait().unwrap(); + let (report, done) = release(&database, Some(&boot), &["proc-survivor".into()]).unwrap(); + assert!(done, "{report}"); + assert!(unsettled_records_at(&database).unwrap().is_empty()); + } + + /// A record whose Runner ran on an earlier boot of the host is released as one that the + /// restart ended, though no gateway said so yet, and whatever runs with its root's PID now + /// (P1R-C05). + #[test] + fn records_release_takes_a_restart_as_the_end() { + let Some(boot) = host_boot_id() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let store = Store::open(&database).unwrap(); + // Its Runner's PID and its root's PID run other processes now. + let earlier = runner_running_on(Some("an-earlier-boot".into())); + let (mut child, root) = live_root(); + sent(&store, &earlier, "proc-rebooted", Some(&root)); + let (report, done) = release(&database, Some(&boot), &["proc-rebooted".into()]).unwrap(); + assert!(done, "{report}"); + assert_eq!(ids(&report, "released"), ["proc-rebooted"]); + let record = store.process_record("proc-rebooted").unwrap().unwrap(); + assert_eq!( + (record.phase, record.lost), + (ProcessPhase::Unknown, Some(ProcessLoss::HostRebooted)) + ); + assert!(record.released_at_ms.is_some()); + child.kill().unwrap(); + child.wait().unwrap(); + } + + /// The records commands need the operations database, and release needs one that exists. + #[test] + fn records_commands_need_their_database() { + let refusal = run(&Command::Records(RecordsCommand::List), None).unwrap_err(); + assert!(refusal.contains("--operations-db"), "{refusal}"); + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("missing.sqlite"); + let refusal = release(&missing, None, &["proc-1".into()]).unwrap_err(); + assert!(refusal.contains("does not exist"), "{refusal}"); + assert!(!missing.exists()); + } +} diff --git a/crates/server/src/records.rs b/crates/server/src/records.rs index 6387472..c110a87 100644 --- a/crates/server/src/records.rs +++ b/crates/server/src/records.rs @@ -5,20 +5,26 @@ //! ended: the Runner is the authority on live processes. Once a record says how an execution //! ended, the gateway acknowledges it to the Runner, which until then keeps the ended process //! for a controller (P1R-C04). +//! +//! When the Runner no longer holds a process, or the gateway lost the Runner, the records answer +//! what the Runner cannot, with why (P1R-C05): an execution whose end was recorded is reported +//! as it ended, and one whose Runner was lost before it ended as `unknown`, never as an end. use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; -use codespace_domain::ProcessTermination; use codespace_domain::{ - ApprovalId, ErrorBody, ErrorCode, ExecCommandParams, ProcessId, ProcessState, + ApprovalId, ErrorBody, ErrorCode, ExecCommandParams, LostReason, ProcessId, ProcessState, + ProcessStatusResult, ProcessTermination, ReadProcessResult, }; use codespace_runner::{ - Runner, RunnerIdentity, RunnerInventoryEntry, RunnerProcessRoot, RunnerProcessStatus, - RuntimeBackend, ShellRelease, + process_signal, Runner, RunnerError, RunnerIdentity, RunnerInventoryEntry, RunnerProcessRoot, + RunnerProcessStatus, RuntimeBackend, ShellRelease, }; use codespace_store::{ - NewProcessRecord, ProcessEnd, RecordedParticipation, RootProcess, RunnerRef, Store, + NewProcessRecord, ProcessEnd, ProcessLoss, ProcessOutcome, ProcessPhase, ProcessRecord, + RecordedParticipation, RootProcess, RunnerRef, Store, }; use tokio::sync::{mpsc, watch}; @@ -87,6 +93,8 @@ pub struct ProcessRecords { runner: RunnerRef, /// Bumped after each end that the records follow from an exit event (P1R-C04). ends: watch::Sender, + /// Set once the gateway lost the Runner (P1R-C05). + lost: AtomicBool, } impl ProcessRecords { @@ -120,6 +128,7 @@ impl ProcessRecords { nonce: identity.nonce.clone(), }, ends: watch::Sender::new(0), + lost: AtomicBool::new(false), }); let following = records.clone(); let Exits(mut exited) = exits; @@ -155,6 +164,92 @@ impl ProcessRecords { records } + /// Follow the loss of the Runner these records are of (P1R-C05): once `lost` says why the + /// gateway lost it, see [`Self::runner_lost`]. + pub fn follow_loss(self: &Arc, mut lost: watch::Receiver>) { + let records = self.clone(); + tokio::spawn(async move { + loop { + if let Some(reason) = lost.borrow_and_update().clone() { + records.runner_lost(&reason); + return; + } + if lost.changed().await.is_err() { + return; + } + } + }); + } + + /// The gateway lost the Runner these records are of, for `reason` (P1R-C05): what became of + /// each execution sent to it that had not ended cannot be learned any more, and some may + /// still run, so each of their records becomes `unknown` and keeps its workspace held until + /// an operator releases it. An end the Runner reported before still ends its record. + pub(crate) fn runner_lost(&self, reason: &str) { + self.lost.store(true, Ordering::SeqCst); + match self + .store + .record_runner_lost(&self.runner, ProcessLoss::RunnerLost) + { + Ok(processes) => tracing::warn!( + ?processes, + reason, + "the independent Runner was lost before these processes ended: what became of \ + them cannot be learned, so their records say unknown and their workspaces stay \ + held until an operator finds them and releases them" + ), + Err(err) => tracing::error!( + reason, + error = %err.message, + "the independent Runner was lost, and the records of its processes could not say \ + so: their workspaces stay held" + ), + } + } + + /// The record to answer from about `process_id`, which the Runner did not answer about but + /// with `err` (P1R-C05): when it no longer holds the process, or the gateway lost it, and + /// the record says that the execution was sent. None when the Runner remains the one to + /// ask, as for an execution of its own that has not ended as far as its record says. + pub(crate) fn recorded(&self, process_id: &ProcessId, err: &RunnerError) -> Option { + let lost_runner = match err.as_execution().map(|body| body.code) { + Some(ErrorCode::ProcessNotFound) => false, + Some(ErrorCode::RunnerDisconnected) if self.lost.load(Ordering::SeqCst) => true, + _ => return None, + }; + let record = match self.store.process_record(&process_id.0) { + Ok(Some(record)) => record, + Ok(None) => return None, + Err(err) => { + tracing::warn!( + process_id = %process_id.0, + error = %err.message, + "the process record could not be read" + ); + return None; + } + }; + if record.outcome == Some(ProcessOutcome::NotStarted) { + return None; + } + let ours = record.runner == self.runner; + let loss_since = || ProcessLoss::of(&record.runner, self.runner.boot_id.as_deref()); + let reason = match (record.lost, record.phase) { + (Some(loss), _) => lost_reason(loss), + (None, ProcessPhase::Terminal) if ours && lost_runner => LostReason::RunnerLost, + // Its end is recorded: only what the Runner retained is gone. + (None, ProcessPhase::Terminal) => match loss_since() { + ProcessLoss::HostRebooted => LostReason::HostRebooted, + ProcessLoss::RunnerLost => LostReason::RetentionExpired, + }, + (None, _) if ours && lost_runner => LostReason::RunnerLost, + (None, _) if ours => return None, + // Another Runner answers at its Runner's socket. + (None, _) => lost_reason(loss_since()), + }; + Some(Recorded { record, reason }) + } + /// The Runner these records are of. pub(crate) fn runner(&self) -> &RunnerRef { &self.runner @@ -310,3 +405,86 @@ impl ProcessRecords { } } } + +/// An execution as its record tells it, when its Runner cannot (P1R-C05). +pub(crate) struct Recorded { + record: ProcessRecord, + reason: LostReason, +} + +impl Recorded { + /// The furthest output offset the gateway knows of. + fn output_total(&self) -> u64 { + self.record.output_cursor.unwrap_or(0) + } + + /// The execution's status: as its record says it ended, with nothing of its output + /// retained. One whose Runner was lost before it ended is `unknown`, never an end; after + /// a host restart, it ended, and how is not known. + pub(crate) fn status(&self) -> ProcessStatusResult { + let record = &self.record; + let unknown_end = ( + ProcessState::Exited, + Some(ProcessTermination::Unknown), + None, + None, + ); + let (state, termination, exit_code, signal) = match (record.phase, record.outcome) { + (ProcessPhase::Terminal, Some(ProcessOutcome::Ended(termination))) => match termination + { + ProcessTermination::Exited => ( + ProcessState::Exited, + Some(ProcessTermination::Exited), + record.exit_code, + None, + ), + ProcessTermination::Signaled => match record.signal { + Some(number) => ( + ProcessState::Exited, + Some(ProcessTermination::Signaled), + None, + Some(process_signal(number)), + ), + None => unknown_end, + }, + termination => (ProcessState::Exited, Some(termination), None, None), + }, + (ProcessPhase::Terminal, _) => unknown_end, + _ if self.reason == LostReason::HostRebooted => unknown_end, + _ => (ProcessState::Unknown, None, None, None), + }; + ProcessStatusResult { + process_id: ProcessId(record.process_id.clone()), + state, + exit_code, + termination, + signal, + output_total: self.output_total(), + output_retained_from: self.output_total(), + eof: true, + lost_reason: Some(self.reason), + coordination: None, + } + } + + /// A read from `cursor`: nothing of the output is retained, and nothing more will be. + pub(crate) fn read(&self, cursor: u64) -> ReadProcessResult { + ReadProcessResult { + process_id: ProcessId(self.record.process_id.clone()), + cursor: cursor.max(self.output_total()), + chunk: String::new(), + eof: true, + output_lost: true, + retained_from: self.output_total(), + lost_reason: Some(self.reason), + coordination: None, + } + } +} + +fn lost_reason(loss: ProcessLoss) -> LostReason { + match loss { + ProcessLoss::RunnerLost => LostReason::RunnerLost, + ProcessLoss::HostRebooted => LostReason::HostRebooted, + } +} diff --git a/crates/server/src/recovery.rs b/crates/server/src/recovery.rs index 43304dc..ee8999d 100644 --- a/crates/server/src/recovery.rs +++ b/crates/server/src/recovery.rs @@ -16,11 +16,14 @@ //! | none | holds it running | any | the workspace stays blocked | //! | anything else | | | the workspace stays blocked: a failed lookup is asked again, a mismatch waits for an operator | //! -//! A Runner proves that it never received a request only for itself: a record sent to another -//! Runner keeps its workspace blocked. An approval left `resuming` is consumed with the process -//! the Runner created for it, given back when the Runner proved that it created none, and -//! otherwise left as it is. A workspace that waits is reconciled again after each exit, and -//! one that waits for an answer also after a growing pause. +//! A Runner proves that it never received a request only for itself. A record of an execution +//! sent to another Runner that has not ended means that Runner was lost, since another one +//! answers at its socket (P1R-C05): the record becomes `unknown`, with why, and like every +//! record whose Runner was lost it keeps its workspace blocked until an operator releases it +//! (D8). An approval left `resuming` is consumed with the process the Runner created for it, +//! given back when the Runner proved that it created none, and otherwise left as it is. A +//! workspace that waits is reconciled again after each exit, and one that waits for an answer +//! also after a growing pause. use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak}; @@ -31,7 +34,10 @@ use codespace_domain::{ OperationResumeResult, ProcessId, ProcessState, }; use codespace_runner::{Recalled, RunnerClaim, RunnerProcessStatus, UdsRunner}; -use codespace_store::{ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, Store}; +use codespace_store::{ + ProcessLoss, ProcessOutcome, ProcessPhase, ProcessRecord, RecordedParticipation, RunnerRef, + Store, +}; use tokio::sync::watch; use crate::records::{Leases, ProcessRecords}; @@ -278,6 +284,7 @@ impl Recovery { ) -> BTreeMap { let mut bars = Bars::default(); let ours = self.records.runner(); + let lost = self.lose_other_runners(records); let mut decisions: Vec> = Vec::with_capacity(records.len()); let mut asked = Vec::new(); for (index, record) in records.iter().enumerate() { @@ -285,16 +292,24 @@ impl Recovery { ProcessPhase::Terminal => Some(Decision::Settled { started: record.outcome != Some(ProcessOutcome::NotStarted), }), - // Only the Runner that was sent it can prove what became of it (P1R-C05). - _ if record.runner != *ours => Some(Decision::Blocked(format!( - "process `{}` was sent to another Runner, which this gateway cannot ask \ - what became of it", - record.process_id - ))), - ProcessPhase::Unknown => Some(Decision::Blocked(format!( - "the record of process `{}` says that what became of it cannot be learned", - record.process_id + ProcessPhase::Unknown => Some(Decision::Blocked(awaits_release( + record, + record.lost.unwrap_or(ProcessLoss::RunnerLost), ))), + // Only the Runner that was sent it can prove what became of it, and that Runner + // is lost (P1R-C05). + _ if record.runner != *ours => Some(match lost.get(&record.process_id) { + Some(Ok(loss)) => Decision::Blocked(awaits_release(record, *loss)), + Some(Err(err)) => Decision::Pending(format!( + "the Runner that process `{}` was sent to is lost, and its record could \ + not say so: {err}", + record.process_id + )), + None => Decision::Pending(format!( + "the record of process `{}` was not marked lost", + record.process_id + )), + }), ProcessPhase::Dispatching | ProcessPhase::Running => { asked.push(index); None @@ -462,6 +477,53 @@ impl Recovery { bars.0 } + /// The Runners other than this gateway's that `records` sent executions to which have not + /// ended are lost: another Runner answers at their socket (P1R-C05). Each of their records + /// that had not ended becomes `unknown`, for a restart of the host when the lost Runner ran + /// on an earlier boot. Returns, for each such record, its loss, or why it could not be + /// recorded. + fn lose_other_runners( + &self, + records: &[ProcessRecord], + ) -> HashMap> { + let ours = self.records.runner(); + let mut runners: Vec<&RunnerRef> = Vec::new(); + for record in records { + if matches!( + record.phase, + ProcessPhase::Dispatching | ProcessPhase::Running + ) && record.runner != *ours + && !runners.contains(&&record.runner) + { + runners.push(&record.runner); + } + } + let mut lost = HashMap::new(); + for runner in runners { + let loss = ProcessLoss::of(runner, ours.boot_id.as_deref()); + let marked = self + .store + .record_runner_lost(runner, loss) + .map_err(|err| err.message); + if let Ok(processes) = &marked { + if !processes.is_empty() { + tracing::warn!( + runner_pid = runner.pid, + loss = loss.as_str(), + ?processes, + "an earlier independent Runner was lost before these processes ended: \ + another Runner answers at its socket, so what became of them cannot be \ + learned; their workspaces stay blocked until an operator releases them" + ); + } + } + for record in records.iter().filter(|record| record.runner == *runner) { + lost.insert(record.process_id.clone(), marked.clone().map(|_| loss)); + } + } + lost + } + /// Act on `decision` for `record`, and return what it says of the execution's start. fn apply( &self, @@ -669,6 +731,23 @@ impl Bars { } } +/// Why `record`, whose Runner was lost for `loss`, keeps its workspace blocked until an +/// operator releases it (P1R-C05, D8). +fn awaits_release(record: &ProcessRecord, loss: ProcessLoss) -> String { + let lost = match loss { + ProcessLoss::RunnerLost => { + "was sent to a Runner that was lost, so what became of it cannot be learned and it \ + may still run" + } + ProcessLoss::HostRebooted => "was sent to a Runner that the host's restart ended", + }; + format!( + "process `{}` {lost}; an operator confirms that nothing of it runs and releases its \ + record (`codespace-mcp records`)", + record.process_id + ) +} + /// What became of an execution that takes no part in the resource authority, as its Runner /// recalled it. fn decide(record: &ProcessRecord, recalled: &Recalled) -> Decision { diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index 71284c8..9af5fd7 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -277,6 +277,13 @@ fn link_observer() -> LinkObserver { stop, and this gateway refuses every request that needs the Runner with \ RUNNER_DISCONNECTED until it is restarted" ), + LinkEvent::RunnerLost(reason) => tracing::error!( + %reason, + "lost the independent Runner: what became of its processes cannot be learned and some \ + may still run, so their records become unknown and keep their workspaces until an \ + operator finds the processes and releases them (`codespace-mcp records list`); this \ + gateway refuses every request that needs the Runner with RUNNER_DISCONNECTED" + ), }) } diff --git a/crates/server/tests/independent_runner.rs b/crates/server/tests/independent_runner.rs index 6d18d7f..b70e19b 100644 --- a/crates/server/tests/independent_runner.rs +++ b/crates/server/tests/independent_runner.rs @@ -3,9 +3,11 @@ //! leaves or stops the Runner as its exit intent says. Each gateway records its executions in //! the operations database, which it is the only writer of (P1R-C02), takes control of the //! Runner with a Claim for that database (P1R-C03), and reconciles what earlier controllers -//! left before it starts work where they did (P1R-C04). The tests with a Runner need the -//! worker binary (`CODESPACE_RUNTIME_BIN`) and return early without it, as the gateway's other -//! worker tests do. +//! left before it starts work where they did (P1R-C04). A gateway whose Runner is lost answers +//! for its executions from their records, and an operator releases what they hold once nothing +//! of them runs (P1R-C05). The tests with a Runner need the worker binary +//! (`CODESPACE_RUNTIME_BIN`) and return early without it, as the gateway's other worker tests +//! do. use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Stdio}; @@ -190,6 +192,21 @@ async fn refused(mut command: Command) -> String { String::from_utf8_lossy(&output.stderr).into_owned() } +/// What `codespace-mcp records` prints with `args` for the operations database, and whether +/// it did all it was asked (P1R-C05). +async fn records(setup: &Setup, args: &[&str]) -> (serde_json::Value, bool) { + let args = [&["records"], args].concat(); + let output = timeout( + Duration::from_secs(30), + setup.gateway_command(&args).stdin(Stdio::null()).output(), + ) + .await + .expect("the records command ends") + .expect("run the records command"); + let report = serde_json::from_slice(&output.stdout).unwrap_or_else(|_| panic!("{output:?}")); + (report, output.status.success()) +} + /// The lines a process writes to `stderr`, read for as long as it writes them. fn log_lines(stderr: tokio::process::ChildStderr) -> mpsc::UnboundedReceiver { let (lines, logged) = mpsc::unbounded_channel(); @@ -267,6 +284,34 @@ impl Gateway { body["process_id"].as_str().unwrap().to_string() } + async fn read(&self, process_id: &str, cursor: u64) -> serde_json::Value { + let read = self + .client + .call_tool( + CallToolRequestParams::new(TOOL_READ_PROCESS) + .with_arguments(object!({ "process_id": process_id, "cursor": cursor })), + ) + .await + .expect("read_process"); + payload(&read) + } + + /// Start `command` once new work in `demo` is no longer refused while the gateway + /// reconciles (P1R-C04), and return its process id. + async fn started_once_reconciled(&self, command: &[&str]) -> String { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let started = self.exec(command).await; + let body = payload(started.as_ref().expect("exec")); + if body["code"] != "RECOVERY_PENDING" { + assert_eq!(body["dispatch_status"], "confirmed", "{body}"); + return body["process_id"].as_str().unwrap().to_string(); + } + assert!(Instant::now() < deadline, "not reconciled: {body}"); + sleep(Duration::from_millis(40)).await; + } + } + async fn status(&self, process_id: &str) -> serde_json::Value { let status = self .client @@ -863,3 +908,168 @@ async fn a_sandboxed_command_cannot_reach_the_runner_socket() { let status = exited(&mut runner).await; assert!(status.success(), "{status:?}"); } + +/// The record of `process_id` once `done` holds of it. +async fn record_once( + setup: &Setup, + process_id: &str, + done: impl Fn(&codespace_store::ProcessRecord) -> bool, +) -> codespace_store::ProcessRecord { + let store = Store::open(&setup.database()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match store.process_record(process_id).unwrap() { + Some(record) if done(&record) => return record, + record => assert!(Instant::now() < deadline, "{record:?}"), + } + sleep(Duration::from_millis(20)).await; + } +} + +/// An independent Runner killed with SIGKILL is lost (P1R-C05). Its gateway answers for the +/// executions it was sent from their records, one that had not ended as `unknown` and one that +/// ended as it ended, each with `runner_lost`, and refuses new work. Where no command sandbox +/// ties it to the Runner, a pipe process of the killed Runner keeps running: nothing ended it, +/// and `records list` shows it. `records release` waits for the gateway to end and for that +/// process, then releases the record, and a new Runner's gateway starts new work there. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_killed_runners_processes_stay_held_until_an_operator_releases_them() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let gateway = setup.gateway("detach").await; + let ended = gateway + .started(&["/bin/sh", "-c", "echo ended; exit 5"]) + .await; + gateway.ended(&ended).await; + gateway.output(&ended, "ended").await; + record_once(&setup, &ended, |record| { + record.phase == ProcessPhase::Terminal + }) + .await; + let beat = setup.ws.join("beat"); + let running = gateway + .started(&[ + "/bin/sh", + "-c", + "i=0; while :; do i=$((i+1)); echo $i > beat; sleep 0.1; done", + ]) + .await; + assert!(still_beating(&beat).await); + + runner.kill().await.expect("SIGKILL the Runner"); + let deadline = Instant::now() + Duration::from_secs(15); + let status = loop { + let status = gateway.status(&running).await; + if status["state"] == "unknown" { + break status; + } + assert!( + Instant::now() < deadline, + "the Runner is not lost: {status}" + ); + sleep(Duration::from_millis(40)).await; + }; + assert_eq!(status["lost_reason"], "runner_lost", "{status}"); + assert_eq!(status["eof"], true, "{status}"); + assert!(status.get("exit_code").is_none(), "{status}"); + let read = gateway.read(&running, 0).await; + assert_eq!( + (&read["chunk"], &read["eof"], &read["output_lost"]), + (&"".into(), &true.into(), &true.into()), + "{read}" + ); + assert_eq!(read["lost_reason"], "runner_lost", "{read}"); + let status = gateway.status(&ended).await; + assert_eq!( + ( + &status["state"], + &status["exit_code"], + &status["lost_reason"] + ), + (&"exited".into(), &5.into(), &"runner_lost".into()), + "{status}" + ); + let new_work = error_body(&gateway.exec(&["/bin/echo", "new"]).await); + assert_eq!(new_work["code"], "RUNNER_DISCONNECTED", "{new_work}"); + assert!( + new_work["message"] + .as_str() + .unwrap() + .contains("lost its independent Runner"), + "{new_work}" + ); + // The command sandbox ends its processes with the Runner; without it, nothing does. + let survives = !codespace_runner::linux_sandbox_available(); + assert_eq!(still_beating(&beat).await, survives); + + // The list is read while the gateway runs. + let (listed, done) = records(&setup, &["list"]).await; + assert!(done, "{listed}"); + let held = listed["records"].as_array().unwrap(); + assert_eq!(held.len(), 1, "{listed}"); + assert_eq!( + (&held[0]["process_id"], &held[0]["phase"], &held[0]["lost"]), + ( + &running.as_str().into(), + &"unknown".into(), + &"runner_lost".into() + ), + "{listed}" + ); + assert_eq!(held[0]["runner"]["this_boot"], true, "{listed}"); + assert_eq!(held[0]["root"]["running"], survives, "{listed}"); + let root_pid = held[0]["root"]["pid"].as_u64().unwrap(); + let refusal = refused(setup.gateway_command(&["records", "release", &running])).await; + assert!(refusal.contains("while no gateway runs"), "{refusal}"); + let shut = gateway.shut_down().await; + assert!(shut.success(), "{shut:?}"); + + if survives { + let (report, done) = records(&setup, &["release", &running]).await; + assert!(!done, "{report}"); + assert!( + report["refused"][0]["reason"] + .as_str() + .unwrap() + .contains("still runs"), + "{report}" + ); + let killed = std::process::Command::new("kill") + .args(["-KILL", &root_pid.to_string()]) + .status() + .unwrap(); + assert!(killed.success()); + assert!(!still_beating(&beat).await); + } + let deadline = Instant::now() + Duration::from_secs(15); + let report = loop { + // An exited root that its new parent has not reaped yet runs no more. + let (report, done) = records(&setup, &["release", &running]).await; + if done { + break report; + } + assert!(Instant::now() < deadline, "{report}"); + sleep(Duration::from_millis(100)).await; + }; + assert_eq!(report["released"], serde_json::json!([running]), "{report}"); + let (listed, _) = records(&setup, &["list"]).await; + assert_eq!(listed["records"], serde_json::json!([]), "{listed}"); + + let mut runner = setup.start_runner(&bin).await; + let next = setup.gateway("stop").await; + let process_id = next.started_once_reconciled(&["/bin/echo", "after"]).await; + next.output(&process_id, "after").await; + let status = next.status(&running).await; + assert_eq!( + (&status["state"], &status["lost_reason"]), + (&"unknown".into(), &"runner_lost".into()), + "{status}" + ); + let shut = next.shut_down().await; + assert!(shut.success(), "{shut:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 26726e8..2ac6cae 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -29,8 +29,8 @@ use sha2::{Digest, Sha256}; pub use approvals::{ApprovalRecord, ResumeClaim}; pub use coord::CreateIntent; pub use processes::{ - NewProcessRecord, ProcessEnd, ProcessOutcome, ProcessPhase, ProcessRecord, - RecordedParticipation, RootProcess, RunnerRef, + unsettled_records_at, NewProcessRecord, ProcessEnd, ProcessLoss, ProcessOutcome, ProcessPhase, + ProcessRecord, RecordedParticipation, RootProcess, RunnerRef, }; use resource::AcquireOutcome; pub use resource::{LockMode, Resource, ResourceGuard}; @@ -251,6 +251,7 @@ impl Store { .map_err(|e| e.to_string())?; migrate_operations(&conn)?; migrate_approvals(&conn)?; + processes::migrate(&conn)?; // Last, so a database whose upgrade was cut short is upgraded again. if version < SCHEMA_VERSION { conn.execute_batch(&format!("PRAGMA user_version = {SCHEMA_VERSION}")) @@ -1464,6 +1465,102 @@ mod tests { ); } + /// The `processes` table of a CodeSpace from before P1R-C05, which records no loss. + const PROCESSES_BEFORE_LOSS: &str = "CREATE TABLE processes ( + process_id TEXT PRIMARY KEY, + runner_boot_id TEXT, + runner_pid INTEGER NOT NULL, + runner_started_at_ms INTEGER NOT NULL, + runner_nonce TEXT NOT NULL, + controller_epoch INTEGER, + workspace_id TEXT NOT NULL, + approval_id TEXT, + attempt_id TEXT, + participation TEXT NOT NULL, + tty INTEGER NOT NULL, + argv_digest TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ('dispatching', 'running', 'terminal', 'unknown')), + deadline_at_ms INTEGER, + root_pid INTEGER, + root_started TEXT, + termination TEXT, + exit_code INTEGER, + signal INTEGER, + output_cursor INTEGER, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + INSERT INTO processes (process_id, runner_boot_id, runner_pid, runner_started_at_ms, + runner_nonce, controller_epoch, workspace_id, participation, tty, argv_digest, phase, + root_pid, root_started, created_at_ms, updated_at_ms) + VALUES ('proc-1', 'boot-a', 4242, 1760000000000, 'nonce-a', 2, 'demo', 'off', 0, + 'sha256:00', 'running', 777, 'linux:100', 1, 1); + PRAGMA user_version = 1;"; + + /// Process records from before losses were recorded (P1R-C05) can be listed without + /// anything being written to the database, and gain the loss columns, empty, once a gateway + /// opens it: each record stays as it was, and so does the schema version. + #[test] + fn records_from_before_losses_are_listed_as_they_are_and_gain_the_columns() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ops.sqlite"); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch(PROCESSES_BEFORE_LOSS) + .unwrap(); + let before = std::fs::read(&path).unwrap(); + let listed = unsettled_records_at(&path).unwrap(); + assert_eq!(listed.len(), 1, "{listed:?}"); + let earlier = &listed[0]; + assert_eq!(earlier.process_id, "proc-1"); + assert_eq!(earlier.phase, ProcessPhase::Running); + assert_eq!(earlier.runner, runner()); + assert_eq!((earlier.lost, earlier.released_at_ms), (None, None)); + assert_eq!(std::fs::read(&path).unwrap(), before, "listing wrote to it"); + + let store = Store::open(&path).unwrap(); + let conn = rusqlite::Connection::open(&path).unwrap(); + let columns = table_columns(&conn, "processes").unwrap(); + assert!(columns.ends_with(&["lost".to_string(), "released_at_ms".to_string()])); + assert_eq!(user_version(&path), SCHEMA_VERSION); + assert_eq!( + store.process_record("proc-1").unwrap().as_ref(), + Some(earlier) + ); + assert_eq!(unsettled_records_at(&path).unwrap(), listed); + // A database brought up to date is opened again as it is. + drop(store); + let store = Store::open(&path).unwrap(); + assert_eq!(store.unsettled_records().unwrap(), listed); + } + + /// Listing records reads only a database it knows: one that does not exist, or that a newer + /// CodeSpace wrote, is refused, and one without process records has none (P1R-C05). + #[test] + fn listing_records_reads_only_a_database_it_knows() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("missing.sqlite"); + let err = unsettled_records_at(&missing).unwrap_err(); + assert!(err.contains("does not exist"), "{err}"); + assert!(!missing.exists()); + + let newer = dir.path().join("newer.sqlite"); + dispatch(&Store::open(&newer).unwrap(), "proc-1"); + rusqlite::Connection::open(&newer) + .unwrap() + .execute_batch(&format!("PRAGMA user_version = {}", SCHEMA_VERSION + 1)) + .unwrap(); + let err = unsettled_records_at(&newer).unwrap_err(); + assert!(err.contains("newer than this CodeSpace's"), "{err}"); + + let plain = dir.path().join("plain.sqlite"); + rusqlite::Connection::open(&plain) + .unwrap() + .execute_batch("CREATE TABLE operations (operation_id TEXT PRIMARY KEY);") + .unwrap(); + assert!(unsettled_records_at(&plain).unwrap().is_empty()); + } + /// A database keeps its identity across reopenings, and another database has its own /// (P1R-C03). The schema version stays. #[test] diff --git a/crates/store/src/processes.rs b/crates/store/src/processes.rs index 46e4a49..77c700c 100644 --- a/crates/store/src/processes.rs +++ b/crates/store/src/processes.rs @@ -5,6 +5,10 @@ //! the authority on live processes. //! //! A record keeps the argv's digest, never the argv, its environment or a secret. +//! +//! When the Runner an execution was sent to is lost, what became of the execution cannot be +//! learned any more (P1R-C05): a record that had not ended becomes `unknown`, with why, and +//! holds its workspace until an operator confirms that nothing of it runs and releases it. use codespace_domain::{ApprovalId, ErrorBody, ErrorCode, ProcessTermination}; use rusqlite::{params, Connection, OptionalExtension}; @@ -34,10 +38,31 @@ pub(crate) const SCHEMA: &str = "CREATE TABLE IF NOT EXISTS processes ( signal INTEGER, output_cursor INTEGER, created_at_ms INTEGER NOT NULL, - updated_at_ms INTEGER NOT NULL + updated_at_ms INTEGER NOT NULL, + lost TEXT, + released_at_ms INTEGER ); CREATE INDEX IF NOT EXISTS idx_processes_workspace_phase ON processes(workspace_id, phase);"; +/// The columns that a database from before P1R-C05 lacks: added, empty, when it is opened. A +/// CodeSpace from before them leaves them alone, so the schema version stays. +const LOSS_COLUMNS: [(&str, &str); 2] = [("lost", "TEXT"), ("released_at_ms", "INTEGER")]; + +/// Add the columns that a `processes` table from before P1R-C05 lacks. +pub(crate) fn migrate(conn: &Connection) -> Result<(), String> { + let columns = crate::table_columns(conn, "processes")?; + for (name, kind) in LOSS_COLUMNS { + if !columns.iter().any(|column| column == name) { + conn.execute( + &format!("ALTER TABLE processes ADD COLUMN {name} {kind}"), + [], + ) + .map_err(|err| err.to_string())?; + } + } + Ok(()) +} + /// The Runner an execution was sent to, by the identity its `Hello` states (P1R-C01). A Runner /// with an earlier Runner's PID, after a restart or a reboot, is another Runner. #[derive(Debug, Clone, PartialEq, Eq)] @@ -57,12 +82,13 @@ pub enum ProcessPhase { Running, /// The Runner reported that it ended, or answered that it did not start it. Terminal, - /// What became of it cannot be learned any more (P1R-C05). Nothing records it yet. + /// What became of it cannot be learned any more: the Runner it was sent to was lost before + /// it ended (P1R-C05). The record says why. Unknown, } impl ProcessPhase { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Self::Dispatching => "dispatching", Self::Running => "running", @@ -82,6 +108,44 @@ impl ProcessPhase { } } +/// Why what became of an execution cannot be learned any more (P1R-C05). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessLoss { + /// The Runner it was sent to was lost: nothing accepts connections at the Runner's socket, + /// or another Runner answers there. A process of a killed Runner can outlive it, so the + /// execution may still run. + RunnerLost, + /// The host restarted since the Runner it was sent to started, which ended it. + HostRebooted, +} + +impl ProcessLoss { + pub fn as_str(self) -> &'static str { + match self { + Self::RunnerLost => "runner_lost", + Self::HostRebooted => "host_rebooted", + } + } + + fn parse(raw: &str) -> Option { + Some(match raw { + "runner_lost" => Self::RunnerLost, + "host_rebooted" => Self::HostRebooted, + _ => return None, + }) + } + + /// Why an execution sent to the Runner `lost` cannot be followed any more, on the host + /// whose current boot is `boot_id`: a Runner of another boot ended with that boot. Without + /// both boots, nothing proves a restart. + pub fn of(lost: &RunnerRef, boot_id: Option<&str>) -> Self { + match (lost.boot_id.as_deref(), boot_id) { + (Some(then), Some(now)) if then != now => Self::HostRebooted, + _ => Self::RunnerLost, + } + } +} + /// How an execution takes part in the resource authority. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RecordedParticipation { @@ -90,7 +154,7 @@ pub enum RecordedParticipation { } impl RecordedParticipation { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Self::Off => "off", Self::Required => "required", @@ -217,6 +281,12 @@ pub struct ProcessRecord { pub output_cursor: Option, pub created_at_ms: u64, pub updated_at_ms: u64, + /// Why what became of the execution cannot be learned, once its Runner was lost before it + /// ended (P1R-C05). + pub lost: Option, + /// When an operator released the workspace that the record held after its Runner was lost, + /// in milliseconds since the Unix epoch (P1R-C05, D8). + pub released_at_ms: Option, } impl ProcessRecord { @@ -271,6 +341,12 @@ impl ProcessRecord { self.output_cursor = Some(cursor); } } + + /// Whether the record still holds its workspace although what became of its execution + /// cannot be learned: it waits for an operator (P1R-C05). + pub fn awaits_release(&self) -> bool { + self.phase == ProcessPhase::Unknown && self.released_at_ms.is_none() + } } impl Store { @@ -361,7 +437,7 @@ impl Store { /// (P1R-C04): every execution sent to another Runner, or to `runner` under an earlier Claim /// or before Claims were recorded, that has not ended as far as its record says, and every /// one whose approval is still being resumed. What the gateway sends under its own Claim is - /// not. Oldest first. + /// not, nor what an operator released (P1R-C05). Oldest first. pub fn records_to_reconcile( &self, runner: &RunnerRef, @@ -374,6 +450,7 @@ impl Store { WHERE (phase IN ('dispatching', 'running', 'unknown') OR approval_id IN (SELECT approval_id FROM approvals WHERE state = 'resuming')) + AND released_at_ms IS NULL AND NOT (runner_boot_id IS ?1 AND runner_pid = ?2 AND runner_started_at_ms = ?3 AND runner_nonce = ?4 AND controller_epoch IS NOT NULL AND controller_epoch >= ?5) @@ -420,6 +497,80 @@ impl Store { Ok(records) } + /// The Runner `lost` was lost (P1R-C05): what became of each execution sent to it that had + /// not ended, as far as its record says, cannot be learned any more. Each such record + /// becomes `unknown` for `loss`, and holds its workspace until an operator releases it. + /// Returns their IDs, oldest first; a record that ended stays as it is. + pub fn record_runner_lost( + &self, + lost: &RunnerRef, + loss: ProcessLoss, + ) -> Result, ErrorBody> { + let mut conn = self.conn.lock().expect("sqlite mutex"); + let tx = conn.transaction().map_err(record_err)?; + let identity = params![ + lost.boot_id, + lost.pid, + to_sql(lost.started_at_ms), + lost.nonce + ]; + let unended = "runner_boot_id IS ?1 AND runner_pid = ?2 AND runner_started_at_ms = ?3 + AND runner_nonce = ?4 AND phase IN ('dispatching', 'running')"; + let ids = { + let mut statement = tx + .prepare(&format!( + "SELECT process_id FROM processes WHERE {unended} + ORDER BY created_at_ms, process_id" + )) + .map_err(record_err)?; + let rows = statement + .query_map(identity, |row| row.get::<_, String>(0)) + .map_err(record_err)?; + rows.collect::, _>>().map_err(record_err)? + }; + tx.execute( + &format!( + "UPDATE processes SET phase = 'unknown', lost = ?5, updated_at_ms = ?6 + WHERE {unended}" + ), + params![ + lost.boot_id, + lost.pid, + to_sql(lost.started_at_ms), + lost.nonce, + loss.as_str(), + to_sql(now_ms()), + ], + ) + .map_err(record_err)?; + tx.commit().map_err(record_err)?; + Ok(ids) + } + + /// An operator confirmed that nothing of the execution `process_id` runs any more, whose + /// Runner was lost, and releases the workspace its record holds (P1R-C05, D8). The record + /// stays `unknown`: how the execution ended was never learned. Returns whether it was + /// released now: not when it is not waiting for a release, or already was released. + pub fn release_record(&self, process_id: &str) -> Result { + let conn = self.conn.lock().expect("sqlite mutex"); + let now = to_sql(now_ms()); + let released = conn + .execute( + "UPDATE processes SET released_at_ms = ?2, updated_at_ms = ?2 + WHERE process_id = ?1 AND phase = 'unknown' AND released_at_ms IS NULL", + params![process_id, now], + ) + .map_err(record_err)?; + Ok(released == 1) + } + + /// The records that may still hold a workspace (P1R-C05): every execution whose record has + /// not ended and was not released, whichever Runner it was sent to, oldest first. + pub fn unsettled_records(&self) -> Result, ErrorBody> { + let conn = self.conn.lock().expect("sqlite mutex"); + unsettled(&conn, RECORD_SELECT) + } + /// Change the record of an execution that `runner` was sent, and write it back when it /// changed. Records of executions sent to another Runner are never changed. fn change_record( @@ -481,7 +632,73 @@ pub(crate) static CRASH_BEFORE_COMMIT: std::sync::atomic::AtomicBool = const RECORD_SELECT: &str = "SELECT process_id, runner_boot_id, runner_pid, runner_started_at_ms, runner_nonce, controller_epoch, workspace_id, approval_id, attempt_id, participation, tty, argv_digest, phase, deadline_at_ms, root_pid, root_started, termination, exit_code, signal, - output_cursor, created_at_ms, updated_at_ms FROM processes"; + output_cursor, created_at_ms, updated_at_ms, lost, released_at_ms FROM processes"; + +/// [`RECORD_SELECT`] for a table from before P1R-C05, which records no loss. +const RECORD_SELECT_BEFORE_LOSS: &str = "SELECT * FROM (SELECT process_id, runner_boot_id, + runner_pid, runner_started_at_ms, runner_nonce, controller_epoch, workspace_id, approval_id, + attempt_id, participation, tty, argv_digest, phase, deadline_at_ms, root_pid, root_started, + termination, exit_code, signal, output_cursor, created_at_ms, updated_at_ms, + NULL AS lost, NULL AS released_at_ms FROM processes)"; + +/// The records that may still hold a workspace, read with `select`. +fn unsettled(conn: &Connection, select: &str) -> Result, ErrorBody> { + let mut statement = conn + .prepare(&format!( + "{select} + WHERE phase IN ('dispatching', 'running', 'unknown') AND released_at_ms IS NULL + ORDER BY created_at_ms, process_id" + )) + .map_err(record_err)?; + let rows = statement.query_map([], map_record).map_err(record_err)?; + let mut records = Vec::new(); + for row in rows { + records.push(row.map_err(record_err)??); + } + Ok(records) +} + +/// The records that may still hold a workspace in the operations database at `database`, read +/// without writing anything to it, so that an operator can look while a gateway records into +/// it (P1R-C05). A database without process records has none; one that a newer CodeSpace wrote +/// is refused. +pub fn unsettled_records_at(database: &std::path::Path) -> Result, String> { + use rusqlite::OpenFlags; + if !database.exists() { + return Err(format!( + "the operations database {} does not exist", + database.display() + )); + } + let conn = Connection::open_with_flags( + database, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(|err| format!("cannot open {}: {err}", database.display()))?; + conn.busy_timeout(std::time::Duration::from_secs(5)) + .map_err(|err| err.to_string())?; + let version = crate::schema_version(&conn)?; + if version > crate::SCHEMA_VERSION { + return Err(format!( + "the operations database has schema version {version}, newer than this \ + CodeSpace's {}: read it with the CodeSpace that wrote it", + crate::SCHEMA_VERSION + )); + } + let columns = crate::table_columns(&conn, "processes")?; + if columns.is_empty() { + return Ok(Vec::new()); + } + let select = if LOSS_COLUMNS + .iter() + .all(|(name, _)| columns.iter().any(|column| column == name)) + { + RECORD_SELECT + } else { + RECORD_SELECT_BEFORE_LOSS + }; + unsettled(&conn, select).map_err(|err| err.message) +} fn load(conn: &Connection, process_id: &str) -> Result, ErrorBody> { conn.query_row( @@ -501,6 +718,7 @@ fn map_record(row: &rusqlite::Row<'_>) -> rusqlite::Result = row.get(14)?; let root_started: Option = row.get(15)?; let termination: Option = row.get(16)?; + let lost: Option = row.get(22)?; let unreadable = |what: &str, value: &str| { Err(ErrorBody::new( ErrorCode::Internal, @@ -520,6 +738,13 @@ fn map_record(row: &rusqlite::Row<'_>) -> rusqlite::Result return Ok(unreadable("termination", &raw)), }, }; + let lost = match lost { + None => None, + Some(raw) => match ProcessLoss::parse(&raw) { + Some(loss) => Some(loss), + None => return Ok(unreadable("loss", &raw)), + }, + }; Ok(Ok(ProcessRecord { runner: RunnerRef { boot_id: row.get(1)?, @@ -546,6 +771,8 @@ fn map_record(row: &rusqlite::Row<'_>) -> rusqlite::Result>(19)?.map(from_sql), created_at_ms: from_sql(row.get(20)?), updated_at_ms: from_sql(row.get(21)?), + lost, + released_at_ms: row.get::<_, Option>(23)?.map(from_sql), process_id, })) } @@ -1139,6 +1366,132 @@ mod tests { ); } + /// Once a Runner is lost, each record of what was sent to it that had not ended becomes + /// `unknown`, with why, and holds its workspace: a gateway reconciles it, and it is listed, + /// until an operator releases it (P1R-C05). A record that ended, and one of another Runner, + /// stay as they are. An end the lost Runner reported still ends its record. + #[test] + fn a_lost_runners_unended_records_wait_for_a_release() { + let store = Store::memory().unwrap(); + let runner = runner(); + let other = RunnerRef { + nonce: "nonce-b".into(), + ..runner.clone() + }; + dispatch_under(&store, "proc-dispatching", Some(1), None); + dispatch_under(&store, "proc-running", Some(1), None); + store + .record_running(&runner, "proc-running", Some(&root()), None) + .unwrap(); + dispatch_under(&store, "proc-ended", Some(1), None); + store + .record_end(&runner, "proc-ended", &exited(0, 3)) + .unwrap(); + dispatch_to(&store, &other, "proc-other", Some(1), None); + let ended = record(&store, "proc-ended"); + + assert_eq!( + store + .record_runner_lost(&runner, ProcessLoss::RunnerLost) + .unwrap(), + ["proc-dispatching", "proc-running"] + ); + for process_id in ["proc-dispatching", "proc-running"] { + let lost = record(&store, process_id); + assert_eq!(lost.phase, ProcessPhase::Unknown, "{process_id}"); + assert_eq!(lost.lost, Some(ProcessLoss::RunnerLost), "{process_id}"); + assert_eq!(lost.released_at_ms, None); + assert!(lost.awaits_release()); + } + assert_eq!(record(&store, "proc-running").root, Some(root())); + assert_eq!(record(&store, "proc-ended"), ended); + assert_eq!( + record(&store, "proc-other").phase, + ProcessPhase::Dispatching + ); + // Nothing is left to lose. + assert!(store + .record_runner_lost(&runner, ProcessLoss::HostRebooted) + .unwrap() + .is_empty()); + assert_eq!( + record(&store, "proc-running").lost, + Some(ProcessLoss::RunnerLost) + ); + + let current = RunnerRef { + nonce: "nonce-c".into(), + ..runner.clone() + }; + assert_eq!( + ids(&store.unsettled_records().unwrap()), + ["proc-dispatching", "proc-other", "proc-running"] + ); + assert_eq!( + ids(&store.records_to_reconcile(¤t, 1).unwrap()), + ["proc-dispatching", "proc-other", "proc-running"] + ); + + // Only a record that waits for it is released, once. + assert!(store.release_record("proc-running").unwrap()); + assert!(!store.release_record("proc-running").unwrap()); + for process_id in ["proc-ended", "proc-other", "proc-unrecorded"] { + assert!(!store.release_record(process_id).unwrap(), "{process_id}"); + } + let released = record(&store, "proc-running"); + assert_eq!(released.phase, ProcessPhase::Unknown); + assert!(released.released_at_ms.is_some()); + assert!(!released.awaits_release()); + assert_eq!( + ids(&store.unsettled_records().unwrap()), + ["proc-dispatching", "proc-other"] + ); + assert_eq!( + ids(&store.records_to_reconcile(¤t, 1).unwrap()), + ["proc-dispatching", "proc-other"] + ); + + // The lost Runner reported this end before it was lost: the record ends. + assert!(store + .record_end(&runner, "proc-dispatching", &exited(2, 1)) + .unwrap()); + let ended = record(&store, "proc-dispatching"); + assert_eq!(ended.phase, ProcessPhase::Terminal); + assert_eq!( + (ended.exit_code, ended.lost), + (Some(2), Some(ProcessLoss::RunnerLost)) + ); + assert!(!store.release_record("proc-dispatching").unwrap()); + assert_eq!(ids(&store.unsettled_records().unwrap()), ["proc-other"]); + } + + /// A lost Runner of an earlier boot of the host ended with that boot; without both boots, + /// nothing proves a restart (P1R-C05). + #[test] + fn a_loss_names_a_restart_only_across_two_known_boots() { + let then = runner(); + assert_eq!( + ProcessLoss::of(&then, Some("boot-b")), + ProcessLoss::HostRebooted + ); + assert_eq!( + ProcessLoss::of(&then, Some("boot-a")), + ProcessLoss::RunnerLost + ); + assert_eq!(ProcessLoss::of(&then, None), ProcessLoss::RunnerLost); + let unbooted = RunnerRef { + boot_id: None, + ..then + }; + assert_eq!( + ProcessLoss::of(&unbooted, Some("boot-b")), + ProcessLoss::RunnerLost + ); + for loss in [ProcessLoss::RunnerLost, ProcessLoss::HostRebooted] { + assert_eq!(ProcessLoss::parse(loss.as_str()), Some(loss)); + } + } + /// An approval that a gateway left `resuming` is consumed with the process the Runner /// created, or given back when it created none, once; never while a resume of it is in /// progress (P1R-C04). diff --git a/docs/agent-integration.md b/docs/agent-integration.md index eecc022..a85c924 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -85,7 +85,7 @@ The response contains a server-issued `process_id` and `dispatch_status`. `confi } ``` -Each result has `chunk`, `cursor`, `eof`, `output_lost`, and `retained_from`. Output combines stdout/stderr without preserving their identity. EOF means output collection is complete; it is not a successful exit status. Judge termination with `process_status`: `state` is `running` or `exited`. After exit, `termination` is `exited`, `signaled`, `timeout`, `terminated`, or `unknown`. `exit_code` is present only when `termination` is `exited`. `signaled` means that a signal CodeSpace did not send ended the process, for example a crash or a kill from another process. `signal` then gives its `number` and, when known, its `name`, such as `SIGKILL`. Signal numbers differ between operating systems. A kill that CodeSpace requested is reported as `timeout` or `terminated` instead. `exited` does not rule out a signal: a `tty: true` process reports a signal death as `exited`, with `exit_code` 1, the same as a command that failed with status 1, or, when CodeSpace launches it through DevGuard (participation `required`), with 128 plus the signal number, such as 143 for SIGTERM. Under the Linux command sandbox the status is that of the sandbox helper ([runner isolation](runner-isolation.md)). `signaled`, `timeout`, `terminated`, and `unknown` are not success, even when `eof` is true. The last 256 KiB are retained. If `output_lost` is true, the retained window is not the complete log. Do not claim a build or test passed solely from EOF or an incomplete log. If success cannot be established from a reliable task-specific result, report it as unverified. +Each result has `chunk`, `cursor`, `eof`, `output_lost`, and `retained_from`. Output combines stdout/stderr without preserving their identity. EOF means output collection is complete; it is not a successful exit status. Judge termination with `process_status`: `state` is `running` or `exited`, or, after an [independent Runner](runner-isolation.md#runner-loss) was lost, `unknown`. After exit, `termination` is `exited`, `signaled`, `timeout`, `terminated`, or `unknown`. `exit_code` is present only when `termination` is `exited`. `signaled` means that a signal CodeSpace did not send ended the process, for example a crash or a kill from another process. `signal` then gives its `number` and, when known, its `name`, such as `SIGKILL`. Signal numbers differ between operating systems. A kill that CodeSpace requested is reported as `timeout` or `terminated` instead. `exited` does not rule out a signal: a `tty: true` process reports a signal death as `exited`, with `exit_code` 1, the same as a command that failed with status 1, or, when CodeSpace launches it through DevGuard (participation `required`), with 128 plus the signal number, such as 143 for SIGTERM. Under the Linux command sandbox the status is that of the sandbox helper ([runner isolation](runner-isolation.md)). `signaled`, `timeout`, `terminated`, and `unknown` are not success, even when `eof` is true. The last 256 KiB are retained. If `output_lost` is true, the retained window is not the complete log. `lost_reason`, when present, says why output, or what became of the process, is missing: `ring_overflow` when only the most recent output is kept, and with an independent Runner `retention_expired` when the Runner no longer keeps an ended process, `runner_lost` when the Runner that ran it was lost, and `host_rebooted` when the host restarted since it started. Such an answer comes from the gateway's record and has no output. With `runner_lost`, `state` may be `unknown`: the process may still run, and that is not an end. Report the command as unverified and do not start it again: its workspace stays refused until an operator settles it. Do not claim a build or test passed solely from EOF or an incomplete log. If success cannot be established from a reliable task-specific result, report it as unverified. ```json { @@ -150,7 +150,7 @@ Interactive input and cancellation use the same handle: A live command occupies the workspace. Wait for it to end or terminate it before applying a patch or starting another command. Reads and searches remain available. Long-lived development servers therefore require a workflow that stops them before edits. -`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. While that gateway claims its Runner again after a lost connection, it refuses requests with `RUNNER_RECONNECTING`, which sent nothing and consumed no approval: retry after a pause. Then, in a workspace where work was started before, it refuses new commands, patches and approval resumes with `RECOVERY_PENDING` until it has [reconciled](runner-isolation.md#reconciliation) that workspace with the Runner; its message says whether retrying after a pause helps. A process it finds running there holds the workspace as before. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. +`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. While that gateway claims its Runner again after a lost connection, it refuses requests with `RUNNER_RECONNECTING`, which sent nothing and consumed no approval: retry after a pause. Then, in a workspace where work was started before, it refuses new commands, patches and approval resumes with `RECOVERY_PENDING` until it has [reconciled](runner-isolation.md#reconciliation) that workspace with the Runner; its message says whether retrying after a pause helps. A process it finds running there holds the workspace as before. If the gateway loses the Runner itself, it answers `process_status` and `read_process` for that Runner's processes from its records, with `lost_reason`, refuses other requests that need the Runner with `RUNNER_DISCONNECTED`, and keeps their workspaces refused, with `RECOVERY_PENDING` in the next gateway, until an operator releases them ([Runner loss](runner-isolation.md#runner-loss)). If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. ## Confirm a held mutation diff --git a/docs/architecture.md b/docs/architecture.md index 3d5f386..3032687 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,7 +51,7 @@ If helper application fails, the Runner attempts per-file snapshot restoration. ## Process lifetime -MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead, and a gateway that lost its connection to it claims it again and reconciles what earlier controllers left before it starts new work where they did. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. +MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead, and a gateway that lost its connection to it claims it again and reconciles what earlier controllers left before it starts new work where they did. When the Runner itself is lost, the gateway answers for its executions from their records, and their workspaces wait for an operator ([Runner loss](runner-isolation.md#runner-loss)). See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. ## Planned execution coordination diff --git a/docs/devguard-integration.md b/docs/devguard-integration.md index df13e4b..6c2b2c7 100644 --- a/docs/devguard-integration.md +++ b/docs/devguard-integration.md @@ -338,7 +338,7 @@ Default participation remains `off`; `required` is an explicit operator choice i ## Initial Gateway recovery scope -**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. The third unit, P1R-C03, adds the [Claim](runner-isolation.md#claim): one connection at a time controls the Runner, under a controller epoch that each request changing a process or the workspace carries, and a gateway that lost its connection claims the Runner again instead of being restarted. While no gateway controls the Runner, any process of the same user that can reach its socket can claim it, an exposure the [Runner's documentation](runner-isolation.md#claim-exposure) states. The fourth unit, P1R-C04, adds [reconciliation](runner-isolation.md#reconciliation): after each Claim, the gateway compares its records, what the Runner holds and remembers, and for a `required` execution its DevGuard attempt, which the Runner looks up, and refuses new executions, patches and approval resumes with `RECOVERY_PENDING` only in the workspaces where an earlier controller left work, until it has reconciled them. A process found running holds its workspace again, an approval left resuming is consumed with the process the Runner created for it or granted again when the Runner proves that it created none, and a workspace whose accounts do not agree, or whose record names another Runner, stays refused for an operator. Runner loss comes in the next unit, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. +**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. The second unit, P1R-C02, adds [process records](runner-isolation.md#process-records): the gateway records each execution in its operations database before it sends the execution to the Runner and follows it to its end, as the database's only writer. The third unit, P1R-C03, adds the [Claim](runner-isolation.md#claim): one connection at a time controls the Runner, under a controller epoch that each request changing a process or the workspace carries, and a gateway that lost its connection claims the Runner again instead of being restarted. While no gateway controls the Runner, any process of the same user that can reach its socket can claim it, an exposure the [Runner's documentation](runner-isolation.md#claim-exposure) states. The fourth unit, P1R-C04, adds [reconciliation](runner-isolation.md#reconciliation): after each Claim, the gateway compares its records, what the Runner holds and remembers, and for a `required` execution its DevGuard attempt, which the Runner looks up, and refuses new executions, patches and approval resumes with `RECOVERY_PENDING` only in the workspaces where an earlier controller left work, until it has reconciled them. A process found running holds its workspace again, an approval left resuming is consumed with the process the Runner created for it or granted again when the Runner proves that it created none, and a workspace whose accounts do not agree, or whose record names another Runner, stays refused for an operator. The fifth unit, P1R-C05, adds [Runner loss](runner-isolation.md#runner-loss): when the Runner is killed, crashes or stops, or the host restarts, the gateway answers `process_status` and `read_process` for its executions from their records, with a `lost_reason`, and each execution that had not ended becomes `unknown` and holds its workspace until an operator finds its processes and releases its record with `codespace-mcp records` ([Operations](operations.md#runner-loss)). The operator releases a `required` execution's record only once DevGuard no longer charges its attempt, which only DevGuard's administrator quiescence shows, since DevGuard answers a lookup only to the attempt's own instance; until then DevGuard keeps the lost Runner's instance, so a new Runner cannot register as the `codespace` consumer's one instance. The mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. P1-RECOVERY introduces an operator-selected independent Runner mode and explicit capability. Existing modes keep their current shutdown/disconnect contracts. InProcess shares the Gateway PID and is excluded from live Gateway restart recovery. diff --git a/docs/error-codes.md b/docs/error-codes.md index 0b759ce..e80646a 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -54,9 +54,9 @@ Errors use uppercase identifiers and a message. An `operation_id` may be present | `APPROVAL_NOT_FOUND` | Unknown confirmation-hold id | | `APPROVAL_CONFLICT` | Hold is still pending, already decided, or a resume is already in progress | | `APPROVAL_AMBIGUOUS` | Resume was interrupted and the terminal result is not on disk. Includes `approval_id`; patch cases may also include `operation_id` | -| `RECOVERY_PENDING` | A controller before this gateway's Claim started work in this workspace on its [independent Runner](runner-isolation.md#reconciliation), this gateway's own connection before a lost connection included, and the gateway has not reconciled what it left there yet. The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts there until then. The approval stays as it was. The message says why: when the gateway waits for an answer it tries again by itself, so retry after a pause; when the records, the Runner and DevGuard do not agree, or a record names another Runner, retrying does not help until an operator resolves it. Not occupancy | +| `RECOVERY_PENDING` | A controller before this gateway's Claim started work in this workspace on its [independent Runner](runner-isolation.md#reconciliation), this gateway's own connection before a lost connection included, and the gateway has not reconciled what it left there yet. The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts there until then. The approval stays as it was. The message says why: when the gateway waits for an answer it tries again by itself, so retry after a pause; when the records, the Runner and DevGuard do not agree, retrying does not help until an operator resolves it, and when a record is of a lost Runner, until an operator confirms that nothing of it runs and [releases it](operations.md#runner-loss). Not occupancy | | `RUNNER_RECONNECTING` | The gateway's connection to its [independent Runner](runner-isolation.md#independent-runner) was lost, and the gateway is claiming the Runner again. The request was not sent: nothing was started, reserved or recorded for it, and an approval it would resume is not consumed. The Runner's processes keep running; retry after a pause | -| `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and cannot claim it again: nothing accepts connections at its socket, another Runner answers there, or the Runner refused the Claim for a reason that waiting cannot end. The request was not sent, and the Runner's processes keep running; restart the gateway to reach them | +| `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and cannot claim it again: nothing accepts connections at its socket, another Runner answers there, or the Runner refused the Claim for a reason that waiting cannot end. The request was not sent. When the Runner refused the Claim, its processes keep running; restart the gateway to reach them. Otherwise the gateway lost the Runner: what became of its processes cannot be learned and some may still run, `process_status` and `read_process` answer for them from the gateway's records with `lost_reason`, and an operator finds them and releases their workspaces ([Runner loss](operations.md#runner-loss)) | | `STALE_CONTROLLER` | The [independent Runner](runner-isolation.md#claim) refused a request sent under a Claim that another Claim replaced, or from a connection that does not control it. Nothing the request asked for was done, and an approval it resumed is granted again. Retrying sends the request under the gateway's current Claim | | `INTERNAL` | Scheduler or store invariant failed, or a gateway with an [independent Runner](runner-isolation.md#process-records) could not record an execution, which it then did not start. Not occupancy; do not retry as `WORKSPACE_BUSY` | diff --git a/docs/execution-substrate.md b/docs/execution-substrate.md index 6f4617d..3b62dbb 100644 --- a/docs/execution-substrate.md +++ b/docs/execution-substrate.md @@ -41,7 +41,7 @@ The operator registry maps `read-only` and `workspace-write` to effective permis Gateway fills workspace-root cwd, runner-local environment defaults, time/output limits, PTY choice, and policy into the internal Runner request. Only `tty` is exposed as a spawn-time terminal option. `tty_size` is not a spawn argument. A running PTY is resized with `process_resize`. Public calls do not accept arbitrary cwd/env/timeout overrides. See [operations](operations.md) for defaults and [Agent Loop integration](agent-integration.md) for result handling. -`exec_command` returns dispatch identity (`process_id`, `dispatch_status`). `process_status` reports `running` or `exited` plus termination metadata. `process_resize` changes the size of a running PTY. `read_process` reports output including `output_lost` and `retained_from`. EOF is not success. After handle eviction the next lookup is `PROCESS_NOT_FOUND`, not a new state. Pipe-backed resize is `PROCESS_NOT_TTY`; an exited handle is `PROCESS_NOT_RUNNING`. On Linux sandbox, the wait status is that of the managed child (the helper argv); it is not documented as identical to the user argv. +`exec_command` returns dispatch identity (`process_id`, `dispatch_status`). `process_status` reports `running` or `exited` plus termination metadata, or `unknown` after an [independent Runner](runner-isolation.md#runner-loss) was lost, with `lost_reason`. `process_resize` changes the size of a running PTY. `read_process` reports output including `output_lost`, `retained_from` and, when output is missing, `lost_reason`. EOF is not success. After handle eviction the next lookup is `PROCESS_NOT_FOUND`, not a new state, except that a gateway with an independent Runner answers from the execution's record, with `lost_reason` `retention_expired`. Pipe-backed resize is `PROCESS_NOT_TTY`; an exited handle is `PROCESS_NOT_RUNNING`. On Linux sandbox, the wait status is that of the managed child (the helper argv); it is not documented as identical to the user argv. A workspace mutation lease prevents simultaneous patch/exec mutations. FIFO order starts when an eligible request attempts to acquire the resource, not when the MCP message arrives. Request-owned patch and exec work on the same workspace waits in an in-memory per-resource FIFO until the current request releases the lease. When an exec becomes a live process, trailing waiters fail with `WORKSPACE_BUSY` and later arrivals are rejected immediately. Queue saturation is `RESOURCE_QUEUE_FULL`. That wait is not a durable or thread-keyed queue. Read and find remain available while a command runs, so filesystem I/O must reject symlink races at open time rather than rely on a prior path check. Runner file operations use `codespace-fs`; patch execution uses the separate patch helper. `operation_status` exposes the recorded patch ledger (`kind` is `patch`); live commands stay on `process_id` and are not recovered through that lookup. diff --git a/docs/ko/agent-integration.md b/docs/ko/agent-integration.md index c0b7c2d..d572904 100644 --- a/docs/ko/agent-integration.md +++ b/docs/ko/agent-integration.md @@ -85,7 +85,7 @@ MCP ํด๋ผ์ด์–ธํŠธ SDK๋กœ stdio ๋˜๋Š” Streamable HTTP๋ฅผ ์ดˆ๊ธฐํ™”ํ•˜๊ณ , ์ดˆ } ``` -๊ฒฐ๊ณผ์—๋Š” `chunk`, `cursor`, `eof`, `output_lost`, `retained_from`์ด ์žˆ์Šต๋‹ˆ๋‹ค. stdout/stderr๋Š” ๊ตฌ๋ถ„ ์—†์ด ํ•ฉ์ณ์ง‘๋‹ˆ๋‹ค. EOF๋Š” ์ถœ๋ ฅ ์ˆ˜์ง‘์ด ๋๋‚ฌ๋‹ค๋Š” ๋œป์ด๋ฉฐ ์„ฑ๊ณต ์ข…๋ฃŒ๋ฅผ ๋‚˜ํƒ€๋‚ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ข…๋ฃŒ๋Š” `process_status`๋กœ ํŒ์ •ํ•ฉ๋‹ˆ๋‹ค. `state`๋Š” `running` ๋˜๋Š” `exited`์ž…๋‹ˆ๋‹ค. ์ข…๋ฃŒ ํ›„ `termination`์€ `exited`ยท`signaled`ยท`timeout`ยท`terminated`ยท`unknown` ์ค‘ ํ•˜๋‚˜์ž…๋‹ˆ๋‹ค. `exit_code`๋Š” `termination`์ด `exited`์ผ ๋•Œ๋งŒ ์žˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. `signaled`๋Š” CodeSpace๊ฐ€ ๋ณด๋‚ด์ง€ ์•Š์€ ์‹ ํ˜ธ๋กœ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚ฌ๋‹ค๋Š” ๋œป์ž…๋‹ˆ๋‹ค. ํฌ๋ž˜์‹œ๋‚˜ ๋‹ค๋ฅธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋ณด๋‚ธ kill์ด ์—ฌ๊ธฐ์— ํ•ด๋‹นํ•ฉ๋‹ˆ๋‹ค. ์ด๋•Œ `signal`์—๋Š” ์‹ ํ˜ธ ๋ฒˆํ˜ธ `number`์™€, ์•Œ ์ˆ˜ ์žˆ๋Š” ๊ฒฝ์šฐ `SIGKILL` ๊ฐ™์€ ์ด๋ฆ„ `name`์ด ๋“ค์–ด๊ฐ‘๋‹ˆ๋‹ค. ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋Š” ์šด์˜์ฒด์ œ๋งˆ๋‹ค ๋‹ค๋ฆ…๋‹ˆ๋‹ค. CodeSpace๊ฐ€ ์š”์ฒญํ•œ kill์€ `signaled`๊ฐ€ ์•„๋‹ˆ๋ผ `timeout`์ด๋‚˜ `terminated`๋กœ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค. `exited`๋ผ๊ณ  ํ•ด์„œ ์‹ ํ˜ธ๋กœ ๋๋‚˜์ง€ ์•Š์•˜๋‹ค๋Š” ๋œป์€ ์•„๋‹™๋‹ˆ๋‹ค. `tty: true` ํ”„๋กœ์„ธ์Šค๋Š” ์‹ ํ˜ธ๋กœ ๋๋‚˜๋„ `exited`๋กœ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค. `exit_code`๋Š” 1์ด์–ด์„œ ์ƒํƒœ 1๋กœ ์‹คํŒจํ•œ ๋ช…๋ น๊ณผ ๊ตฌ๋ณ„๋˜์ง€ ์•Š๊ณ , CodeSpace๊ฐ€ DevGuard๋ฅผ ๊ฑฐ์ณ ๋„์šด ํ”„๋กœ์„ธ์Šค(participation `required`)๋ผ๋ฉด 128์— ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋ฅผ ๋”ํ•œ ๊ฐ’(SIGTERM์ด๋ฉด 143)์ž…๋‹ˆ๋‹ค. Linux ๋ช…๋ น ์ƒŒ๋“œ๋ฐ•์Šค์—์„œ๋Š” ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ์˜ ์ข…๋ฃŒ ์ƒํƒœ๊ฐ€ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค([๋Ÿฌ๋„ˆ ๊ฒฉ๋ฆฌ](runner-isolation.md)). `signaled`ยท`timeout`ยท`terminated`ยท`unknown`์€ `eof`๊ฐ€ ์ฐธ์ด์–ด๋„ ์„ฑ๊ณต์ด ์•„๋‹™๋‹ˆ๋‹ค. ๋งˆ์ง€๋ง‰ 256 KiB๋งŒ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. `output_lost`๊ฐ€ ์ฐธ์ด๋ฉด ๋ณด๊ด€ ์ฐฝ์ด ์ „์ฒด ๋กœ๊ทธ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. EOF๋‚˜ ๋ถˆ์™„์ „ํ•œ ๋กœ๊ทธ๋งŒ์œผ๋กœ ๋นŒ๋“œยทํ…Œ์ŠคํŠธ ์„ฑ๊ณต์„ ์„ ์–ธํ•˜์ง€ ๋งˆ์„ธ์š”. ์‹ ๋ขฐํ•  ์ˆ˜ ์žˆ๋Š” ์ž‘์—…๋ณ„ ๊ฒฐ๊ณผ๋กœ ์„ฑ๊ณต์„ ํ™•์ธํ•  ์ˆ˜ ์—†๋‹ค๋ฉด ๋ฏธ๊ฒ€์ฆ์œผ๋กœ ๋ณด๊ณ ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. +๊ฒฐ๊ณผ์—๋Š” `chunk`, `cursor`, `eof`, `output_lost`, `retained_from`์ด ์žˆ์Šต๋‹ˆ๋‹ค. stdout/stderr๋Š” ๊ตฌ๋ถ„ ์—†์ด ํ•ฉ์ณ์ง‘๋‹ˆ๋‹ค. EOF๋Š” ์ถœ๋ ฅ ์ˆ˜์ง‘์ด ๋๋‚ฌ๋‹ค๋Š” ๋œป์ด๋ฉฐ ์„ฑ๊ณต ์ข…๋ฃŒ๋ฅผ ๋‚˜ํƒ€๋‚ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ข…๋ฃŒ๋Š” `process_status`๋กœ ํŒ์ •ํ•ฉ๋‹ˆ๋‹ค. `state`๋Š” `running` ๋˜๋Š” `exited`์ด๊ณ , [๋…๋ฆฝ Runner](runner-isolation.md#runner-loss)๋ฅผ ์žƒ์€ ๋’ค์—๋Š” `unknown`์ผ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ์ข…๋ฃŒ ํ›„ `termination`์€ `exited`ยท`signaled`ยท`timeout`ยท`terminated`ยท`unknown` ์ค‘ ํ•˜๋‚˜์ž…๋‹ˆ๋‹ค. `exit_code`๋Š” `termination`์ด `exited`์ผ ๋•Œ๋งŒ ์žˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. `signaled`๋Š” CodeSpace๊ฐ€ ๋ณด๋‚ด์ง€ ์•Š์€ ์‹ ํ˜ธ๋กœ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚ฌ๋‹ค๋Š” ๋œป์ž…๋‹ˆ๋‹ค. ํฌ๋ž˜์‹œ๋‚˜ ๋‹ค๋ฅธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋ณด๋‚ธ kill์ด ์—ฌ๊ธฐ์— ํ•ด๋‹นํ•ฉ๋‹ˆ๋‹ค. ์ด๋•Œ `signal`์—๋Š” ์‹ ํ˜ธ ๋ฒˆํ˜ธ `number`์™€, ์•Œ ์ˆ˜ ์žˆ๋Š” ๊ฒฝ์šฐ `SIGKILL` ๊ฐ™์€ ์ด๋ฆ„ `name`์ด ๋“ค์–ด๊ฐ‘๋‹ˆ๋‹ค. ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋Š” ์šด์˜์ฒด์ œ๋งˆ๋‹ค ๋‹ค๋ฆ…๋‹ˆ๋‹ค. CodeSpace๊ฐ€ ์š”์ฒญํ•œ kill์€ `signaled`๊ฐ€ ์•„๋‹ˆ๋ผ `timeout`์ด๋‚˜ `terminated`๋กœ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค. `exited`๋ผ๊ณ  ํ•ด์„œ ์‹ ํ˜ธ๋กœ ๋๋‚˜์ง€ ์•Š์•˜๋‹ค๋Š” ๋œป์€ ์•„๋‹™๋‹ˆ๋‹ค. `tty: true` ํ”„๋กœ์„ธ์Šค๋Š” ์‹ ํ˜ธ๋กœ ๋๋‚˜๋„ `exited`๋กœ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค. `exit_code`๋Š” 1์ด์–ด์„œ ์ƒํƒœ 1๋กœ ์‹คํŒจํ•œ ๋ช…๋ น๊ณผ ๊ตฌ๋ณ„๋˜์ง€ ์•Š๊ณ , CodeSpace๊ฐ€ DevGuard๋ฅผ ๊ฑฐ์ณ ๋„์šด ํ”„๋กœ์„ธ์Šค(participation `required`)๋ผ๋ฉด 128์— ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋ฅผ ๋”ํ•œ ๊ฐ’(SIGTERM์ด๋ฉด 143)์ž…๋‹ˆ๋‹ค. Linux ๋ช…๋ น ์ƒŒ๋“œ๋ฐ•์Šค์—์„œ๋Š” ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ์˜ ์ข…๋ฃŒ ์ƒํƒœ๊ฐ€ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค([๋Ÿฌ๋„ˆ ๊ฒฉ๋ฆฌ](runner-isolation.md)). `signaled`ยท`timeout`ยท`terminated`ยท`unknown`์€ `eof`๊ฐ€ ์ฐธ์ด์–ด๋„ ์„ฑ๊ณต์ด ์•„๋‹™๋‹ˆ๋‹ค. ๋งˆ์ง€๋ง‰ 256 KiB๋งŒ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. `output_lost`๊ฐ€ ์ฐธ์ด๋ฉด ๋ณด๊ด€ ์ฐฝ์ด ์ „์ฒด ๋กœ๊ทธ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. `lost_reason`์ด ์žˆ์œผ๋ฉด ์ถœ๋ ฅ์ด๋‚˜ ํ”„๋กœ์„ธ์Šค์˜ ๊ฒฐ๊ณผ๊ฐ€ ์™œ ๋น ์กŒ๋Š”์ง€ ์•Œ๋ ค ์ค๋‹ˆ๋‹ค. `ring_overflow`๋Š” ๊ฐ€์žฅ ์ตœ๊ทผ ์ถœ๋ ฅ๋งŒ ๋ณด๊ด€ํ–ˆ๋‹ค๋Š” ๋œป์ž…๋‹ˆ๋‹ค. ๋…๋ฆฝ Runner์—์„œ `retention_expired`๋Š” Runner๊ฐ€ ๋๋‚œ ํ”„๋กœ์„ธ์Šค๋ฅผ ๋”๋Š” ๋ณด๊ด€ํ•˜์ง€ ์•Š๋Š”๋‹ค๋Š” ๋œป์ด๊ณ , `runner_lost`๋Š” ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์‹คํ–‰ํ•œ Runner๋ฅผ ์žƒ์—ˆ๋‹ค๋Š” ๋œป, `host_rebooted`๋Š” ํ”„๋กœ์„ธ์Šค๊ฐ€ ์‹œ์ž‘๋œ ๋’ค ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ–ˆ๋‹ค๋Š” ๋œป์ž…๋‹ˆ๋‹ค. ์ด๋Ÿฐ ๋‹ต์€ ๊ฒŒ์ดํŠธ์›จ์ด์˜ ๊ธฐ๋ก์—์„œ ์˜ค๋ฉฐ ์ถœ๋ ฅ์ด ์—†์Šต๋‹ˆ๋‹ค. `runner_lost`์ด๋ฉด `state`๊ฐ€ `unknown`์ผ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ํ”„๋กœ์„ธ์Šค๊ฐ€ ์•„์ง ์‹คํ–‰ ์ค‘์ผ ์ˆ˜ ์žˆ์œผ๋ฉฐ, ์ด๋Š” ์ข…๋ฃŒ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ๋ช…๋ น์„ ๋ฏธ๊ฒ€์ฆ์œผ๋กœ ๋ณด๊ณ ํ•˜๊ณ  ๋‹ค์‹œ ์‹œ์ž‘ํ•˜์ง€ ๋งˆ์„ธ์š”. ์šด์˜์ž๊ฐ€ ์ •๋ฆฌํ•  ๋•Œ๊นŒ์ง€ ๊ทธ workspace๋Š” ๊ณ„์† ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค. EOF๋‚˜ ๋ถˆ์™„์ „ํ•œ ๋กœ๊ทธ๋งŒ์œผ๋กœ ๋นŒ๋“œยทํ…Œ์ŠคํŠธ ์„ฑ๊ณต์„ ์„ ์–ธํ•˜์ง€ ๋งˆ์„ธ์š”. ์‹ ๋ขฐํ•  ์ˆ˜ ์žˆ๋Š” ์ž‘์—…๋ณ„ ๊ฒฐ๊ณผ๋กœ ์„ฑ๊ณต์„ ํ™•์ธํ•  ์ˆ˜ ์—†๋‹ค๋ฉด ๋ฏธ๊ฒ€์ฆ์œผ๋กœ ๋ณด๊ณ ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ```json { @@ -150,7 +150,7 @@ MCP ํด๋ผ์ด์–ธํŠธ SDK๋กœ stdio ๋˜๋Š” Streamable HTTP๋ฅผ ์ดˆ๊ธฐํ™”ํ•˜๊ณ , ์ดˆ ์‹คํ–‰ ์ค‘์ธ ๋ช…๋ น์€ ์ž‘์—… ๊ณต๊ฐ„์„ ์ ์œ ํ•ฉ๋‹ˆ๋‹ค. ํŒจ์น˜๋ฅผ ์ ์šฉํ•˜๊ฑฐ๋‚˜ ๋‹ค๋ฅธ ๋ช…๋ น์„ ์‹œ์ž‘ํ•˜๋ ค๋ฉด ๊ธฐ์กด ๋ช…๋ น์ด ๋๋‚  ๋•Œ๊นŒ์ง€ ๊ธฐ๋‹ค๋ฆฌ๊ฑฐ๋‚˜ ์ข…๋ฃŒํ•˜์„ธ์š”. ์ฝ๊ธฐ์™€ ๊ฒ€์ƒ‰์€ ๊ณ„์† ๊ฐ€๋Šฅํ•ฉ๋‹ˆ๋‹ค. ๊ฐœ๋ฐœ ์„œ๋ฒ„๋ฅผ ์˜ค๋ž˜ ์‹คํ–‰ํ•˜๋Š” ์ž‘์—… ํ๋ฆ„์ด๋ผ๋ฉด ์ˆ˜์ • ์ „์— ์„œ๋ฒ„๋ฅผ ๋ฉˆ์ถ”๋Š” ์ ˆ์ฐจ๊ฐ€ ํ•„์š”ํ•ฉ๋‹ˆ๋‹ค. -`workspace_info.execution.process.capabilities.lifetime`์€ ์†Œ์œ ์ž๊ฐ€ MCP ์„ธ์…˜์ด ์•„๋‹ˆ๋ผ ๋Ÿฌ๋„ˆ ์ธ์Šคํ„ด์Šค์ž„์„ ์•Œ๋ฆฝ๋‹ˆ๋‹ค. Streamable HTTP๋‚˜ MCP ํด๋ผ์ด์–ธํŠธ ๋Š๊น€์€ ํ”„๋กœ์„ธ์Šค๋ฅผ ์œ ์ง€ํ•˜๋ฏ€๋กœ, ์ €์žฅํ•œ `process_id`๋กœ ๋‹ค์‹œ ์—ฐ๊ฒฐํ•˜๋ฉด ๋ฉ๋‹ˆ๋‹ค. UDS worker ์—ฐ๊ฒฐ์ด ๋Š๊ธฐ๊ฑฐ๋‚˜ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ข…๋ฃŒ๋˜๋ฉด(stdio EOF ํฌํ•จ) ์†Œ์œ  ์„œ๋ธŒํŠธ๋ฆฌ๊ฐ€ ์ข…๋ฃŒ๋˜๊ณ  ํ•ธ๋“ค์ด ์‚ฌ๋ผ์ง‘๋‹ˆ๋‹ค. ์žฌ์‹œ์ž‘์€ `process_id`๋ฅผ ๋ณต๊ตฌํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹Œ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)์—์„œ๋Š” `runner_disconnect`๊ฐ€ `keep_running`์ด๊ณ , `gateway_shutdown`์€ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ detachํ•˜๋ฉด `keep_running`, Runner๋ฅผ ์ค‘์ง€ํ•˜๋ฉด `terminate`์ž…๋‹ˆ๋‹ค. ๊ทธ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ํ•˜๋Š” ์ผ์€ `initialize.instructions`์—์„œ ํ™•์ธํ•˜์„ธ์š”. ๊ทธ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์—ฐ๊ฒฐ์„ ์žƒ์€ ๋’ค Runner๋ฅผ ๋‹ค์‹œ Claimํ•˜๋Š” ๋™์•ˆ ์š”์ฒญ์„ `RUNNER_RECONNECTING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ์ด ๊ฑฐ๋ถ€๋Š” ์•„๋ฌด๊ฒƒ๋„ ๋ณด๋‚ด์ง€ ์•Š์•˜๊ณ  ์Šน์ธ๋„ ์†Œ๋ชจํ•˜์ง€ ์•Š์•˜์œผ๋ฏ€๋กœ, ์ž ์‹œ ๋’ค ์žฌ์‹œ๋„ํ•˜์„ธ์š”. ๊ทธ ๋’ค ์ด์ „์— ์ž‘์—…์„ ์‹œ์ž‘ํ•œ workspace์—์„œ๋Š” Runner์™€ ๊ทธ workspace๋ฅผ [๋Œ€์กฐ](runner-isolation.md#reconciliation)ํ•  ๋•Œ๊นŒ์ง€ ์ƒˆ ๋ช…๋ น, ํŒจ์น˜, ์Šน์ธ ์žฌ๊ฐœ๋ฅผ `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ์ž ์‹œ ๋’ค ์žฌ์‹œ๋„ํ•˜๋ฉด ๋˜๋Š”์ง€๋Š” ๊ทธ ๋ฉ”์‹œ์ง€๊ฐ€ ์•Œ๋ ค ์ค๋‹ˆ๋‹ค. ๊ทธ๊ณณ์—์„œ ์‹คํ–‰ ์ค‘์œผ๋กœ ํ™•์ธํ•œ ํ”„๋กœ์„ธ์Šค๋Š” ์ง€๊ธˆ์ฒ˜๋Ÿผ workspace๋ฅผ ์ ์œ ํ•ฉ๋‹ˆ๋‹ค. spawn ์‘๋‹ต์„ ๋ฐ›๊ธฐ ์ „์— `process_id`๋ฅผ ์žƒ์œผ๋ฉด `process_status`๋กœ ์ฐพ์ง€ ๋ง๊ณ  ๋ช…๋ น์„ ์ค‘๋ณต ์‹œ์ž‘ํ•˜์ง€ ๋งˆ์„ธ์š”. +`workspace_info.execution.process.capabilities.lifetime`์€ ์†Œ์œ ์ž๊ฐ€ MCP ์„ธ์…˜์ด ์•„๋‹ˆ๋ผ ๋Ÿฌ๋„ˆ ์ธ์Šคํ„ด์Šค์ž„์„ ์•Œ๋ฆฝ๋‹ˆ๋‹ค. Streamable HTTP๋‚˜ MCP ํด๋ผ์ด์–ธํŠธ ๋Š๊น€์€ ํ”„๋กœ์„ธ์Šค๋ฅผ ์œ ์ง€ํ•˜๋ฏ€๋กœ, ์ €์žฅํ•œ `process_id`๋กœ ๋‹ค์‹œ ์—ฐ๊ฒฐํ•˜๋ฉด ๋ฉ๋‹ˆ๋‹ค. UDS worker ์—ฐ๊ฒฐ์ด ๋Š๊ธฐ๊ฑฐ๋‚˜ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ข…๋ฃŒ๋˜๋ฉด(stdio EOF ํฌํ•จ) ์†Œ์œ  ์„œ๋ธŒํŠธ๋ฆฌ๊ฐ€ ์ข…๋ฃŒ๋˜๊ณ  ํ•ธ๋“ค์ด ์‚ฌ๋ผ์ง‘๋‹ˆ๋‹ค. ์žฌ์‹œ์ž‘์€ `process_id`๋ฅผ ๋ณต๊ตฌํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹Œ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)์—์„œ๋Š” `runner_disconnect`๊ฐ€ `keep_running`์ด๊ณ , `gateway_shutdown`์€ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ detachํ•˜๋ฉด `keep_running`, Runner๋ฅผ ์ค‘์ง€ํ•˜๋ฉด `terminate`์ž…๋‹ˆ๋‹ค. ๊ทธ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ํ•˜๋Š” ์ผ์€ `initialize.instructions`์—์„œ ํ™•์ธํ•˜์„ธ์š”. ๊ทธ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์—ฐ๊ฒฐ์„ ์žƒ์€ ๋’ค Runner๋ฅผ ๋‹ค์‹œ Claimํ•˜๋Š” ๋™์•ˆ ์š”์ฒญ์„ `RUNNER_RECONNECTING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ์ด ๊ฑฐ๋ถ€๋Š” ์•„๋ฌด๊ฒƒ๋„ ๋ณด๋‚ด์ง€ ์•Š์•˜๊ณ  ์Šน์ธ๋„ ์†Œ๋ชจํ•˜์ง€ ์•Š์•˜์œผ๋ฏ€๋กœ, ์ž ์‹œ ๋’ค ์žฌ์‹œ๋„ํ•˜์„ธ์š”. ๊ทธ ๋’ค ์ด์ „์— ์ž‘์—…์„ ์‹œ์ž‘ํ•œ workspace์—์„œ๋Š” Runner์™€ ๊ทธ workspace๋ฅผ [๋Œ€์กฐ](runner-isolation.md#reconciliation)ํ•  ๋•Œ๊นŒ์ง€ ์ƒˆ ๋ช…๋ น, ํŒจ์น˜, ์Šน์ธ ์žฌ๊ฐœ๋ฅผ `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ์ž ์‹œ ๋’ค ์žฌ์‹œ๋„ํ•˜๋ฉด ๋˜๋Š”์ง€๋Š” ๊ทธ ๋ฉ”์‹œ์ง€๊ฐ€ ์•Œ๋ ค ์ค๋‹ˆ๋‹ค. ๊ทธ๊ณณ์—์„œ ์‹คํ–‰ ์ค‘์œผ๋กœ ํ™•์ธํ•œ ํ”„๋กœ์„ธ์Šค๋Š” ์ง€๊ธˆ์ฒ˜๋Ÿผ workspace๋ฅผ ์ ์œ ํ•ฉ๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ Runner ์ž์ฒด๋ฅผ ์žƒ์œผ๋ฉด ๊ทธ Runner์˜ ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•œ `process_status`์™€ `read_process`์—๋Š” ๊ธฐ๋ก์œผ๋กœ `lost_reason`๊ณผ ํ•จ๊ป˜ ๋‹ตํ•˜๊ณ , Runner๊ฐ€ ํ•„์š”ํ•œ ๋‹ค๋ฅธ ์š”์ฒญ์€ `RUNNER_DISCONNECTED`๋กœ ๊ฑฐ๋ถ€ํ•˜๋ฉฐ, ์šด์˜์ž๊ฐ€ ๊ทธ ๊ธฐ๋ก์„ ํ’€ ๋•Œ๊นŒ์ง€ ๊ทธ workspace๋ฅผ ๊ณ„์† ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ๋‹ค์Œ ๊ฒŒ์ดํŠธ์›จ์ด์—์„œ๋Š” `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค([Runner ์†์‹ค](runner-isolation.md#runner-loss)). spawn ์‘๋‹ต์„ ๋ฐ›๊ธฐ ์ „์— `process_id`๋ฅผ ์žƒ์œผ๋ฉด `process_status`๋กœ ์ฐพ์ง€ ๋ง๊ณ  ๋ช…๋ น์„ ์ค‘๋ณต ์‹œ์ž‘ํ•˜์ง€ ๋งˆ์„ธ์š”. ## ๋ณด๋ฅ˜๋œ ๋ณ€๊ฒฝ ํ™•์ธ diff --git a/docs/ko/architecture.md b/docs/ko/architecture.md index 83f79c9..82dae0e 100644 --- a/docs/ko/architecture.md +++ b/docs/ko/architecture.md @@ -58,7 +58,7 @@ UDS worker์™€ ๊ธฐ๋ณธ ๋Ÿฌ๋„ˆ๋Š” ๋ชจ๋‘ ๊ฐ™์€ ํ˜ธ์ŠคํŠธ์—์„œ ์‹คํ–‰ํ•ฉ๋‹ˆ๋‹ค. ## ํ”„๋กœ์„ธ์Šค ์ˆ˜๋ช… -MCP ์š”์ฒญ์ด ๋๋‚˜๋„ ๊ด€๋ฆฌ ์ค‘์ธ ํ”„๋กœ์„ธ์Šค๋Š” ์œ ์ง€๋ฉ๋‹ˆ๋‹ค. ํด๋ผ์ด์–ธํŠธ๋Š” `process_id`๋กœ ํ›„์† ํ˜ธ์ถœ์„ ์ˆ˜ํ–‰ํ•ฉ๋‹ˆ๋‹ค. ์„œ๋ฒ„ ์žฌ์‹œ์ž‘ ํ›„์—๋Š” ํ•ธ๋“ค์ด ์‚ฌ๋ผ์ง‘๋‹ˆ๋‹ค. UDS ๋ชจ๋“œ์—์„œ๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ worker๋ฅผ ๊ด€๋ฆฌํ•˜๋ฏ€๋กœ ๋‚ด๋ถ€ ์—ฐ๊ฒฐ ์ข…๋ฃŒ๋‚˜ ์„œ๋ฒ„ ์ข…๋ฃŒ ์‹œ worker์™€ ์ž์‹ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์ข…๋ฃŒ๋˜๋ฉฐ ์žฌ์ ‘์†์€ ์ง€์›ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹Œ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)๋Š” ์ด์™€ ๋‹ฌ๋ฆฌ ๊ฒŒ์ดํŠธ์›จ์ด๋ณด๋‹ค ์˜ค๋ž˜ ์‚ด๊ณ , Runner์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์€ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” Runner๋ฅผ ๋‹ค์‹œ Claimํ•œ ๋’ค ์•ž์„  controller๋“ค์ด ์ž‘์—…์„ ์‹œ์ž‘ํ•œ ๊ณณ์—์„œ๋Š” ๊ทธ๋“ค์ด ๋‚จ๊ธด ๊ฒƒ์„ ๋Œ€์กฐํ•˜๊ณ  ๋‚˜์„œ ์ƒˆ ์ž‘์—…์„ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. [๋Ÿฌ๋„ˆ ๊ฒฉ๋ฆฌ](runner-isolation.md)์—์„œ ์ „์†ก ๊ฒฝ๊ณ„์™€ ๊ฒฉ๋ฆฌ ๊ฒฝ๊ณ„๋ฅผ ๊ตฌ๋ถ„ํ•ด ์„ค๋ช…ํ•ฉ๋‹ˆ๋‹ค. +MCP ์š”์ฒญ์ด ๋๋‚˜๋„ ๊ด€๋ฆฌ ์ค‘์ธ ํ”„๋กœ์„ธ์Šค๋Š” ์œ ์ง€๋ฉ๋‹ˆ๋‹ค. ํด๋ผ์ด์–ธํŠธ๋Š” `process_id`๋กœ ํ›„์† ํ˜ธ์ถœ์„ ์ˆ˜ํ–‰ํ•ฉ๋‹ˆ๋‹ค. ์„œ๋ฒ„ ์žฌ์‹œ์ž‘ ํ›„์—๋Š” ํ•ธ๋“ค์ด ์‚ฌ๋ผ์ง‘๋‹ˆ๋‹ค. UDS ๋ชจ๋“œ์—์„œ๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ worker๋ฅผ ๊ด€๋ฆฌํ•˜๋ฏ€๋กœ ๋‚ด๋ถ€ ์—ฐ๊ฒฐ ์ข…๋ฃŒ๋‚˜ ์„œ๋ฒ„ ์ข…๋ฃŒ ์‹œ worker์™€ ์ž์‹ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์ข…๋ฃŒ๋˜๋ฉฐ ์žฌ์ ‘์†์€ ์ง€์›ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹Œ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)๋Š” ์ด์™€ ๋‹ฌ๋ฆฌ ๊ฒŒ์ดํŠธ์›จ์ด๋ณด๋‹ค ์˜ค๋ž˜ ์‚ด๊ณ , Runner์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์€ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” Runner๋ฅผ ๋‹ค์‹œ Claimํ•œ ๋’ค ์•ž์„  controller๋“ค์ด ์ž‘์—…์„ ์‹œ์ž‘ํ•œ ๊ณณ์—์„œ๋Š” ๊ทธ๋“ค์ด ๋‚จ๊ธด ๊ฒƒ์„ ๋Œ€์กฐํ•˜๊ณ  ๋‚˜์„œ ์ƒˆ ์ž‘์—…์„ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. Runner ์ž์ฒด๋ฅผ ์žƒ์œผ๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ทธ ์‹คํ–‰๋“ค์— ๋Œ€ํ•ด ๊ธฐ๋ก์œผ๋กœ ๋‹ตํ•˜๊ณ , ๊ทธ workspace๋Š” ์šด์˜์ž๋ฅผ ๊ธฐ๋‹ค๋ฆฝ๋‹ˆ๋‹ค([Runner ์†์‹ค](runner-isolation.md#runner-loss)). [๋Ÿฌ๋„ˆ ๊ฒฉ๋ฆฌ](runner-isolation.md)์—์„œ ์ „์†ก ๊ฒฝ๊ณ„์™€ ๊ฒฉ๋ฆฌ ๊ฒฝ๊ณ„๋ฅผ ๊ตฌ๋ถ„ํ•ด ์„ค๋ช…ํ•ฉ๋‹ˆ๋‹ค. ## ๊ณ„ํš๋œ ์‹คํ–‰ ์กฐ์ • ๊ตฌ์กฐ diff --git a/docs/ko/devguard-integration.md b/docs/ko/devguard-integration.md index 8570780..12d7543 100644 --- a/docs/ko/devguard-integration.md +++ b/docs/ko/devguard-integration.md @@ -338,7 +338,7 @@ contract/client pin, daemon/helper artifact์™€ CodeSpace Runner wire๋Š” ๊ฐ๊ฐ ## ์ตœ์ดˆ Gateway ๋ณต๊ตฌ ๋ฒ”์œ„ -**์ƒํƒœ, P1-RECOVERY(์†Œ์œ ์ž ์ง€์‹œ 2026-10-14):** P1-RECOVERY๋Š” DevGuard์˜ [P1-RECOVERY ๊ณ„ํš](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#์žฌ๊ธฐ์ค€-2026-10-14)์—์„œ ๋‹ค์‹œ ์žก์€ ๊ธฐ์ค€๋Œ€๋กœ ๋‹จ์œ„๋งˆ๋‹ค PR ํ•˜๋‚˜๋กœ ๊ตฌํ˜„ํ•œ๋‹ค. ์ฒซ ๋‹จ์œ„ P1R-C01์€ ์•„์ง ์ง€์›ํ•˜์ง€ ์•Š๋Š” opt-in ๋ชจ๋“œ์ธ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)๋ฅผ ๋”ํ•œ๋‹ค. ์šด์˜์ž๋‚˜ ํ•˜๋„ค์Šค๊ฐ€ Runner๋ฅผ ์‹œ์ž‘ํ•˜๊ณ , Runner์˜ ํ”„๋กœ์„ธ์Šค, ์ถœ๋ ฅ, timeout์€ ๋ชจ๋“  gateway๋ณด๋‹ค ์˜ค๋ž˜ ์‚ฐ๋‹ค. gateway๋Š” ์ข…๋ฃŒ ์˜๋„๋Œ€๋กœ Runner์—์„œ detachํ•˜๊ฑฐ๋‚˜ Runner๋ฅผ ์ค‘์ง€ํ•œ๋‹ค. ๋‘ ๋ฒˆ์งธ ๋‹จ์œ„ P1R-C02๋Š” [์‹คํ–‰ ๊ธฐ๋ก](runner-isolation.md#process-records)์„ ๋”ํ•œ๋‹ค. gateway๋Š” ์ž‘์—… ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค์— ์“ฐ๋Š” ์œ ์ผํ•œ gateway๋กœ์„œ ๊ฐ ์‹คํ–‰์„ Runner์— ๋ณด๋‚ด๊ธฐ ์ „์— ๊ธฐ๋กํ•˜๊ณ  ๋๋‚  ๋•Œ๊นŒ์ง€ ๋”ฐ๋ผ๊ฐ„๋‹ค. ์„ธ ๋ฒˆ์งธ ๋‹จ์œ„ P1R-C03์€ [Claim](runner-isolation.md#claim)์„ ๋”ํ•œ๋‹ค. ํ•œ ๋ฒˆ์— ํ•œ ์—ฐ๊ฒฐ๋งŒ controller epoch ์•„๋ž˜์—์„œ Runner๋ฅผ ์ œ์–ดํ•˜๊ณ , ํ”„๋กœ์„ธ์Šค๋‚˜ workspace๋ฅผ ๋ฐ”๊พธ๋Š” ์š”์ฒญ์€ ๋ชจ๋‘ ๊ทธ epoch๋ฅผ ํ•จ๊ป˜ ๋ณด๋‚ธ๋‹ค. ์—ฐ๊ฒฐ์„ ์žƒ์€ gateway๋Š” ์žฌ์‹œ์ž‘ํ•˜์ง€ ์•Š๊ณ  Runner๋ฅผ ๋‹ค์‹œ Claimํ•œ๋‹ค. Runner๋ฅผ ์ œ์–ดํ•˜๋Š” gateway๊ฐ€ ์—†๋Š” ๋™์•ˆ์—๋Š” Runner์˜ ์†Œ์ผ“์— ๋‹ฟ์„ ์ˆ˜ ์žˆ๋Š” ๊ฐ™์€ ์‚ฌ์šฉ์ž์˜ ํ”„๋กœ์„ธ์Šค๋ผ๋ฉด ๋ฌด์—‡์ด๋“  Runner๋ฅผ Claimํ•  ์ˆ˜ ์žˆ์œผ๋ฉฐ, ์ด ๋…ธ์ถœ์€ [Runner ๋ฌธ์„œ](runner-isolation.md#claim-exposure)์— ์ ์—ˆ๋‹ค. ๋„ค ๋ฒˆ์งธ ๋‹จ์œ„ P1R-C04๋Š” [๋Œ€์กฐ](runner-isolation.md#reconciliation)๋ฅผ ๋”ํ•œ๋‹ค. gateway๋Š” Claimํ•  ๋•Œ๋งˆ๋‹ค ์ž๊ธฐ ๊ธฐ๋ก, Runner๊ฐ€ ๊ฐ€์ง„ ๊ฒƒ๊ณผ ๊ธฐ์–ตํ•˜๋Š” ๊ฒƒ, `required` ์‹คํ–‰์ด๋ฉด Runner๊ฐ€ ์กฐํšŒํ•œ DevGuard attempt๋ฅผ ๋งž์ถฐ ๋ณด๊ณ , ์•ž์„  controller๊ฐ€ ์ž‘์—…์„ ๋‚จ๊ธด workspace์—์„œ๋งŒ ๋Œ€์กฐ๋ฅผ ๋งˆ์น  ๋•Œ๊นŒ์ง€ ์ƒˆ ์‹คํ–‰, ํŒจ์น˜, ์Šน์ธ ์žฌ๊ฐœ๋ฅผ `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•œ๋‹ค. ์‹คํ–‰ ์ค‘์œผ๋กœ ํ™•์ธํ•œ ํ”„๋กœ์„ธ์Šค๋Š” ๋‹ค์‹œ workspace๋ฅผ ์ ์œ ํ•˜๊ณ , resuming์œผ๋กœ ๋‚จ์€ ์Šน์ธ์€ Runner๊ฐ€ ๊ทธ ์Šน์ธ์„ ์œ„ํ•ด ๋งŒ๋“  ํ”„๋กœ์„ธ์Šค๋กœ consumed๊ฐ€ ๋˜๊ฑฐ๋‚˜, Runner๊ฐ€ ์•„๋ฌด๊ฒƒ๋„ ๋งŒ๋“ค์ง€ ์•Š์•˜์Œ์ด ์ฆ๋ช…๋˜๋ฉด ๋‹ค์‹œ granted๊ฐ€ ๋œ๋‹ค. ์„ธ ๊ฐ€์ง€๊ฐ€ ๋งž์ง€ ์•Š๊ฑฐ๋‚˜ ๊ธฐ๋ก์ด ๋‹ค๋ฅธ Runner๋ฅผ ๊ฐ€๋ฆฌํ‚ค๋Š” workspace๋Š” ์šด์˜์ž๊ฐ€ ํŒ๋‹จํ•  ๋•Œ๊นŒ์ง€ ๊ณ„์† ๊ฑฐ๋ถ€๋œ๋‹ค. Runner ์†์‹ค ์ฒ˜๋ฆฌ๋Š” ๋‹ค์Œ ๋‹จ์œ„์—์„œ ๋”ํ•˜๋ฉฐ, ๋งˆ์ง€๋ง‰ ๋‹จ์œ„๊ฐ€ qualificationํ•œ ๋’ค์—์•ผ ์ด ๋ชจ๋“œ๋ฅผ ์ง€์›ํ•œ๋‹ค. ์ด ๋ชจ๋“œ์—์„œ๋Š” Runner๊ฐ€ DevGuard์˜ ์‹คํ–‰ ์†Œ์œ ์ž์ด๋ฉฐ, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--์‹คํ–‰-์†Œ์œ ์ž์ธ-runner์˜-๋‹จ์ผ-๋“ฑ๋ก)์˜ ๋ณด์™„๋Œ€๋กœ ์†Œ๋น„์ž secret์„ ์ž๊ธฐ ์ž๊ฒฉ ํŒŒ์ผ์—์„œ ์ง์ ‘ ์ฝ๋Š”๋‹ค. ๊ทธ gateway๋Š” Runner๊ฐ€ ๋“ฑ๋กํ•˜๊ณ  launch helper๋กœ ์‹คํ–‰ํ•œ๋‹ค๊ณ  ์•Œ๋ฆด ๋•Œ๋งŒ `required` workspace๋ฅผ ๋ฐ›์•„๋“ค์ธ๋‹ค. ์ด ์ ˆ์˜ ๋‚˜๋จธ์ง€๋Š” ๊ณ„ํšํ•œ ๋ฒ”์œ„๋‹ค. +**์ƒํƒœ, P1-RECOVERY(์†Œ์œ ์ž ์ง€์‹œ 2026-10-14):** P1-RECOVERY๋Š” DevGuard์˜ [P1-RECOVERY ๊ณ„ํš](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#์žฌ๊ธฐ์ค€-2026-10-14)์—์„œ ๋‹ค์‹œ ์žก์€ ๊ธฐ์ค€๋Œ€๋กœ ๋‹จ์œ„๋งˆ๋‹ค PR ํ•˜๋‚˜๋กœ ๊ตฌํ˜„ํ•œ๋‹ค. ์ฒซ ๋‹จ์œ„ P1R-C01์€ ์•„์ง ์ง€์›ํ•˜์ง€ ์•Š๋Š” opt-in ๋ชจ๋“œ์ธ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)๋ฅผ ๋”ํ•œ๋‹ค. ์šด์˜์ž๋‚˜ ํ•˜๋„ค์Šค๊ฐ€ Runner๋ฅผ ์‹œ์ž‘ํ•˜๊ณ , Runner์˜ ํ”„๋กœ์„ธ์Šค, ์ถœ๋ ฅ, timeout์€ ๋ชจ๋“  gateway๋ณด๋‹ค ์˜ค๋ž˜ ์‚ฐ๋‹ค. gateway๋Š” ์ข…๋ฃŒ ์˜๋„๋Œ€๋กœ Runner์—์„œ detachํ•˜๊ฑฐ๋‚˜ Runner๋ฅผ ์ค‘์ง€ํ•œ๋‹ค. ๋‘ ๋ฒˆ์งธ ๋‹จ์œ„ P1R-C02๋Š” [์‹คํ–‰ ๊ธฐ๋ก](runner-isolation.md#process-records)์„ ๋”ํ•œ๋‹ค. gateway๋Š” ์ž‘์—… ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค์— ์“ฐ๋Š” ์œ ์ผํ•œ gateway๋กœ์„œ ๊ฐ ์‹คํ–‰์„ Runner์— ๋ณด๋‚ด๊ธฐ ์ „์— ๊ธฐ๋กํ•˜๊ณ  ๋๋‚  ๋•Œ๊นŒ์ง€ ๋”ฐ๋ผ๊ฐ„๋‹ค. ์„ธ ๋ฒˆ์งธ ๋‹จ์œ„ P1R-C03์€ [Claim](runner-isolation.md#claim)์„ ๋”ํ•œ๋‹ค. ํ•œ ๋ฒˆ์— ํ•œ ์—ฐ๊ฒฐ๋งŒ controller epoch ์•„๋ž˜์—์„œ Runner๋ฅผ ์ œ์–ดํ•˜๊ณ , ํ”„๋กœ์„ธ์Šค๋‚˜ workspace๋ฅผ ๋ฐ”๊พธ๋Š” ์š”์ฒญ์€ ๋ชจ๋‘ ๊ทธ epoch๋ฅผ ํ•จ๊ป˜ ๋ณด๋‚ธ๋‹ค. ์—ฐ๊ฒฐ์„ ์žƒ์€ gateway๋Š” ์žฌ์‹œ์ž‘ํ•˜์ง€ ์•Š๊ณ  Runner๋ฅผ ๋‹ค์‹œ Claimํ•œ๋‹ค. Runner๋ฅผ ์ œ์–ดํ•˜๋Š” gateway๊ฐ€ ์—†๋Š” ๋™์•ˆ์—๋Š” Runner์˜ ์†Œ์ผ“์— ๋‹ฟ์„ ์ˆ˜ ์žˆ๋Š” ๊ฐ™์€ ์‚ฌ์šฉ์ž์˜ ํ”„๋กœ์„ธ์Šค๋ผ๋ฉด ๋ฌด์—‡์ด๋“  Runner๋ฅผ Claimํ•  ์ˆ˜ ์žˆ์œผ๋ฉฐ, ์ด ๋…ธ์ถœ์€ [Runner ๋ฌธ์„œ](runner-isolation.md#claim-exposure)์— ์ ์—ˆ๋‹ค. ๋„ค ๋ฒˆ์งธ ๋‹จ์œ„ P1R-C04๋Š” [๋Œ€์กฐ](runner-isolation.md#reconciliation)๋ฅผ ๋”ํ•œ๋‹ค. gateway๋Š” Claimํ•  ๋•Œ๋งˆ๋‹ค ์ž๊ธฐ ๊ธฐ๋ก, Runner๊ฐ€ ๊ฐ€์ง„ ๊ฒƒ๊ณผ ๊ธฐ์–ตํ•˜๋Š” ๊ฒƒ, `required` ์‹คํ–‰์ด๋ฉด Runner๊ฐ€ ์กฐํšŒํ•œ DevGuard attempt๋ฅผ ๋งž์ถฐ ๋ณด๊ณ , ์•ž์„  controller๊ฐ€ ์ž‘์—…์„ ๋‚จ๊ธด workspace์—์„œ๋งŒ ๋Œ€์กฐ๋ฅผ ๋งˆ์น  ๋•Œ๊นŒ์ง€ ์ƒˆ ์‹คํ–‰, ํŒจ์น˜, ์Šน์ธ ์žฌ๊ฐœ๋ฅผ `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•œ๋‹ค. ์‹คํ–‰ ์ค‘์œผ๋กœ ํ™•์ธํ•œ ํ”„๋กœ์„ธ์Šค๋Š” ๋‹ค์‹œ workspace๋ฅผ ์ ์œ ํ•˜๊ณ , resuming์œผ๋กœ ๋‚จ์€ ์Šน์ธ์€ Runner๊ฐ€ ๊ทธ ์Šน์ธ์„ ์œ„ํ•ด ๋งŒ๋“  ํ”„๋กœ์„ธ์Šค๋กœ consumed๊ฐ€ ๋˜๊ฑฐ๋‚˜, Runner๊ฐ€ ์•„๋ฌด๊ฒƒ๋„ ๋งŒ๋“ค์ง€ ์•Š์•˜์Œ์ด ์ฆ๋ช…๋˜๋ฉด ๋‹ค์‹œ granted๊ฐ€ ๋œ๋‹ค. ์„ธ ๊ฐ€์ง€๊ฐ€ ๋งž์ง€ ์•Š๊ฑฐ๋‚˜ ๊ธฐ๋ก์ด ๋‹ค๋ฅธ Runner๋ฅผ ๊ฐ€๋ฆฌํ‚ค๋Š” workspace๋Š” ์šด์˜์ž๊ฐ€ ํŒ๋‹จํ•  ๋•Œ๊นŒ์ง€ ๊ณ„์† ๊ฑฐ๋ถ€๋œ๋‹ค. ๋‹ค์„ฏ ๋ฒˆ์งธ ๋‹จ์œ„ P1R-C05๋Š” [Runner ์†์‹ค](runner-isolation.md#runner-loss) ์ฒ˜๋ฆฌ๋ฅผ ๋”ํ•œ๋‹ค. Runner๊ฐ€ kill๋˜๊ฑฐ๋‚˜, ํฌ๋ž˜์‹œํ•˜๊ฑฐ๋‚˜, ๋ฉˆ์ถ”๊ฑฐ๋‚˜, ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•˜๋ฉด gateway๋Š” ๊ทธ ์‹คํ–‰๋“ค์— ๋Œ€ํ•œ `process_status`์™€ `read_process`์— ๊ธฐ๋ก์œผ๋กœ `lost_reason`๊ณผ ํ•จ๊ป˜ ๋‹ตํ•˜๊ณ , ๋๋‚˜์ง€ ์•Š์€ ์‹คํ–‰์€ `unknown`์ด ๋˜์–ด ์šด์˜์ž๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฐพ๊ณ  `codespace-mcp records`๋กœ ๊ธฐ๋ก์„ ํ’€ ๋•Œ๊นŒ์ง€ workspace๋ฅผ ์ ์œ ํ•œ๋‹ค([์šด์˜](operations.md#runner-loss)). `required` ์‹คํ–‰์˜ ๊ธฐ๋ก์€ DevGuard๊ฐ€ ๊ทธ attempt๋ฅผ ๋”๋Š” ๊ณผ๊ธˆํ•˜์ง€ ์•Š์„ ๋•Œ์—๋งŒ ์šด์˜์ž๊ฐ€ ํ‘ผ๋‹ค. DevGuard๋Š” attempt๋ฅผ ์†Œ์œ ํ•œ ์ธ์Šคํ„ด์Šค์˜ ์กฐํšŒ์—๋งŒ ๋‹ตํ•˜๋ฏ€๋กœ, ์ด๋Š” DevGuard ๊ด€๋ฆฌ์ž์˜ quiescence๋กœ๋งŒ ํ™•์ธํ•  ์ˆ˜ ์žˆ๋‹ค. ๊ทธ๋•Œ๊นŒ์ง€ DevGuard๋Š” ์žƒ์€ Runner์˜ ์ธ์Šคํ„ด์Šค๋ฅผ ์œ ์ง€ํ•˜๋ฏ€๋กœ, ์ƒˆ Runner๋Š” `codespace` ์†Œ๋น„์ž์˜ ์œ ์ผํ•œ ์ธ์Šคํ„ด์Šค๋กœ ๋“ฑ๋กํ•  ์ˆ˜ ์—†๋‹ค. ๋งˆ์ง€๋ง‰ ๋‹จ์œ„๊ฐ€ qualificationํ•œ ๋’ค์—์•ผ ์ด ๋ชจ๋“œ๋ฅผ ์ง€์›ํ•œ๋‹ค. ์ด ๋ชจ๋“œ์—์„œ๋Š” Runner๊ฐ€ DevGuard์˜ ์‹คํ–‰ ์†Œ์œ ์ž์ด๋ฉฐ, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--์‹คํ–‰-์†Œ์œ ์ž์ธ-runner์˜-๋‹จ์ผ-๋“ฑ๋ก)์˜ ๋ณด์™„๋Œ€๋กœ ์†Œ๋น„์ž secret์„ ์ž๊ธฐ ์ž๊ฒฉ ํŒŒ์ผ์—์„œ ์ง์ ‘ ์ฝ๋Š”๋‹ค. ๊ทธ gateway๋Š” Runner๊ฐ€ ๋“ฑ๋กํ•˜๊ณ  launch helper๋กœ ์‹คํ–‰ํ•œ๋‹ค๊ณ  ์•Œ๋ฆด ๋•Œ๋งŒ `required` workspace๋ฅผ ๋ฐ›์•„๋“ค์ธ๋‹ค. ์ด ์ ˆ์˜ ๋‚˜๋จธ์ง€๋Š” ๊ณ„ํšํ•œ ๋ฒ”์œ„๋‹ค. P1-RECOVERY๋Š” ์šด์˜์ž๊ฐ€ ์„ ํƒํ•˜๋Š” ๋…๋ฆฝ Runner ๋ชจ๋“œ์™€ ๋ช…์‹œ์ ์ธ capability๋ฅผ ๋„์ž…ํ•œ๋‹ค. ๊ธฐ์กด ๋ชจ๋“œ์˜ ์ข…๋ฃŒยท์—ฐ๊ฒฐ ์†Œ์‹ค ๊ณ„์•ฝ์€ ์œ ์ง€ํ•œ๋‹ค. InProcess๋Š” Gateway์™€ PID๋ฅผ ๊ณต์œ ํ•˜๋ฏ€๋กœ Gateway ์žฌ์‹œ์ž‘ ์ค‘ live ๋ณต๊ตฌ ๋Œ€์ƒ์—์„œ ์ œ์™ธํ•œ๋‹ค. diff --git a/docs/ko/error-codes.md b/docs/ko/error-codes.md index f3ff0fe..30a1716 100644 --- a/docs/ko/error-codes.md +++ b/docs/ko/error-codes.md @@ -57,9 +57,9 @@ | `APPROVAL_NOT_FOUND` | ์•Œ ์ˆ˜ ์—†๋Š” ํ™•์ธ ํ™€๋“œ ID | | `APPROVAL_CONFLICT` | ํ™€๋“œ๊ฐ€ ์•„์ง ๋Œ€๊ธฐ ์ค‘์ด๊ฑฐ๋‚˜ ์ด๋ฏธ ๊ฒฐ์ •๋˜์—ˆ๊ฑฐ๋‚˜, ์žฌ๊ฐœ๊ฐ€ ์ด๋ฏธ ์ง„ํ–‰ ์ค‘์ž„ | | `APPROVAL_AMBIGUOUS` | ์žฌ๊ฐœ๊ฐ€ ์ค‘๋‹จ๋˜์–ด ๋‹จ๋ง ๊ฒฐ๊ณผ๊ฐ€ ๋””์Šคํฌ์— ์—†์Œ. `approval_id` ํฌํ•จ. ํŒจ์น˜๋Š” `operation_id`๋„ ์žˆ์„ ์ˆ˜ ์žˆ์Œ | -| `RECOVERY_PENDING` | ์ด ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ Claimํ•˜๊ธฐ ์ „์˜ controller๊ฐ€ [๋…๋ฆฝ Runner](runner-isolation.md#reconciliation)์—์„œ ์ด workspace์˜ ์ž‘์—…์„ ์‹œ์ž‘ํ–ˆ๊ณ , ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๊ทธ controller๊ฐ€ ๋‚จ๊ธด ๊ฒƒ์„ ์•„์ง ๋Œ€์กฐํ•˜์ง€ ์•Š์Œ. ์—ฐ๊ฒฐ์„ ์žƒ๊ธฐ ์ „์˜ ์ด ๊ฒŒ์ดํŠธ์›จ์ด ์ž์‹ ๋„ ํฌํ•จํ•จ. ๋‚จ์€ ํ”„๋กœ์„ธ์Šค๋Š” ์กฐํšŒ, ์ถœ๋ ฅ ์ฝ๊ธฐ, stdin ์“ฐ๊ธฐ, ํฌ๊ธฐ ๋ณ€๊ฒฝ, ์ข…๋ฃŒ๊ฐ€ ๊ฐ€๋Šฅํ•˜์ง€๋งŒ, ๋Œ€์กฐ๋ฅผ ๋งˆ์น˜๊ธฐ ์ „์—๋Š” ๊ทธ workspace์—์„œ ์‹คํ–‰, ํŒจ์น˜, ์Šน์ธ ์žฌ๊ฐœ๋ฅผ ์‹œ์ž‘ํ•˜์ง€ ์•Š์Œ. ์Šน์ธ์€ ๊ทธ๋Œ€๋กœ ๋‚จ์Œ. ๋ฉ”์‹œ์ง€๊ฐ€ ์ด์œ ๋ฅผ ์•Œ๋ฆผ. ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๋‹ต์„ ๊ธฐ๋‹ค๋ฆฌ๋Š” ์ค‘์ด๋ฉด ์Šค์Šค๋กœ ๋‹ค์‹œ ์‹œ๋„ํ•˜๋ฏ€๋กœ ์ž ์‹œ ๋’ค์— ์žฌ์‹œ๋„ํ•จ. ๊ธฐ๋ก, Runner, DevGuard๊ฐ€ ์„œ๋กœ ๋งž์ง€ ์•Š๊ฑฐ๋‚˜ ๊ธฐ๋ก์ด ๋‹ค๋ฅธ Runner๋ฅผ ๊ฐ€๋ฆฌํ‚ค๋ฉด ์šด์˜์ž๊ฐ€ ํ•ด๊ฒฐํ•˜๊ธฐ ์ „์—๋Š” ์žฌ์‹œ๋„ํ•ด๋„ ์†Œ์šฉ์—†์Œ. ์ ์œ ๊ฐ€ ์•„๋‹˜ | +| `RECOVERY_PENDING` | ์ด ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ Claimํ•˜๊ธฐ ์ „์˜ controller๊ฐ€ [๋…๋ฆฝ Runner](runner-isolation.md#reconciliation)์—์„œ ์ด workspace์˜ ์ž‘์—…์„ ์‹œ์ž‘ํ–ˆ๊ณ , ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๊ทธ controller๊ฐ€ ๋‚จ๊ธด ๊ฒƒ์„ ์•„์ง ๋Œ€์กฐํ•˜์ง€ ์•Š์Œ. ์—ฐ๊ฒฐ์„ ์žƒ๊ธฐ ์ „์˜ ์ด ๊ฒŒ์ดํŠธ์›จ์ด ์ž์‹ ๋„ ํฌํ•จํ•จ. ๋‚จ์€ ํ”„๋กœ์„ธ์Šค๋Š” ์กฐํšŒ, ์ถœ๋ ฅ ์ฝ๊ธฐ, stdin ์“ฐ๊ธฐ, ํฌ๊ธฐ ๋ณ€๊ฒฝ, ์ข…๋ฃŒ๊ฐ€ ๊ฐ€๋Šฅํ•˜์ง€๋งŒ, ๋Œ€์กฐ๋ฅผ ๋งˆ์น˜๊ธฐ ์ „์—๋Š” ๊ทธ workspace์—์„œ ์‹คํ–‰, ํŒจ์น˜, ์Šน์ธ ์žฌ๊ฐœ๋ฅผ ์‹œ์ž‘ํ•˜์ง€ ์•Š์Œ. ์Šน์ธ์€ ๊ทธ๋Œ€๋กœ ๋‚จ์Œ. ๋ฉ”์‹œ์ง€๊ฐ€ ์ด์œ ๋ฅผ ์•Œ๋ฆผ. ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๋‹ต์„ ๊ธฐ๋‹ค๋ฆฌ๋Š” ์ค‘์ด๋ฉด ์Šค์Šค๋กœ ๋‹ค์‹œ ์‹œ๋„ํ•˜๋ฏ€๋กœ ์ž ์‹œ ๋’ค์— ์žฌ์‹œ๋„ํ•จ. ๊ธฐ๋ก, Runner, DevGuard๊ฐ€ ์„œ๋กœ ๋งž์ง€ ์•Š์œผ๋ฉด ์šด์˜์ž๊ฐ€ ํ•ด๊ฒฐํ•˜๊ธฐ ์ „์—๋Š”, ๊ธฐ๋ก์ด ์žƒ์€ Runner์˜ ๊ฒƒ์ด๋ฉด ์šด์˜์ž๊ฐ€ ๊ทธ์ค‘ ์•„๋ฌด๊ฒƒ๋„ ์‹คํ–‰ ์ค‘์ด์ง€ ์•Š์Œ์„ ํ™•์ธํ•˜๊ณ  [๊ธฐ๋ก์„ ํ’€๊ธฐ](operations.md#runner-loss) ์ „์—๋Š” ์žฌ์‹œ๋„ํ•ด๋„ ์†Œ์šฉ์—†์Œ. ์ ์œ ๊ฐ€ ์•„๋‹˜ | | `RUNNER_RECONNECTING` | ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์—ˆ๊ณ  Runner๋ฅผ ๋‹ค์‹œ Claimํ•˜๋Š” ์ค‘์ž„. ์š”์ฒญ์€ ๋ณด๋‚ด์ง€ ์•Š์•˜์Œ. ๊ทธ ์š”์ฒญ์„ ์œ„ํ•ด ์‹œ์ž‘ํ•˜๊ฑฐ๋‚˜ ์˜ˆ์•ฝํ•˜๊ฑฐ๋‚˜ ๊ธฐ๋กํ•œ ๊ฒƒ์€ ์—†๊ณ , ์žฌ๊ฐœํ•˜๋ ค๋˜ ์Šน์ธ์€ ์†Œ๋น„๋˜์ง€ ์•Š์Œ. Runner์˜ ํ”„๋กœ์„ธ์Šค๋Š” ๊ณ„์† ์‹คํ–‰๋จ. ์ž ์‹œ ๋’ค ์žฌ์‹œ๋„ํ•  ๊ฒƒ | -| `RUNNER_DISCONNECTED` | ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์—ˆ๊ณ  ๋‹ค์‹œ Claimํ•  ์ˆ˜ ์—†์Œ. ์†Œ์ผ“์—์„œ ์—ฐ๊ฒฐ์„ ๋ฐ›์ง€ ์•Š๊ฑฐ๋‚˜, ๋‹ค๋ฅธ Runner๊ฐ€ ์‘๋‹ตํ•˜๊ฑฐ๋‚˜, ๊ธฐ๋‹ค๋ ค๋„ ํ’€๋ฆฌ์ง€ ์•Š๋Š” ์ด์œ ๋กœ Runner๊ฐ€ Claim์„ ๊ฑฐ๋ถ€ํ•จ. ์š”์ฒญ์€ ๋ณด๋‚ด์ง€ ์•Š์•˜๊ณ  Runner์˜ ํ”„๋กœ์„ธ์Šค๋Š” ๊ณ„์† ์‹คํ–‰๋จ. ๊ทธ ํ”„๋กœ์„ธ์Šค์— ๋‹ฟ์œผ๋ ค๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด๋ฅผ ์žฌ์‹œ์ž‘ํ•  ๊ฒƒ | +| `RUNNER_DISCONNECTED` | ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์—ˆ๊ณ  ๋‹ค์‹œ Claimํ•  ์ˆ˜ ์—†์Œ. ์†Œ์ผ“์—์„œ ์—ฐ๊ฒฐ์„ ๋ฐ›์ง€ ์•Š๊ฑฐ๋‚˜, ๋‹ค๋ฅธ Runner๊ฐ€ ์‘๋‹ตํ•˜๊ฑฐ๋‚˜, ๊ธฐ๋‹ค๋ ค๋„ ํ’€๋ฆฌ์ง€ ์•Š๋Š” ์ด์œ ๋กœ Runner๊ฐ€ Claim์„ ๊ฑฐ๋ถ€ํ•จ. ์š”์ฒญ์€ ๋ณด๋‚ด์ง€ ์•Š์•˜์Œ. Runner๊ฐ€ Claim์„ ๊ฑฐ๋ถ€ํ–ˆ๋‹ค๋ฉด ๊ทธ ํ”„๋กœ์„ธ์Šค๋Š” ๊ณ„์† ์‹คํ–‰๋˜๋ฉฐ, ๊ทธ ํ”„๋กœ์„ธ์Šค์— ๋‹ฟ์œผ๋ ค๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด๋ฅผ ์žฌ์‹œ์ž‘ํ•  ๊ฒƒ. ๊ทธ ๋ฐ–์˜ ๊ฒฝ์šฐ๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ Runner๋ฅผ ์žƒ์€ ๊ฒƒ์ž„. ๊ทธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์–ด๋–ป๊ฒŒ ๋˜์—ˆ๋Š”์ง€ ์•Œ ์ˆ˜ ์—†๊ณ  ์ผ๋ถ€๋Š” ์•„์ง ์‹คํ–‰ ์ค‘์ผ ์ˆ˜ ์žˆ์Œ. `process_status`์™€ `read_process`๋Š” ๊ฒŒ์ดํŠธ์›จ์ด์˜ ๊ธฐ๋ก์œผ๋กœ `lost_reason`๊ณผ ํ•จ๊ป˜ ๋‹ตํ•˜๋ฉฐ, ์šด์˜์ž๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฐพ๊ณ  workspace๋ฅผ ํ’‚([Runner ์†์‹ค](operations.md#runner-loss)) | | `STALE_CONTROLLER` | [๋…๋ฆฝ Runner](runner-isolation.md#claim)๊ฐ€ ๋‹ค๋ฅธ Claim์œผ๋กœ ๋ฐ”๋€ Claim ์•„๋ž˜์—์„œ ๋ณด๋‚ธ ์š”์ฒญ, ๋˜๋Š” Runner๋ฅผ ์ œ์–ดํ•˜์ง€ ์•Š๋Š” ์—ฐ๊ฒฐ์˜ ์š”์ฒญ์„ ๊ฑฐ๋ถ€ํ•จ. ์š”์ฒญ์ด ์š”๊ตฌํ•œ ์ผ์€ ํ•˜๋‚˜๋„ ํ•˜์ง€ ์•Š์•˜๊ณ , ์š”์ฒญ์ด ์žฌ๊ฐœํ•œ ์Šน์ธ์€ ๋‹ค์‹œ ์Šน์ธ๋œ ์ƒํƒœ๊ฐ€ ๋จ. ์žฌ์‹œ๋„ํ•˜๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด์˜ ํ˜„์žฌ Claim ์•„๋ž˜์—์„œ ์š”์ฒญ์„ ๋ณด๋ƒ„ | | `INTERNAL` | ์Šค์ผ€์ค„๋Ÿฌยท์Šคํ† ์–ด ๋ถˆ๋ณ€์กฐ๊ฑด ์‹คํŒจ, ๋˜๋Š” [๋…๋ฆฝ Runner](runner-isolation.md#process-records)์— ์—ฐ๊ฒฐํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์‹คํ–‰์„ ๊ธฐ๋กํ•˜์ง€ ๋ชปํ•ด ์‹œ์ž‘ํ•˜์ง€ ์•Š์Œ. ์ ์œ ๊ฐ€ ์•„๋‹ˆ๋ฏ€๋กœ `WORKSPACE_BUSY`๋กœ ์žฌ์‹œ๋„ํ•˜์ง€ ๋ง ๊ฒƒ | diff --git a/docs/ko/execution-substrate.md b/docs/ko/execution-substrate.md index 78e6a69..02c0428 100644 --- a/docs/ko/execution-substrate.md +++ b/docs/ko/execution-substrate.md @@ -46,7 +46,7 @@ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๋‚ด๋ถ€ Runner ์š”์ฒญ์— ์ž‘์—… ๊ณต๊ฐ„ ๋ฃจํŠธ cwd, ๋Ÿฌ๋„ˆ ํ™˜๊ฒฝ ๊ธฐ๋ณธ๊ฐ’, ์‹œ๊ฐ„ยท์ถœ๋ ฅ ์ œํ•œ, PTY ์„ ํƒ, ์ •์ฑ…์„ ์ฑ„์›๋‹ˆ๋‹ค. spawn ์‹œ ๊ณต๊ฐœ๋œ ํ„ฐ๋ฏธ๋„ ์˜ต์…˜์€ `tty`๋ฟ์ž…๋‹ˆ๋‹ค. `tty_size`๋Š” spawn ์ธ์ž๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ์‹คํ–‰ ์ค‘์ธ PTY๋Š” `process_resize`๋กœ ํฌ๊ธฐ๋ฅผ ๋ฐ”๊ฟ‰๋‹ˆ๋‹ค. ๊ณต๊ฐœ ํ˜ธ์ถœ์€ ์ž„์˜์˜ cwdยทํ™˜๊ฒฝ๋ณ€์ˆ˜ยท์ œํ•œ ์‹œ๊ฐ„ ๋ณ€๊ฒฝ์„ ๋ฐ›์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ [์šด์˜](operations.md), ๊ฒฐ๊ณผ ์ฒ˜๋ฆฌ๋Š” [Agent Loop ์—ฐ๋™](agent-integration.md)์„ ์ฐธ๊ณ ํ•˜์„ธ์š”. -`exec_command`๋Š” ๋””์ŠคํŒจ์น˜ ์‹๋ณ„(`process_id`, `dispatch_status`)๋งŒ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. ์ข…๋ฃŒ ํŒ์ •์€ `process_status`์˜ `running`/`exited`์™€ termination ๋ฉ”ํƒ€๋ฐ์ดํ„ฐ๋ฅผ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค. ์‹คํ–‰ ์ค‘์ธ PTY ํฌ๊ธฐ๋Š” `process_resize`๋กœ ๋ฐ”๊ฟ‰๋‹ˆ๋‹ค. `read_process`๋Š” `output_lost`์™€ `retained_from`์„ ํฌํ•จํ•œ ์ถœ๋ ฅ์„ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. EOF๋Š” ์„ฑ๊ณต์ด ์•„๋‹™๋‹ˆ๋‹ค. ํ•ธ๋“ค์ด ๋งŒ๋ฃŒ๋œ ๋’ค์˜ ์กฐํšŒ๋Š” ์ƒˆ ์ƒํƒœ๊ฐ€ ์•„๋‹ˆ๋ผ `PROCESS_NOT_FOUND`์ž…๋‹ˆ๋‹ค. ํŒŒ์ดํ”„ ํ”„๋กœ์„ธ์Šค์˜ ํฌ๊ธฐ ๋ณ€๊ฒฝ์€ `PROCESS_NOT_TTY`, ์ข…๋ฃŒ๋œ ํ•ธ๋“ค์€ `PROCESS_NOT_RUNNING`์ž…๋‹ˆ๋‹ค. Linux ์ƒŒ๋“œ๋ฐ•์Šค์—์„œ wait ์ƒํƒœ๋Š” ๊ด€๋ฆฌ ์ž์‹(ํ—ฌํผ argv)์˜ ์ฝ”๋“œ์ด๋ฉฐ, ์‚ฌ์šฉ์ž argv์™€ ๋™์ผํ•˜๋‹ค๊ณ  ๋ฌธ์„œํ™”ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. +`exec_command`๋Š” ๋””์ŠคํŒจ์น˜ ์‹๋ณ„(`process_id`, `dispatch_status`)๋งŒ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. ์ข…๋ฃŒ ํŒ์ •์€ `process_status`์˜ `running`/`exited`์™€ termination ๋ฉ”ํƒ€๋ฐ์ดํ„ฐ๋ฅผ ์‚ฌ์šฉํ•˜๋ฉฐ, [๋…๋ฆฝ Runner](runner-isolation.md#runner-loss)๋ฅผ ์žƒ์€ ๋’ค์—๋Š” `lost_reason`๊ณผ ํ•จ๊ป˜ `unknown`์ด ์˜ฌ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ์‹คํ–‰ ์ค‘์ธ PTY ํฌ๊ธฐ๋Š” `process_resize`๋กœ ๋ฐ”๊ฟ‰๋‹ˆ๋‹ค. `read_process`๋Š” `output_lost`์™€ `retained_from`, ์ถœ๋ ฅ์ด ๋น ์กŒ์œผ๋ฉด `lost_reason`์„ ํฌํ•จํ•œ ์ถœ๋ ฅ์„ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. EOF๋Š” ์„ฑ๊ณต์ด ์•„๋‹™๋‹ˆ๋‹ค. ํ•ธ๋“ค์ด ๋งŒ๋ฃŒ๋œ ๋’ค์˜ ์กฐํšŒ๋Š” ์ƒˆ ์ƒํƒœ๊ฐ€ ์•„๋‹ˆ๋ผ `PROCESS_NOT_FOUND`์ž…๋‹ˆ๋‹ค. ๋‹ค๋งŒ ๋…๋ฆฝ Runner์— ์—ฐ๊ฒฐํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ทธ ์‹คํ–‰์˜ ๊ธฐ๋ก์œผ๋กœ `lost_reason` `retention_expired`์™€ ํ•จ๊ป˜ ๋‹ตํ•ฉ๋‹ˆ๋‹ค. ํŒŒ์ดํ”„ ํ”„๋กœ์„ธ์Šค์˜ ํฌ๊ธฐ ๋ณ€๊ฒฝ์€ `PROCESS_NOT_TTY`, ์ข…๋ฃŒ๋œ ํ•ธ๋“ค์€ `PROCESS_NOT_RUNNING`์ž…๋‹ˆ๋‹ค. Linux ์ƒŒ๋“œ๋ฐ•์Šค์—์„œ wait ์ƒํƒœ๋Š” ๊ด€๋ฆฌ ์ž์‹(ํ—ฌํผ argv)์˜ ์ฝ”๋“œ์ด๋ฉฐ, ์‚ฌ์šฉ์ž argv์™€ ๋™์ผํ•˜๋‹ค๊ณ  ๋ฌธ์„œํ™”ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ž‘์—… ๊ณต๊ฐ„ ์ž ๊ธˆ์€ ํŒจ์น˜์™€ ๋ช…๋ น์ด ๋™์‹œ์— ํŒŒ์ผ์„ ๋ณ€๊ฒฝํ•˜์ง€ ๋ชปํ•˜๊ฒŒ ํ•ฉ๋‹ˆ๋‹ค. FIFO ์ˆœ์„œ๋Š” MCP ๋ฉ”์‹œ์ง€ ๋„์ฐฉ์ด ์•„๋‹ˆ๋ผ, ์ ๊ฒฉ ์š”์ฒญ์ด ์ž์›์„ `acquire()`ํ•  ๋•Œ ์‹œ์ž‘๋ฉ๋‹ˆ๋‹ค. ๊ฐ™์€ ์ž‘์—… ๊ณต๊ฐ„์—์„œ ์š”์ฒญ์ด ์†Œ์œ ํ•œ ํŒจ์น˜ยทexec๋Š” ํ˜„์žฌ ์š”์ฒญ์ด ์ž„๋Œ€๋ฅผ ๋†“์„ ๋•Œ๊นŒ์ง€ ๋ฉ”๋ชจ๋ฆฌ FIFO๋กœ ๊ธฐ๋‹ค๋ฆฝ๋‹ˆ๋‹ค. exec๊ฐ€ ๋ผ์ด๋ธŒ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋˜๋ฉด ์ด๋ฏธ ์ค„ ์„œ ์žˆ๋˜ ๋Œ€๊ธฐ์ž์™€ ์ดํ›„ ๋„์ฐฉ์€ `WORKSPACE_BUSY`๋กœ ๊ฑฐ์ ˆ๋ฉ๋‹ˆ๋‹ค. ํ๊ฐ€ ๊ฐ€๋“ ์ฐจ๋ฉด `RESOURCE_QUEUE_FULL`์ž…๋‹ˆ๋‹ค. ์ด ๋Œ€๊ธฐ๋Š” SQLite๋‚˜ ์Šค๋ ˆ๋“œ ํ‚ค ํ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ๋ช…๋ น ์‹คํ–‰ ์ค‘์—๋„ ์ฝ๊ธฐ์™€ ๊ฒ€์ƒ‰์€ ๊ฐ€๋Šฅํ•˜๋ฏ€๋กœ ํŒŒ์ผ I/O๋Š” ์‚ฌ์ „ ๊ฒฝ๋กœ ๊ฒ€์‚ฌ์—๋งŒ ์˜์กดํ•˜์ง€ ์•Š๊ณ  ํŒŒ์ผ์„ ์—ฌ๋Š” ์‹œ์ ์˜ ์‹ฌ๋ณผ๋ฆญ ๋งํฌ ๋ณ€๊ฒฝ๋„ ๊ฑฐ๋ถ€ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. Runner ํŒŒ์ผ ์ž‘์—…์€ `codespace-fs`, ํŒจ์น˜ ์ ์šฉ์€ ๋ณ„๋„ ํŒจ์น˜ ๋„์šฐ๋ฏธ๋ฅผ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค. `operation_status`๋Š” ๊ธฐ๋ก๋œ ํŒจ์น˜ ์›์žฅ(`kind`๋Š” `patch`)์„ ์กฐํšŒํ•˜๋ฉฐ, ์‹คํ–‰ ์ค‘์ธ ๋ช…๋ น์€ `process_id`๋กœ๋งŒ ๋‹ค๋ฃจ๊ณ  ์ด ์กฐํšŒ๋กœ ๋ณต๊ตฌํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. diff --git a/docs/ko/operations.md b/docs/ko/operations.md index 18951ae..95da557 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -115,7 +115,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -worker๋Š” ๊ฐ™์€ ํ˜ธ์ŠคํŠธ์—์„œ ์‹คํ–‰ํ•˜๋Š” ๋ณ„๋„ ํ”„๋กœ์„ธ์Šค์ด๋ฉฐ ์ปจํ…Œ์ด๋„ˆ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ „์šฉ ์†Œ์ผ“ ๋””๋ ‰ํ„ฐ๋ฆฌ๋ฅผ ๋งŒ๋“ค๊ณ  ์ž์‹ ํ”„๋กœ์„ธ์Šค๋ฅผ ๊ด€๋ฆฌํ•ฉ๋‹ˆ๋‹ค. worker ์—ฐ๊ฒฐ์ด ๋Š๊ธฐ๊ฑฐ๋‚˜ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ข…๋ฃŒ๋˜๋ฉด ํ•ด๋‹น worker์˜ ํ”„๋กœ์„ธ์Šค๋„ ์ข…๋ฃŒ๋ฉ๋‹ˆ๋‹ค. ์žฌ์ ‘์†๊ณผ ํ”„๋กœ์„ธ์Šค ๋ณต๊ตฌ๋Š” ์ง€์›ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. MCP/HTTP ํด๋ผ์ด์–ธํŠธ ๋Š๊น€์€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฃฝ์ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ `in-process`์ž…๋‹ˆ๋‹ค. ํ”„๋กœ์„ธ์Šค๊ฐ€ ๊ฒŒ์ดํŠธ์›จ์ด๋ณด๋‹ค ์˜ค๋ž˜ ์‚ด๊ณ , ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ž‘์—… ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค์— ๊ธฐ๋กํ•˜๋ฉฐ, ์—ฐ๊ฒฐ์„ ์žƒ์€ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๋‹ค์‹œ Claimํ•œ ๋’ค ์ž๊ธฐ ๊ธฐ๋ก๊ณผ ๋Œ€์กฐํ•˜๋Š” [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)๋Š” ์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. +worker๋Š” ๊ฐ™์€ ํ˜ธ์ŠคํŠธ์—์„œ ์‹คํ–‰ํ•˜๋Š” ๋ณ„๋„ ํ”„๋กœ์„ธ์Šค์ด๋ฉฐ ์ปจํ…Œ์ด๋„ˆ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ „์šฉ ์†Œ์ผ“ ๋””๋ ‰ํ„ฐ๋ฆฌ๋ฅผ ๋งŒ๋“ค๊ณ  ์ž์‹ ํ”„๋กœ์„ธ์Šค๋ฅผ ๊ด€๋ฆฌํ•ฉ๋‹ˆ๋‹ค. worker ์—ฐ๊ฒฐ์ด ๋Š๊ธฐ๊ฑฐ๋‚˜ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ข…๋ฃŒ๋˜๋ฉด ํ•ด๋‹น worker์˜ ํ”„๋กœ์„ธ์Šค๋„ ์ข…๋ฃŒ๋ฉ๋‹ˆ๋‹ค. ์žฌ์ ‘์†๊ณผ ํ”„๋กœ์„ธ์Šค ๋ณต๊ตฌ๋Š” ์ง€์›ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. MCP/HTTP ํด๋ผ์ด์–ธํŠธ ๋Š๊น€์€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฃฝ์ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ `in-process`์ž…๋‹ˆ๋‹ค. ํ”„๋กœ์„ธ์Šค๊ฐ€ ๊ฒŒ์ดํŠธ์›จ์ด๋ณด๋‹ค ์˜ค๋ž˜ ์‚ด๊ณ , ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ž‘์—… ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค์— ๊ธฐ๋กํ•˜๋ฉฐ, ์—ฐ๊ฒฐ์„ ์žƒ์€ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๋‹ค์‹œ Claimํ•œ ๋’ค ์ž๊ธฐ ๊ธฐ๋ก๊ณผ ๋Œ€์กฐํ•˜๋Š” [๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)๋Š” ์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ์ด๋Ÿฐ Runner ์ž์ฒด๋ฅผ ์žƒ์œผ๋ฉด ์šด์˜์ž๊ฐ€ `codespace-mcp records`๋กœ ๊ทธ Runner๊ฐ€ ์‹คํ–‰ํ•˜๋˜ ๊ฒƒ์„ ์ •๋ฆฌํ•ฉ๋‹ˆ๋‹ค([Runner ์†์‹ค](#runner-loss)). @@ -130,8 +130,8 @@ worker๋Š” ๊ฐ™์€ ํ˜ธ์ŠคํŠธ์—์„œ ์‹คํ–‰ํ•˜๋Š” ๋ณ„๋„ ํ”„๋กœ์„ธ์Šค์ด๋ฉฐ ์ปจํ…Œ | `CODESPACE_OPERATIONS_DB` ๋ฏธ์„ค์ • | ํŒจ์น˜ ์ž‘์—…๊ณผ ์ง€์‹œ ํ๋ฅผ ๋ฉ”๋ชจ๋ฆฌ์— ๋ณด๊ด€ํ•˜๋ฉฐ ์žฌ์‹œ์ž‘ ์‹œ ์‚ฌ๋ผ์ง. ๋…๋ฆฝ Runner์— ์—ฐ๊ฒฐํ•˜๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์ด ์„ค์ • ์—†์ด ์‹œ์ž‘ํ•˜์ง€ ์•Š์Œ | | `CODESPACE_PROCESS_TIMEOUT_SECS` | ์–‘์˜ ์ •์ˆ˜. ๊ธฐ๋ณธ 30์ดˆ. ๋Ÿฌ๋„ˆ ํ™˜๊ฒฝ์— ์„ค์ • | | `CODESPACE_MAX_PROCESSES` | ๋Ÿฌ๋„ˆ ์ „์ฒด์˜ ์‹คํ–‰ ์ค‘ ํ”„๋กœ์„ธ์Šค ๊ธฐ๋ณธ ์ƒํ•œ 8๊ฐœ. ๋„˜๋Š” ์š”์ฒญ์€ ํ”„๋กœ์„ธ์Šค๋ฅผ ๋งŒ๋“ค๊ธฐ ์ „์— ๊ฑฐ๋ถ€. ์ž‘์—… ๊ณต๊ฐ„๋ณ„ ์ ์œ  ๊ทœ์น™๋„ ์ ์šฉ | -| ํ”„๋กœ์„ธ์Šค ์ถœ๋ ฅ | ๋งˆ์ง€๋ง‰ 256 KiB ๋ณด๊ด€. stdout/stderr๋ฅผ ํ•ฉ์นจ. `read_process`๋Š” `output_lost`์™€ `retained_from`์„ ๋ณด๊ณ ํ•˜๊ณ , ์ข…๋ฃŒ๋Š” `process_status`๋กœ ์กฐํšŒ. ์‹คํ–‰ ์ค‘์ธ PTY ํฌ๊ธฐ๋Š” `process_resize`๋กœ ๋ณ€๊ฒฝ | -| ์ข…๋ฃŒ๋œ ํ•ธ๋“ค | ๊ธฐ๋ณธ ์ตœ๋Œ€ 15๋ถ„, ์ตœ๋Œ€ 64๊ฐœ ๋ณด๊ด€. ์˜๊ตฌ ์ €์žฅํ•˜์ง€ ์•Š์Œ. ๋…๋ฆฝ Runner๋Š” ๋๋‚œ ํ”„๋กœ์„ธ์Šค์˜ ์ƒํƒœ์™€ ์ถœ๋ ฅ์„ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ํ™•์ธํ•  ๋•Œ๊นŒ์ง€ ๋ณด๊ด€ํ•˜๋ฉฐ, ํ•ธ๋“ค์„ ๋†“์€ ๋’ค์—๋Š” ์ตœ๋Œ€ 64๊ฐœ๊นŒ์ง€ ๋ณด๊ด€ํ•จ([๋๋‚œ ํ”„๋กœ์„ธ์Šค](runner-isolation.md#reconciliation)) | +| ํ”„๋กœ์„ธ์Šค ์ถœ๋ ฅ | ๋งˆ์ง€๋ง‰ 256 KiB ๋ณด๊ด€. stdout/stderr๋ฅผ ํ•ฉ์นจ. `read_process`๋Š” `output_lost`์™€ `retained_from`์„ ๋ณด๊ณ ํ•˜๊ณ , ์ถœ๋ ฅ์ด ๋น ์ง„ ์ด์œ ๋Š” `lost_reason`์ด ์•Œ๋ ค ์คŒ. ์ข…๋ฃŒ๋Š” `process_status`๋กœ ์กฐํšŒ. ์‹คํ–‰ ์ค‘์ธ PTY ํฌ๊ธฐ๋Š” `process_resize`๋กœ ๋ณ€๊ฒฝ | +| ์ข…๋ฃŒ๋œ ํ•ธ๋“ค | ๊ธฐ๋ณธ ์ตœ๋Œ€ 15๋ถ„, ์ตœ๋Œ€ 64๊ฐœ ๋ณด๊ด€. ์˜๊ตฌ ์ €์žฅํ•˜์ง€ ์•Š์Œ. ๋…๋ฆฝ Runner๋Š” ๋๋‚œ ํ”„๋กœ์„ธ์Šค์˜ ์ƒํƒœ์™€ ์ถœ๋ ฅ์„ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ํ™•์ธํ•  ๋•Œ๊นŒ์ง€ ๋ณด๊ด€ํ•˜๋ฉฐ, ํ•ธ๋“ค์„ ๋†“์€ ๋’ค์—๋Š” ์ตœ๋Œ€ 64๊ฐœ๊นŒ์ง€ ๋ณด๊ด€ํ•จ([๋๋‚œ ํ”„๋กœ์„ธ์Šค](runner-isolation.md#reconciliation)). ๊ทธ ๋’ค์—๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ถœ๋ ฅ ์—†์ด ์‹คํ–‰ ๊ธฐ๋ก์œผ๋กœ ๋‹ตํ•˜๋ฉฐ `lost_reason`์€ `retention_expired` | ๋กœ๊ทธ์™€ ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค๋Š” ํ† ํฐยท๊ฒŒ์ดํŠธ์›จ์ด ์„ค์ •๊ณผ ๊ฐ™์ด ๊ด€๋ฆฌ ๋Œ€์ƒ ์ž‘์—… ๊ณต๊ฐ„ ๋ฐ–์— ๋‘์„ธ์š”. stderr ๋กœ๊ทธ์˜ ๋ณด๊ด€ยท์ˆœํ™˜์€ ์šด์˜์ž๊ฐ€ ๊ด€๋ฆฌํ•ฉ๋‹ˆ๋‹ค. Bearer ํ† ํฐ์„ ๋กœ๊ทธ๋‚˜ ์ปค๋ฐ‹์— ๋„ฃ์ง€ ๋งˆ์„ธ์š”. ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค๋ฅผ ์‚ญ์ œํ•˜๋ฉด ํŒจ์น˜ ์ค‘๋ณต ์‹คํ–‰ ๋ฐฉ์ง€ ๊ธฐ๋ก๊ณผ ํ™•์ธ ํ™€๋“œ ํ–‰๋„ ์‚ฌ๋ผ์ง‘๋‹ˆ๋‹ค. @@ -141,6 +141,43 @@ worker๋Š” ๊ฐ™์€ ํ˜ธ์ŠคํŠธ์—์„œ ์‹คํ–‰ํ•˜๋Š” ๋ณ„๋„ ํ”„๋กœ์„ธ์Šค์ด๋ฉฐ ์ปจํ…Œ ํŒจ์น˜ ์‘๋‹ต์„ ๋ฐ›์ง€ ๋ชปํ–ˆ๋‹ค๋ฉด `operation_id` ๋˜๋Š” `operation_key` ์ค‘ ํ•˜๋‚˜๋งŒ ์ง€์ •ํ•ด `operation_status`๋ฅผ ์กฐํšŒํ•ฉ๋‹ˆ๋‹ค. ์žฌ์‹œ์ž‘ ํ›„ ๋ฏธ์™„๋ฃŒ ๊ธฐ๋ก์€ `unknown`์ด ๋˜๋ฏ€๋กœ ํŒŒ์ผ์„ ํ™•์ธํ•œ ๋’ค ๋‹ค์Œ ํ–‰๋™์„ ๊ฒฐ์ •ํ•˜์„ธ์š”. ํ”„๋กœ์„ธ์Šค๋Š” `process_id`๋กœ ๊ด€๋ฆฌํ•˜๋ฉฐ `operation_status`๋กœ ๋ณต๊ตฌํ•  ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค. ํ”„๋กœ์„ธ์Šค๊ฐ€ ์ฃฝ์„ ๋•Œ `queued`์ด๋˜ ํ™•์ธ ํ™€๋“œ๋Š” ๋‹ค์‹œ acquireํ•ฉ๋‹ˆ๋‹ค. `resuming`์ด๋˜ ํ™€๋“œ๋Š” ํŒจ์น˜ ์›์žฅ์—์„œ ๋ณต๊ตฌํ•˜๊ฑฐ๋‚˜, ์ €์žฅ๋œ ๋‹จ๋ง ๊ฒฐ๊ณผ๋ฅผ ์žฌํ˜„ํ•˜๊ฑฐ๋‚˜, `APPROVAL_AMBIGUOUS`๋ฅผ ๋ฐ˜ํ™˜ํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. exec๋Š” ๋‹ค์‹œ spawnํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋…๋ฆฝ Runner์— ์—ฐ๊ฒฐํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” `resuming`์œผ๋กœ ๋‚จ์€ exec ํ™€๋“œ๋ฅผ [๋Œ€์กฐ](runner-isolation.md#reconciliation)ํ•  ๋•Œ ์ •๋ฆฌํ•ฉ๋‹ˆ๋‹ค. Runner๊ฐ€ ๊ทธ ํ™€๋“œ๋ฅผ ์œ„ํ•ด ๋งŒ๋“  ํ”„๋กœ์„ธ์Šค๋กœ consumed๊ฐ€ ๋˜๊ฑฐ๋‚˜, Runner๊ฐ€ ์•„๋ฌด๊ฒƒ๋„ ๋งŒ๋“ค์ง€ ์•Š์•˜์Œ์ด ์ฆ๋ช…๋˜๋ฉด ๋‹ค์‹œ granted๊ฐ€ ๋ฉ๋‹ˆ๋‹ค. [์žฌ์‹œ๋„์™€ ๋ณต๊ตฌ ๊ทœ์น™](agent-integration.md)์„ ์ฐธ๊ณ ํ•˜์„ธ์š”. + + +## Runner ์†์‹ค + +[๋…๋ฆฝ Runner](runner-isolation.md#independent-runner)์— ์—ฐ๊ฒฐํ•œ ๊ฒŒ์ดํŠธ์›จ์ด์—๋งŒ ํ•ด๋‹นํ•˜๋ฉฐ, ์ด ๋ชจ๋“œ๋Š” ์•„์ง ์ง€์›ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ทธ Runner๊ฐ€ kill๋˜๊ฑฐ๋‚˜, ํฌ๋ž˜์‹œํ•˜๊ฑฐ๋‚˜, ๋ฉˆ์ถ”๊ฑฐ๋‚˜, ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•˜๋ฉด Runner๊ฐ€ ์‹คํ–‰ํ•˜๋˜ ๊ฒƒ์„ Runner์—์„œ ๋”๋Š” ์•Œ์•„๋‚ผ ์ˆ˜ ์—†๊ณ , ๊ทธ ํ”„๋กœ์„ธ์Šค๋Š” Runner๋ณด๋‹ค ์˜ค๋ž˜ ์‚ด ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค([Runner ์†์‹ค](runner-isolation.md#runner-loss)). ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ทธ ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•ด ๊ธฐ๋ก์œผ๋กœ `lost_reason`๊ณผ ํ•จ๊ป˜ `unknown`์ด๋ผ๊ณ  ๋‹ตํ•ฉ๋‹ˆ๋‹ค. ๊ทธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์ ์œ ํ•˜๋˜ workspace๋Š” ์šด์˜์ž๊ฐ€ ๊ทธ์ค‘ ์•„๋ฌด๊ฒƒ๋„ ์‹คํ–‰ ์ค‘์ด์ง€ ์•Š์Œ์„ ํ™•์ธํ•˜๊ณ  ๊ธฐ๋ก์„ ํ’€ ๋•Œ๊นŒ์ง€ ๊ณ„์† ๊ฑฐ๋ถ€๋˜๋ฉฐ, ๋‹ค์Œ ๊ฒŒ์ดํŠธ์›จ์ด์—์„œ๋Š” `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค. CodeSpace๋Š” ์ด ํ”„๋กœ์„ธ์Šค๋ฅผ ํ•˜๋‚˜๋„ ์ข…๋ฃŒํ•˜์ง€ ์•Š๊ณ , ์•„๋ฌด๊ฒƒ๋„ ๋‹ค์‹œ ์‹œ์ž‘ํ•˜์ง€ ์•Š์œผ๋ฉฐ, ์ด ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•ด DevGuard์— ๋ฌป์ง€๋„ ์•Š์Šต๋‹ˆ๋‹ค. + +1. **๊ธฐ๋ก์„ ๋‚˜์—ดํ•ฉ๋‹ˆ๋‹ค.** ์•„์ง workspace๋ฅผ ์ ์œ ํ•  ์ˆ˜ ์žˆ๋Š” ๊ธฐ๋ก์„ ๋ณด์—ฌ ์ค๋‹ˆ๋‹ค. ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค๋ฅผ ์ฝ๊ธฐ๋งŒ ํ•˜๋ฏ€๋กœ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์‹คํ–‰ ์ค‘์ด์–ด๋„ ๋ฉ๋‹ˆ๋‹ค. + + ```bash + ./dist/codespace-mcp records list --operations-db "$CODESPACE_OPERATIONS_DB" + ``` + + ๊ฒฐ๊ณผ๋Š” JSON์ž…๋‹ˆ๋‹ค. ๊ฐ ๊ธฐ๋ก์—๋Š” `process_id`์™€ `workspace_id`, `dispatching`ยท`running`ยท`unknown` ์ค‘ ํ•˜๋‚˜์ธ `phase`, ๊ฒŒ์ดํŠธ์›จ์ด๋‚˜ release๊ฐ€ ์†์‹ค๋กœ ํ‘œ์‹œํ–ˆ์œผ๋ฉด `runner_lost` ๋˜๋Š” `host_rebooted`์ด๊ณ  ๊ทธ ์ „์—๋Š” null์ธ `lost`, `participation`๊ณผ DevGuard `attempt_id`, ๊ทธ ์‹คํ–‰์ด ์žฌ๊ฐœํ•œ `approval_id`, `tty`๊ฐ€ ์žˆ์Šต๋‹ˆ๋‹ค. `runner`์—๋Š” Runner์˜ PID์™€ ๋ถ€ํŒ…, ํ˜ธ์ŠคํŠธ์˜ ์ด๋ฒˆ ๋ถ€ํŒ…์—์„œ ์‹คํ–‰๋˜์—ˆ๋Š”์ง€๋ฅผ ๋œปํ•˜๋Š” `this_boot`, ๊ทธ PID๋ฅผ ๊ฐ€์ง„ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์ง€๊ธˆ ์‹คํ–‰ ์ค‘์ธ์ง€๋ฅผ ๋œปํ•˜๋Š” `pid_running`์ด ์žˆ์Šต๋‹ˆ๋‹ค. ์ด ํ”„๋กœ์„ธ์Šค๋Š” ๊ทธ PID๋ฅผ ์ด์–ด๋ฐ›์€ ๋‹ค๋ฅธ ํ”„๋กœ์„ธ์Šค์ผ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. `root`์—๋Š” root ํ”„๋กœ์„ธ์Šค์™€, ๋ฐ”๋กœ ๊ทธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์•„์ง ์‹คํ–‰ ์ค‘์ธ์ง€๋ฅผ ๋œปํ•˜๋Š” `running`์ด ์žˆ์œผ๋ฉฐ, ๊ธฐ๋ก์œผ๋กœ ์•Œ ์ˆ˜ ์—†์œผ๋ฉด `running`์€ null์ž…๋‹ˆ๋‹ค. + +2. **์•„์ง ์‹คํ–‰ ์ค‘์ธ ๊ฒƒ์„ ์ฐพ์Šต๋‹ˆ๋‹ค.** ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ–ˆ๋‹ค๋ฉด(`host_rebooted`์ด๊ฑฐ๋‚˜ `this_boot`๊ฐ€ false) ์•„๋ฌด๊ฒƒ๋„ ์‹คํ–‰ ์ค‘์ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ทธ๋ ‡์ง€ ์•Š๋‹ค๋ฉด ๋‹ค์Œ์„ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. + - `running`์ด true์ด๋ฉด root ํ”„๋กœ์„ธ์Šค๊ฐ€ ์‹คํ–‰ ์ค‘์ž…๋‹ˆ๋‹ค. `ps -o pid,pgid,sid,lstart,command -p `๋Š” ํ”„๋กœ์„ธ์Šค๋ฅผ ๊ทธ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน, ์„ธ์…˜๊ณผ ํ•จ๊ป˜ ๋ณด์—ฌ ์ค๋‹ˆ๋‹ค. + - pipe ํ”„๋กœ์„ธ์Šค์™€ ๊ทธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์‹œ์ž‘ํ•œ ๊ฒƒ์€, ์Šค์Šค๋กœ ๋ฒ—์–ด๋‚˜์ง€ ์•Š๋Š” ํ•œ ์ž์‹ ์ด ์‹œ์ž‘๋œ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน์— ๋‚จ์Šต๋‹ˆ๋‹ค. ๊ทธ ๊ทธ๋ฃน์€ Runner์˜ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน์ด๊ฑฐ๋‚˜, ๋„คํŠธ์›Œํฌ๋ฅผ ํ—ˆ์šฉํ•œ ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ์™€ DevGuard์˜ launch helper์ฒ˜๋Ÿผ root๊ฐ€ ์ด๋„๋Š” ๊ทธ๋ฃน์ž…๋‹ˆ๋‹ค. `pgrep -g `๋Š” ๊ทธ๋ฃน์„, `pgrep -P `๋Š” ํ”„๋กœ์„ธ์Šค์˜ ์ž์‹์„ ๋‚˜์—ดํ•˜๊ณ , `lsof -d cwd`๋Š” ์–ด๋–ค ํ”„๋กœ์„ธ์Šค๊ฐ€ workspace ๋””๋ ‰ํ„ฐ๋ฆฌ์—์„œ ์ž‘์—…ํ•˜๋Š”์ง€ ๋ณด์—ฌ ์ค๋‹ˆ๋‹ค. Runner๊ฐ€ ๋๋‚œ ๋’ค ์ถœ๋ ฅ์— ์“ด ํ”„๋กœ์„ธ์Šค๋Š”, SIGPIPE๋ฅผ ๋ฌด์‹œํ•˜์ง€ ์•Š์•˜๋‹ค๋ฉด ๊ทธ ์‹ ํ˜ธ๋กœ ๋๋‚ฌ์„ ๊ฐ€๋Šฅ์„ฑ์ด ํฝ๋‹ˆ๋‹ค. + - `tty: true` ํ”„๋กœ์„ธ์Šค๋Š” ์ž๊ธฐ ์„ธ์…˜์„ ์ด๋Œ๋ฉฐ, `pgrep -s `๊ฐ€ ๊ทธ ์„ธ์…˜์„ ๋‚˜์—ดํ•ฉ๋‹ˆ๋‹ค. ์ด ํ”„๋กœ์„ธ์Šค๋Š” Runner๊ฐ€ ๋๋‚  ๋•Œ ๋ฐ›์•˜์„ ๊ฐ€๋Šฅ์„ฑ์ด ํฐ SIGHUP์„ ๋ฌด์‹œํ•œ ๊ฒฝ์šฐ์—๋งŒ Runner๋ณด๋‹ค ์˜ค๋ž˜ ์‚ฝ๋‹ˆ๋‹ค. + - Runner๋ฅผ ์žƒ์—ˆ์„ ๋•Œ ์•„์ง `dispatching`์ด์–ด์„œ root๊ฐ€ ์—†๋Š” ๊ธฐ๋ก์€ ํ”„๋กœ์„ธ์Šค๋ฅผ ์‹œ์ž‘ํ–ˆ์„ ์ˆ˜๋„, ์‹œ์ž‘ํ•˜์ง€ ์•Š์•˜์„ ์ˆ˜๋„ ์žˆ์Šต๋‹ˆ๋‹ค. ์œ„์™€ ๊ฐ™์ด ์ฐพ์•„๋ด…๋‹ˆ๋‹ค. + - Linux ๋ช…๋ น ์ƒŒ๋“œ๋ฐ•์Šค์—์„œ ๋„คํŠธ์›Œํฌ๋ฅผ ํ—ˆ์šฉํ•˜์ง€ ์•Š์€ workspace์˜ ๋ช…๋ น์€ Runner์™€ ํ•จ๊ป˜ ๋๋‚˜์ง€๋งŒ, ๋„คํŠธ์›Œํฌ๋ฅผ ํ—ˆ์šฉํ•œ workspace์˜ ๋ช…๋ น์€ ๊ทธ๋ ‡์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋‚˜๋จธ์ง€๋Š” Runner์˜ ์„œ๋น„์Šค๊ฐ€ ๋ฉˆ์ถœ ๋•Œ ์„œ๋น„์Šค ๊ด€๋ฆฌ์ž๊ฐ€ ์ด๋ฏธ ์ข…๋ฃŒํ–ˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. systemd๋Š” ๊ธฐ๋ณธ๊ฐ’ `KillMode=control-group`์— ๋”ฐ๋ผ ์„œ๋น„์Šค์˜ control group์„, launchd๋Š” job์ด `AbandonProcessGroup`์„ ์„ค์ •ํ•˜์ง€ ์•Š์•˜๋‹ค๋ฉด job์˜ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน์„ ์ข…๋ฃŒํ•ฉ๋‹ˆ๋‹ค. ๊ฐ€์ •ํ•˜์ง€ ๋ง๊ณ  ํ™•์ธํ•˜์„ธ์š”. + +3. **์•„์ง ์‹คํ–‰ ์ค‘์ธ ๊ฒƒ์„ ์ข…๋ฃŒํ•ฉ๋‹ˆ๋‹ค.** ๋๊นŒ์ง€ ์‹คํ–‰ํ•˜์ง€ ์•Š๊ธฐ๋กœ ์ •ํ–ˆ๋‹ค๋ฉด `kill -TERM `๋ฅผ ๋ณด๋‚ด๊ณ , ๋๋‚˜์ง€ ์•Š์œผ๋ฉด `kill -KILL `๋ฅผ ๋ณด๋ƒ…๋‹ˆ๋‹ค. deadline์„ ์ง€ํ‚ค๊ฒŒ ํ•˜๋˜ Runner๊ฐ€ ์—†์œผ๋ฏ€๋กœ ์ด์ œ deadline์ด ์ด ํ”„๋กœ์„ธ์Šค๋ฅผ ๋๋‚ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + +4. **`required` ๊ธฐ๋ก์€ DevGuard๊ฐ€ ๊ทธ attempt๋ฅผ ๋”๋Š” ๊ณผ๊ธˆํ•˜์ง€ ์•Š์„ ๋•Œ๊นŒ์ง€ ๊ธฐ๋‹ค๋ฆฝ๋‹ˆ๋‹ค.** ์žƒ์€ Runner๋Š” DevGuard์˜ ์‹คํ–‰ ์†Œ์œ ์ž์˜€๊ณ , DevGuard๋Š” attempt์˜ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚ฌ์Œ์„ ๊ด€์ฐฐํ•˜๊ฑฐ๋‚˜ ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•  ๋•Œ๊นŒ์ง€ ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•ฉ๋‹ˆ๋‹ค. macOS์—์„œ `devguard admission --open`์€ DevGuard์˜ `quiescence`๋ฅผ ์ถœ๋ ฅํ•ฉ๋‹ˆ๋‹ค. `charged_attempts`๋Š” ๊ณผ๊ธˆ ์ค‘์ธ attempt ์ˆ˜์ด๊ณ , `attempts`๋Š” ๊ทธ key๋ฅผ ์ตœ๋Œ€ 16๊ฐœ ๋ณด์—ฌ ์ค๋‹ˆ๋‹ค. ๊ทธ key ์ค‘ ํ•˜๋‚˜๊ฐ€ ๊ธฐ๋ก์˜ `attempt_id`๋ฅผ ๊ฐ€์กŒ๊ฑฐ๋‚˜ `charged_attempts`๊ฐ€ ๋‚˜์—ด๋œ key ์ˆ˜๋ณด๋‹ค ํฌ๋ฉด DevGuard๊ฐ€ ์•„์ง ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•˜๊ณ  ์žˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ์ด ๋ช…๋ น์€ admission์ด ๋‹ซํ˜€ ์žˆ์œผ๋ฉด ๋‹ค์‹œ ์—ฝ๋‹ˆ๋‹ค. ์ค‘๋‹จ๋œ DevGuard upgrade๊ฐ€ admission์„ ๋‹ซํžŒ ์ฑ„๋กœ ๋‚จ๊ธฐ๋ฏ€๋กœ, ๊ทธ๋Ÿฐ upgrade๋Š” [DevGuard ์šด์˜ ๋ฌธ์„œ](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/operations.md#์ œ๊ณต-๋ช…๋ น)๋Œ€๋กœ ๋จผ์ € ๋งˆ์น˜์„ธ์š”. DevGuard์—๋Š” quiescence๋ฅผ ์ฝ๊ธฐ๋งŒ ํ•˜๋Š” ๋ช…๋ น์ด ์—†์Šต๋‹ˆ๋‹ค. [DevGuard ๋Œ€์กฐ ๊ณ„์•ฝ](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/contracts.md#๋Œ€์กฐ)์ด ์„ค๋ช…ํ•˜๋“ฏ DevGuard๊ฐ€ attempt์˜ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ถ”์ ํ•˜์ง€ ๋ชปํ•˜๊ฒŒ ๋˜๋ฉด, ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋ชจ๋‘ ๋๋‚œ ๋’ค์—๋„ ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•  ๋•Œ๊นŒ์ง€ ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•ฉ๋‹ˆ๋‹ค. DevGuard๊ฐ€ attempt๋ฅผ ๊ณผ๊ธˆํ•˜๋Š” ๋™์•ˆ์—๋Š” `required` ๊ธฐ๋ก์„ ํ’€์ง€ ๋งˆ์„ธ์š”. + +5. **๊ฒŒ์ดํŠธ์›จ์ด๋ฅผ ๋ฉˆ์ถ”๊ณ  ๊ธฐ๋ก์„ ํ’‰๋‹ˆ๋‹ค.** + + ```bash + ./dist/codespace-mcp records release --operations-db "$CODESPACE_OPERATIONS_DB" ... + ``` + + ์‹คํ–‰ ์ค‘์ธ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” lock ํŒŒ์ผ์„ ์žก๊ณ  ์žˆ์œผ๋ฏ€๋กœ, ์ด ๋ช…๋ น์€ ๊ทธ ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค์—์„œ ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์‹คํ–‰ ์ค‘์ด์ง€ ์•Š์„ ๋•Œ๋งŒ ๊ธฐ๋ก์„ ๋ฐ”๊ฟ‰๋‹ˆ๋‹ค. Runner๋ฅผ ์žƒ์€ ๊ธฐ๋ก์„ ๊ทธ root ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋”๋Š” ์‹คํ–‰ ์ค‘์ด์ง€ ์•Š์„ ๋•Œ ํ’€๊ณ , `released`, `already_released`, ๊ทธ๋ฆฌ๊ณ  ๊ฐ๊ฐ ์ด์œ ๊ฐ€ ๋ถ™์€ `refused`๋ฅผ JSON์œผ๋กœ ์ถœ๋ ฅํ•ฉ๋‹ˆ๋‹ค. ํ•˜๋‚˜๋ผ๋„ ๊ฑฐ๋ถ€ํ•˜๋ฉด ์ข…๋ฃŒ ์ƒํƒœ 1๋กœ ๋๋‚ฉ๋‹ˆ๋‹ค. ๋๋‚œ ๊ธฐ๋ก์€ workspace๋ฅผ ์ ์œ ํ•˜์ง€ ์•Š์œผ๋ฏ€๋กœ ๊ฑฐ๋ถ€ํ•˜๊ณ , root ํ”„๋กœ์„ธ์Šค๊ฐ€ ์•„์ง ์‹คํ–‰ ์ค‘์ธ ๊ธฐ๋ก๊ณผ Runner๊ฐ€ ์•„์ง ์‹คํ–‰ ์ค‘์ผ ์ˆ˜ ์žˆ๋Š” ๊ธฐ๋ก๋„ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์‹คํ–‰๋˜์ง€ ์•Š๋Š” ๋™์•ˆ ๋ฉˆ์ถ˜ Runner๋Š” ์ž๊ธฐ ๊ธฐ๋ก์„ `running`์œผ๋กœ ๋‚จ๊น๋‹ˆ๋‹ค. ์ด ๋ช…๋ น์€ ๊ทธ Runner์˜ PID๋ฅผ ๊ฐ€์ง„ ํ”„๋กœ์„ธ์Šค๊ฐ€ ํ•˜๋‚˜๋„ ์—†์„ ๋•Œ๋งŒ ๊ทธ ๊ธฐ๋ก์„ ๊ทธ Runner์˜ ๋๋‚˜์ง€ ์•Š์€ ๋‹ค๋ฅธ ๊ธฐ๋ก๊ณผ ํ•จ๊ป˜ ์†์‹ค๋กœ ํ‘œ์‹œํ•ฉ๋‹ˆ๋‹ค. ์ด ๋ช…๋ น์€ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ข…๋ฃŒํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ํ‘ผ ๊ธฐ๋ก์€ `unknown`์œผ๋กœ ๋‚จ๊ณ , ๊ทธ ์‹คํ–‰์ด ์žฌ๊ฐœํ•œ ์Šน์ธ์— ๋Œ€ํ•ด `operation_resume`์€ `APPROVAL_AMBIGUOUS`๋กœ ๋‹ตํ•ฉ๋‹ˆ๋‹ค. ๊ทธ ๋ช…๋ น์€ ๋‹ค์‹œ ์‹คํ–‰๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + +6. **Runner์™€ ๊ฒŒ์ดํŠธ์›จ์ด๋ฅผ ๋‹ค์‹œ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค.** ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ฐ workspace๋ฅผ ๋Œ€์กฐํ•˜๋ฉฐ, ๊ธฐ๋ก์ด ๋ชจ๋‘ ๋๋‚ฌ๊ฑฐ๋‚˜ ํ’€๋ฆฐ workspace๋Š” ๋‹ค์‹œ ์ƒˆ ์ž‘์—…์„ ๋ฐ›์Šต๋‹ˆ๋‹ค. DevGuard๋ฅผ ์“ฐ๋ฉด ์ƒˆ Runner๋Š” `codespace` ์†Œ๋น„์ž์˜ ์ธ์Šคํ„ด์Šค ํ•˜๋‚˜๋กœ ๋“ฑ๋กํ•˜๋Š”๋ฐ, DevGuard๋Š” ์žƒ์€ Runner์˜ ์ธ์Šคํ„ด์Šค๋ฅผ retireํ•œ ๋’ค์—๋งŒ, ์ฆ‰ ๊ทธ Runner์˜ attempt๊ฐ€ ๋”๋Š” ๊ณผ๊ธˆ๋˜์ง€ ์•Š์„ ๋•Œ๋งŒ ์ด๋ฅผ ํ—ˆ์šฉํ•ฉ๋‹ˆ๋‹ค. ๊ทธ๋•Œ๊นŒ์ง€ `workspace_info`๋Š” `resource_authority.registration.state`๋ฅผ `refused`๋กœ ๋ณด๊ณ ํ•˜๊ณ  `required` workspace๋Š” ์‹คํ–‰๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. `off` workspace๋Š” ์˜ํ–ฅ์„ ๋ฐ›์ง€ ์•Š์Šต๋‹ˆ๋‹ค. DevGuard๋Š” ์†Œ๋น„์ž์˜ ์ธ์Šคํ„ด์Šค๊ฐ€ ํ•˜๋‚˜๋ผ๋„ retire๋˜์ง€ ์•Š์€ ๋™์•ˆ ๊ทธ ์†Œ๋น„์ž์˜ ์ธ์Šคํ„ด์Šค ํ•œ๋„๋ฅผ ๋ฐ”๊พธ์ง€ ๋ชปํ•˜๊ฒŒ ํ•ฉ๋‹ˆ๋‹ค. + +`RECOVERY_PENDING`์ด ๊ธฐ๋ก, Runner, DevGuard๊ฐ€ ์„œ๋กœ ๋งž์ง€ ์•Š๋Š”๋‹ค๊ณ  ์•Œ๋ฆฌ๋Š” workspace๋Š” ๊ทธ Runner๊ฐ€ ์‹คํ–‰ ์ค‘์ธ ๋™์•ˆ `records release`๋กœ ์ •๋ฆฌ๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ทธ Runner๋ฅผ ๋ฉˆ์ถ”๋ฉด ๊ทธ Runner๊ฐ€ ์ ์œ ํ•˜๋˜ ๋ชจ๋“  workspace์—์„œ ๋๋‚˜์ง€ ์•Š์€ ๊ธฐ๋ก์ด ์žƒ์€ Runner์˜ ๊ธฐ๋ก์ด ๋˜๊ณ , ๊ทธ ๋’ค์—๋Š” ์œ„ ๋‹จ๊ณ„๋ฅผ ๋”ฐ๋ฆ…๋‹ˆ๋‹ค. + @@ -158,5 +195,6 @@ worker๋Š” ๊ฐ™์€ ํ˜ธ์ŠคํŠธ์—์„œ ์‹คํ–‰ํ•˜๋Š” ๋ณ„๋„ ํ”„๋กœ์„ธ์Šค์ด๋ฉฐ ์ปจํ…Œ | Linux์—์„œ ์ƒŒ๋“œ๋ฐ•์Šค๊ฐ€ ์—†๋‹ค๊ณ  ํ‘œ์‹œ | ๋„์šฐ๋ฏธ ์œ„์น˜, bubblewrap, ๋„ค์ž„์ŠคํŽ˜์ด์Šค ์ง€์› ํ™•์ธ | | `enabled` ๋ช…๋ น ์‹œ์ž‘ ์‹คํŒจ | ์‚ฌ์šฉ ๊ฐ€๋Šฅํ•œ Linux ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ ํ•„์š” | | HTTP ์š”์ฒญ ๊ฑฐ๋ถ€ | `/mcp` ๊ฒฝ๋กœ, Bearer ํ—ค๋”, Host ๊ฒ€์ฆ | +| ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ Runner๋ฅผ ์žƒ์—ˆ๋‹ค๊ณ  ์•Œ๋ฆฌ๋Š” `RUNNER_DISCONNECTED`, ๋˜๋Š” `codespace-mcp records`๋ฅผ ์•Œ๋ ค ์ฃผ๋Š” `RECOVERY_PENDING` | [Runner ์†์‹ค](#runner-loss) ์ ˆ์ฐจ๋ฅผ ๋”ฐ๋ฅด๊ธฐ | ๋กœ์ปฌ ์ „์†ก ํ…Œ์ŠคํŠธ๋Š” ์‹ค์ œ ChatGPT ์—ฐ๊ฒฐ, ๋ชจ๋“  ํŒจํ‚ค์ง€ ๊ด€๋ฆฌ์ž์˜ ํ”„๋ก์‹œ ํ˜ธํ™˜์„ฑ, ์ปค๋„ยท์ปจํ…Œ์ด๋„ˆ ํƒˆ์ถœ ๋ฐฉ์–ด๊นŒ์ง€ ๊ฒ€์ฆํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. diff --git a/docs/ko/runner-isolation.md b/docs/ko/runner-isolation.md index cda18a3..54f4e3c 100644 --- a/docs/ko/runner-isolation.md +++ b/docs/ko/runner-isolation.md @@ -23,7 +23,7 @@ ## ๋…๋ฆฝ Runner -> **์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค.** ๋…๋ฆฝ Runner์™€ ๊ทธ ์‹คํ–‰์˜ ๊ธฐ๋ก, ํ•œ ๊ฒŒ์ดํŠธ์›จ์ด์—์„œ ๋‹ค์Œ ๊ฒŒ์ดํŠธ์›จ์ด๋กœ ์ œ์–ด๊ถŒ์„ ๋„˜๊ธฐ๋Š” Claim, ๊ทธ๋ฆฌ๊ณ  Claim๋งˆ๋‹ค ๋’ค๋”ฐ๋ฅด๋Š” ๋Œ€์กฐ๋Š” ๊ฒŒ์ดํŠธ์›จ์ด ๋ณต๊ตฌ์˜ ์ฒซ ๋‹จ๊ณ„์ž…๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์—ฐ๊ฒฐ์„ ์žƒ์œผ๋ฉด Runner๋ฅผ ๋‹ค์‹œ Claimํ•˜๊ณ , ์ด์ „ controller๊ฐ€ ์ž‘์—…์„ ์‹œ์ž‘ํ•œ ๊ณณ์—์„œ๋Š” ๊ทธ controller๊ฐ€ ๋‚จ๊ธด ๊ฒƒ์„ ๋Œ€์กฐํ•œ ๋’ค์—์•ผ ์ƒˆ ์ž‘์—…์„ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. Runner ์ž์ฒด๋ฅผ ์žƒ์€ ๋’ค์˜ ์ฒ˜๋ฆฌ๋Š” ๋‹ค์Œ ๋‹จ๊ณ„์—์„œ ๋”ํ•˜๋ฉฐ, ์ด ๋ชจ๋“œ๋Š” qualification์„ ๋งˆ์นœ ๋’ค์—์•ผ ์ง€์›ํ•ฉ๋‹ˆ๋‹ค. +> **์•„์ง ์ง€์›ํ•˜๋Š” ๋ชจ๋“œ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค.** ๋…๋ฆฝ Runner์™€ ๊ทธ ์‹คํ–‰์˜ ๊ธฐ๋ก, ํ•œ ๊ฒŒ์ดํŠธ์›จ์ด์—์„œ ๋‹ค์Œ ๊ฒŒ์ดํŠธ์›จ์ด๋กœ ์ œ์–ด๊ถŒ์„ ๋„˜๊ธฐ๋Š” Claim, ๊ทธ๋ฆฌ๊ณ  Claim๋งˆ๋‹ค ๋’ค๋”ฐ๋ฅด๋Š” ๋Œ€์กฐ๋Š” ๊ฒŒ์ดํŠธ์›จ์ด ๋ณต๊ตฌ์˜ ์ฒซ ๋‹จ๊ณ„์ž…๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์—ฐ๊ฒฐ์„ ์žƒ์œผ๋ฉด Runner๋ฅผ ๋‹ค์‹œ Claimํ•˜๊ณ , ์ด์ „ controller๊ฐ€ ์ž‘์—…์„ ์‹œ์ž‘ํ•œ ๊ณณ์—์„œ๋Š” ๊ทธ controller๊ฐ€ ๋‚จ๊ธด ๊ฒƒ์„ ๋Œ€์กฐํ•œ ๋’ค์—์•ผ ์ƒˆ ์ž‘์—…์„ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. Runner ์ž์ฒด๋ฅผ ์žƒ์œผ๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ทธ ์‹คํ–‰๋“ค์— ๋Œ€ํ•ด ๊ธฐ๋ก์œผ๋กœ ๋‹ตํ•˜๊ณ , ์šด์˜์ž๊ฐ€ ํ’€ ๋•Œ๊นŒ์ง€ ๊ทธ workspace๋ฅผ ์ ์œ ํ•œ ์ฑ„๋กœ ๋‘ก๋‹ˆ๋‹ค([Runner ์†์‹ค](#runner-loss)). ์ด ๋ชจ๋“œ๋Š” qualification์„ ๋งˆ์นœ ๋’ค์—์•ผ ์ง€์›ํ•ฉ๋‹ˆ๋‹ค. ๋…๋ฆฝ Runner๋Š” ์ž์‹ ์—๊ฒŒ ์—ฐ๊ฒฐํ•˜๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๋ณด๋‹ค ์˜ค๋ž˜ ์‚ฝ๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ์ข…๋ฃŒ๋˜๊ฑฐ๋‚˜ ์žฌ์‹œ์ž‘๋˜๊ฑฐ๋‚˜ ๊ฐ•์ œ๋กœ ์ข…๋ฃŒ๋˜์–ด๋„ ํ”„๋กœ์„ธ์Šค์™€ ๊ทธ ์ถœ๋ ฅ, timeout์€ ๊ณ„์†๋ฉ๋‹ˆ๋‹ค. ์šด์˜์ž๋‚˜ ํ•˜๋„ค์Šค๊ฐ€ ๋…๋ฆฝ Runner๋ฅผ ์‹œ์ž‘ํ•˜๊ณ , ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์—ฐ๊ฒฐ๋งŒ ํ•ฉ๋‹ˆ๋‹ค. @@ -57,27 +57,46 @@ Claim์„ ๋ฐ›์•„๋“ค์ผ ๋•Œ๋งˆ๋‹ค Runner์˜ controller epoch๊ฐ€ ํ•˜๋‚˜ ์˜ฌ๋ผ๊ฐ‘ ๋‹ค๋ฅธ ์—ฐ๊ฒฐ์ด Runner๋ฅผ ์ œ์–ดํ•˜๋Š” ๋™์•ˆ์ด๋‚˜ ๊ทธ ์—ฐ๊ฒฐ์˜ ์š”์ฒญ์ด ์ฒ˜๋ฆฌ๋˜๋Š” ๋™์•ˆ ์‹œ์ž‘ํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ฐ„๊ฒฉ์„ ๋Š˜๋ ค ๊ฐ€๋ฉฐ ์ตœ๋Œ€ 30์ดˆ ๋™์•ˆ ๋‹ค์‹œ ์š”์ฒญํ•˜๊ณ , ๊ทธ๋ž˜๋„ ์•ˆ ๋˜๋ฉด ์‹œ์ž‘์„ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ๋‹ค๋ฅธ ์ž‘์—… ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค๋ฅผ ์“ฐ๋Š” ๊ฒฝ์šฐ์ฒ˜๋Ÿผ ๋‹ค๋ฅธ ์ด์œ ๋กœ ๊ฑฐ๋ถ€๋œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๋ฐ”๋กœ ์‹œ์ž‘์„ ๊ฑฐ๋ถ€ํ•˜๋ฉฐ, Runner๋Š” ๊ทธ๋Œ€๋กœ ๋‚จ์Šต๋‹ˆ๋‹ค. -**์—ฐ๊ฒฐ์„ ์žƒ์—ˆ์„ ๋•Œ.** Runner์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์—ˆ๊ฑฐ๋‚˜, ํ˜ธ์ถœ์ด deadline์„ ๋„˜๊ฒจ ์—ฐ๊ฒฐ์„ ๋‹ซ์€ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๋‹ค์‹œ ์—ฐ๊ฒฐํ•ด ๊ฐ™์€ Runner๋ฅผ Claimํ•ฉ๋‹ˆ๋‹ค. ๋จผ์ € ์†Œ์ผ“์—์„œ ์‘๋‹ตํ•˜๋Š” Runner๊ฐ€ ์‹œ์ž‘ํ•  ๋•Œ ์ฐพ์€ Runner์™€ ๊ฐ™์€ ์‹๋ณ„ ์ •๋ณด๋ฅผ ๊ฐ€์กŒ๋Š”์ง€ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. ๋ฐ”๋กœ ํ•œ ๋ฒˆ ์‹œ๋„ํ•˜๊ณ , ์‹คํŒจํ•˜๋ฉด 100๋ฐ€๋ฆฌ์ดˆ๋ฅผ ๊ธฐ๋‹ค๋ฆฐ ๋’ค ๋‹ค์‹œ ์‹œ๋„ํ•˜๋ฉฐ, ์‹คํŒจํ•  ๋•Œ๋งˆ๋‹ค ๊ธฐ๋‹ค๋ฆฌ๋Š” ์‹œ๊ฐ„์„ ๋‘ ๋ฐฐ์”ฉ ์ตœ๋Œ€ 5์ดˆ๊นŒ์ง€ ๋Š˜๋ฆฝ๋‹ˆ๋‹ค. ๊ทธ๋™์•ˆ์—๋Š” Runner๊ฐ€ ํ•„์š”ํ•œ ์š”์ฒญ์„ ๋ชจ๋‘ `RUNNER_RECONNECTING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ๊ทธ ์š”์ฒญ์„ ์œ„ํ•ด์„œ๋Š” ์•„๋ฌด๊ฒƒ๋„ ๋ณด๋‚ด๊ฑฐ๋‚˜ ์˜ˆ์•ฝํ•˜๊ฑฐ๋‚˜ ๊ธฐ๋กํ•˜์ง€ ์•Š์œผ๋ฉฐ, ์žฌ๊ฐœํ•˜๋ ค๋˜ ์Šน์ธ์€ ์†Œ๋น„๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์†Œ์ผ“์—์„œ ๋” ์ด์ƒ ์—ฐ๊ฒฐ์„ ๋ฐ›์ง€ ์•Š๊ฑฐ๋‚˜, ๋‹ค๋ฅธ Runner๊ฐ€ ์‘๋‹ตํ•˜๊ฑฐ๋‚˜, ๊ธฐ๋‹ค๋ ค๋„ ํ’€๋ฆฌ์ง€ ์•Š๋Š” ์ด์œ ๋กœ Runner๊ฐ€ Claim์„ ๊ฑฐ๋ถ€ํ•˜๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์‹œ๋„๋ฅผ ๋ฉˆ์ถ”๊ณ  ๊ทธ๋Ÿฐ ์š”์ฒญ์„ ๋ชจ๋‘ `RUNNER_DISCONNECTED`๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ๊ทธ๋Ÿฌ๋ฉด Runner์˜ ํ”„๋กœ์„ธ์Šค์— ๋‹ค์‹œ ๋‹ฟ์œผ๋ ค๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด๋ฅผ ์žฌ์‹œ์ž‘ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ์—ฐ๊ฒฐ์„ ์žƒ์„ ๋•Œ ์ด๋ฏธ ๋ณด๋‚ธ ์š”์ฒญ์€ ์ง€๊ธˆ์ฒ˜๋Ÿผ ๋ถˆํ™•์‹คํ•œ ๊ฒฐ๊ณผ๋ฅผ ๋ฐ›์Šต๋‹ˆ๋‹ค. ์‹คํ–‰์ด๋ผ๋ฉด `dispatch_status: unknown`์ž…๋‹ˆ๋‹ค. Runner๊ฐ€ ์ „ํ•˜์ง€ ๋ชปํ•œ ์ข…๋ฃŒ ์ด๋ฒคํŠธ๋Š” ๋‹ค์Œ controller์—๊ฒŒ ๊ฐ€๊ณ , ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ฐ ์ข…๋ฃŒ๋ฅผ ํ•œ ๋ฒˆ๋งŒ ์ „๋‹ฌํ•ฉ๋‹ˆ๋‹ค. +**์—ฐ๊ฒฐ์„ ์žƒ์—ˆ์„ ๋•Œ.** Runner์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์—ˆ๊ฑฐ๋‚˜, ํ˜ธ์ถœ์ด deadline์„ ๋„˜๊ฒจ ์—ฐ๊ฒฐ์„ ๋‹ซ์€ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๋‹ค์‹œ ์—ฐ๊ฒฐํ•ด ๊ฐ™์€ Runner๋ฅผ Claimํ•ฉ๋‹ˆ๋‹ค. ๋จผ์ € ์†Œ์ผ“์—์„œ ์‘๋‹ตํ•˜๋Š” Runner๊ฐ€ ์‹œ์ž‘ํ•  ๋•Œ ์ฐพ์€ Runner์™€ ๊ฐ™์€ ์‹๋ณ„ ์ •๋ณด๋ฅผ ๊ฐ€์กŒ๋Š”์ง€ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. ๋ฐ”๋กœ ํ•œ ๋ฒˆ ์‹œ๋„ํ•˜๊ณ , ์‹คํŒจํ•˜๋ฉด 100๋ฐ€๋ฆฌ์ดˆ๋ฅผ ๊ธฐ๋‹ค๋ฆฐ ๋’ค ๋‹ค์‹œ ์‹œ๋„ํ•˜๋ฉฐ, ์‹คํŒจํ•  ๋•Œ๋งˆ๋‹ค ๊ธฐ๋‹ค๋ฆฌ๋Š” ์‹œ๊ฐ„์„ ๋‘ ๋ฐฐ์”ฉ ์ตœ๋Œ€ 5์ดˆ๊นŒ์ง€ ๋Š˜๋ฆฝ๋‹ˆ๋‹ค. ๊ทธ๋™์•ˆ์—๋Š” Runner๊ฐ€ ํ•„์š”ํ•œ ์š”์ฒญ์„ ๋ชจ๋‘ `RUNNER_RECONNECTING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ๊ทธ ์š”์ฒญ์„ ์œ„ํ•ด์„œ๋Š” ์•„๋ฌด๊ฒƒ๋„ ๋ณด๋‚ด๊ฑฐ๋‚˜ ์˜ˆ์•ฝํ•˜๊ฑฐ๋‚˜ ๊ธฐ๋กํ•˜์ง€ ์•Š์œผ๋ฉฐ, ์žฌ๊ฐœํ•˜๋ ค๋˜ ์Šน์ธ์€ ์†Œ๋น„๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์†Œ์ผ“์—์„œ ๋” ์ด์ƒ ์—ฐ๊ฒฐ์„ ๋ฐ›์ง€ ์•Š๊ฑฐ๋‚˜ ๋‹ค๋ฅธ Runner๊ฐ€ ์‘๋‹ตํ•˜๋ฉด Runner๋ฅผ [์žƒ์€](#runner-loss) ๊ฒƒ์ž…๋‹ˆ๋‹ค. ๊ธฐ๋‹ค๋ ค๋„ ํ’€๋ฆฌ์ง€ ์•Š๋Š” ์ด์œ ๋กœ Runner๊ฐ€ Claim์„ ๊ฑฐ๋ถ€ํ•  ๋•Œ๋„ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์‹œ๋„๋ฅผ ๋ฉˆ์ถฅ๋‹ˆ๋‹ค. ์–ด๋Š ์ชฝ์ด๋“  ๊ทธ๋Ÿฐ ์š”์ฒญ์„ ๋ชจ๋‘ `RUNNER_DISCONNECTED`๋กœ ๊ฑฐ๋ถ€ํ•˜๋ฉฐ, Runner์˜ ํ”„๋กœ์„ธ์Šค์— ๋‹ค์‹œ ๋‹ฟ์œผ๋ ค๋ฉด ๊ฒŒ์ดํŠธ์›จ์ด๋ฅผ ์žฌ์‹œ์ž‘ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ์—ฐ๊ฒฐ์„ ์žƒ์„ ๋•Œ ์ด๋ฏธ ๋ณด๋‚ธ ์š”์ฒญ์€ ์ง€๊ธˆ์ฒ˜๋Ÿผ ๋ถˆํ™•์‹คํ•œ ๊ฒฐ๊ณผ๋ฅผ ๋ฐ›์Šต๋‹ˆ๋‹ค. ์‹คํ–‰์ด๋ผ๋ฉด `dispatch_status: unknown`์ž…๋‹ˆ๋‹ค. Runner๊ฐ€ ์ „ํ•˜์ง€ ๋ชปํ•œ ์ข…๋ฃŒ ์ด๋ฒคํŠธ๋Š” ๋‹ค์Œ controller์—๊ฒŒ ๊ฐ€๊ณ , ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ฐ ์ข…๋ฃŒ๋ฅผ ํ•œ ๋ฒˆ๋งŒ ์ „๋‹ฌํ•ฉ๋‹ˆ๋‹ค. **์ด์ „ controller๊ฐ€ ์žˆ์—ˆ์„ ๋•Œ.** ์ƒˆ๋กœ ์‹œ์ž‘ํ•œ Runner๋ฅผ ์ฒ˜์Œ Claimํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๊ด€๋ฆฌ worker์™€ ๊ฐ™์ด ๋™์ž‘ํ•ฉ๋‹ˆ๋‹ค. ํ•œ controller๊ฐ€ ์–ด๋–ค workspace์—์„œ ์ž‘์—…(์‹คํ–‰, ํŒจ์น˜, DevGuard๋ฅผ ์“ฐ๋ฉด ์ค€๋น„๋‚˜ launch)์„ ์‹œ์ž‘ํ•œ ๋’ค์—๋Š”, ๊ทธ ๋’ค์˜ controller๋“ค์ด Runner์˜ ํ”„๋กœ์„ธ์Šค๋ฅผ ๋„˜๊ฒจ๋ฐ›์Šต๋‹ˆ๋‹ค. ์—ฐ๊ฒฐ์„ ์žƒ์—ˆ๋‹ค๊ฐ€ Runner๋ฅผ ๋‹ค์‹œ Claimํ•œ ๊ฐ™์€ ๊ฒŒ์ดํŠธ์›จ์ด๋„ ๋งˆ์ฐฌ๊ฐ€์ง€์ด๋ฉฐ, Claim์˜ ์‘๋‹ต์€ ์•ž์„  controller๊ฐ€ ์ž‘์—…์„ ์‹œ์ž‘ํ•œ workspace๋ฅผ ๋ชจ๋‘ ์•Œ๋ฆฝ๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๋„˜๊ฒจ๋ฐ›์€ ํ”„๋กœ์„ธ์Šค๋ฅผ `process_id`๋กœ ๋ฐ”๋กœ ์กฐํšŒํ•˜๊ณ , ์ถœ๋ ฅ์„ ์ฝ๊ณ , stdin์— ์“ฐ๊ณ , ํฌ๊ธฐ๋ฅผ ๋ฐ”๊พธ๊ณ , ์ข…๋ฃŒํ•  ์ˆ˜ ์žˆ์œผ๋ฉฐ, DevGuard๊ฐ€ ์‘๋‹ตํ•˜์ง€ ๋ชปํ•  ๋•Œ๋„ ๋งˆ์ฐฌ๊ฐ€์ง€์ž…๋‹ˆ๋‹ค. Claim์ด ์•Œ๋ฆฐ workspace, Runner๊ฐ€ ์‹คํ–‰ ์ค‘์ธ ํ”„๋กœ์„ธ์Šค๋ฅผ ๊ฐ€์ง„ workspace, [๋Œ€์กฐ](#reconciliation)ํ•  ๊ธฐ๋ก์ด ๋‚จ์€ workspace์—์„œ๋Š” ๋จผ์ € ์•ž์„  controller๋“ค์ด ๋‚จ๊ธด ๊ฒƒ์„ ๋Œ€์กฐํ•ฉ๋‹ˆ๋‹ค. ๊ทธ๋•Œ๊นŒ์ง€๋Š” ๊ทธ workspace์˜ ์ƒˆ ์‹คํ–‰, ํŒจ์น˜, ์Šน์ธ ์žฌ๊ฐœ๋ฅผ ์Šน์ธ์„ ๊ฐ€์ ธ๊ฐ€๊ธฐ ์ „์— `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•˜๋ฏ€๋กœ ์Šน์ธ์€ ๊ทธ๋Œ€๋กœ ๋‚จ์Šต๋‹ˆ๋‹ค. Runner๋„ Claim์ด ์•Œ๋ฆฐ workspace์—์„œ๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๊ทธ workspace๋ฅผ ๋Œ€์กฐํ–ˆ๋‹ค๊ณ  ์•Œ๋ฆด ๋•Œ๊นŒ์ง€ ์ด๋Ÿฐ ์š”์ฒญ์„ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. ๋‹ค๋ฅธ workspace๋Š” ๋ฐ”๋กœ ์ƒˆ ์ž‘์—…์„ ๋ฐ›์Šต๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๋Š” Runner์˜ ์‹๋ณ„ ์ •๋ณด(ํ”„๋กœ์„ธ์Šค ID, ๋ถ€ํŒ…๊ณผ ์‹œ์ž‘ ์‹œ๊ฐ), Claim์˜ epoch, ์•ž์„  controller๊ฐ€ ์ž‘์—…์„ ์‹œ์ž‘ํ•œ workspace, ๋„˜๊ฒจ๋ฐ›์€ ํ”„๋กœ์„ธ์Šค ์ˆ˜, ๊ทธ ๋ชฉ๋ก์— ๋ฐ˜์˜๋œ ๋งˆ์ง€๋ง‰ ์ข…๋ฃŒ ์ด๋ฒคํŠธ์˜ ๋ฒˆํ˜ธ๋ฅผ ๋กœ๊ทธ์— ๋‚จ๊น๋‹ˆ๋‹ค. -**๋Œ€์กฐ.** ๊ฒŒ์ดํŠธ์›จ์ด๋Š” Claimํ•  ๋•Œ๋งˆ๋‹ค, `terminal`์ด ์•„๋‹ˆ๋ฉด์„œ ์ด Runner์— ์ด๋ฒˆ Claim์ด๋‚˜ ๊ทธ ๋’ค์˜ Claim์œผ๋กœ ๋ณด๋‚ธ ๊ฒƒ์ด ์•„๋‹Œ ๊ธฐ๋ก๊ณผ, ์•„์ง `resuming`์ธ ์Šน์ธ์˜ ๊ธฐ๋ก์„ ๋Œ€์กฐํ•ฉ๋‹ˆ๋‹ค. ๊ฐ ์‹คํ–‰์— ๋Œ€ํ•ด ์„ธ ๊ฐ€์ง€๋ฅผ ๋งž์ถฐ ๋ด…๋‹ˆ๋‹ค. ๊ทธ ์‹คํ–‰์˜ [๊ธฐ๋ก](#process-records), ๊ธฐ๋ก์ด ๊ฐ€๋ฆฌํ‚ค๋Š” Runner๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•ด ๊ฐ€์ง„ ๊ฒƒ์ด๋‚˜ ๊ธฐ์–ตํ•˜๋Š” ๊ฒƒ, ๊ทธ๋ฆฌ๊ณ  DevGuard๋ฅผ ์“ฐ๋ฉด `required` workspace์˜ ์‹คํ–‰์— ๋Œ€ํ•ด Runner๊ฐ€ DevGuard๋กœ ์กฐํšŒํ•œ attempt์ž…๋‹ˆ๋‹ค. Runner๋Š” ์‹œ์ž‘ํ•˜๋ผ๊ณ  ๋ฐ›์€ `process_id`๋งˆ๋‹ค, ํ”„๋กœ์„ธ์Šค๋ฅผ ๋งŒ๋“ค์—ˆ๋Š”์ง€ ์•„๋‹ˆ๋ฉด ์š”์ฒญ์„ ๋ฐ›๊ณ ๋„ ์•„๋ฌด๊ฒƒ๋„ ๋งŒ๋“ค์ง€ ์•Š์•˜๋Š”์ง€๋ฅผ ID๋งŒ์œผ๋กœ ์ˆ˜๋ช… ๋‚ด๋‚ด ๊ธฐ์–ตํ•ฉ๋‹ˆ๋‹ค. ์š”์ฒญ์„ ๋ฐ›์€ ์ ์ด ์—†๋‹ค๋Š” ์ฆ๋ช…์€ Runner ์ž์‹ ์— ๋Œ€ํ•ด์„œ๋งŒ ํ•  ์ˆ˜ ์žˆ์œผ๋ฏ€๋กœ, ์ค‘์ง€๋˜์—ˆ๊ฑฐ๋‚˜ ์žƒ์€ Runner๋ฅผ ํฌํ•จํ•ด ๋‹ค๋ฅธ Runner๋ฅผ ๊ฐ€๋ฆฌํ‚ค๋Š” ๊ธฐ๋ก์€ ์ด๋ ‡๊ฒŒ ์ •๋ฆฌํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. +**๋Œ€์กฐ.** ๊ฒŒ์ดํŠธ์›จ์ด๋Š” Claimํ•  ๋•Œ๋งˆ๋‹ค, `terminal`์ด ์•„๋‹ˆ๋ฉด์„œ ์ด Runner์— ์ด๋ฒˆ Claim์ด๋‚˜ ๊ทธ ๋’ค์˜ Claim์œผ๋กœ ๋ณด๋‚ธ ๊ฒƒ์ด ์•„๋‹Œ ๊ธฐ๋ก๊ณผ, ์•„์ง `resuming`์ธ ์Šน์ธ์˜ ๊ธฐ๋ก์„ ๋Œ€์กฐํ•ฉ๋‹ˆ๋‹ค. ๊ฐ ์‹คํ–‰์— ๋Œ€ํ•ด ์„ธ ๊ฐ€์ง€๋ฅผ ๋งž์ถฐ ๋ด…๋‹ˆ๋‹ค. ๊ทธ ์‹คํ–‰์˜ [๊ธฐ๋ก](#process-records), ๊ธฐ๋ก์ด ๊ฐ€๋ฆฌํ‚ค๋Š” Runner๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•ด ๊ฐ€์ง„ ๊ฒƒ์ด๋‚˜ ๊ธฐ์–ตํ•˜๋Š” ๊ฒƒ, ๊ทธ๋ฆฌ๊ณ  DevGuard๋ฅผ ์“ฐ๋ฉด `required` workspace์˜ ์‹คํ–‰์— ๋Œ€ํ•ด Runner๊ฐ€ DevGuard๋กœ ์กฐํšŒํ•œ attempt์ž…๋‹ˆ๋‹ค. Runner๋Š” ์‹œ์ž‘ํ•˜๋ผ๊ณ  ๋ฐ›์€ `process_id`๋งˆ๋‹ค, ํ”„๋กœ์„ธ์Šค๋ฅผ ๋งŒ๋“ค์—ˆ๋Š”์ง€ ์•„๋‹ˆ๋ฉด ์š”์ฒญ์„ ๋ฐ›๊ณ ๋„ ์•„๋ฌด๊ฒƒ๋„ ๋งŒ๋“ค์ง€ ์•Š์•˜๋Š”์ง€๋ฅผ ID๋งŒ์œผ๋กœ ์ˆ˜๋ช… ๋‚ด๋‚ด ๊ธฐ์–ตํ•ฉ๋‹ˆ๋‹ค. ์š”์ฒญ์„ ๋ฐ›์€ ์ ์ด ์—†๋‹ค๋Š” ์ฆ๋ช…์€ Runner ์ž์‹ ์— ๋Œ€ํ•ด์„œ๋งŒ ํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ๋‹ค๋ฅธ Runner๋ฅผ ๊ฐ€๋ฆฌํ‚ค๋ฉด์„œ ๋๋‚˜์ง€ ์•Š์€ ๊ธฐ๋ก์€ ๊ทธ ์†Œ์ผ“์—์„œ ๋‹ค๋ฅธ Runner๊ฐ€ ์‘๋‹ตํ•˜๋ฏ€๋กœ ๊ทธ Runner๋ฅผ [์žƒ์—ˆ๋‹ค](#runner-loss)๋Š” ๋œป์ด๊ณ , ์šด์˜์ž๋งŒ ์ •๋ฆฌํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. | ๊ธฐ๋ก | ๊ธฐ๋ก์ด ๊ฐ€๋ฆฌํ‚ค๋Š” Runner | DevGuard attempt(`required`๋งŒ) | ๊ฒฐ๊ณผ | | --- | --- | --- | --- | | `dispatching` ๋˜๋Š” `running` | ํ”„๋กœ์„ธ์Šค๋ฅผ ์‹คํ–‰ ์ค‘์œผ๋กœ ๊ฐ€์ง | launch๋จ | ๊ธฐ๋ก์€ `running`์ด ๋˜๊ณ , ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋‹ค์‹œ workspace๋ฅผ ์ ์œ ํ•ฉ๋‹ˆ๋‹ค. ๊ทธ workspace์˜ ์ƒˆ ์ž‘์—…์€ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚˜๊ฑฐ๋‚˜ ์ข…๋ฃŒ๋  ๋•Œ๊นŒ์ง€ `WORKSPACE_BUSY`๋ฅผ ๋ฐ›์Šต๋‹ˆ๋‹ค | | `dispatching` ๋˜๋Š” `running` | ๋๋‚œ ํ”„๋กœ์„ธ์Šค๋ฅผ ๊ฐ€์ง, ๋˜๋Š” ๋งŒ๋“ค์—ˆ์ง€๋งŒ ๋”๋Š” ๊ฐ–์ง€ ์•Š์Œ | scope๊ฐ€ ๋๋‚˜ ํ•ด์ œ๋จ, ๋˜๋Š” ์‹œ์ž‘ ์ „์— ๋๋‚จ | ๊ธฐ๋ก์€ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์–ด๋–ป๊ฒŒ ๋๋‚ฌ๋Š”์ง€ ๋‚จ๊ธฐ๋ฉฐ, Runner๊ฐ€ ๋”๋Š” ๊ฐ–์ง€ ์•Š์œผ๋ฉด ์ข…๋ฃŒ ๋ฐฉ์‹์€ `unknown`์ž…๋‹ˆ๋‹ค. workspace ์ ์œ ๋Š” ํ•ด์ œ๋ฉ๋‹ˆ๋‹ค | | `dispatching` | ์š”์ฒญ์„ ๋ฐ›์€ ์  ์—†์Œ, ๋˜๋Š” ๋ฐ›๊ณ  ์•„๋ฌด๊ฒƒ๋„ ๋งŒ๋“ค์ง€ ์•Š์Œ | ์—†์Œ, ๋˜๋Š” ์‹œ์ž‘ ์ „์— ๋๋‚จ | ๊ธฐ๋ก์€ ์‹คํ–‰์ด ์‹œ์ž‘๋˜์ง€ ์•Š์•˜๋‹ค๊ณ  ๋‚จ๊ธฐ๊ณ , workspace ์ ์œ ๋Š” ํ•ด์ œ๋ฉ๋‹ˆ๋‹ค | +| `dispatching`, `running` ๋˜๋Š” `unknown` | ๋‹ค๋ฅธ Runner์ด๋ฉฐ, ์žƒ์—ˆ์Œ | ๋ฌด์—‡์ด๋“  | ๊ธฐ๋ก์€ ์ด์œ ์™€ ํ•จ๊ป˜ `unknown`์ด ๋˜๊ณ , ์šด์˜์ž๊ฐ€ ๊ธฐ๋ก์„ ํ’€ ๋•Œ๊นŒ์ง€ workspace๋Š” ๊ณ„์† ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค | | ์—†์Œ | ํ”„๋กœ์„ธ์Šค๋ฅผ ์‹คํ–‰ ์ค‘์œผ๋กœ ๊ฐ€์ง | ๋ฌด์—‡์ด๋“  | workspace๋Š” ๊ณ„์† ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค | | ๊ทธ ๋ฐ–์˜ ๋ชจ๋“  ๊ฒฝ์šฐ | | | workspace๋Š” ๊ณ„์† ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค | -Runner๋‚˜ DevGuard์— ๋ฌป์ง€ ๋ชปํ–ˆ๊ฑฐ๋‚˜ DevGuard๊ฐ€ attempt๋ฅผ ์•„์ง ์ •๋ฆฌํ•˜์ง€ ์•Š์•„ ๋‹ต์„ ๊ธฐ๋‹ค๋ฆฌ๋Š” workspace๋Š” 500๋ฐ€๋ฆฌ์ดˆ ๋’ค์— ๋‹ค์‹œ ๋Œ€์กฐํ•˜๋ฉฐ, ๋‹ค์‹œ ์‹œ๋„ํ•  ๋•Œ๋งˆ๋‹ค ๊ธฐ๋‹ค๋ฆฌ๋Š” ์‹œ๊ฐ„์„ ๋‘ ๋ฐฐ์”ฉ ์ตœ๋Œ€ 30์ดˆ๊นŒ์ง€ ๋Š˜๋ฆฝ๋‹ˆ๋‹ค. ์„ธ ๊ฐ€์ง€๊ฐ€ ์„œ๋กœ ๋งž์ง€ ์•Š๊ฑฐ๋‚˜ ๊ธฐ๋ก์ด ๋‹ค๋ฅธ Runner๋ฅผ ๊ฐ€๋ฆฌํ‚ค๋Š” workspace๋Š” ์šด์˜์ž๊ฐ€ ํ•ด๊ฒฐํ•  ๋•Œ๊นŒ์ง€ ๊ณ„์† ๊ฑฐ๋ถ€๋˜๋ฉฐ, ๊ทธ ์ ˆ์ฐจ๋Š” ๋‹ค์Œ ๋‹จ๊ณ„์—์„œ ๋”ํ•ฉ๋‹ˆ๋‹ค. Runner๊ฐ€ ์‹คํ–‰ ์ค‘์ธ ํ”„๋กœ์„ธ์Šค๋ฅผ ๊ฐ€์กŒ์ง€๋งŒ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์„ค๋ช…ํ•˜๋Š” ๊ธฐ๋ก์ด ์—†๋Š” workspace๋Š” ๊ทธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚˜๊ฑฐ๋‚˜ ์ข…๋ฃŒ๋  ๋•Œ๊นŒ์ง€ ๊ณ„์† ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์ข…๋ฃŒ๊ฐ€ ์žˆ์„ ๋•Œ๋งˆ๋‹ค ๋‹ค์‹œ ๋Œ€์กฐํ•ฉ๋‹ˆ๋‹ค. ๊ณ„์† ๊ฑฐ๋ถ€ํ•˜๋Š” workspace๋Š” ๊ทธ ์ด์œ ์™€ ํ•จ๊ป˜ ๋กœ๊ทธ์— ๋‚จ๊ธฐ๊ณ , ๊ทธ ์ด์œ ๋Š” `RECOVERY_PENDING`์—๋„ ๋“ค์–ด๊ฐ‘๋‹ˆ๋‹ค. ๋Œ€์กฐ๋ฅผ ๋งˆ์นœ workspace๋„ ๋กœ๊ทธ์— ๋‚จ๊น๋‹ˆ๋‹ค. ๋Œ€์กฐํ•  ์ˆ˜ ์—†๋Š” workspace์˜ ์ ์œ ๋Š” ํ’€์ง€ ์•Š์Šต๋‹ˆ๋‹ค. +Runner๋‚˜ DevGuard์— ๋ฌป์ง€ ๋ชปํ–ˆ๊ฑฐ๋‚˜ DevGuard๊ฐ€ attempt๋ฅผ ์•„์ง ์ •๋ฆฌํ•˜์ง€ ์•Š์•„ ๋‹ต์„ ๊ธฐ๋‹ค๋ฆฌ๋Š” workspace๋Š” 500๋ฐ€๋ฆฌ์ดˆ ๋’ค์— ๋‹ค์‹œ ๋Œ€์กฐํ•˜๋ฉฐ, ๋‹ค์‹œ ์‹œ๋„ํ•  ๋•Œ๋งˆ๋‹ค ๊ธฐ๋‹ค๋ฆฌ๋Š” ์‹œ๊ฐ„์„ ๋‘ ๋ฐฐ์”ฉ ์ตœ๋Œ€ 30์ดˆ๊นŒ์ง€ ๋Š˜๋ฆฝ๋‹ˆ๋‹ค. ์„ธ ๊ฐ€์ง€๊ฐ€ ์„œ๋กœ ๋งž์ง€ ์•Š๋Š” workspace๋Š” ์šด์˜์ž๊ฐ€ ํ•ด๊ฒฐํ•  ๋•Œ๊นŒ์ง€, ์žƒ์€ Runner์˜ ๊ธฐ๋ก์ด ์žˆ๋Š” workspace๋Š” ์šด์˜์ž๊ฐ€ ๊ทธ ๊ธฐ๋ก์„ ํ’€ ๋•Œ๊นŒ์ง€ ๊ณ„์† ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค([์šด์˜](operations.md#runner-loss)). Runner๊ฐ€ ์‹คํ–‰ ์ค‘์ธ ํ”„๋กœ์„ธ์Šค๋ฅผ ๊ฐ€์กŒ์ง€๋งŒ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ์„ค๋ช…ํ•˜๋Š” ๊ธฐ๋ก์ด ์—†๋Š” workspace๋Š” ๊ทธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚˜๊ฑฐ๋‚˜ ์ข…๋ฃŒ๋  ๋•Œ๊นŒ์ง€ ๊ณ„์† ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์ข…๋ฃŒ๊ฐ€ ์žˆ์„ ๋•Œ๋งˆ๋‹ค ๋‹ค์‹œ ๋Œ€์กฐํ•ฉ๋‹ˆ๋‹ค. ๊ณ„์† ๊ฑฐ๋ถ€ํ•˜๋Š” workspace๋Š” ๊ทธ ์ด์œ ์™€ ํ•จ๊ป˜ ๋กœ๊ทธ์— ๋‚จ๊ธฐ๊ณ , ๊ทธ ์ด์œ ๋Š” `RECOVERY_PENDING`์—๋„ ๋“ค์–ด๊ฐ‘๋‹ˆ๋‹ค. ๋Œ€์กฐ๋ฅผ ๋งˆ์นœ workspace๋„ ๋กœ๊ทธ์— ๋‚จ๊น๋‹ˆ๋‹ค. ๋Œ€์กฐํ•  ์ˆ˜ ์—†๋Š” workspace์˜ ์ ์œ ๋Š” ํ’€์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์•ž์„  controller๊ฐ€ `resuming`์œผ๋กœ ๋‚จ๊ธด ์Šน์ธ์€ Runner๊ฐ€ ๊ทธ ์Šน์ธ์„ ์œ„ํ•ด ๋งŒ๋“  ์‹คํ–‰์œผ๋กœ consumed๊ฐ€ ๋˜๋ฉฐ, ๊ทธ ๋’ค์˜ `operation_resume`์€ ๊ทธ ์‹คํ–‰์˜ `process_id`๋ฅผ `dispatch_status: confirmed`์™€ ํ•จ๊ป˜ ๋Œ๋ ค์ค๋‹ˆ๋‹ค. Runner๊ฐ€ ์•„๋ฌด๊ฒƒ๋„ ๋งŒ๋“ค์ง€ ์•Š์•˜์Œ์ด ์ฆ๋ช…๋˜๋ฉด ์Šน์ธ์€ ๋‹ค์‹œ granted๊ฐ€ ๋˜์–ด ํ•œ ๋ฒˆ ๋” ์žฌ๊ฐœํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ๊ทธ ๋ฐ–์˜ ๊ฒฝ์šฐ์—๋Š” `resuming`์œผ๋กœ ๋‚จ๊ณ , `operation_resume`์€ ์ง€๊ธˆ์ฒ˜๋Ÿผ `APPROVAL_AMBIGUOUS`๋ฅผ ๋Œ๋ ค์ค๋‹ˆ๋‹ค. ์‹คํ–‰์ด ์‹œ์ž‘๋˜์—ˆ์„ ์ˆ˜๋„ ์žˆ๋Š” ์Šน์ธ์€ ๋‹ค์‹œ granted๊ฐ€ ๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. -**๋๋‚œ ํ”„๋กœ์„ธ์Šค.** ๋…๋ฆฝ Runner๋Š” ๋๋‚œ ํ”„๋กœ์„ธ์Šค์™€ ๊ทธ ์ƒํƒœ, ์ถœ๋ ฅ์„ controller๊ฐ€ ๊ทธ ๊ธฐ๋ก์— ํ”„๋กœ์„ธ์Šค๊ฐ€ ์–ด๋–ป๊ฒŒ ๋๋‚ฌ๋Š”์ง€ ๋‚จ๊ฒผ๋‹ค๊ณ  ํ™•์ธํ•  ๋•Œ๊นŒ์ง€ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. ๊ทธ๋ž˜์„œ ์–ด๋–ค ๊ฒŒ์ดํŠธ์›จ์ด๋„ ๋ฐ›์ง€ ๋ชปํ•œ ์ข…๋ฃŒ๋„ ๋‹ค์Œ controller์—๊ฒŒ ๊ฐ‘๋‹ˆ๋‹ค. ํ”„๋กœ์„ธ์Šค์˜ ํ•ธ๋“ค์€ ๊ด€๋ฆฌ worker์™€ ๊ฐ™์ด 15๋ถ„์ด ์ง€๋‚˜๊ฑฐ๋‚˜ ๋๋‚œ ํ”„๋กœ์„ธ์Šค๊ฐ€ 64๊ฐœ๋ฅผ ๋„˜์œผ๋ฉด ๋†“๊ณ , ํ•ธ๋“ค์„ ๋†“์€ ๋๋‚œ ํ”„๋กœ์„ธ์Šค๋Š” 64๊ฐœ๊นŒ์ง€ ๋ณด๊ด€ํ•˜๋ฉฐ ๊ฐ€์žฅ ์˜ค๋ž˜๋œ ๊ฒƒ๋ถ€ํ„ฐ ๋ฒ„๋ฆฝ๋‹ˆ๋‹ค. ๋ฒ„๋ฆฐ ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•ด์„œ๋Š” Runner๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ๋งŒ๋“ค์—ˆ๋‹ค๋Š” ๊ฒƒ๋งŒ ๊ธฐ์–ตํ•ฉ๋‹ˆ๋‹ค. +**๋๋‚œ ํ”„๋กœ์„ธ์Šค.** ๋…๋ฆฝ Runner๋Š” ๋๋‚œ ํ”„๋กœ์„ธ์Šค์™€ ๊ทธ ์ƒํƒœ, ์ถœ๋ ฅ์„ controller๊ฐ€ ๊ทธ ๊ธฐ๋ก์— ํ”„๋กœ์„ธ์Šค๊ฐ€ ์–ด๋–ป๊ฒŒ ๋๋‚ฌ๋Š”์ง€ ๋‚จ๊ฒผ๋‹ค๊ณ  ํ™•์ธํ•  ๋•Œ๊นŒ์ง€ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. ๊ทธ๋ž˜์„œ ์–ด๋–ค ๊ฒŒ์ดํŠธ์›จ์ด๋„ ๋ฐ›์ง€ ๋ชปํ•œ ์ข…๋ฃŒ๋„ ๋‹ค์Œ controller์—๊ฒŒ ๊ฐ‘๋‹ˆ๋‹ค. ํ”„๋กœ์„ธ์Šค์˜ ํ•ธ๋“ค์€ ๊ด€๋ฆฌ worker์™€ ๊ฐ™์ด 15๋ถ„์ด ์ง€๋‚˜๊ฑฐ๋‚˜ ๋๋‚œ ํ”„๋กœ์„ธ์Šค๊ฐ€ 64๊ฐœ๋ฅผ ๋„˜์œผ๋ฉด ๋†“๊ณ , ํ•ธ๋“ค์„ ๋†“์€ ๋๋‚œ ํ”„๋กœ์„ธ์Šค๋Š” 64๊ฐœ๊นŒ์ง€ ๋ณด๊ด€ํ•˜๋ฉฐ ๊ฐ€์žฅ ์˜ค๋ž˜๋œ ๊ฒƒ๋ถ€ํ„ฐ ๋ฒ„๋ฆฝ๋‹ˆ๋‹ค. ๋ฒ„๋ฆฐ ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•ด์„œ๋Š” Runner๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ ๋งŒ๋“ค์—ˆ๋‹ค๋Š” ๊ฒƒ๋งŒ ๊ธฐ์–ตํ•ฉ๋‹ˆ๋‹ค. Runner๊ฐ€ ๋”๋Š” ๊ฐ–์ง€ ์•Š๋Š” ๋๋‚œ ํ”„๋กœ์„ธ์Šค์— ๋Œ€ํ•ด์„œ๋Š” `process_status`์™€ `read_process`๊ฐ€ ๊ทธ ๊ธฐ๋ก์œผ๋กœ ๋‹ตํ•˜๋ฉฐ, `lost_reason`์€ `retention_expired`์ž…๋‹ˆ๋‹ค. + + + +**Runner ์†์‹ค.** Runner์™€์˜ ์—ฐ๊ฒฐ์„ ์žƒ์€ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์†Œ์ผ“์—์„œ ๋” ์ด์ƒ ์—ฐ๊ฒฐ์„ ๋ฐ›์ง€ ์•Š๊ฑฐ๋‚˜, ๊ทธ ์†Œ์ผ“์˜ worker๊ฐ€ ๋‹ค๋ฅธ ์‹๋ณ„ ์ •๋ณด๋ฅผ ์•Œ๋ฆฌ๊ฑฐ๋‚˜ ์•„๋ฌด ์‹๋ณ„ ์ •๋ณด๋„ ์•Œ๋ฆฌ์ง€ ์•Š์œผ๋ฉด Runner๋ฅผ ์žƒ์—ˆ๋‹ค๊ณ  ๋ด…๋‹ˆ๋‹ค. Runner๊ฐ€ ๊ฐ•์ œ๋กœ ์ข…๋ฃŒ๋˜์—ˆ๊ฑฐ๋‚˜, ๋น„์ •์ƒ ์ข…๋ฃŒํ–ˆ๊ฑฐ๋‚˜, ์ค‘์ง€๋˜์—ˆ๊ณ , ๋‹ค๋ฅธ Runner๋กœ ๋ฐ”๋€Œ์—ˆ์„ ์ˆ˜๋„ ์žˆ์Šต๋‹ˆ๋‹ค. ๊ทธ๋•Œ๋ถ€ํ„ฐ๋Š” ๊ทธ Runner์— ๋ณด๋‚ธ ์‹คํ–‰์ด ์–ด๋–ป๊ฒŒ ๋˜์—ˆ๋Š”์ง€ ์•Œ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค. ๊ฐ•์ œ๋กœ ์ข…๋ฃŒ๋œ Runner๋Š” ์•„๋ฌด๊ฒƒ๋„ ๋๋‚ด์ง€ ์•Š์œผ๋ฏ€๋กœ, ๊ทธ ํŒŒ์ดํ”„ ํ”„๋กœ์„ธ์Šค๋Š” ์ž์‹ ์ด ์‹œ์ž‘ํ•œ ํ”„๋กœ์„ธ์Šค์™€ ํ•จ๊ป˜, ์ž์‹ ์ด ์‹œ์ž‘๋œ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน ์•ˆ์—์„œ ๊ณ„์† ์‹คํ–‰๋ฉ๋‹ˆ๋‹ค. ๊ทธ ๊ทธ๋ฃน์€ Runner์˜ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน์ด๊ฑฐ๋‚˜, ๋„คํŠธ์›Œํฌ๋ฅผ ํ—ˆ์šฉํ•œ ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ์™€ DevGuard์˜ launch helper์ฒ˜๋Ÿผ root๊ฐ€ ์ด๋„๋Š” ๊ทธ๋ฃน์ž…๋‹ˆ๋‹ค. ๊ทธ ๋’ค ์ถœ๋ ฅ์— ์“ฐ๋Š” ํŒŒ์ดํ”„ ํ”„๋กœ์„ธ์Šค๋Š” SIGPIPE๋ฅผ ๋ฐ›์„ ๊ฐ€๋Šฅ์„ฑ์ด ํฌ๊ณ , `tty: true` ํ”„๋กœ์„ธ์Šค๋Š” Runner๊ฐ€ ๋๋‚˜๋ฉฐ ํ„ฐ๋ฏธ๋„์ด ๋‹ซํž ๋•Œ SIGHUP์„ ๋ฐ›์„ ๊ฐ€๋Šฅ์„ฑ์ด ํฌ์ง€๋งŒ, ์–ด๋Š ์ชฝ์ด๋“  ์ด๋ฅผ ๋ฌด์‹œํ•  ์ˆ˜ ์žˆ์œผ๋ฏ€๋กœ ๋๋‚ฌ๋‹ค๊ณ  ๋ณด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. Linux ๋ช…๋ น ์ƒŒ๋“œ๋ฐ•์Šค ์•ˆ์—์„œ ๋„คํŠธ์›Œํฌ๋ฅผ ํ—ˆ์šฉํ•˜์ง€ ์•Š์€ workspace์˜ ๋ช…๋ น์€, ์ƒŒ๋“œ๋ฐ•์Šค๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค๋ฅผ Runner์˜ ์ˆ˜๋ช…์— ๋ฌถ์œผ๋ฏ€๋กœ ๊ทธ ๋ช…๋ น์„ ์‹œ์ž‘ํ•œ Runner์™€ ํ•จ๊ป˜ ๋๋‚ฉ๋‹ˆ๋‹ค. ์ด ๊ฒฝ์šฐ์˜ ์‹œํ—˜์ด ์ด๋ฅผ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. ๋„คํŠธ์›Œํฌ๋ฅผ ํ—ˆ์šฉํ•˜๋ฉด ๋ช…๋ น๊ณผ ๊ทธ ๋„คํŠธ์›Œํฌ ํ”„๋ก์‹œ๋ฅผ ์‹คํ–‰ํ•˜๋Š” ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ๊ฐ€ Runner์— ๋ฌถ์ด์ง€ ์•Š์œผ๋ฏ€๋กœ, ๋ช…๋ น์ด Runner๋ณด๋‹ค ์˜ค๋ž˜ ์‚ด ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์†์‹ค๊ณผ ๊ทธ์— ํ•ด๋‹นํ•˜๋Š” ํ”„๋กœ์„ธ์Šค๋ฅผ ๋กœ๊ทธ์— ๋‚จ๊ธฐ๊ณ , Runner๊ฐ€ ํ•„์š”ํ•œ ์š”์ฒญ์„ ๋ชจ๋‘ `RUNNER_DISCONNECTED`๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. + +์žƒ์€ Runner์— ๋ณด๋‚ธ ๊ฒƒ ์ค‘ ๋๋‚˜์ง€ ์•Š์€ ๊ธฐ๋ก์€ ๋ชจ๋‘ ์ด์œ ์™€ ํ•จ๊ป˜ `unknown`์ด ๋˜๊ณ  workspace๋ฅผ ๊ณ„์† ์ ์œ ํ•ฉ๋‹ˆ๋‹ค. Runner๋ฅผ ์žƒ์€ ๊ฒŒ์ดํŠธ์›จ์ด์—์„œ๋„, [๋Œ€์กฐ](#reconciliation)ํ•˜๋ฉฐ ๋‹ค๋ฅธ Runner์˜ ๊ธฐ๋ก์„ ์ฐพ๋Š” ๋‹ค์Œ ๊ฒŒ์ดํŠธ์›จ์ด์—์„œ๋„ ๊ทธ๋ ‡์Šต๋‹ˆ๋‹ค. Runner๊ฐ€ ํ˜ธ์ŠคํŠธ์˜ ์ด์ „ ๋ถ€ํŒ…์—์„œ ์‹คํ–‰๋œ ๊ธฐ๋ก์€ ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•ด ์‹คํ–‰์ด ๋๋‚ฌ๋‹ค๊ณ  ๋‚จ๊น๋‹ˆ๋‹ค. Runner๊ฐ€ ์†์‹ค ์ „์— ์•Œ๋ฆฐ ์ข…๋ฃŒ๋Š” ๊ทธ๋Œ€๋กœ ๊ทธ ๊ธฐ๋ก์„ ๋๋ƒ…๋‹ˆ๋‹ค. ์ด๋Ÿฐ ์‹คํ–‰์€ ๋‹ค์‹œ ์‹œ์ž‘ํ•˜์ง€ ์•Š์œผ๋ฉฐ, ๋‹ค์Œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์šด์˜์ž๊ฐ€ ๊ทธ ์‹คํ–‰์ด ํ•˜๋‚˜๋„ ๋‚จ์•„ ์žˆ์ง€ ์•Š์Œ์„ ํ™•์ธํ•˜๊ณ  ๊ธฐ๋ก์„ ํ’€ ๋•Œ๊นŒ์ง€ ๊ทธ workspace์˜ ์ƒˆ ์ž‘์—…์„ `RECOVERY_PENDING`์œผ๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค([์šด์˜](operations.md#runner-loss)). ์ด ์˜ค๋ฅ˜๋Š” ํ”„๋กœ์„ธ์Šค์™€ `codespace-mcp records`๋ฅผ ์•Œ๋ ค ์ค๋‹ˆ๋‹ค. ์ด๋Ÿฐ ์‹คํ–‰์ด ์žฌ๊ฐœํ•œ ์Šน์ธ์€ `resuming`์œผ๋กœ ๋‚จ๊ณ , ๊ทธ ๊ธฐ๋ก์„ ํ’€๋ฉด `operation_resume`์€ ๊ทธ ์Šน์ธ์— ๋Œ€ํ•ด `APPROVAL_AMBIGUOUS`๋ฅผ ๋Œ๋ ค์ค๋‹ˆ๋‹ค. ๊ธฐ๋ก์˜ deadline๊ณผ argv digest๋Š” ๊ทธ๋Œ€๋กœ์ž…๋‹ˆ๋‹ค. + +Runner๊ฐ€ ๋”๋Š” ๊ฐ–์ง€ ์•Š๋Š” ์‹คํ–‰, ๋˜๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ Runner์™€ ํ•จ๊ป˜ ์žƒ์€ ์‹คํ–‰์— ๋Œ€ํ•ด์„œ๋Š” `process_status`์™€ `read_process`๊ฐ€ ๊ธฐ๋ก์œผ๋กœ ๋‹ตํ•ฉ๋‹ˆ๋‹ค. + +| ๊ธฐ๋ก | `state` | `lost_reason` | +| --- | --- | --- | +| ๋๋‚ฌ๊ณ  Runner๊ฐ€ ๋”๋Š” ๊ฐ–์ง€ ์•Š์Œ | ๊ธฐ๋ก๋œ ์ข…๋ฃŒ ๋ฐฉ์‹, ์ข…๋ฃŒ ์ฝ”๋“œ, ์‹ ํ˜ธ์™€ ํ•จ๊ป˜ `exited` | `retention_expired` | +| ๋๋‚ฌ๊ณ  ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ Runner๋ฅผ ์žƒ์Œ | ๊ธฐ๋ก๋Œ€๋กœ `exited` | `runner_lost` | +| ํ˜ธ์ŠคํŠธ ์žฌ์‹œ์ž‘ ์ „์— ๋๋‚จ | ๊ธฐ๋ก๋Œ€๋กœ `exited` | `host_rebooted` | +| ๋๋‚˜์ง€ ์•Š์•˜๊ณ , ์ด๋ฒˆ ๋ถ€ํŒ…์—์„œ Runner๋ฅผ ์žƒ์Œ | `unknown`: ํ”„๋กœ์„ธ์Šค๊ฐ€ ์•„์ง ์‹คํ–‰ ์ค‘์ผ ์ˆ˜ ์žˆ์œผ๋ฉฐ, ์ข…๋ฃŒ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค | `runner_lost` | +| ๋๋‚˜์ง€ ์•Š์•˜๊ณ , Runner๊ฐ€ ์ด์ „ ๋ถ€ํŒ…์—์„œ ์‹คํ–‰๋จ | ์ข…๋ฃŒ ๋ฐฉ์‹ `unknown`๊ณผ ํ•จ๊ป˜ `exited` | `host_rebooted` | + +์ด๋Ÿฐ ๋‹ต์—๋Š” ์ถœ๋ ฅ์ด ์—†์Šต๋‹ˆ๋‹ค. `read_process`๋Š” ๋นˆ `chunk`์™€ ํ•จ๊ป˜ `eof`์™€ `output_lost`๋ฅผ true๋กœ ๋Œ๋ ค์ฃผ๊ณ , `retained_from`๊ณผ cursor๋Š” ๊ฒŒ์ดํŠธ์›จ์ด๊ฐ€ ๋ณธ ๊ฐ€์žฅ ๋จผ ์ถœ๋ ฅ ์œ„์น˜์— ๋‘ก๋‹ˆ๋‹ค. ์š”์ฒญํ•œ cursor๊ฐ€ ๋” ๋ฉ€๋ฉด ๊ทธ cursor์— ๋‘ก๋‹ˆ๋‹ค. Runner๊ฐ€ ํ”„๋กœ์„ธ์Šค๋Š” ๊ฐ–๊ณ  ์žˆ์ง€๋งŒ ์ถœ๋ ฅ์˜ ์ผ๋ถ€๋งŒ ๊ฐ–๊ณ  ์žˆ๋Š” ๋™์•ˆ์—๋Š” `lost_reason`์ด `ring_overflow`์ด๊ณ , `read_process`์—์„œ๋Š” `output_lost`๊ฐ€, `process_status`์—์„œ๋Š” 0๋ณด๋‹ค ํฐ `output_retained_from`์ด ํ•จ๊ป˜ ์˜ต๋‹ˆ๋‹ค. Runner ์†์‹ค ๋’ค์—๋Š” ์ถœ๋ ฅ์„ ๋˜์‚ด๋ฆฌ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. diff --git a/docs/ko/security-model.md b/docs/ko/security-model.md index 6654d88..81b90ae 100644 --- a/docs/ko/security-model.md +++ b/docs/ko/security-model.md @@ -44,7 +44,7 @@ Linux ๋„์šฐ๋ฏธ๊ฐ€ ์—†์œผ๋ฉด restricted ์ •์ฑ…์˜ ์ž‘์—… ๊ณต๊ฐ„์—์„œ๋„ ๋น„๊ฒฉ ## ์ž‘์—…๊ณผ ๋ณต๊ตฌ์˜ ์•ˆ์ „์„ฑ -๋ฒ„์ „ ๊ฒ€์‚ฌ๋Š” ์˜ˆ์ƒํ•˜์ง€ ๋ชปํ•œ ํŒŒ์ผ ๋ฒ„์ „์— ํŒจ์น˜๋ฅผ ์ ์šฉํ•˜๋Š” ๊ฒƒ์„ ๋ง‰์Šต๋‹ˆ๋‹ค. ์ž‘์—… ํ‚ค๋Š” ํŒจ์น˜ ์ค‘๋ณต ์š”์ฒญ์„ ๊ตฌ๋ถ„ํ•˜๋ฉฐ ์ธ์ฆ ํ† ํฐ์ด ์•„๋‹™๋‹ˆ๋‹ค. ์ž‘์—… ๊ณต๊ฐ„ ์ ์œ ๋Š” ํŒŒ์ผ์„ ๋ณ€๊ฒฝํ•  ์ˆ˜ ์žˆ๋Š” ํŒจ์น˜ยท๋ช…๋ น ์‹คํ–‰์„ ์ง๋ ฌํ™”ํ•ฉ๋‹ˆ๋‹ค. ์š”์ฒญ ์†Œ์œ  ์ถฉ๋Œ์€ `acquire()`์—์„œ ์‹œ์ž‘ํ•˜๋Š” ์ธ๋ฉ”๋ชจ๋ฆฌ FIFO์—์„œ ๊ธฐ๋‹ค๋ฆฌ๊ณ , ๋ผ์ด๋ธŒ ํ”„๋กœ์„ธ์Šค๋Š” ๋’ค ๋Œ€๊ธฐ์ž์™€ ์ƒˆ ์š”์ฒญ์— `WORKSPACE_BUSY`๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. ์ ์œ  ์ƒํƒœ๋Š” ์žฌ์‹œ์ž‘ ํ›„์—๋„ ๋‚จ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์˜์†์ ์ธ ํ”„๋กœ์„ธ์Šค ๋ณต๊ตฌ๋Š” ์—†์Šต๋‹ˆ๋‹ค. ๋‹ค๋งŒ [๋…๋ฆฝ Runner](runner-isolation.md#reconciliation)์— ์—ฐ๊ฒฐํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๋Œ€์กฐํ•  ๋•Œ ์‹คํ–‰ ์ค‘์œผ๋กœ ํ™•์ธํ•œ ํ”„๋กœ์„ธ์Šค๋งˆ๋‹ค ์ ์œ ๋ฅผ ๋ณต์›ํ•˜๋ฉฐ, ๊ทธ ์ „์—๋Š” ๊ทธ workspace์—์„œ ์ƒˆ ์ž‘์—…์„ ์‹œ์ž‘ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. +๋ฒ„์ „ ๊ฒ€์‚ฌ๋Š” ์˜ˆ์ƒํ•˜์ง€ ๋ชปํ•œ ํŒŒ์ผ ๋ฒ„์ „์— ํŒจ์น˜๋ฅผ ์ ์šฉํ•˜๋Š” ๊ฒƒ์„ ๋ง‰์Šต๋‹ˆ๋‹ค. ์ž‘์—… ํ‚ค๋Š” ํŒจ์น˜ ์ค‘๋ณต ์š”์ฒญ์„ ๊ตฌ๋ถ„ํ•˜๋ฉฐ ์ธ์ฆ ํ† ํฐ์ด ์•„๋‹™๋‹ˆ๋‹ค. ์ž‘์—… ๊ณต๊ฐ„ ์ ์œ ๋Š” ํŒŒ์ผ์„ ๋ณ€๊ฒฝํ•  ์ˆ˜ ์žˆ๋Š” ํŒจ์น˜ยท๋ช…๋ น ์‹คํ–‰์„ ์ง๋ ฌํ™”ํ•ฉ๋‹ˆ๋‹ค. ์š”์ฒญ ์†Œ์œ  ์ถฉ๋Œ์€ `acquire()`์—์„œ ์‹œ์ž‘ํ•˜๋Š” ์ธ๋ฉ”๋ชจ๋ฆฌ FIFO์—์„œ ๊ธฐ๋‹ค๋ฆฌ๊ณ , ๋ผ์ด๋ธŒ ํ”„๋กœ์„ธ์Šค๋Š” ๋’ค ๋Œ€๊ธฐ์ž์™€ ์ƒˆ ์š”์ฒญ์— `WORKSPACE_BUSY`๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. ์ ์œ  ์ƒํƒœ๋Š” ์žฌ์‹œ์ž‘ ํ›„์—๋„ ๋‚จ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์˜์†์ ์ธ ํ”„๋กœ์„ธ์Šค ๋ณต๊ตฌ๋Š” ์—†์Šต๋‹ˆ๋‹ค. ๋‹ค๋งŒ [๋…๋ฆฝ Runner](runner-isolation.md#reconciliation)์— ์—ฐ๊ฒฐํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ๋Œ€์กฐํ•  ๋•Œ ์‹คํ–‰ ์ค‘์œผ๋กœ ํ™•์ธํ•œ ํ”„๋กœ์„ธ์Šค๋งˆ๋‹ค ์ ์œ ๋ฅผ ๋ณต์›ํ•˜๋ฉฐ, ๊ทธ ์ „์—๋Š” ๊ทธ workspace์—์„œ ์ƒˆ ์ž‘์—…์„ ์‹œ์ž‘ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ทธ Runner๋ฅผ ์žƒ์œผ๋ฉด ๋๋‚˜์ง€ ์•Š์€ ์‹คํ–‰๋งˆ๋‹ค, ์šด์˜์ž๊ฐ€ ๊ทธ์ค‘ ์•„๋ฌด๊ฒƒ๋„ ์‹คํ–‰ ์ค‘์ด์ง€ ์•Š์Œ์„ ํ™•์ธํ•˜๊ณ  ํ”„๋กœ์„ธ์Šค ํ•˜๋‚˜์”ฉ ํ’€ ๋•Œ๊นŒ์ง€ workspace๋ฅผ ์ ์œ ํ•ฉ๋‹ˆ๋‹ค. ์•„๋ฌด๊ฒƒ๋„ ๋‹ค์‹œ ์‹คํ–‰ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค([Runner ์†์‹ค](operations.md#runner-loss)). `CODESPACE_OPERATIONS_DB`์˜ `approvals` ํ™•์ธ ํ™€๋“œ๋Š” ํ–‰์ด `pending`ยท`granted`ยท`queued`ยท`resuming`์ธ ๋™์•ˆ V4A ํŒจ์น˜๋‚˜ exec argv๋ฅผ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. ํŒจ์น˜ ์›์žฅ์€ ํŒจ์น˜ ๋ณธ๋ฌธ์ด ์•„๋‹ˆ๋ผ ํ•ด์‹œ๋ฅผ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. `denied` ๋˜๋Š” `consumed` ๋’ค์—๋Š” ํ™€๋“œ ๋ณธ๋ฌธ์„ digest ๋ฉ”ํƒ€(๋„๊ตฌ, ์ž‘์—… ๊ณต๊ฐ„, fingerprint)๋กœ ๋ฐ”๊ฟ‰๋‹ˆ๋‹ค. [๋…๋ฆฝ Runner](runner-isolation.md#process-records)์— ์—ฐ๊ฒฐํ•œ ๊ฒŒ์ดํŠธ์›จ์ด๋Š” ์‹คํ–‰๋„ ์—ฌ๊ธฐ์— ๊ธฐ๋กํ•˜๋Š”๋ฐ(`processes`), argv๋Š” SHA-256 digest๋กœ๋งŒ ๋‚จ๊ธฐ๊ณ  argv ์ž์ฒด์™€ ๊ทธ ํ™˜๊ฒฝ, secret์€ ๋‚จ๊ธฐ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋˜ Runner๋ฅผ Claimํ•  ๋•Œ ๋ฐํžˆ๋Š” ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค์˜ ์ž„์˜ ์‹๋ณ„์ž(`store_identity`)๋ฅผ ์—ฌ๊ธฐ์— ๋‘ก๋‹ˆ๋‹ค. ์ด ์‹๋ณ„์ž๋Š” ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค๋ฅผ ์„œ๋กœ ๊ตฌ๋ณ„ํ•  ๋ฟ secret์ด ์•„๋‹™๋‹ˆ๋‹ค. ์ด ๋ฐ์ดํ„ฐ๋ฒ ์ด์Šค๋Š” ํ† ํฐยท๊ฒŒ์ดํŠธ์›จ์ด ์„ค์ •๊ณผ ๊ฐ™์ด ์ž‘์—… ๊ณต๊ฐ„ ๋ฃจํŠธ ๋ฐ–์— ๋‘์„ธ์š”. ํ™€๋“œ๋Š” ์›Œํฌํ”Œ๋กœ ์ผ์‹œ์ •์ง€์ด๋ฉฐ ๊ฐ™์€ MCP ํ˜ธ์ถœ์ž๊ฐ€ grantํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ๊ฒฉ๋ฆฌ ๊ฒฝ๊ณ„๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. diff --git a/docs/operations.md b/docs/operations.md index bdc646d..0ecd487 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -109,7 +109,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, whose gateway records them in the operations database, and which a gateway claims again after a lost connection and then reconciles with its records, is not a supported mode yet. +The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, whose gateway records them in the operations database, and which a gateway claims again after a lost connection and then reconciles with its records, is not a supported mode yet; when such a Runner itself is lost, an operator settles what it ran with `codespace-mcp records` ([Runner loss](#runner-loss)). @@ -122,8 +122,8 @@ The worker runs on the same host and is not a container. The gateway creates a p | `CODESPACE_OPERATIONS_DB` unset | In-memory patch operations and instruction queue; lost on restart. A gateway with an independent Runner does not start without it | | `CODESPACE_PROCESS_TIMEOUT_SECS` | Positive integer; default 30 seconds; set in the runner environment | | `CODESPACE_MAX_PROCESSES` | Default 8 live processes across the runner; a request over it is refused before any process is created; workspace occupancy still applies | -| Process output | Last 256 KiB retained; stdout/stderr combined; `read_process` reports `output_lost` and `retained_from`; `process_status` reports termination; `process_resize` resizes a running PTY | -| Completed handles | Default retention up to 15 minutes and 64 completed entries; not durable. An independent Runner keeps an ended process's status and output until its gateway acknowledges them, at most 64 once their handles are released ([ended processes](runner-isolation.md#reconciliation)) | +| Process output | Last 256 KiB retained; stdout/stderr combined; `read_process` reports `output_lost` and `retained_from`, and `lost_reason` says why output is missing; `process_status` reports termination; `process_resize` resizes a running PTY | +| Completed handles | Default retention up to 15 minutes and 64 completed entries; not durable. An independent Runner keeps an ended process's status and output until its gateway acknowledges them, at most 64 once their handles are released ([ended processes](runner-isolation.md#reconciliation)); after that, its gateway answers from the process record, without output, with `lost_reason` `retention_expired` | Store logs and the database outside the managed workspace, with tokens and gateway configuration. Rotate stderr capture yourself. Do not log Bearer tokens or commit real credentials. Deleting the database also deletes patch idempotency records and confirmation-hold rows. @@ -133,6 +133,43 @@ Store logs and the database outside the managed workspace, with tokens and gatew After losing a patch response, query `operation_status` with exactly one of `operation_id` or `operation_key`. An unfinished record becomes `unknown` after restart; inspect files before deciding what to do. Processes use `process_id` and cannot be recovered through `operation_status`. A confirmation hold that was `queued` when the process died can be resumed and will reacquire. A hold that was `resuming` may recover a patch from that ledger, replay a stored terminal result, or return `APPROVAL_AMBIGUOUS`; exec is not respawned. A gateway with an independent Runner settles an exec hold left `resuming` when it [reconciles](runner-isolation.md#reconciliation): consumed with the process that the Runner created for it, or granted again when the Runner proves that it created none. See [retry and recovery rules](agent-integration.md). + + +## Runner loss + +This applies to a gateway with an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet. When that Runner is killed, crashes or stops, or the host restarts, what it was running can no longer be learned from it, and its processes may outlive it ([Runner loss](runner-isolation.md#runner-loss)). The gateway answers for them from their records, as `unknown` with a `lost_reason`, and each workspace they held stays refused, with `RECOVERY_PENDING` in the next gateway, until you confirm that nothing of them runs and release their records. CodeSpace ends none of these processes, starts nothing again, and asks DevGuard nothing about them. + +1. **List the records** that may still hold a workspace. This only reads the database, so it also works while a gateway runs: + + ```bash + ./dist/codespace-mcp records list --operations-db "$CODESPACE_OPERATIONS_DB" + ``` + + It prints JSON. Each record has its `process_id` and `workspace_id`; its `phase`, `dispatching`, `running` or `unknown`; `lost`, which is `runner_lost` or `host_rebooted` once a gateway or a release marked it lost, and null before; its `participation` and DevGuard `attempt_id`; the `approval_id` it resumed; `tty`; its `runner`, with the Runner's PID and boot, `this_boot`, whether it ran on this boot of the host, and `pid_running`, whether a process with that PID runs now, which may be another process that took the PID; and its `root` process, with `running`, whether that very process still runs, or null when the record cannot tell. + +2. **Find what still runs** of each record. After a restart of the host, `host_rebooted` or `this_boot` false, nothing does. Otherwise: + - The root process runs when `running` is true. `ps -o pid,pgid,sid,lstart,command -p ` shows a process with its process group and session. + - A pipe process, and what it starts, stays in the process group it started in unless it leaves it: the Runner's own, or one that the root leads, as a sandbox helper with network enabled and DevGuard's launch helper do. `pgrep -g ` lists a group, `pgrep -P ` a process's children, and `lsof -d cwd` shows which processes work in the workspace's directories. One that wrote to its output since the Runner's end most likely ended with SIGPIPE, unless it ignores that signal. + - A `tty: true` process leads its own session, which `pgrep -s ` lists, and outlives the Runner only if it ignored the SIGHUP that the Runner's end most likely sent it. + - A record without a root, still `dispatching` when the Runner was lost, may or may not have started its process: look for it as above. + - Under the Linux command sandbox, commands in a workspace without network end with the Runner; with network enabled, they do not. A service manager may have ended the rest when the Runner's service stopped, as systemd does for a service's control group with its default `KillMode=control-group`, and launchd for a job's process group unless the job sets `AbandonProcessGroup`. Check rather than assume. + +3. **End what still runs**, once you decide that it should not finish: `kill -TERM `, then `kill -KILL ` if it does not exit. No deadline ends it any more, since the Runner that enforced the deadline is gone. + +4. **For a `required` record, wait until DevGuard no longer charges its attempt.** The lost Runner was DevGuard's execution owner, and DevGuard keeps the attempt charged until it observes that the attempt's processes ended, or until the host restarts. On macOS, `devguard admission --open` prints DevGuard's `quiescence`: `charged_attempts`, how many attempts it charges, and `attempts`, up to 16 of their keys. While a key there has the record's `attempt_id`, or `charged_attempts` is larger than the number of keys listed, DevGuard may still charge it. That command also reopens admission when it is closed, as an interrupted DevGuard upgrade leaves it, so complete such an upgrade first as [DevGuard's operations guide](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/operations.md#available-commands) says. DevGuard has no command that only reads its quiescence. When DevGuard lost track of an attempt's processes, as its [reconciliation contract](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/contracts.md#reconciliation) describes, it keeps the attempt charged until the host restarts, even after they all ended. Do not release a `required` record while DevGuard charges its attempt. + +5. **Stop the gateway and release the records:** + + ```bash + ./dist/codespace-mcp records release --operations-db "$CODESPACE_OPERATIONS_DB" ... + ``` + + It changes records only while no gateway runs on the database, since a running gateway holds its lock file. It releases each record whose Runner was lost once the record's root process no longer runs, prints JSON with `released`, `already_released` and `refused`, each refusal with its reason, and exits with status 1 when it refused any. It refuses a record that ended, which holds no workspace, one whose root process still runs, and one whose Runner may still run: a Runner that stopped while no gateway ran left its records `running`, and they are marked lost, with every other unended record of that Runner, only once no process has that Runner's PID. It ends no process. A released record stays `unknown`, and `operation_resume` answers an approval it resumed with `APPROVAL_AMBIGUOUS`: that command never runs again. + +6. **Start a Runner and the gateway again.** The gateway reconciles each workspace, and one whose records all ended or were released takes new work again. With DevGuard, the new Runner registers as the `codespace` consumer's one instance, which DevGuard allows only once it has retired the lost Runner's instance, when none of that Runner's attempts is charged any more. Until then `workspace_info` reports `resource_authority.registration.state` as `refused`, and `required` workspaces do not run; `off` workspaces are not affected. DevGuard does not let a consumer's instance limit change while one of its instances is not retired. + +A workspace where `RECOVERY_PENDING` says that the records, the Runner and DevGuard do not agree is not settled by `records release` while its Runner runs. Stopping that Runner makes its unended records records of a lost Runner, in every workspace it held, and the steps above then apply. + @@ -148,5 +185,6 @@ After losing a patch response, query `operation_status` with exactly one of `ope | Linux reports no sandbox | Helper location, bubblewrap, namespace support; see isolation guide | | `enabled` command fails before starting | A working Linux sandbox helper is required | | HTTP rejects the request | `/mcp` path, Bearer header, and Host validation | +| `RUNNER_DISCONNECTED` saying that the gateway lost its Runner, or `RECOVERY_PENDING` naming `codespace-mcp records` | Follow [Runner loss](#runner-loss) | Local transport tests do not establish a live ChatGPT connection, compatibility with every package manager through the proxy, or protection against kernel/container escapes. diff --git a/docs/runner-isolation.md b/docs/runner-isolation.md index 6cec195..334ba0c 100644 --- a/docs/runner-isolation.md +++ b/docs/runner-isolation.md @@ -20,7 +20,7 @@ Managed process lifetime is owned by the runner instance, not the MCP connection ## Independent Runner -> **Not a supported mode yet.** The independent Runner, the records of its executions, the Claim that hands its control from one gateway to the next and the reconciliation after each Claim are the first steps of Gateway recovery. A gateway claims the Runner again when its connection is lost, and starts new work where an earlier controller started work only once it has reconciled what that controller left. A later step adds what follows the loss of the Runner itself, and the mode is supported only once it is qualified. +> **Not a supported mode yet.** The independent Runner, the records of its executions, the Claim that hands its control from one gateway to the next and the reconciliation after each Claim are the first steps of Gateway recovery. A gateway claims the Runner again when its connection is lost, and starts new work where an earlier controller started work only once it has reconciled what that controller left. When the Runner itself is lost, the gateway answers for its executions from their records and keeps their workspaces until an operator releases them ([Runner loss](#runner-loss)). The mode is supported only once it is qualified. An independent Runner outlives the gateways that connect to it: its processes, their output and their timeouts carry on when a gateway exits, restarts or is killed. An operator or a harness starts it, and a gateway only connects to it: @@ -54,27 +54,46 @@ Each granted Claim raises the Runner's controller epoch by one. The epoch never A gateway that starts while another connection controls the Runner, or while that connection's requests run, asks again with a growing pause for up to 30 seconds, and then refuses to start. A gateway refused for another reason, such as another operations database, refuses to start at once and leaves the Runner as it was. -**A lost connection.** A gateway whose connection to the Runner is lost, or that closed it after a call passed its deadline, connects and claims the same Runner again. It first checks that the Runner at the socket has the identity it found at its start. It tries at once, waits 100 milliseconds after the first failed try, and twice as long after each further one, up to 5 seconds. Meanwhile it refuses each request that needs the Runner with `RUNNER_RECONNECTING`: nothing is sent, reserved or recorded for the request, and an approval it would resume is not consumed. When nothing accepts connections at the socket any more, another Runner answers there, or the Runner refuses the Claim for a reason that waiting cannot end, the gateway stops trying and refuses every such request with `RUNNER_DISCONNECTED`; it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection was lost keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. An exit event that the Runner could not deliver goes to its next controller, and the gateway hands each exit on once. +**A lost connection.** A gateway whose connection to the Runner is lost, or that closed it after a call passed its deadline, connects and claims the same Runner again. It first checks that the Runner at the socket has the identity it found at its start. It tries at once, waits 100 milliseconds after the first failed try, and twice as long after each further one, up to 5 seconds. Meanwhile it refuses each request that needs the Runner with `RUNNER_RECONNECTING`: nothing is sent, reserved or recorded for the request, and an approval it would resume is not consumed. When nothing accepts connections at the socket any more, or another Runner answers there, the Runner is [lost](#runner-loss). When the Runner refuses the Claim for a reason that waiting cannot end, the gateway stops trying too. Either way it refuses every such request with `RUNNER_DISCONNECTED`, and it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection was lost keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. An exit event that the Runner could not deliver goes to its next controller, and the gateway hands each exit on once. **After an earlier controller.** The first gateway that claims a newly started Runner works as it does with a managed worker. Once a controller has started work in a workspace (an execution, a patch, or with DevGuard a preparation or a launch), every later controller inherits the Runner's processes, the same gateway included once it claimed the Runner again after a lost connection, and the Claim's answer names each workspace where a controller before it started work. The gateway can query, read, write to, resize and terminate the inherited processes with their `process_id` at once, also while DevGuard cannot answer. In each workspace that the Claim names, where the Runner holds a running process, or that a record left to [reconcile](#reconciliation) names, it first reconciles what the earlier controllers left. Until then it refuses new executions, patches and approval resumes there with `RECOVERY_PENDING`, before it takes up an approval, so the approval stays as it was. The Runner also refuses them in each workspace that the Claim names, until the gateway tells it that the workspace is reconciled. Other workspaces take new work at once. The gateway logs the Runner's identity (its process ID, boot and start time), the epoch of its Claim, the workspaces where a controller before it started work, how many processes it inherited, and the number of the last exit event that list reflects. -**Reconciliation.** After each Claim, the gateway reconciles each record that is not `terminal` and was not sent to this Runner under this Claim or a later one, and each record of an approval that is still `resuming`. It compares three accounts of each execution: its [record](#process-records), what the Runner that the record names holds or remembers of the process, and with DevGuard, for an execution in a `required` workspace, its attempt, which the Runner looks up with DevGuard. For its whole life, the Runner remembers each `process_id` that it was asked to start, by its ID alone: whether it created the process, or received the request and created nothing. It proves that it never received a request only for itself: a record that names another Runner, one that was stopped or lost included, is not settled this way. +**Reconciliation.** After each Claim, the gateway reconciles each record that is not `terminal` and was not sent to this Runner under this Claim or a later one, and each record of an approval that is still `resuming`. It compares three accounts of each execution: its [record](#process-records), what the Runner that the record names holds or remembers of the process, and with DevGuard, for an execution in a `required` workspace, its attempt, which the Runner looks up with DevGuard. For its whole life, the Runner remembers each `process_id` that it was asked to start, by its ID alone: whether it created the process, or received the request and created nothing. It proves that it never received a request only for itself: a record that names another Runner and has not ended means that that Runner is [lost](#runner-loss), since another Runner answers at its socket, and only an operator settles it. | Record | The Runner that the record names | DevGuard attempt (`required` only) | Outcome | | --- | --- | --- | --- | | `dispatching` or `running` | holds the process running | launched | the record says `running`, and the process holds its workspace again: new work there returns `WORKSPACE_BUSY` until it exits or is terminated | | `dispatching` or `running` | holds the process ended, or created it and no longer holds it | released after its scope ended, or ended before it started | the record says how the process ended, with the termination `unknown` when the Runner no longer holds it, and the workspace is released | | `dispatching` | never received the request, or received it and created nothing | none, or ended before it started | the record says that the execution did not start, and the workspace is released | +| `dispatching`, `running` or `unknown` | is another Runner, which is lost | any | the record says `unknown`, with why, and the workspace stays refused until an operator releases the record | | none | holds a process running | any | the workspace stays refused | | anything else | | | the workspace stays refused | -A workspace that waits for an answer, because the Runner or DevGuard could not be asked or DevGuard has not settled an attempt yet, is reconciled again after 500 milliseconds, and twice as long after each further try, up to 30 seconds. A workspace where the accounts do not agree, or whose record names another Runner, stays refused until an operator resolves it, with a procedure that a later step adds; a workspace where the Runner holds a running process that no record accounts for stays refused until that process exits or is terminated. The gateway reconciles again after each exit too. It logs each workspace that stays refused with the reason, which `RECOVERY_PENDING` also states, and each workspace once it is reconciled. It does not release a workspace that it cannot reconcile. +A workspace that waits for an answer, because the Runner or DevGuard could not be asked or DevGuard has not settled an attempt yet, is reconciled again after 500 milliseconds, and twice as long after each further try, up to 30 seconds. A workspace where the accounts do not agree stays refused until an operator resolves it, and one with a record of a lost Runner until an operator releases that record ([Operations](operations.md#runner-loss)); a workspace where the Runner holds a running process that no record accounts for stays refused until that process exits or is terminated. The gateway reconciles again after each exit too. It logs each workspace that stays refused with the reason, which `RECOVERY_PENDING` also states, and each workspace once it is reconciled. It does not release a workspace that it cannot reconcile. An approval that an earlier controller left `resuming` is consumed with the execution that the Runner created for it, and `operation_resume` then returns that execution's `process_id` with `dispatch_status: confirmed`. When the Runner proved that it created none, the approval is granted again and can be resumed once more. Otherwise it stays `resuming`, and `operation_resume` returns `APPROVAL_AMBIGUOUS` as before: an approval whose execution may have started is never granted again. -**Ended processes.** An independent Runner keeps each ended process, its status and its output, until a controller acknowledges that its record says how the process ended, so an exit that no gateway received reaches the next controller. It still releases a process's handle after 15 minutes, or beyond 64 ended processes, as a managed worker does, and keeps at most 64 ended processes whose handles it released, dropping the oldest first. Of a dropped process, the Runner remembers only that it created it. +**Ended processes.** An independent Runner keeps each ended process, its status and its output, until a controller acknowledges that its record says how the process ended, so an exit that no gateway received reaches the next controller. It still releases a process's handle after 15 minutes, or beyond 64 ended processes, as a managed worker does, and keeps at most 64 ended processes whose handles it released, dropping the oldest first. Of a dropped process, the Runner remembers only that it created it. Once the Runner no longer keeps an ended process, `process_status` and `read_process` answer from its record, with `lost_reason` `retention_expired`. + + + +**Runner loss.** A gateway whose connection to the Runner is lost finds the Runner lost when nothing accepts connections at its socket any more, or when the worker there states another identity, or none: the Runner was killed, crashed or stopped, and perhaps replaced. What became of the executions it was sent cannot be learned from then on. A killed Runner ends nothing, so its pipe processes keep running, with the processes they started, in the process group they started in: the Runner's own, or one that their root leads, as a sandbox helper with network enabled and DevGuard's launch helper do. One that writes to its output afterwards most likely receives SIGPIPE, and a `tty: true` process most likely receives SIGHUP once the Runner's end closes its terminal, but either may ignore it, so neither is taken as ended. Under the Linux command sandbox, a command in a workspace without network ends with the Runner that started it, since the sandbox ties its processes to the Runner's life, as the tests of this case check. With network enabled, the sandbox helper that runs the command and its network proxy is not tied to the Runner, so the command can outlive it. The gateway logs the loss and the processes it concerns, and refuses every request that needs the Runner with `RUNNER_DISCONNECTED`. + +Each record of what was sent to the lost Runner that had not ended becomes `unknown`, with why, and keeps its workspace: in the gateway that lost the Runner, and in the next gateway, which finds records of another Runner as it [reconciles](#reconciliation). A record whose Runner ran on an earlier boot of the host says that the host restarted, which ended it. An end that the Runner reported before it was lost still ends its record. Nothing starts again for such an execution, and the next gateway refuses new work in its workspace with `RECOVERY_PENDING`, which names the process and `codespace-mcp records`, until an operator confirms that nothing of it runs and releases the record ([Operations](operations.md#runner-loss)). An approval that such an execution resumed stays `resuming`, and once its record is released, `operation_resume` returns `APPROVAL_AMBIGUOUS` for it. The record's deadline and argv digest stay as they were. + +`process_status` and `read_process` answer for an execution that the Runner no longer holds, or that the gateway lost with the Runner, from its record: + +| The record | `state` | `lost_reason` | +| --- | --- | --- | +| ended, and the Runner no longer keeps it | `exited`, with the recorded termination, exit code and signal | `retention_expired` | +| ended, and the gateway lost the Runner | `exited`, as recorded | `runner_lost` | +| ended before a restart of the host | `exited`, as recorded | `host_rebooted` | +| not ended, and the Runner was lost on this boot | `unknown`: the process may still run, and this is not an end | `runner_lost` | +| not ended, and the Runner ran on an earlier boot | `exited`, with the termination `unknown` | `host_rebooted` | + +Such an answer has no output: `read_process` returns an empty `chunk` with `eof` and `output_lost` true, and `retained_from` and the cursor at the furthest output offset a gateway saw, or at the cursor asked for when that is further. While the Runner keeps a process but not all of its output, `lost_reason` is `ring_overflow`, with `output_lost` in `read_process` and an `output_retained_from` above 0 in `process_status`. Output is never restored after a Runner loss. diff --git a/docs/security-model.md b/docs/security-model.md index 8abc7de..6dafbe0 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -34,7 +34,7 @@ The operator-registered root is a trust anchor. Keep tokens, the operations data ## Operation and recovery safety -Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery, except that a gateway with an [independent Runner](runner-isolation.md#reconciliation) restores the occupancy of each process that it finds running when it reconciles, and starts no new work in a workspace before that. +Version checks prevent applying a patch to an unexpected file version. Operation keys detect duplicate patch requests; they are not authorization tokens. Workspace occupancy serializes mutating patch and exec work. Request-owned conflicts wait in an in-memory FIFO that starts at acquire. A live process returns `WORKSPACE_BUSY` for trailing waiters and new arrivals. Occupancy is not durable. There is no durable process recovery, except that a gateway with an [independent Runner](runner-isolation.md#reconciliation) restores the occupancy of each process that it finds running when it reconciles, and starts no new work in a workspace before that. When its Runner is lost, each execution that had not ended keeps its workspace until an operator confirms that nothing of it runs and releases it, one process at a time; nothing is run again ([Runner loss](operations.md#runner-loss)). Confirmation holds in `CODESPACE_OPERATIONS_DB` (`approvals`) store the V4A patch or exec argv while the row is `pending`, `granted`, `queued`, or `resuming`. The patch operations ledger stores hashes, not patch text. After `denied` or `consumed`, the hold body is replaced with digest metadata (tool, workspace, fingerprint). A gateway with an [independent Runner](runner-isolation.md#process-records) also records its executions there (`processes`), each with the SHA-256 digest of its argv, never the argv, its environment or a secret, and keeps there the database's random identity (`store_identity`), which it names when it claims the Runner: it tells one database from another and is not a secret. Keep this database outside the workspace root, with tokens and gateway configuration. The hold is a workflow pause: the same MCP caller can grant it. It is not an isolation boundary. diff --git a/docs/translations.json b/docs/translations.json index dbf73e1..36dc2af 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -78,8 +78,8 @@ "์ตœ์†Œ-๋„์ž…-์กฐ๊ฑด", "์ตœ์ดˆ-gateway-๋ณต๊ตฌ-๋ฒ”์œ„" ], - "source_sha256": "73d4de8918976b98bd92f2b1811d0872d9603edf23ac847b59eb9e74a9f36514", - "translation_sha256": "9616be59db270581e1035460a8bef34bf06815f46c2db4d79eb87f2bd66d5a6d" + "source_sha256": "77bada859adefafb165febb3273f179c6135fcb5b6f757396dd639eb22c3ca0f", + "translation_sha256": "319176abbe5b1e8029493d5e3a637e712287a40c518068c6cd0e8b5e38f7b40b" }, { "id": "cs-rg-qualification", @@ -257,8 +257,8 @@ "์žฌ์‹œ๋„์™€-๋ณต๊ตฌ", "ํŒŒ์ผ-์ฝ๊ธฐ์™€-ํŒจ์น˜" ], - "source_sha256": "c0257f9844525cb9a2dbbca6fa6ee4dbfa376de877cb8a647d55e4a476031a2d", - "translation_sha256": "095fe129c42f71fab2a7ec47bdbffdd81f0c6141e616e454c46e4f2f3134833b" + "source_sha256": "a3a4705eb2a425b423aa936b0c3f9b448fbb217f58ec4bbbb8ca7031333ff8d2", + "translation_sha256": "6cb63bc559a6b7e9f6fc9c1fa425113c67a08970faef1f84eb98094ef41e6b10" }, { "id": "operations", @@ -284,6 +284,8 @@ "register-a-workspace", "reproduce-the-mvp-flow", "run-the-gateway", + "runner-loss", + "runner-์†์‹ค", "start-the-server", "troubleshooting", "verify-the-first-connection", @@ -310,8 +312,8 @@ "์ž‘์—…-๊ณต๊ฐ„-๋“ฑ๋ก", "์ฒซ-์—ฐ๊ฒฐ-ํ™•์ธ" ], - "source_sha256": "3b42dfbca4a44361af4476134e59d02af7313f5093590057b0e81ca35b3f42b5", - "translation_sha256": "b44f15c0a1c415e6015e4d76ca6421a648ef61572b2b959c9e6faa21d8b6b35c" + "source_sha256": "b9e516baca30cf936c2cb98cec8992649f4be9eca36e0c381b03f63502f05d2d", + "translation_sha256": "9121e8c5474640e694bcb509a0a679827a23b4ee76c973885c36cd331c688293" }, { "id": "chatgpt-connector", @@ -400,8 +402,8 @@ "ํ˜„์žฌ-์‹คํ–‰-๊ตฌ์กฐ", "ํ™•์žฅ-์‹œ-์œ ์ง€ํ• -๊ฒฝ๊ณ„" ], - "source_sha256": "9d03ae6de59d281f2f992bd39e4bf32516946ea0b490de3e0325d2f78cb0574d", - "translation_sha256": "10d6fffe680c807b9b7dd22148032e2529ea452d1d527ac2ec6f1cade693aa0b" + "source_sha256": "686f0cd9906db317058fa6128080362e74da25bbdd8863095ac849a566d99bb8", + "translation_sha256": "02d127df0e182eab1c9c504194143ad9f282162ffed7ca3a8cf53cc3d3d75d6b" }, { "id": "execution-substrate", @@ -465,8 +467,8 @@ "ํ™•์ธ-ํ™€๋“œ", "ํ›…๊ณผ-์Šคํ‚ฌ" ], - "source_sha256": "93e4ef27d771f9bec226af62ae66a785962aea6030050164195d9d9bcadc82ce", - "translation_sha256": "512a93517c56dd59aebae0271025658fd3c17a9f318bf3e6217b061245eb2b40" + "source_sha256": "366ffeb1ad5ef8112a9dcddc57486851147faf6f257e1bb95be2a764f4a9579a", + "translation_sha256": "23aa9948c2ca43fb207ec9843e9b4a93dfb1c23e8023e8d3c8afa6be8da133df" }, { "id": "protocol-compatibility", @@ -576,8 +578,8 @@ "ํ”„๋กœ์„ธ์Šค-์ •์ง์„ฑ", "ํ”„๋กœํ•„-mvp" ], - "source_sha256": "cdb542f72d794730735b7a71cf5c45bfb3aa012d3d6f637196dc28dc41249516", - "translation_sha256": "5bd6ab3149c3c72b7d94a96910ff412be67163ad73b0df1b0c8d12b2b9d109cf" + "source_sha256": "c9ca06af27ed21c9cce3be7528c9d5aa5cdbe03db171dc02ae24880ffc755859", + "translation_sha256": "80cd04a917bbbaf09f0c9f3191fa16b6ef79c1222cf65428a5d0801c5435e70f" }, { "id": "runner-isolation", @@ -604,6 +606,7 @@ "process-records", "reconciliation", "runner-isolation", + "runner-loss", "what-the-compose-fixture-does", "แ„‹แ…ตแ„’แ…ฎ-แ„‘แ…ณแ„…แ…ฉแ„‰แ…ฆแ„‰แ…ณ-แ„‡แ…ฎแ†ซแ„…แ…ต", "๋„คํŠธ์›Œํฌ์™€-ํŒŒ์ผ-์ ‘๊ทผ-๋ฒ”์œ„", @@ -613,8 +616,8 @@ "์ปจํ…Œ์ด๋„ˆ-์‹คํ—˜-๊ตฌ์„ฑ๊ณผ-๊ฒ€์ฆ", "ํ˜ธ์ŠคํŠธ์™€-worker-์‹คํ–‰" ], - "source_sha256": "e76149aba184f9de7f6e11441db5f9f519db95f20b62eb367de67e49381f98d2", - "translation_sha256": "950575f22d943f0904b7a805b4cd81f154f218a14e219bc16e92e8449d6ff1f7" + "source_sha256": "ada35d6e3907abe03287e595acf28acfda9ba546ede51d5d65ce4115db67268a", + "translation_sha256": "3336edb2d573475407bb95ff5b8220b3195836e42debb39f1feff0ae61c5a429" }, { "id": "error-codes", @@ -642,8 +645,8 @@ "์ „์†ก-์‹คํŒจ", "์ „์†ก-์‹คํŒจ-์ž‘์—…-์—†์Œ" ], - "source_sha256": "5892534235706ee8d0d3a9bf18a1d36b18578441567f3eba92f69d91fbce2010", - "translation_sha256": "45046341089415fcc7d64e850629e6605ece6d1ab48b3a05e9579e32f8f56bbd" + "source_sha256": "cc10d7c2915b2a339723dfcf81a02201c45aa6396292b2d24a06631561bd45bd", + "translation_sha256": "0f20fb1831799f4aae566f1dffff42f20030718959918f819cdcea469974dba7" }, { "id": "codex-reuse", diff --git a/scripts/validate-upstream.py b/scripts/validate-upstream.py index f9e138f..855603f 100644 --- a/scripts/validate-upstream.py +++ b/scripts/validate-upstream.py @@ -88,13 +88,15 @@ def stages(): # against the worker, the fixture authority and the helper built here. With them, the # independent Runner (P1R-C01): its identity, socket directory and wire, its Claim with # the peer credentials that macOS reads through `getpeereid` (P1R-C03), the gateway's - # settings, refusals and reconnect, then the worker binary and the gateway binary - # against it, with the default worker built again over the DevGuard one. + # settings, refusals and reconnect, the operator's records commands, which read the + # host's boot session and whether a process still runs as itself (P1R-C05), then the + # worker binary and the gateway binary against it, with the default worker built again + # over the DevGuard one. + devguard_binaries() + [cargo('test', 'devguard'), cargo('test', 'codex-runtime', '--features', 'devguard', '--bin', 'codespace-codex-runtime'), cargo('test', 'root', '-p', 'codespace-runner', '--features', 'devguard', '--lib', '--', 'registration', 'wire', 'admission', 'launch', 'independent', 'lifecycle'), - cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--lib', '--', 'devguard', 'independent'), + cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--lib', '--', 'devguard', 'independent', 'operator'), cargo('build', 'codex-runtime', '--bin', 'codespace-codex-runtime'), cargo('test', 'codex-runtime', '--test', 'runtime_binary', '--', 'independent'), cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--test', 'independent_runner')], From 8dbab2b55bb5aee9fa4aa59d3658c7b7511a9dbc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 11:27:13 +0000 Subject: [PATCH 2/3] docs: the runbook's step for a DevGuard release that reports no quiescence (P1R-C05) devguard admission --open prints quiescence as null when the serving release does not report it. Nothing then tells whether DevGuard still charges a lost Runner's attempt, so the runbook keeps a required record until the host restarts, which ends the attempt in DevGuard too. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/ko/operations.md | 2 +- docs/operations.md | 2 +- docs/translations.json | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ko/operations.md b/docs/ko/operations.md index 95da557..4c596f6 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -164,7 +164,7 @@ worker๋Š” ๊ฐ™์€ ํ˜ธ์ŠคํŠธ์—์„œ ์‹คํ–‰ํ•˜๋Š” ๋ณ„๋„ ํ”„๋กœ์„ธ์Šค์ด๋ฉฐ ์ปจํ…Œ 3. **์•„์ง ์‹คํ–‰ ์ค‘์ธ ๊ฒƒ์„ ์ข…๋ฃŒํ•ฉ๋‹ˆ๋‹ค.** ๋๊นŒ์ง€ ์‹คํ–‰ํ•˜์ง€ ์•Š๊ธฐ๋กœ ์ •ํ–ˆ๋‹ค๋ฉด `kill -TERM `๋ฅผ ๋ณด๋‚ด๊ณ , ๋๋‚˜์ง€ ์•Š์œผ๋ฉด `kill -KILL `๋ฅผ ๋ณด๋ƒ…๋‹ˆ๋‹ค. deadline์„ ์ง€ํ‚ค๊ฒŒ ํ•˜๋˜ Runner๊ฐ€ ์—†์œผ๋ฏ€๋กœ ์ด์ œ deadline์ด ์ด ํ”„๋กœ์„ธ์Šค๋ฅผ ๋๋‚ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. -4. **`required` ๊ธฐ๋ก์€ DevGuard๊ฐ€ ๊ทธ attempt๋ฅผ ๋”๋Š” ๊ณผ๊ธˆํ•˜์ง€ ์•Š์„ ๋•Œ๊นŒ์ง€ ๊ธฐ๋‹ค๋ฆฝ๋‹ˆ๋‹ค.** ์žƒ์€ Runner๋Š” DevGuard์˜ ์‹คํ–‰ ์†Œ์œ ์ž์˜€๊ณ , DevGuard๋Š” attempt์˜ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚ฌ์Œ์„ ๊ด€์ฐฐํ•˜๊ฑฐ๋‚˜ ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•  ๋•Œ๊นŒ์ง€ ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•ฉ๋‹ˆ๋‹ค. macOS์—์„œ `devguard admission --open`์€ DevGuard์˜ `quiescence`๋ฅผ ์ถœ๋ ฅํ•ฉ๋‹ˆ๋‹ค. `charged_attempts`๋Š” ๊ณผ๊ธˆ ์ค‘์ธ attempt ์ˆ˜์ด๊ณ , `attempts`๋Š” ๊ทธ key๋ฅผ ์ตœ๋Œ€ 16๊ฐœ ๋ณด์—ฌ ์ค๋‹ˆ๋‹ค. ๊ทธ key ์ค‘ ํ•˜๋‚˜๊ฐ€ ๊ธฐ๋ก์˜ `attempt_id`๋ฅผ ๊ฐ€์กŒ๊ฑฐ๋‚˜ `charged_attempts`๊ฐ€ ๋‚˜์—ด๋œ key ์ˆ˜๋ณด๋‹ค ํฌ๋ฉด DevGuard๊ฐ€ ์•„์ง ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•˜๊ณ  ์žˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ์ด ๋ช…๋ น์€ admission์ด ๋‹ซํ˜€ ์žˆ์œผ๋ฉด ๋‹ค์‹œ ์—ฝ๋‹ˆ๋‹ค. ์ค‘๋‹จ๋œ DevGuard upgrade๊ฐ€ admission์„ ๋‹ซํžŒ ์ฑ„๋กœ ๋‚จ๊ธฐ๋ฏ€๋กœ, ๊ทธ๋Ÿฐ upgrade๋Š” [DevGuard ์šด์˜ ๋ฌธ์„œ](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/operations.md#์ œ๊ณต-๋ช…๋ น)๋Œ€๋กœ ๋จผ์ € ๋งˆ์น˜์„ธ์š”. DevGuard์—๋Š” quiescence๋ฅผ ์ฝ๊ธฐ๋งŒ ํ•˜๋Š” ๋ช…๋ น์ด ์—†์Šต๋‹ˆ๋‹ค. [DevGuard ๋Œ€์กฐ ๊ณ„์•ฝ](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/contracts.md#๋Œ€์กฐ)์ด ์„ค๋ช…ํ•˜๋“ฏ DevGuard๊ฐ€ attempt์˜ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ถ”์ ํ•˜์ง€ ๋ชปํ•˜๊ฒŒ ๋˜๋ฉด, ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋ชจ๋‘ ๋๋‚œ ๋’ค์—๋„ ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•  ๋•Œ๊นŒ์ง€ ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•ฉ๋‹ˆ๋‹ค. DevGuard๊ฐ€ attempt๋ฅผ ๊ณผ๊ธˆํ•˜๋Š” ๋™์•ˆ์—๋Š” `required` ๊ธฐ๋ก์„ ํ’€์ง€ ๋งˆ์„ธ์š”. +4. **`required` ๊ธฐ๋ก์€ DevGuard๊ฐ€ ๊ทธ attempt๋ฅผ ๋”๋Š” ๊ณผ๊ธˆํ•˜์ง€ ์•Š์„ ๋•Œ๊นŒ์ง€ ๊ธฐ๋‹ค๋ฆฝ๋‹ˆ๋‹ค.** ์žƒ์€ Runner๋Š” DevGuard์˜ ์‹คํ–‰ ์†Œ์œ ์ž์˜€๊ณ , DevGuard๋Š” attempt์˜ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚ฌ์Œ์„ ๊ด€์ฐฐํ•˜๊ฑฐ๋‚˜ ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•  ๋•Œ๊นŒ์ง€ ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•ฉ๋‹ˆ๋‹ค. macOS์—์„œ `devguard admission --open`์€ DevGuard์˜ `quiescence`๋ฅผ ์ถœ๋ ฅํ•ฉ๋‹ˆ๋‹ค. `charged_attempts`๋Š” ๊ณผ๊ธˆ ์ค‘์ธ attempt ์ˆ˜์ด๊ณ , `attempts`๋Š” ๊ทธ key๋ฅผ ์ตœ๋Œ€ 16๊ฐœ ๋ณด์—ฌ ์ค๋‹ˆ๋‹ค. ๊ทธ key ์ค‘ ํ•˜๋‚˜๊ฐ€ ๊ธฐ๋ก์˜ `attempt_id`๋ฅผ ๊ฐ€์กŒ๊ฑฐ๋‚˜ `charged_attempts`๊ฐ€ ๋‚˜์—ด๋œ key ์ˆ˜๋ณด๋‹ค ํฌ๋ฉด DevGuard๊ฐ€ ์•„์ง ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•˜๊ณ  ์žˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ์ด ๋ช…๋ น์€ admission์ด ๋‹ซํ˜€ ์žˆ์œผ๋ฉด ๋‹ค์‹œ ์—ฝ๋‹ˆ๋‹ค. ์ค‘๋‹จ๋œ DevGuard upgrade๊ฐ€ admission์„ ๋‹ซํžŒ ์ฑ„๋กœ ๋‚จ๊ธฐ๋ฏ€๋กœ, ๊ทธ๋Ÿฐ upgrade๋Š” [DevGuard ์šด์˜ ๋ฌธ์„œ](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/operations.md#์ œ๊ณต-๋ช…๋ น)๋Œ€๋กœ ๋จผ์ € ๋งˆ์น˜์„ธ์š”. DevGuard์—๋Š” quiescence๋ฅผ ์ฝ๊ธฐ๋งŒ ํ•˜๋Š” ๋ช…๋ น์ด ์—†์Šต๋‹ˆ๋‹ค. `quiescence`๊ฐ€ null๋กœ ์ถœ๋ ฅ๋˜๋ฉด ์‹คํ–‰ ์ค‘์ธ DevGuard release๊ฐ€ ์ด๋ฅผ ๋ณด๊ณ ํ•˜์ง€ ์•Š๋Š” ๊ฒƒ์ด๋ฏ€๋กœ DevGuard๊ฐ€ ๊ทธ attempt๋ฅผ ์•„์ง ๊ณผ๊ธˆํ•˜๋Š”์ง€ ์•Œ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค. ์ด๋•Œ๋Š” ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•  ๋•Œ๊นŒ์ง€ ๊ธฐ๋ก์„ ๊ทธ๋Œ€๋กœ ๋‘์„ธ์š”. ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•˜๋ฉด DevGuard์—์„œ๋„ ๊ทธ attempt๊ฐ€ ๋๋‚ฉ๋‹ˆ๋‹ค. [DevGuard ๋Œ€์กฐ ๊ณ„์•ฝ](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/ko/contracts.md#๋Œ€์กฐ)์ด ์„ค๋ช…ํ•˜๋“ฏ DevGuard๊ฐ€ attempt์˜ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ถ”์ ํ•˜์ง€ ๋ชปํ•˜๊ฒŒ ๋˜๋ฉด, ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋ชจ๋‘ ๋๋‚œ ๋’ค์—๋„ ํ˜ธ์ŠคํŠธ๊ฐ€ ์žฌ์‹œ์ž‘ํ•  ๋•Œ๊นŒ์ง€ ๊ทธ attempt๋ฅผ ๊ณผ๊ธˆํ•ฉ๋‹ˆ๋‹ค. DevGuard๊ฐ€ attempt๋ฅผ ๊ณผ๊ธˆํ•˜๋Š” ๋™์•ˆ์—๋Š” `required` ๊ธฐ๋ก์„ ํ’€์ง€ ๋งˆ์„ธ์š”. 5. **๊ฒŒ์ดํŠธ์›จ์ด๋ฅผ ๋ฉˆ์ถ”๊ณ  ๊ธฐ๋ก์„ ํ’‰๋‹ˆ๋‹ค.** diff --git a/docs/operations.md b/docs/operations.md index 0ecd487..f0bb7fa 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -156,7 +156,7 @@ This applies to a gateway with an [independent Runner](runner-isolation.md#indep 3. **End what still runs**, once you decide that it should not finish: `kill -TERM `, then `kill -KILL ` if it does not exit. No deadline ends it any more, since the Runner that enforced the deadline is gone. -4. **For a `required` record, wait until DevGuard no longer charges its attempt.** The lost Runner was DevGuard's execution owner, and DevGuard keeps the attempt charged until it observes that the attempt's processes ended, or until the host restarts. On macOS, `devguard admission --open` prints DevGuard's `quiescence`: `charged_attempts`, how many attempts it charges, and `attempts`, up to 16 of their keys. While a key there has the record's `attempt_id`, or `charged_attempts` is larger than the number of keys listed, DevGuard may still charge it. That command also reopens admission when it is closed, as an interrupted DevGuard upgrade leaves it, so complete such an upgrade first as [DevGuard's operations guide](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/operations.md#available-commands) says. DevGuard has no command that only reads its quiescence. When DevGuard lost track of an attempt's processes, as its [reconciliation contract](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/contracts.md#reconciliation) describes, it keeps the attempt charged until the host restarts, even after they all ended. Do not release a `required` record while DevGuard charges its attempt. +4. **For a `required` record, wait until DevGuard no longer charges its attempt.** The lost Runner was DevGuard's execution owner, and DevGuard keeps the attempt charged until it observes that the attempt's processes ended, or until the host restarts. On macOS, `devguard admission --open` prints DevGuard's `quiescence`: `charged_attempts`, how many attempts it charges, and `attempts`, up to 16 of their keys. While a key there has the record's `attempt_id`, or `charged_attempts` is larger than the number of keys listed, DevGuard may still charge it. That command also reopens admission when it is closed, as an interrupted DevGuard upgrade leaves it, so complete such an upgrade first as [DevGuard's operations guide](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/operations.md#available-commands) says. DevGuard has no command that only reads its quiescence. When it prints `quiescence` as null, the serving DevGuard release does not report it, and nothing tells whether DevGuard still charges the attempt: keep the record until the host restarts, which ends the attempt in DevGuard too. When DevGuard lost track of an attempt's processes, as its [reconciliation contract](https://github.com/novelKR/DevGuard/blob/7cd86e4c56cdd6c6067ceda183b8f3f82075fcaa/docs/contracts.md#reconciliation) describes, it keeps the attempt charged until the host restarts, even after they all ended. Do not release a `required` record while DevGuard charges its attempt. 5. **Stop the gateway and release the records:** diff --git a/docs/translations.json b/docs/translations.json index 36dc2af..feb7e7e 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -312,8 +312,8 @@ "์ž‘์—…-๊ณต๊ฐ„-๋“ฑ๋ก", "์ฒซ-์—ฐ๊ฒฐ-ํ™•์ธ" ], - "source_sha256": "b9e516baca30cf936c2cb98cec8992649f4be9eca36e0c381b03f63502f05d2d", - "translation_sha256": "9121e8c5474640e694bcb509a0a679827a23b4ee76c973885c36cd331c688293" + "source_sha256": "5950758a72f81664dce25afd7c281c9201fab11fa4ae4d78461bdaeb00ce978b", + "translation_sha256": "09ca3ed836f6b34f0528e9f750882af013389b47a50047cae4bd512a64250355" }, { "id": "chatgpt-connector", From 5deb102057ac86f46983ce4df56f0948a5741887 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 12:15:32 +0000 Subject: [PATCH 3/3] store: let go of the writer lock when it is dropped (P1R-C05) Loops of this unit's operator tests found `records release` refused right after a test let go of the lock. Closing the lock's descriptor does not let go of its flock while a child that another thread is forking holds a copy of the descriptor, until that child runs its program. The lock now unlocks when it is dropped, and a test keeps a copy of the descriptor open across the drop. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/store/src/writer.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/store/src/writer.rs b/crates/store/src/writer.rs index 3b41754..9e9f80d 100644 --- a/crates/store/src/writer.rs +++ b/crates/store/src/writer.rs @@ -18,7 +18,7 @@ const RETRY: Duration = Duration::from_millis(100); /// ends, however it ends. #[derive(Debug)] pub struct WriterLock { - _file: File, + file: File, path: PathBuf, } @@ -88,7 +88,7 @@ impl WriterLock { .and_then(|()| file.seek(SeekFrom::Start(0))) .and_then(|_| file.write_all(format!("{}\n", std::process::id()).as_bytes())) .map_err(|err| format!("cannot write {}: {err}", path.display()))?; - Ok(Self { _file: file, path }) + Ok(Self { file, path }) } pub fn path(&self) -> &Path { @@ -96,6 +96,15 @@ impl WriterLock { } } +impl Drop for WriterLock { + fn drop(&mut self) { + // Let go at once: a child that another thread is forking holds a copy of this + // descriptor, and with it the lock, until it runs its program. + // SAFETY: flock unlocks the open file description behind a descriptor this lock owns. + unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; + } +} + /// Take the lock if no other open file holds it. fn try_lock(file: &File) -> std::io::Result { loop { @@ -194,6 +203,20 @@ mod tests { ); } + /// A writer lets go when it is dropped, though a copy of its descriptor lives on, as in a + /// child that another thread forked and that has not run its program yet. + #[test] + fn a_writer_lets_go_though_a_copy_of_its_descriptor_lives_on() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("ops.sqlite"); + let first = WriterLock::acquire(&database, Duration::ZERO, not_waiting).unwrap(); + let copy = first.file.try_clone().unwrap(); + drop(first); + let second = WriterLock::acquire(&database, Duration::ZERO, not_waiting).unwrap(); + drop(second); + drop(copy); + } + /// A gateway that is killed while it holds the lock lets go of it with its process: the /// next one never has to remove a stale lock file. #[test]