From c22e8230439be159071a943753c669e172f19e87 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:42:46 +0000 Subject: [PATCH] runner, server: opt-in independent Runner that outlives its gateways (P1R-C01) The worker takes `--lifecycle managed|independent`. `managed`, the default, is unchanged. An independent Runner is started by an operator or a harness and prepares its own 0700 socket directory. It serves one gateway connection at a time: its processes, their output, their timeouts and their exit events outlive each connection, and it accepts the next connection only after every request of the previous one has ended, a preparation or a launch included. A gateway leaves it with Detach or ends it with Stop; a connection that ends without either is a loss, and the processes keep running. SIGTERM or SIGINT stops the Runner, which waits up to 5 seconds for its processes. Hello states the lifecycle, the Runner's identity (boot id, pid, start time and a nonce), whether an earlier connection controlled it and how many processes it inherits. The worker protocol is now 8. The gateway takes `--runner-lifecycle independent` only with `--runner uds`, `--runner-socket` and `--runner-exit-intent detach|stop`, and refuses a Runner whose lifecycle differs. At shutdown it detaches or stops the Runner as the exit intent says, and SIGUSR1 always detaches. It releases no lease when the connection ends and does not reconnect: later requests fail with RUNNER_DISCONNECTED. A gateway that inherits an earlier controller's processes can query, read, write to, resize and terminate them, but refuses new executions, patches and approval resumes with RECOVERY_PENDING, checked before an approval is claimed so that the approval stays as it was. An independent gateway exits without waiting for a pending stdin read, so a signal ends it while its client keeps stdin open. With DevGuard, the independent Runner reads its consumer secret from `--devguard-credential-file`, and workspaces that require participation run only when it states that it launches. The MCP instructions and workspace_info's lifetime values follow the mode. The docs describe the mode as not supported yet, with what each event does in each lifecycle, and the CS-RG adoption step keeps naming the qualified tree. The macos-core validation stage runs the new tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/codex-runtime/Cargo.toml | 2 +- .../src/bin/codespace-codex-runtime.rs | 329 +++- crates/codex-runtime/tests/runtime_binary.rs | 225 ++- crates/domain/src/error.rs | 14 + crates/runner/src/lib.rs | 23 +- crates/runner/src/lifecycle.rs | 151 ++ crates/runner/src/process.rs | 27 + crates/runner/src/socket.rs | 88 ++ crates/runner/src/uds.rs | 114 +- crates/runner/src/wire.rs | 1402 +++++++++++++++-- crates/runner/tests/uds_runner.rs | 2 +- crates/server/src/config.rs | 156 ++ crates/server/src/devguard.rs | 29 +- crates/server/src/main.rs | 169 +- crates/server/src/mcp.rs | 432 ++++- crates/server/src/runtime.rs | 69 +- crates/server/tests/independent_runner.rs | 446 ++++++ docs/agent-integration.md | 2 +- docs/architecture.md | 2 +- docs/codex-reuse.md | 2 +- docs/devguard-integration.md | 11 +- docs/error-codes.md | 2 + docs/ko/agent-integration.md | 2 +- docs/ko/architecture.md | 2 +- docs/ko/codex-reuse.md | 2 +- docs/ko/devguard-integration.md | 11 +- docs/ko/error-codes.md | 2 + docs/ko/operations.md | 2 +- docs/ko/runner-isolation.md | 39 +- docs/operations.md | 2 +- docs/runner-isolation.md | 39 +- docs/translations.json | 30 +- scripts/validate-upstream.py | 12 +- 33 files changed, 3557 insertions(+), 283 deletions(-) create mode 100644 crates/runner/src/lifecycle.rs create mode 100644 crates/server/tests/independent_runner.rs diff --git a/crates/codex-runtime/Cargo.toml b/crates/codex-runtime/Cargo.toml index 5e6febb..09e43a8 100644 --- a/crates/codex-runtime/Cargo.toml +++ b/crates/codex-runtime/Cargo.toml @@ -23,7 +23,7 @@ devguard = ["codespace-runner/devguard"] codespace-runner = { path = "../runner" } codex-process-hardening = { path = "../../third_party/codex/codex-rs/process-hardening" } codex-uds = { path = "../../third_party/codex/codex-rs/uds" } -tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "io-util", "fs", "sync", "process", "time"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "io-util", "fs", "sync", "process", "time", "signal"] } [patch.crates-io] tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "0e5b2d73aa18dd9f0a50ee9ff199d5aef7594186" } diff --git a/crates/codex-runtime/src/bin/codespace-codex-runtime.rs b/crates/codex-runtime/src/bin/codespace-codex-runtime.rs index d7d40c5..66016c8 100644 --- a/crates/codex-runtime/src/bin/codespace-codex-runtime.rs +++ b/crates/codex-runtime/src/bin/codespace-codex-runtime.rs @@ -2,7 +2,8 @@ //! newly allocated unique dir when run standalone). Parent chmod is the //! gateway's job: this process never calls //! `prepare_private_socket_directory`. One `InProcessRunner` per -//! process. P0 is 1:1 — serve one connection, then exit. +//! process. By default (`--lifecycle managed`) it is 1:1 — serve one +//! connection, then exit. //! //! Built with the `devguard` feature, the worker is CodeSpace's execution owner in DevGuard //! (CSRG-U2). A gateway started with `--devguard register` passes, after the socket, the @@ -11,15 +12,25 @@ //! before anything else, registers itself in the background and answers the gateway's //! `Registration` requests from then on. With `--devguard-launch-helper` too, it launches the //! governed executions it prepares through that helper, as their helper's parent (CSRG-U4). +//! +//! With `--lifecycle independent` (P1R-C01) an operator or a harness starts the worker, not a +//! gateway: it is an independent Runner. It prepares its socket's directory itself, created +//! with mode 0700 or accepted only as a private directory of its user, and serves one gateway +//! connection at a time. Its processes, their output and their timeouts outlive each connection +//! until a gateway's `Stop`, or the Runner's own SIGTERM or SIGINT, ends them and the Runner. +//! With DevGuard it reads the consumer secret from `--devguard-credential-file`, as no gateway +//! hands it one. -use std::path::PathBuf; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; #[cfg(not(feature = "devguard"))] use codespace_runner::serve_runner_connection; #[cfg(feature = "devguard")] use codespace_runner::serve_runner_connection_with_registration; use codespace_runner::{ - allocate_private_runner_dir, host_worker, reclaim_leftover_socket, runner_socket_path, + allocate_private_runner_dir, host_worker, prepare_runner_socket_dir, reclaim_leftover_socket, + runner_socket_path, IndependentRunner, RunnerLifecycle, Served, STOP_DRAIN_BOUND, }; fn main() { @@ -33,38 +44,250 @@ fn main() { .block_on(serve()); } +/// The worker's command line: `[SOCKET] [--lifecycle managed|independent] [--devguard-* VALUE]...`. +#[derive(Debug, Default, PartialEq, Eq)] +struct Arguments { + socket: Option, + lifecycle: RunnerLifecycle, + /// The DevGuard settings, in order. A worker built without DevGuard ignores them, as it + /// always has. + devguard: Vec, +} + +fn parse(args: impl IntoIterator) -> Result { + let mut parsed = Arguments::default(); + let mut args = args.into_iter().peekable(); + if args + .peek() + .is_some_and(|first| !first.to_string_lossy().starts_with("--")) + { + parsed.socket = args.next().map(PathBuf::from); + } + while let Some(flag) = args.next() { + let name = flag.to_str().ok_or("a non-UTF-8 argument")?.to_owned(); + let value = args.next().ok_or_else(|| format!("{name} needs a value"))?; + match name.as_str() { + "--lifecycle" => { + parsed.lifecycle = match value.to_str() { + Some("managed") => RunnerLifecycle::Managed, + Some("independent") => RunnerLifecycle::Independent, + _ => return Err("--lifecycle must be managed or independent".into()), + } + } + devguard if devguard.starts_with("--devguard-") => { + parsed.devguard.push(flag); + parsed.devguard.push(value); + } + _ => return Err(format!("unknown argument {name}")), + } + } + Ok(parsed) +} + +/// Report a problem with the worker's settings or socket and exit with `code`. +fn refuse(code: i32, problem: impl std::fmt::Display) -> ! { + eprintln!("codespace-codex-runtime: {problem}"); + std::process::exit(code); +} + async fn serve() { codex_process_hardening::pre_main_hardening(); + let arguments = parse(std::env::args_os().skip(1)).unwrap_or_else(|problem| refuse(2, problem)); // First of all, so the handed descriptor is closed before this process can start a child. #[cfg(feature = "devguard")] - let (registration, launcher) = devguard::start(std::env::args_os().skip(2)); - let socket = match std::env::args().nth(1) { - Some(path) => PathBuf::from(path), - None => match std::env::var_os("CODESPACE_RUNNER_SOCKET") { - Some(path) => PathBuf::from(path), - None => { - let dir = allocate_private_runner_dir(None).expect("private runner dir"); - let socket = runner_socket_path(&dir); - eprintln!("codespace-codex-runtime listening on {}", socket.display()); - socket - } - }, - }; - reclaim_leftover_socket(&socket).expect("runner socket"); - let mut listener = codex_uds::UnixListener::bind(&socket) - .await - .expect("bind runner socket"); - let (runner, events) = host_worker(); - let stream = listener.accept().await.expect("accept runner socket"); - // Keep the listener bound so a live connect-probe sees AddrInUse - // instead of unlinking this rendezvous. - let _listener = listener; - #[cfg(feature = "devguard")] - let _ = - serve_runner_connection_with_registration(stream, runner, events, registration, launcher) - .await; + let (registration, launcher) = devguard::start(arguments.devguard, arguments.lifecycle); #[cfg(not(feature = "devguard"))] - let _ = serve_runner_connection(stream, runner, events).await; + let _ = arguments.devguard; + match arguments.lifecycle { + RunnerLifecycle::Managed => { + let socket = match arguments.socket { + Some(path) => path, + None => match std::env::var_os("CODESPACE_RUNNER_SOCKET") { + Some(path) => PathBuf::from(path), + None => { + let dir = allocate_private_runner_dir(None).expect("private runner dir"); + let socket = runner_socket_path(&dir); + eprintln!("codespace-codex-runtime listening on {}", socket.display()); + socket + } + }, + }; + reclaim_leftover_socket(&socket).expect("runner socket"); + let mut listener = codex_uds::UnixListener::bind(&socket) + .await + .expect("bind runner socket"); + let (runner, events) = host_worker(); + let stream = listener.accept().await.expect("accept runner socket"); + // Keep the listener bound so a live connect-probe sees AddrInUse + // instead of unlinking this rendezvous. + let _listener = listener; + #[cfg(feature = "devguard")] + let _ = serve_runner_connection_with_registration( + stream, + runner, + events, + registration, + launcher, + ) + .await; + #[cfg(not(feature = "devguard"))] + let _ = serve_runner_connection(stream, runner, events).await; + } + RunnerLifecycle::Independent => { + let socket = arguments.socket.unwrap_or_else(|| { + refuse(2, "--lifecycle independent needs the socket's path first") + }); + let (runner, events) = host_worker(); + let independent = IndependentRunner::new(runner, events); + #[cfg(feature = "devguard")] + let independent = independent.with_registration(registration, launcher); + std::process::exit(serve_independent(&socket, independent).await); + } + } +} + +/// An independent Runner's life (P1R-C01): its socket, the connections it serves one at a +/// time, and its stop, after which its socket, and the directory it created for it, are +/// removed. The exit code. +async fn serve_independent(socket: &Path, independent: IndependentRunner) -> i32 { + let dir = match socket.parent() { + Some(dir) if !dir.as_os_str().is_empty() => dir, + _ => refuse(2, "the socket's path needs a directory"), + }; + let created = prepare_runner_socket_dir(dir).unwrap_or_else(|err| refuse(1, err)); + let rendezvous = Rendezvous { + socket: socket.to_path_buf(), + dir: created.then(|| dir.to_path_buf()), + }; + if let Err(err) = reclaim_leftover_socket(socket) { + rendezvous.remove_dir(); + refuse(1, err); + } + let listener = match codex_uds::UnixListener::bind(socket).await { + Ok(listener) => listener, + Err(err) => { + rendezvous.remove_dir(); + refuse(1, format!("bind {}: {err}", socket.display())); + } + }; + let stop = independent.stop_handle(); + let mut terminate = signal(tokio::signal::unix::SignalKind::terminate()); + let mut interrupt = signal(tokio::signal::unix::SignalKind::interrupt()); + let watchdog = Rendezvous { + socket: rendezvous.socket.clone(), + dir: rendezvous.dir.clone(), + }; + tokio::spawn(async move { + tokio::select! { + _ = terminate.recv() => {} + _ = interrupt.recv() => {} + } + // Its own signal stops the Runner as a gateway's `Stop` does. + stop.request(); + // A connection whose gateway no longer reads cannot keep the Runner from stopping: + // its processes are already terminated. + tokio::time::sleep(STOP_DRAIN_BOUND * 2).await; + eprintln!("codespace-codex-runtime: the connection did not end; stopping without it"); + watchdog.remove(); + std::process::exit(1); + }); + eprintln!( + "codespace-codex-runtime: independent Runner {} listening on {}", + std::process::id(), + socket.display() + ); + let listener = std::sync::Arc::new(tokio::sync::Mutex::new(listener)); + let terminated = independent + .run( + move || { + let listener = listener.clone(); + async move { listener.lock().await.accept().await } + }, + |ending| match ending { + Ok(Served::Detached) => { + eprintln!("codespace-codex-runtime: a gateway detached; its processes go on") + } + Ok(Served::Lost) => eprintln!( + "codespace-codex-runtime: a gateway's connection ended without a detach or a \ + stop; its processes go on" + ), + Ok(Served::Stopped { .. }) => {} + Err(err) => eprintln!("codespace-codex-runtime: accept: {err}"), + }, + ) + .await; + eprintln!("codespace-codex-runtime: stopped; {terminated} processes were terminated"); + rendezvous.remove(); + 0 +} + +fn signal(kind: tokio::signal::unix::SignalKind) -> tokio::signal::unix::Signal { + tokio::signal::unix::signal(kind).unwrap_or_else(|err| refuse(1, format!("signals: {err}"))) +} + +/// An independent Runner's socket, and its directory when the Runner created it. +struct Rendezvous { + socket: PathBuf, + dir: Option, +} + +impl Rendezvous { + fn remove(&self) { + let _ = std::fs::remove_file(&self.socket); + self.remove_dir(); + } + + /// Only a directory the Runner created, and only while it is empty. + fn remove_dir(&self) { + if let Some(dir) = &self.dir { + let _ = std::fs::remove_dir(dir); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(list: &[&str]) -> Vec { + list.iter().map(OsString::from).collect() + } + + #[test] + fn the_socket_comes_first_and_the_lifecycle_defaults_to_managed() { + assert_eq!(parse(args(&[])).unwrap(), Arguments::default()); + assert_eq!( + parse(args(&["/run/cs/runner.sock", "--lifecycle", "independent"])).unwrap(), + Arguments { + socket: Some("/run/cs/runner.sock".into()), + lifecycle: RunnerLifecycle::Independent, + devguard: Vec::new(), + } + ); + assert_eq!( + parse(args(&[ + "/run/cs/runner.sock", + "--devguard-consumer", + "codespace", + "--lifecycle", + "managed", + ])) + .unwrap(), + Arguments { + socket: Some("/run/cs/runner.sock".into()), + lifecycle: RunnerLifecycle::Managed, + devguard: args(&["--devguard-consumer", "codespace"]), + } + ); + for wrong in [ + &["--lifecycle", "detached"][..], + &["/run/cs/runner.sock", "--lifecycle"], + &["/run/cs/runner.sock", "--stop", "now"], + &["/run/cs/runner.sock", "/run/cs/other.sock"], + ] { + assert!(parse(args(wrong)).is_err(), "{wrong:?}"); + } + } } #[cfg(feature = "devguard")] @@ -78,7 +301,7 @@ mod devguard { use codespace_runner::registration::{ handoff, Owner, OwnerCredential, OwnerSettings, ResourceOwner, }; - use codespace_runner::OwnerRegistration; + use codespace_runner::{OwnerRegistration, RunnerLifecycle}; /// The worker's DevGuard settings after its socket. None of them is a secret. #[derive(Debug, Default, PartialEq, Eq)] @@ -87,6 +310,9 @@ mod devguard { pub(crate) consumer: Option, pub(crate) generation: Option, pub(crate) credential_fd: Option, + /// An independent Runner's consumer secret, which it reads from this private file + /// itself (P1R-C01). + pub(crate) credential_file: Option, pub(crate) launch_helper: Option, } @@ -110,6 +336,7 @@ mod devguard { .map_err(|_| format!("{flag} needs a number"))?, ) } + "--devguard-credential-file" => parsed.credential_file = Some(value.into()), "--devguard-launch-helper" => parsed.launch_helper = Some(value.into()), _ => return Err(format!("unknown argument {flag}")), } @@ -119,9 +346,12 @@ mod devguard { /// Consume the handed descriptor, then register in the background. No registration when /// the worker was started without DevGuard settings: it then answers that its mode cannot - /// register. A launch helper only with a registration. + /// register. A launch helper only with a registration. A managed worker gets its secret + /// through the descriptor from the gateway that starts it; an independent Runner reads it + /// from its file (P1R-C01). pub(crate) fn start( args: impl IntoIterator, + lifecycle: RunnerLifecycle, ) -> (Option>, Option) { let arguments = match parse(args) { Ok(arguments) => arguments, @@ -130,13 +360,32 @@ mod devguard { std::process::exit(2); } }; - let credential = match arguments.credential_fd { + let credential = match ( + lifecycle, + arguments.credential_fd, + arguments.credential_file, + ) { // SAFETY: the gateway started this process with this dedicated descriptor, which // nothing in this process has taken yet. - Some(fd) => unsafe { handoff::receive(fd) } + (RunnerLifecycle::Managed, Some(fd), None) => unsafe { handoff::receive(fd) } .map(OwnerCredential::Handed) .unwrap_or(OwnerCredential::Unavailable), - None => OwnerCredential::Unavailable, + (RunnerLifecycle::Independent, None, Some(file)) => OwnerCredential::File(file), + (_, None, None) => OwnerCredential::Unavailable, + (RunnerLifecycle::Managed, ..) => { + eprintln!( + "codespace-codex-runtime: a worker its gateway starts gets the consumer \ + secret through --devguard-credential-fd only" + ); + std::process::exit(2); + } + (RunnerLifecycle::Independent, ..) => { + eprintln!( + "codespace-codex-runtime: an independent Runner reads the consumer secret \ + from --devguard-credential-file only" + ); + std::process::exit(2); + } }; let launcher = match arguments.launch_helper.map(Launcher::new) { Some(Err(problem)) => { @@ -197,9 +446,19 @@ mod devguard { consumer: Some("codespace".into()), generation: Some("g1".into()), credential_fd: Some(3), + credential_file: None, launch_helper: Some("/opt/devguard/bin/devguard-launch".into()), } ); + assert_eq!( + parse(args(&[ + "--devguard-credential-file", + "/Users/op/.config/codespace/consumer.secret" + ])) + .unwrap() + .credential_file, + Some("/Users/op/.config/codespace/consumer.secret".into()) + ); assert_eq!(parse(args(&[])).unwrap(), Arguments::default()); for wrong in [ &["--devguard-secret", "x"][..], diff --git a/crates/codex-runtime/tests/runtime_binary.rs b/crates/codex-runtime/tests/runtime_binary.rs index 073855b..c26c649 100644 --- a/crates/codex-runtime/tests/runtime_binary.rs +++ b/crates/codex-runtime/tests/runtime_binary.rs @@ -6,11 +6,11 @@ use std::process::Command; use std::sync::Arc; use std::time::Duration; -use codespace_domain::{ProcessId, Profile, WorkspaceId}; +use codespace_domain::{ErrorCode, ProcessId, ProcessState, Profile, WorkspaceId}; use codespace_policy::Workspace; use codespace_runner::{ - read_frame, write_frame, Runner, RunnerExecRequest, RunnerOp, RunnerOpResult, UdsRunner, - WireEnvelope, WIRE_PROTOCOL, + read_frame, write_frame, Runner, RunnerError, RunnerExecRequest, RunnerLifecycle, RunnerOp, + RunnerOpResult, UdsRunner, WireEnvelope, WIRE_PROTOCOL, }; use tokio::net::UnixStream; @@ -144,7 +144,7 @@ async fn hello_on_live_socket() { match parsed.result { Some(RunnerOpResult::Hello { protocol, .. }) => { assert_eq!(protocol, WIRE_PROTOCOL); - assert_eq!(protocol, 7); + assert_eq!(protocol, 8); } other => panic!("unexpected hello {other:?}"), } @@ -211,3 +211,220 @@ async fn worker_socket_drop_terminates_owned_child() { "owned child must stop updating heartbeat after worker disconnect (after={after:?} later={later:?})" ); } + +/// An independent Runner (P1R-C01) at `socket`, whose directory it prepares itself. +fn spawn_independent(socket: &Path) -> tokio::process::Child { + tokio::process::Command::new(BIN) + .arg(socket) + .args(["--lifecycle", "independent"]) + .kill_on_drop(true) + .spawn() + .expect("spawn an independent Runner") +} + +/// A gateway's connection to the Runner at `socket`, after its handshake. +async fn gateway(socket: &Path) -> UdsRunner { + let runner = UdsRunner::from_stream(wait_connect(socket).await, Arc::new(|_| {})); + runner.handshake().await.expect("hello"); + runner +} + +fn mode(path: &Path) -> u32 { + std::fs::symlink_metadata(path) + .unwrap() + .permissions() + .mode() + & 0o777 +} + +/// A command that counts into `beat` ten times a second. +fn beating(beat: &Path, process_id: &str, tty: bool) -> RunnerExecRequest { + let mut request = RunnerExecRequest::for_host( + vec![ + "/bin/sh".into(), + "-c".into(), + format!( + "i=0; while :; do i=$((i+1)); echo $i > '{}'; sleep 0.1; done", + beat.display() + ), + ], + ProcessId(process_id.into()), + Profile::WorkspaceWrite, + ); + request.timeout_ms = 60_000; + request.tty = tty; + request +} + +/// Whether the count in `beat` still goes up. +async fn still_beating(beat: &Path) -> bool { + let read = || std::fs::read_to_string(beat).unwrap_or_default(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while read().trim().is_empty() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(20)).await; + } + let before = read(); + tokio::time::sleep(Duration::from_millis(600)).await; + before != read() +} + +async fn exited(runner: &mut tokio::process::Child) -> std::process::ExitStatus { + tokio::time::timeout(Duration::from_secs(15), runner.wait()) + .await + .expect("the Runner exits") + .expect("wait for the Runner") +} + +fn demo_workspace(root: &Path) -> Workspace { + Workspace::new( + WorkspaceId("demo".into()), + root.to_path_buf(), + Profile::WorkspaceWrite, + ) +} + +/// An independent Runner keeps its pipe and tty processes after its gateway detaches, the next +/// gateway inherits them but starts nothing, and a stop ends them and the Runner, which removes +/// its socket and the directory it created (P1R-C01). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_independent_runner_outlives_its_gateways_until_one_stops_it() { + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let dir = base.path().join("runner"); + let socket = runner_socket(&dir); + let mut runner = spawn_independent(&socket); + let first = gateway(&socket).await; + assert_eq!(mode(&dir), 0o700); + assert_eq!(mode(base.path()), 0o755); + let hello = first.hello(); + assert_eq!(hello.lifecycle, RunnerLifecycle::Independent); + assert_eq!( + hello.identity.as_ref().map(|identity| identity.pid), + runner.id() + ); + assert!(!hello.previous_controller); + // A second Runner neither takes nor removes the live socket. + let second = Command::new(BIN) + .arg(&socket) + .args(["--lifecycle", "independent"]) + .output() + .expect("second Runner"); + assert!(!second.status.success(), "{second:?}"); + assert!(socket.exists()); + + let ws_dir = tempfile::tempdir().unwrap(); + let ws = demo_workspace(ws_dir.path()); + let pipe_beat = ws_dir.path().join("pipe-beat"); + let tty_beat = ws_dir.path().join("tty-beat"); + first + .exec(&ws, beating(&pipe_beat, "proc-pipe", false)) + .await + .unwrap(); + first + .exec(&ws, beating(&tty_beat, "proc-tty", true)) + .await + .unwrap(); + assert!(still_beating(&pipe_beat).await && still_beating(&tty_beat).await); + first.detach().await.unwrap(); + // No gateway is connected, and both keep running. + assert!(still_beating(&pipe_beat).await && still_beating(&tty_beat).await); + + let next = gateway(&socket).await; + let inherited = next.hello(); + assert_eq!(inherited.identity, hello.identity); + assert!(inherited.previous_controller); + assert_eq!(inherited.inherited_processes, 2); + assert_eq!( + next.process_status(&ProcessId("proc-pipe".into())) + .await + .unwrap() + .state, + ProcessState::Running + ); + match next + .exec( + &ws, + beating(&ws_dir.path().join("new-beat"), "proc-new", false), + ) + .await + { + Err(RunnerError::Execution(body)) => assert_eq!(body.code, ErrorCode::RecoveryPending), + other => panic!("unexpected {other:?}"), + } + assert_eq!(next.stop_runner().await.unwrap(), 2); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); + assert!(!still_beating(&pipe_beat).await); + assert!(!still_beating(&tty_beat).await); + assert!(!socket.exists()); + assert!(!dir.exists()); +} + +/// The Runner's own SIGTERM stops it as a gateway's `Stop` does, and the gateway connected to it +/// refuses its later requests (P1R-C01). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sigterm_stops_an_independent_runner_and_its_processes() { + let base = tempfile::tempdir().unwrap(); + let dir = base.path().join("runner"); + let socket = runner_socket(&dir); + let mut runner = spawn_independent(&socket); + let connected = gateway(&socket).await; + let ws_dir = tempfile::tempdir().unwrap(); + let ws = demo_workspace(ws_dir.path()); + let beat = ws_dir.path().join("beat"); + let process_id = ProcessId("proc-pipe".into()); + connected + .exec(&ws, beating(&beat, &process_id.0, false)) + .await + .unwrap(); + assert!(still_beating(&beat).await); + let pid = runner.id().unwrap().to_string(); + let sent = Command::new("kill").args(["-TERM", &pid]).status().unwrap(); + assert!(sent.success()); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); + assert!(!still_beating(&beat).await); + assert!(!socket.exists()); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + match connected.process_status(&process_id).await { + Err(RunnerError::Execution(body)) if body.code == ErrorCode::RunnerDisconnected => { + break + } + other => assert!(tokio::time::Instant::now() < deadline, "{other:?}"), + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +/// An independent Runner does not start in a socket directory that is not private to its user, +/// nor without a socket or with an unknown lifecycle, and changes nothing (P1R-C01). +#[tokio::test] +async fn an_independent_runner_refuses_a_shared_directory_and_unknown_settings() { + let base = tempfile::tempdir().unwrap(); + let shared = base.path().join("shared"); + std::fs::create_dir(&shared).unwrap(); + std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = runner_socket(&shared); + let refused = Command::new(BIN) + .arg(&socket) + .args(["--lifecycle", "independent"]) + .output() + .unwrap(); + assert!(!refused.status.success(), "{refused:?}"); + assert!( + String::from_utf8_lossy(&refused.stderr).contains("0700"), + "{refused:?}" + ); + assert_eq!(mode(&shared), 0o755); + assert!(!socket.exists()); + let socket = socket.display().to_string(); + for args in [ + &["--lifecycle", "independent"][..], + &[&socket, "--lifecycle", "later"], + &[&socket, "--runner-exit-intent", "stop"], + ] { + let refused = Command::new(BIN).args(args).output().unwrap(); + assert_eq!(refused.status.code(), Some(2), "{args:?}: {refused:?}"); + } +} diff --git a/crates/domain/src/error.rs b/crates/domain/src/error.rs index 24be622..1074bb0 100644 --- a/crates/domain/src/error.rs +++ b/crates/domain/src/error.rs @@ -58,6 +58,16 @@ pub enum ErrorCode { /// so whether it reserved resources is not known. Nothing was started; do not assume the /// attempt does not exist. AdmissionUnknown, + /// An earlier gateway controlled this gateway's independent Runner, and recovering what it + /// left is not available yet (P1R-C01): its processes can be queried, read, written to, + /// resized and terminated, but no new execution, patch or approval resume starts, and an + /// approval being resumed is left as it was. Stopping the Runner and starting a new one ends + /// it. Not occupancy; do not retry as busy. + RecoveryPending, + /// This gateway lost its connection to its independent Runner and does not reconnect + /// (P1R-C01). The request never reached the Runner, whose processes keep running; + /// restarting the gateway reaches them again. + RunnerDisconnected, /// Scheduler or store invariant failed. Not occupancy; do not retry as busy. Internal, } @@ -173,6 +183,10 @@ mod tests { assert_eq!(json, "\"RESOURCE_QUEUE_FULL\""); let json = serde_json::to_string(&ErrorCode::ResourcePolicyUnsupported).unwrap(); assert_eq!(json, "\"RESOURCE_POLICY_UNSUPPORTED\""); + let json = serde_json::to_string(&ErrorCode::RecoveryPending).unwrap(); + assert_eq!(json, "\"RECOVERY_PENDING\""); + let json = serde_json::to_string(&ErrorCode::RunnerDisconnected).unwrap(); + assert_eq!(json, "\"RUNNER_DISCONNECTED\""); let json = serde_json::to_string(&ErrorCode::Internal).unwrap(); assert_eq!(json, "\"INTERNAL\""); } diff --git a/crates/runner/src/lib.rs b/crates/runner/src/lib.rs index 5866fc4..c7aca8b 100644 --- a/crates/runner/src/lib.rs +++ b/crates/runner/src/lib.rs @@ -108,6 +108,7 @@ mod files; mod fork_handlers; #[cfg(feature = "devguard")] pub mod launch; +mod lifecycle; mod linux_sandbox; mod patch_helper; mod patch_verify; @@ -142,19 +143,21 @@ pub fn linux_sandbox_available() -> bool { linux_sandbox::probe() } +pub use lifecycle::{RunnerIdentity, RunnerLifecycle}; #[cfg(feature = "devguard")] pub use registration::OwnerRegistration; pub use socket::{ - allocate_private_runner_dir, is_forbidden_runner_dir, reclaim_leftover_socket, - runner_socket_path, RUNNER_SOCKET_NAME, + allocate_private_runner_dir, is_forbidden_runner_dir, prepare_runner_socket_dir, + reclaim_leftover_socket, runner_socket_path, RUNNER_SOCKET_NAME, }; -pub use uds::{DisconnectHook, UdsRunner, RUNNER_CALL_DEADLINE}; +pub use uds::{runner_disconnected, DisconnectHook, RunnerHello, UdsRunner, RUNNER_CALL_DEADLINE}; pub use watch::{FsWatchEvent, FsWatchKind, WatchClosed, WatchSubscription}; #[cfg(feature = "devguard")] pub use wire::serve_runner_connection_with_registration; pub use wire::{ - host_worker, read_frame, serve_runner_connection, write_frame, RunnerEvent, RunnerOp, - RunnerOpResult, WireEnvelope, WireKind, WIRE_PROTOCOL, + host_worker, read_frame, recovery_pending, serve_runner_connection, write_frame, + IndependentRunner, RunnerEvent, RunnerOp, RunnerOpResult, Served, StopHandle, WireEnvelope, + WireKind, STOP_DRAIN_BOUND, WIRE_PROTOCOL, }; /// Execution-plane API. Control-plane fields (`work_id`, coordination, @@ -318,6 +321,16 @@ impl RuntimeBackend { pub fn in_process(on_release: ShellRelease) -> Self { Self::InProcess(InProcessRunner::new(on_release)) } + + /// The backend is an independent Runner that an earlier gateway controlled, so this gateway + /// starts no new execution, patch or approval resume (P1R-C01): see + /// [`UdsRunner::refuses_new_work`]. + pub fn refuses_new_work(&self) -> bool { + match self { + Self::InProcess(_) => false, + Self::Uds(runner) => runner.refuses_new_work(), + } + } } impl Runner for RuntimeBackend { diff --git a/crates/runner/src/lifecycle.rs b/crates/runner/src/lifecycle.rs new file mode 100644 index 0000000..1125621 --- /dev/null +++ b/crates/runner/src/lifecycle.rs @@ -0,0 +1,151 @@ +//! How a worker's processes relate to its connections (P1R-C01), and which worker process a +//! gateway is talking to. +//! +//! A `managed` worker belongs to the gateway that starts it: it serves that one connection and +//! ends its processes as it exits once the connection closes. An `independent` Runner is +//! started by an operator or a harness and only connected to: it serves one connection at a +//! time, and its processes, their output and their timeouts outlive each connection. + +use std::sync::OnceLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +/// How a worker's processes relate to its connections (P1R-C01). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunnerLifecycle { + /// The gateway that starts the worker owns it: one connection, after which the worker ends + /// its processes and exits. + #[default] + Managed, + /// Started by an operator or a harness, not by a gateway. It serves one connection at a + /// time, and its processes, their output and their timeouts outlive each connection until a + /// `Stop`, or its own SIGTERM or SIGINT, ends them and the Runner. + Independent, +} + +impl RunnerLifecycle { + pub fn as_str(self) -> &'static str { + match self { + Self::Managed => "managed", + Self::Independent => "independent", + } + } +} + +/// Which Runner process this is (P1R-C01): stated in every `Hello`, the same for each +/// connection of one process and different for any other. Nothing in it is a secret. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunnerIdentity { + /// The host's boot session, when the host reports one: Linux's `boot_id`, macOS's + /// `kern.bootsessionuuid`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub boot_id: Option, + pub pid: u32, + /// When the Runner started, in milliseconds since the Unix epoch. + pub started_at_ms: u64, + /// 128 random bits minted when the Runner started, in hex. + pub nonce: String, +} + +impl RunnerIdentity { + /// This process's identity, fixed by the first call: a worker makes it first thing. + pub fn this_process() -> RunnerIdentity { + static IDENTITY: OnceLock = OnceLock::new(); + IDENTITY + .get_or_init(|| RunnerIdentity { + boot_id: boot_id(), + pid: std::process::id(), + started_at_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis() as u64) + .unwrap_or(0), + nonce: nonce(), + }) + .clone() + } +} + +#[cfg(target_os = "linux")] +fn boot_id() -> Option { + std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .ok() + .map(|id| id.trim().to_string()) + .filter(|id| !id.is_empty()) +} + +#[cfg(target_os = "macos")] +fn boot_id() -> Option { + let mut buf = [0u8; 64]; + let mut len = buf.len(); + // SAFETY: `buf` has room for `len` bytes, which sysctlbyname fills and updates; the name is + // a NUL-terminated C string. + let result = unsafe { + libc::sysctlbyname( + c"kern.bootsessionuuid".as_ptr(), + buf.as_mut_ptr().cast(), + &mut len, + std::ptr::null_mut(), + 0, + ) + }; + if result != 0 { + return None; + } + let id = String::from_utf8_lossy(&buf[..len.min(buf.len())]); + let id = id.trim_end_matches('\0').trim(); + (!id.is_empty()).then(|| id.to_string()) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn boot_id() -> Option { + None +} + +/// 128 random bits in hex. A host without `/dev/urandom` gets the time and the process ID +/// instead: the nonce tells Runner processes apart, it guards nothing. +fn nonce() -> String { + use std::io::Read; + let mut bytes = [0u8; 16]; + match std::fs::File::open("/dev/urandom").and_then(|mut random| random.read_exact(&mut bytes)) { + Ok(()) => hex::encode(bytes), + Err(_) => { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_nanos()) + .unwrap_or(0); + format!("{nanos:x}{:x}", std::process::id()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_identity_is_this_process_and_stays_the_same() { + let identity = RunnerIdentity::this_process(); + assert_eq!(identity.pid, std::process::id()); + assert_eq!(identity.nonce.len(), 32, "{identity:?}"); + assert!(identity.started_at_ms > 0); + if cfg!(any(target_os = "linux", target_os = "macos")) { + assert!(identity.boot_id.is_some(), "{identity:?}"); + } + assert_eq!(RunnerIdentity::this_process(), identity); + } + + #[test] + fn lifecycles_cross_the_wire_by_name() { + assert_eq!( + serde_json::to_string(&RunnerLifecycle::Independent).unwrap(), + "\"independent\"" + ); + assert_eq!( + serde_json::from_str::("\"managed\"").unwrap(), + RunnerLifecycle::Managed + ); + assert_eq!(RunnerLifecycle::default(), RunnerLifecycle::Managed); + } +} diff --git a/crates/runner/src/process.rs b/crates/runner/src/process.rs index 6a289f9..9fa2fb0 100644 --- a/crates/runner/src/process.rs +++ b/crates/runner/src/process.rs @@ -1121,6 +1121,33 @@ impl InProcessRunner { } Ok(killed) } + + /// Terminate every process of every workspace, as `terminate_process` does each one: a pipe + /// process alone, a PTY process with its process group. How many were still running that + /// nothing had asked to end yet, through a timeout or a termination. An independent Runner + /// does this when it stops (P1R-C01). + pub fn terminate_all(&self) -> u32 { + let mut map = self.inner.lock().expect("runner"); + self.evict_completed(&mut map); + map.values() + .filter(|slot| { + let unasked = slot + .lifecycle + .lock() + .map(|life| life.kill_intent.is_none()) + .unwrap_or(false); + request_kill(slot).unwrap_or(false) && unasked + }) + .count() as u32 + } + + /// Processes whose output has not ended yet: an independent Runner's next connection + /// inherits them (P1R-C01). + pub fn live_processes(&self) -> u32 { + let mut map = self.inner.lock().expect("runner"); + self.evict_completed(&mut map); + live_count(&map) as u32 + } } /// A pipe child's one reaper (CSRG-U5): no other path waits for it. Timeout and termination diff --git a/crates/runner/src/socket.rs b/crates/runner/src/socket.rs index b2af405..1b85095 100644 --- a/crates/runner/src/socket.rs +++ b/crates/runner/src/socket.rs @@ -52,6 +52,53 @@ pub fn allocate_private_runner_dir(base: Option<&Path>) -> Result { } } +/// The directory of an independent Runner's socket, which the Runner prepares itself (P1R-C01): +/// created with mode 0700 when it does not exist yet, otherwise accepted only as a directory of +/// this user with mode 0700 that is not a symbolic link. Its parent must exist. Never chmods an +/// existing directory. Returns whether it was created. +pub fn prepare_runner_socket_dir(dir: &Path) -> Result { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + if is_forbidden_runner_dir(dir) { + return Err(Error::new( + ErrorKind::InvalidInput, + format!( + "the runner socket's directory must not be /, /tmp, /var/tmp, or $HOME (got {})", + dir.display() + ), + )); + } + let mut builder = DirBuilder::new(); + builder.mode(0o700); + match builder.create(dir) { + Ok(()) => return Ok(true), + Err(err) if err.kind() == ErrorKind::AlreadyExists => {} + Err(err) => return Err(err), + } + let meta = std::fs::symlink_metadata(dir)?; + // SAFETY: geteuid has no preconditions and cannot fail. + let euid = unsafe { libc::geteuid() }; + let mode = meta.permissions().mode() & 0o777; + if !meta.file_type().is_dir() || meta.uid() != euid || mode != 0o700 { + return Err(Error::new( + ErrorKind::PermissionDenied, + format!( + "the runner socket's directory {} must be a directory of this user with mode 0700 \ + (found {}, owner {}, mode {mode:o}); it is never changed", + dir.display(), + if meta.file_type().is_symlink() { + "a symbolic link" + } else if meta.file_type().is_dir() { + "a directory" + } else { + "not a directory" + }, + meta.uid(), + ), + )); + } + Ok(false) +} + /// Connect-probe a rendezvous path. Live listeners are not unlinked. /// `ConnectionRefused` leftover files are removed. Missing paths are ok. pub fn reclaim_leftover_socket(path: &Path) -> Result<()> { @@ -205,4 +252,45 @@ mod tests { let dir = tempfile::tempdir().unwrap(); reclaim_leftover_socket(&dir.path().join("missing.sock")).unwrap(); } + + /// An independent Runner creates its socket's directory, or accepts only a private one of + /// its user, and never changes an existing one (P1R-C01). + #[test] + fn an_independent_runner_prepares_only_a_private_directory() { + let parent = tempfile::tempdir().unwrap(); + std::fs::set_permissions(parent.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let mode = |path: &Path| { + std::fs::symlink_metadata(path) + .unwrap() + .permissions() + .mode() + & 0o777 + }; + let created = parent.path().join("runner"); + assert!(prepare_runner_socket_dir(&created).unwrap()); + assert_eq!(mode(&created), 0o700); + assert_eq!(mode(parent.path()), 0o755); + // The same directory again is accepted as it is. + assert!(!prepare_runner_socket_dir(&created).unwrap()); + let shared = parent.path().join("shared"); + std::fs::create_dir(&shared).unwrap(); + std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(0o750)).unwrap(); + let err = prepare_runner_socket_dir(&shared).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::PermissionDenied, "{err}"); + assert_eq!(mode(&shared), 0o750); + let link = parent.path().join("link"); + std::os::unix::fs::symlink(&created, &link).unwrap(); + let err = prepare_runner_socket_dir(&link).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::PermissionDenied, "{err}"); + let file = parent.path().join("file"); + std::fs::write(&file, "").unwrap(); + assert!(prepare_runner_socket_dir(&file).is_err()); + assert_eq!( + prepare_runner_socket_dir(Path::new("/tmp")) + .unwrap_err() + .kind(), + ErrorKind::InvalidInput + ); + assert!(prepare_runner_socket_dir(&parent.path().join("missing/runner")).is_err()); + } } diff --git a/crates/runner/src/uds.rs b/crates/runner/src/uds.rs index 928f099..816900b 100644 --- a/crates/runner/src/uds.rs +++ b/crates/runner/src/uds.rs @@ -3,13 +3,17 @@ //! `Host` + `UdsRunner` is opt-in **transport** on the same host. It does //! not claim Linux container isolation. P0 is 1:1: one gateway connection //! per worker. Replay is connection-local. +//! +//! An independent Runner (P1R-C01) outlives the gateway's connection. The client never +//! reconnects: once its connection ends, every later request is refused with +//! `RUNNER_DISCONNECTED` and the gateway has to be restarted. use std::collections::HashMap; use std::io; use std::path::Path; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::task::{Context, Poll}; use std::time::Duration; @@ -26,8 +30,8 @@ use crate::wire::{ }; use crate::{ Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, RunnerError, RunnerExecRequest, - RunnerExecResult, RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, RunnerResizeResult, - RunnerWriteStdin, ShellRelease, + RunnerExecResult, RunnerIdentity, RunnerLifecycle, RunnerProcessStatus, RunnerReadProcess, + RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, ShellRelease, }; /// Transport deadline for one RPC. Longer than the default exec timeout @@ -37,11 +41,34 @@ pub const RUNNER_CALL_DEADLINE: Duration = Duration::from_secs(120); pub type DisconnectHook = Arc; +/// What the worker's `Hello` stated about its lifecycle (P1R-C01). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RunnerHello { + pub lifecycle: RunnerLifecycle, + pub identity: Option, + /// An earlier gateway controlled this independent Runner. + pub previous_controller: bool, + /// Processes still running when this connection began. + pub inherited_processes: u32, +} + +/// The refusal of a request to an independent Runner after the gateway's connection to it +/// ended (P1R-C01): the request never left, and the gateway does not reconnect. +pub fn runner_disconnected() -> ErrorBody { + ErrorBody::new( + ErrorCode::RunnerDisconnected, + "this gateway lost its connection to the independent Runner and does not reconnect; \ + the Runner's processes keep running. Restart the gateway to reach them", + ) +} + struct Shared { writer: Mutex, pending: Mutex>>>, alive: AtomicBool, on_disconnect: DisconnectHook, + /// What the worker's `Hello` stated (P1R-C01). + hello: OnceLock, /// The worker's `Hello` stated that it answers `Registration` (CSRG-U2). #[cfg(feature = "devguard")] registration: AtomicBool, @@ -56,6 +83,7 @@ struct Shared { pub struct UdsRunner { shared: Arc, next_id: Arc, + call_deadline: Duration, } impl Clone for UdsRunner { @@ -63,6 +91,7 @@ impl Clone for UdsRunner { Self { shared: self.shared.clone(), next_id: self.next_id.clone(), + call_deadline: self.call_deadline, } } } @@ -103,6 +132,7 @@ impl UdsRunner { pending: Mutex::new(HashMap::new()), alive: AtomicBool::new(true), on_disconnect, + hello: OnceLock::new(), #[cfg(feature = "devguard")] registration: AtomicBool::new(false), #[cfg(feature = "devguard")] @@ -117,12 +147,35 @@ impl UdsRunner { Self { shared, next_id: Arc::new(AtomicU64::new(1)), + call_deadline: RUNNER_CALL_DEADLINE, } } + /// This client with another deadline per call than [`RUNNER_CALL_DEADLINE`]. For tests. + #[doc(hidden)] + pub fn with_call_deadline(mut self, deadline: Duration) -> Self { + self.call_deadline = deadline; + self + } + pub async fn handshake(&self) -> Result<(), RunnerError> { match self.call(RunnerOp::Hello).await { Ok(hello @ RunnerOpResult::Hello { protocol, .. }) if protocol == WIRE_PROTOCOL => { + if let RunnerOpResult::Hello { + lifecycle, + identity, + previous_controller, + inherited_processes, + .. + } = &hello + { + let _ = self.shared.hello.set(RunnerHello { + lifecycle: *lifecycle, + identity: identity.clone(), + previous_controller: *previous_controller, + inherited_processes: *inherited_processes, + }); + } #[cfg(feature = "devguard")] if let RunnerOpResult::Hello { registration, @@ -161,6 +214,52 @@ impl UdsRunner { } } + /// What the worker's `Hello` stated about its lifecycle (P1R-C01); before the handshake, a + /// managed worker's defaults. + pub fn hello(&self) -> RunnerHello { + self.shared.hello.get().cloned().unwrap_or_default() + } + + fn independent(&self) -> bool { + self.shared + .hello + .get() + .is_some_and(|hello| hello.lifecycle == RunnerLifecycle::Independent) + } + + /// The worker is an independent Runner that an earlier gateway controlled (P1R-C01). This + /// gateway then queries, reads, writes to, resizes and terminates the processes it inherits, + /// but starts no new execution, patch or approval resume: the Runner refuses them as well. + pub fn refuses_new_work(&self) -> bool { + self.shared.hello.get().is_some_and(|hello| { + hello.lifecycle == RunnerLifecycle::Independent && hello.previous_controller + }) + } + + /// Leave the independent Runner, whose processes, output and timeouts carry on, and close + /// the connection (P1R-C01). + pub async fn detach(&self) -> Result<(), RunnerError> { + let detached = match self.call(RunnerOp::Detach).await { + Ok(RunnerOpResult::Detached) => Ok(()), + Ok(other) => Err(unexpected(other)), + Err(err) => Err(err), + }; + self.close().await; + detached + } + + /// Stop the independent Runner: it ends every process and exits (P1R-C01). How many + /// processes were still running. + pub async fn stop_runner(&self) -> Result { + let stopped = match self.call(RunnerOp::Stop).await { + Ok(RunnerOpResult::Stopped { terminated }) => Ok(terminated), + Ok(other) => Err(unexpected(other)), + Err(err) => Err(err), + }; + self.close().await; + stopped + } + /// Whether the worker's `Hello` stated that it answers `Registration` (CSRG-U2). #[cfg(feature = "devguard")] pub fn states_registration(&self) -> bool { @@ -182,6 +281,9 @@ impl UdsRunner { } match self.call(RunnerOp::Registration).await { Ok(RunnerOpResult::Registration(info)) => Ok(info), + Err(RunnerError::Execution(body)) if body.code == ErrorCode::RunnerDisconnected => { + Err(ResourceRegistrationState::OwnerUnreachable) + } Err(RunnerError::Execution(_)) => Err(ResourceRegistrationState::UnsupportedMode), _ => Err(ResourceRegistrationState::OwnerUnreachable), } @@ -319,6 +421,10 @@ impl UdsRunner { async fn call(&self, op: RunnerOp) -> Result { if !self.shared.alive.load(Ordering::SeqCst) { + // An independent Runner's connection is not made again (P1R-C01). + if self.independent() { + return Err(RunnerError::execution(runner_disconnected())); + } return Err(RunnerError::before_dispatch("runner socket closed")); } let request_id = format!("rrpc-{}", self.next_id.fetch_add(1, Ordering::SeqCst)); @@ -342,7 +448,7 @@ impl UdsRunner { } return Err(err); } - match timeout(RUNNER_CALL_DEADLINE, rx).await { + match timeout(self.call_deadline, rx).await { Ok(Ok(Ok(response))) => decode_response(response), Ok(Ok(Err(err))) => Err(err), Ok(Err(_)) => Err(RunnerError::ambiguous("runner socket closed")), diff --git a/crates/runner/src/wire.rs b/crates/runner/src/wire.rs index 10f7960..6939396 100644 --- a/crates/runner/src/wire.rs +++ b/crates/runner/src/wire.rs @@ -3,25 +3,36 @@ //! Framing is `u32` big-endian length + JSON. Kinds share one stream: //! request, response, and event. `request_id` (`rrpc-…`) is not an HTTP //! id, `operation_id`, `operation_key`, or `process_id`. +//! +//! A `managed` worker serves the one connection of the gateway that started it. An +//! `independent` Runner (P1R-C01) serves one connection at a time through [`IndependentRunner`]: +//! its processes, their output, their timeouts and their exit events outlive each connection. use std::collections::VecDeque; +use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; +use std::time::Duration; use codespace_domain::{ErrorBody, ErrorCode, FindResult, ProcessId, ReadResult}; use codespace_policy::Workspace; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{mpsc, oneshot, watch, Mutex}; +use tokio::task::JoinSet; +use crate::lifecycle::{RunnerIdentity, RunnerLifecycle}; use crate::{ InProcessRunner, Runner, RunnerApplyPatchRequest, RunnerApplyPatchResult, RunnerError, RunnerExecRequest, RunnerExecResult, RunnerProcessStatus, RunnerReadProcess, RunnerReadResult, RunnerResizeResult, RunnerWriteStdin, ShellRelease, }; -pub const WIRE_PROTOCOL: u32 = 7; +pub const WIRE_PROTOCOL: u32 = 8; const MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; const MAX_REPLAY: usize = 32; +/// How long a stopping independent Runner waits for its terminated processes to be reaped and +/// their output to end before it exits anyway (P1R-C01). +pub const STOP_DRAIN_BOUND: Duration = Duration::from_secs(5); #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -166,6 +177,14 @@ pub enum RunnerOp { request_id: String, }, Hello, + /// The gateway leaves an independent Runner and its processes keep running (P1R-C01). The + /// Runner answers `Detached`, reads nothing more from this connection and serves the next + /// one once this connection's requests have ended. A managed worker refuses it. + Detach, + /// Stop an independent Runner (P1R-C01): it reads nothing more, lets this connection's + /// requests end, terminates every process, waits for them, answers `Stopped` and exits. A + /// managed worker refuses it. + Stop, /// The execution owner's DevGuard registration (CSRG-U2). Sent only to a worker whose /// `Hello` states `registration`; a worker without it would close the connection. #[cfg(feature = "devguard")] @@ -209,6 +228,21 @@ pub enum RunnerOpResult { TerminateWorkspace(u32), Hello { protocol: u32, + /// Whether the worker belongs to the gateway that started it or outlives its + /// connections (P1R-C01). + #[serde(default)] + lifecycle: RunnerLifecycle, + /// Which Runner process answered (P1R-C01). + #[serde(default, skip_serializing_if = "Option::is_none")] + identity: Option, + /// An earlier connection of this independent Runner started work, so this connection + /// may observe and control the processes it inherits but starts nothing (P1R-C01). + #[serde(default)] + previous_controller: bool, + /// Processes still running when this connection began, which it inherits from + /// earlier connections (P1R-C01). + #[serde(default)] + inherited_processes: u32, /// The worker answers `Registration` (CSRG-U2). Left out when false, so a worker built /// without DevGuard says `Hello` as before. #[cfg(feature = "devguard")] @@ -224,6 +258,14 @@ pub enum RunnerOpResult { #[serde(default, skip_serializing_if = "is_false")] launch: bool, }, + /// An independent Runner's answer to `Detach` (P1R-C01). + Detached, + /// An independent Runner's answer to `Stop`, sent once its processes ended or + /// [`STOP_DRAIN_BOUND`] passed (P1R-C01): how many processes were still running when it + /// terminated them. + Stopped { + terminated: u32, + }, #[cfg(feature = "devguard")] Registration(codespace_domain::ResourceAuthorityInfo), #[cfg(feature = "devguard")] @@ -237,6 +279,18 @@ fn is_false(value: &bool) -> bool { !*value } +/// The refusal of new work on a connection to an independent Runner that an earlier connection +/// controlled (P1R-C01): recovering what that controller left is not available yet. +pub fn recovery_pending() -> ErrorBody { + ErrorBody::new( + ErrorCode::RecoveryPending, + "an earlier gateway controlled this independent Runner, and recovering what it left is \ + not available yet: its processes can be queried, read, written to, resized and \ + terminated, but nothing new is started. Stop the Runner and start a new one to start \ + work again", + ) +} + /// The worker's registration, when it was started with one, and its launch helper. #[cfg(feature = "devguard")] #[derive(Clone, Default)] @@ -255,6 +309,31 @@ impl Registrar { } } +/// What a connection's `Hello` states. +fn hello( + registrar: &Registrar, + lifecycle: RunnerLifecycle, + previous_controller: bool, + inherited_processes: u32, +) -> RunnerOpResult { + #[cfg(not(feature = "devguard"))] + let () = *registrar; + RunnerOpResult::Hello { + protocol: WIRE_PROTOCOL, + lifecycle, + identity: Some(RunnerIdentity::this_process()), + previous_controller, + inherited_processes, + #[cfg(feature = "devguard")] + registration: true, + #[cfg(feature = "devguard")] + admission: true, + // A worker states `launch` only with a registration and a launch helper. + #[cfg(feature = "devguard")] + launch: registrar.launches(), + } +} + /// Host worker: `InProcessRunner` plus a `ProcessExited` event stream. pub fn host_worker() -> (InProcessRunner, mpsc::UnboundedReceiver) { let (tx, rx) = mpsc::unbounded_channel(); @@ -308,6 +387,7 @@ where .await } +/// A managed worker's one connection. async fn serve( stream: S, runner: InProcessRunner, @@ -317,8 +397,6 @@ async fn serve( where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - #[cfg(not(feature = "devguard"))] - let () = registrar; let (mut read, write) = tokio::io::split(stream); let write = Arc::new(Mutex::new(write)); let event_write = write.clone(); @@ -334,174 +412,521 @@ where } }); + let hello = hello(®istrar, RunnerLifecycle::Managed, false, 0); let mut cache: VecDeque<(String, WireEnvelope)> = VecDeque::new(); loop { let payload = match read_frame(&mut read).await? { Some(payload) => payload, None => return Ok(()), }; - let request: WireEnvelope = match serde_json::from_slice(&payload) { + let (request_id, op) = match parse_request(&payload) { Ok(request) => request, - Err(err) => { - let response = WireEnvelope::response( - String::new(), - Err(ErrorBody::new( - ErrorCode::InvalidPatch, - format!("invalid runner rpc: {err}"), - )), - ); + Err(response) => { let mut writer = write.lock().await; write_frame(&mut *writer, &response).await?; return Ok(()); } }; - if request.protocol != WIRE_PROTOCOL || request.kind != WireKind::Request { - let response = WireEnvelope::response( - request.request_id.clone().unwrap_or_default(), + #[cfg(feature = "devguard")] + let op = match op { + Some(op) if answered_beside(&op) => { + // Beside the other requests, and never cached: asking again opens a new + // session, and a preparation or a launch is never replayed. A launch is its own + // task, so it settles even if the gateway stops waiting. + let (write, registrar, runner) = (write.clone(), registrar.clone(), runner.clone()); + tokio::spawn(async move { + let result = answer_beside(op, registrar, runner).await; + let response = WireEnvelope::response(request_id, result); + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + }); + continue; + } + op => op, + }; + let response = match op { + Some(RunnerOp::Detach | RunnerOp::Stop) => WireEnvelope::response( + request_id, Err(ErrorBody::new( - ErrorCode::InvalidPatch, - "runner rpc protocol mismatch", + ErrorCode::InvalidCommand, + "this worker belongs to the gateway that started it; only an independent \ + Runner detaches or stops on request", )), - ); - let mut writer = write.lock().await; - write_frame(&mut *writer, &response).await?; - return Ok(()); - } - let request_id = request.request_id.clone().unwrap_or_default(); - #[cfg(feature = "devguard")] - if matches!(request.op, Some(RunnerOp::Registration)) { - // Not cached for replay: asking again opens a new session for the same identity. - let (write, registrar) = (write.clone(), registrar.clone()); - tokio::spawn(async move { - let result = match registrar.registration { - Some(registration) => { - Ok(RunnerOpResult::Registration(registration.report().await)) - } - None => Err(ErrorBody::new( - ErrorCode::ResourcePolicyUnsupported, - "this worker was started without DevGuard settings", + ), + op => answer(&mut cache, &runner, request_id, op, &hello).await, + }; + let mut writer = write.lock().await; + write_frame(&mut *writer, &response).await?; + } +} + +/// A request's id and operation, or the response that refuses it and ends the connection. +fn parse_request(payload: &[u8]) -> Result<(String, Option), Box> { + let request: WireEnvelope = serde_json::from_slice(payload).map_err(|err| { + Box::new(WireEnvelope::response( + String::new(), + Err(ErrorBody::new( + ErrorCode::InvalidPatch, + format!("invalid runner rpc: {err}"), + )), + )) + })?; + if request.protocol != WIRE_PROTOCOL || request.kind != WireKind::Request { + return Err(Box::new(WireEnvelope::response( + request.request_id.clone().unwrap_or_default(), + Err(ErrorBody::new( + ErrorCode::InvalidPatch, + "runner rpc protocol mismatch", + )), + ))); + } + Ok((request.request_id.unwrap_or_default(), request.op)) +} + +/// The response to a request answered in turn: a replay from `cache`, the connection's `hello`, +/// or the runner's answer, which `cache` keeps for replay. +async fn answer( + cache: &mut VecDeque<(String, WireEnvelope)>, + runner: &InProcessRunner, + request_id: String, + op: Option, + hello: &RunnerOpResult, +) -> WireEnvelope { + match op { + Some(RunnerOp::Replay { + request_id: original, + }) => cache + .iter() + .find(|(id, _)| id == &original) + .map(|(_, cached)| { + let mut replayed = cached.clone(); + replayed.request_id = Some(request_id.clone()); + replayed + }) + .unwrap_or_else(|| { + WireEnvelope::response( + request_id.clone(), + Err(ErrorBody::new( + ErrorCode::InvalidPatch, + format!("unknown runner request_id `{original}`"), )), - }; - let response = WireEnvelope::response(request_id, result); - let mut writer = write.lock().await; - let _ = write_frame(&mut *writer, &response).await; - }); - continue; + ) + }), + Some(op) => { + let dispatched = match op { + RunnerOp::Hello => Ok(hello.clone()), + op => dispatch(runner, op).await, + }; + let envelope = WireEnvelope::response(request_id.clone(), dispatched); + if !request_id.is_empty() { + if cache.len() >= MAX_REPLAY { + cache.pop_front(); + } + cache.push_back((request_id, envelope.clone())); + } + envelope } - #[cfg(feature = "devguard")] - if let Some(RunnerOp::Prepare { + None => WireEnvelope::response( + request_id, + Err(ErrorBody::new( + ErrorCode::InvalidPatch, + "runner rpc missing op", + )), + ), + } +} + +/// A `Registration`, `Prepare` or `Launch` request, which runs beside the other requests and is +/// never cached for replay. +#[cfg(feature = "devguard")] +fn answered_beside(op: &RunnerOp) -> bool { + matches!( + op, + RunnerOp::Registration | RunnerOp::Prepare { .. } | RunnerOp::Launch { .. } + ) +} + +#[cfg(feature = "devguard")] +async fn answer_beside( + op: RunnerOp, + registrar: Registrar, + runner: InProcessRunner, +) -> Result { + match op { + RunnerOp::Registration => match registrar.registration { + Some(registration) => Ok(RunnerOpResult::Registration(registration.report().await)), + None => Err(ErrorBody::new( + ErrorCode::ResourcePolicyUnsupported, + "this worker was started without DevGuard settings", + )), + }, + RunnerOp::Prepare { workspace, - request: exec, + request, attempt_id, - }) = request.op - { - // Beside the other requests, and never cached: a preparation is not replayed. - let (write, registrar, runner) = (write.clone(), registrar.clone(), runner.clone()); - tokio::spawn(async move { - let result = match registrar.registration { - Some(registration) => Ok(RunnerOpResult::Prepared( - crate::admission::Preparer::new(registration.owner(), runner) - .prepare_and_dispose(workspace, exec, attempt_id) - .await, - )), - None => Err(ErrorBody::new( - ErrorCode::ResourcePolicyUnsupported, - "this worker was started without DevGuard settings", - )), - }; - let response = WireEnvelope::response(request_id, result); - let mut writer = write.lock().await; - let _ = write_frame(&mut *writer, &response).await; - }); - continue; - } - #[cfg(feature = "devguard")] - if let Some(RunnerOp::Launch { + } => match registrar.registration { + Some(registration) => Ok(RunnerOpResult::Prepared( + crate::admission::Preparer::new(registration.owner(), runner) + .prepare_and_dispose(workspace, request, attempt_id) + .await, + )), + None => Err(ErrorBody::new( + ErrorCode::ResourcePolicyUnsupported, + "this worker was started without DevGuard settings", + )), + }, + RunnerOp::Launch { workspace, - request: exec, + request, attempt_id, - }) = request.op + } => match (registrar.registration, registrar.launcher) { + (Some(registration), Some(launcher)) => Ok(RunnerOpResult::Launched( + crate::admission::Preparer::new(registration.owner(), runner) + .with_launcher(Some(launcher)) + .prepare_and_launch(workspace, request, attempt_id) + .await, + )), + _ => Err(ErrorBody::new( + ErrorCode::ResourcePolicyUnsupported, + "this worker was started without DevGuard settings or a launch helper", + )), + }, + other => unreachable!("{other:?} is answered in turn"), + } +} + +/// A request that starts work an earlier connection's successor could not know about: a +/// process, a change to the workspace or an admission (P1R-C01). The first one accepted makes +/// its connection the independent Runner's controller. +fn starts_work(op: &RunnerOp) -> bool { + match op { + RunnerOp::Exec { .. } | RunnerOp::ApplyPatch { .. } => true, + #[cfg(feature = "devguard")] + RunnerOp::Prepare { .. } | RunnerOp::Launch { .. } => true, + _ => false, + } +} + +/// How an independent Runner's connection ended (P1R-C01). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Served { + /// The gateway detached: the processes keep running. + Detached, + /// The connection ended without `Detach` or `Stop`, or with an unusable request: a loss. + /// The processes keep running; the Runner never infers an intent from it. + Lost, + /// The Runner stopped, through `Stop` or its own stop request: it terminated `terminated` + /// processes and waited for them, and serves no further connection. + Stopped { terminated: u32 }, +} + +/// Asks an independent Runner to stop (P1R-C01), as `Stop` does: from a signal handler, or with +/// no connection open. +#[derive(Clone)] +pub struct StopHandle { + requested: Arc>, + /// How many processes the stop terminated so far. + terminated: Arc, + runner: InProcessRunner, +} + +impl StopHandle { + /// Stop the Runner: every process is terminated now, so no request waits on one, and the + /// Runner reads no further request. + pub fn request(&self) { + self.requested.send_replace(true); + self.terminate(); + } + + /// Terminate the processes still running, and count those nothing had asked to end yet. + fn terminate(&self) -> u32 { + let terminated = self.runner.terminate_all(); + self.terminated.fetch_add(terminated, Ordering::SeqCst) + terminated + } + + pub fn is_requested(&self) -> bool { + *self.requested.borrow() + } + + /// Resolves once a stop has been requested. + pub async fn requested(&self) { + let mut requested = self.requested.subscribe(); + let _ = requested.wait_for(|stop| *stop).await; + } +} + +/// The exit events of an independent Runner's processes, kept between its connections. +struct EventHub { + events: mpsc::UnboundedReceiver, + /// An event whose write failed, which the next connection gets first. + undelivered: Option, +} + +/// Forward exit events to one connection until `stop`, or until a write fails. Events left in +/// the hub go to the next connection. +async fn forward_events( + hub: Arc>, + write: Arc>, + mut stop: oneshot::Receiver<()>, +) where + W: AsyncWrite + Unpin, +{ + let mut hub = hub.lock().await; + loop { + let event = match hub.undelivered.take() { + Some(event) => event, + None => tokio::select! { + biased; + _ = &mut stop => return, + event = hub.events.recv() => match event { + Some(event) => event, + None => return, + }, + }, + }; + let mut writer = write.lock().await; + if write_frame(&mut *writer, &WireEnvelope::event(event.clone())) + .await + .is_err() { - // Beside the other requests, and never cached: a launch is never replayed. The - // launch is its own task, so it settles even if the gateway stops waiting. - let (write, registrar, runner) = (write.clone(), registrar.clone(), runner.clone()); - tokio::spawn(async move { - let result = match (registrar.registration, registrar.launcher) { - (Some(registration), Some(launcher)) => Ok(RunnerOpResult::Launched( - crate::admission::Preparer::new(registration.owner(), runner) - .with_launcher(Some(launcher)) - .prepare_and_launch(workspace, exec, attempt_id) - .await, - )), - _ => Err(ErrorBody::new( - ErrorCode::ResourcePolicyUnsupported, - "this worker was started without DevGuard settings or a launch helper", - )), - }; - let response = WireEnvelope::response(request_id, result); - let mut writer = write.lock().await; - let _ = write_frame(&mut *writer, &response).await; - }); - continue; + hub.undelivered = Some(event); + return; } - let response = match request.op { - Some(RunnerOp::Replay { - request_id: original, - }) => cache - .iter() - .find(|(id, _)| id == &original) - .map(|(_, cached)| { - let mut replayed = cached.clone(); - replayed.request_id = Some(request_id.clone()); - replayed - }) - .unwrap_or_else(|| { - WireEnvelope::response( - request_id.clone(), - Err(ErrorBody::new( - ErrorCode::InvalidPatch, - format!("unknown runner request_id `{original}`"), - )), - ) - }), - Some(op) => { - let dispatched = dispatch(&runner, op).await; - // A worker states `launch` only with a registration and a launch helper. - #[cfg(feature = "devguard")] - let dispatched = dispatched.map(|result| match result { - RunnerOpResult::Hello { - protocol, - registration, - admission, - .. - } => RunnerOpResult::Hello { - protocol, - registration, - admission, - launch: registrar.launches(), - }, - other => other, - }); - let envelope = WireEnvelope::response(request_id.clone(), dispatched); - if !request_id.is_empty() { - if cache.len() >= MAX_REPLAY { - cache.pop_front(); + } +} + +/// An independent Runner (P1R-C01), started by an operator or a harness rather than a gateway. +/// It serves one connection at a time, and its processes, their output, their timeouts and +/// their exit events outlive each connection. A connection becomes the Runner's controller with +/// the first request that starts work; every later connection states `previous_controller` in +/// its `Hello` and may only query, read, write stdin, resize and terminate, until the Runner +/// stops. Recovering what an earlier controller left is later work (P1R-C03, P1R-C04). +pub struct IndependentRunner { + runner: InProcessRunner, + events: Arc>, + registrar: Registrar, + /// A connection has started work. + controlled: bool, + stop: StopHandle, +} + +impl IndependentRunner { + pub fn new(runner: InProcessRunner, events: mpsc::UnboundedReceiver) -> Self { + let stop = StopHandle { + requested: Arc::new(watch::Sender::new(false)), + terminated: Arc::new(AtomicU32::new(0)), + runner: runner.clone(), + }; + Self { + runner, + events: Arc::new(Mutex::new(EventHub { + events, + undelivered: None, + })), + #[cfg(feature = "devguard")] + registrar: Registrar::default(), + #[cfg(not(feature = "devguard"))] + registrar: (), + controlled: false, + stop, + } + } + + /// The Runner's own DevGuard registration, which it keeps across its connections, and its + /// launch helper (CSRG-U2, CSRG-U4). + #[cfg(feature = "devguard")] + pub fn with_registration( + mut self, + registration: Option>, + launcher: Option, + ) -> Self { + self.registrar = Registrar { + registration, + launcher, + }; + self + } + + pub fn stop_handle(&self) -> StopHandle { + self.stop.clone() + } + + /// Serve each connection that `accept` yields, one at a time, until the Runner stops: through + /// `Stop`, or a stop request, which ends a connection being served or the wait for the next + /// one. `served` hears how each connection ended, or why one could not be accepted; a failed + /// accept is tried again after a pause. How many processes the stop terminated. + pub async fn run( + mut self, + mut accept: A, + mut served: impl FnMut(std::io::Result), + ) -> u32 + where + A: FnMut() -> F, + F: std::future::Future>, + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let stop = self.stop.clone(); + loop { + let accepted = tokio::select! { + biased; + () = stop.requested() => { + let terminated = self.stop().await; + served(Ok(Served::Stopped { terminated })); + return terminated; + } + accepted = accept() => accepted, + }; + match accepted { + Ok(stream) => { + let ending = self.serve(stream).await; + served(Ok(ending)); + if let Served::Stopped { terminated } = ending { + return terminated; } - cache.push_back((request_id, envelope.clone())); } - envelope + Err(err) => { + served(Err(err)); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + } + + /// Serve one connection until it detaches, is lost or stops the Runner. Every request it + /// sent has ended when this returns, its `Registration`, `Prepare` and `Launch` requests + /// included, so the next connection starts after them. + pub async fn serve(&mut self, stream: S) -> Served + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let previous_controller = self.controlled; + let hello = hello( + &self.registrar, + RunnerLifecycle::Independent, + previous_controller, + self.runner.live_processes(), + ); + let (mut read, write) = tokio::io::split(stream); + let write = Arc::new(Mutex::new(write)); + let (stop_events, events_stopped) = oneshot::channel(); + let forwarder = tokio::spawn(forward_events( + self.events.clone(), + write.clone(), + events_stopped, + )); + let mut beside: JoinSet<()> = JoinSet::new(); + let mut cache: VecDeque<(String, WireEnvelope)> = VecDeque::new(); + let stop = self.stop.clone(); + let ending = loop { + let frame = tokio::select! { + biased; + () = stop.requested() => break Ending::Stop(None), + frame = read_frame(&mut read) => frame, + }; + let Ok(Some(payload)) = frame else { + break Ending::Lost; + }; + let (request_id, op) = match parse_request(&payload) { + Ok(request) => request, + Err(response) => { + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + break Ending::Lost; + } + }; + let response = match op { + Some(RunnerOp::Detach) => break Ending::Detach(request_id), + Some(RunnerOp::Stop) => { + self.stop.request(); + break Ending::Stop(Some(request_id)); + } + Some(op) if starts_work(&op) && previous_controller => { + WireEnvelope::response(request_id, Err(recovery_pending())) + } + op => { + if op.as_ref().is_some_and(starts_work) { + self.controlled = true; + } + #[cfg(feature = "devguard")] + let op = match op { + Some(op) if answered_beside(&op) => { + let (write, registrar, runner) = + (write.clone(), self.registrar.clone(), self.runner.clone()); + beside.spawn(async move { + let result = answer_beside(op, registrar, runner).await; + let response = WireEnvelope::response(request_id, result); + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + }); + continue; + } + op => op, + }; + answer(&mut cache, &self.runner, request_id, op, &hello).await + } + }; + let mut writer = write.lock().await; + if write_frame(&mut *writer, &response).await.is_err() { + break Ending::Lost; } - None => WireEnvelope::response( - request_id, - Err(ErrorBody::new( - ErrorCode::InvalidPatch, - "runner rpc missing op", - )), - ), }; - let mut writer = write.lock().await; - write_frame(&mut *writer, &response).await?; + if let Ending::Detach(request_id) = &ending { + let response = WireEnvelope::response(request_id.clone(), Ok(RunnerOpResult::Detached)); + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + } + // The connection's requests end before anything else: no later connection, and no + // stop, overlaps them. + while beside.join_next().await.is_some() {} + let served = match ending { + Ending::Detach(_) => Served::Detached, + Ending::Lost => Served::Lost, + Ending::Stop(request_id) => { + let terminated = self.end_processes().await; + if let Some(request_id) = request_id { + let response = WireEnvelope::response( + request_id, + Ok(RunnerOpResult::Stopped { terminated }), + ); + let mut writer = write.lock().await; + let _ = write_frame(&mut *writer, &response).await; + } + Served::Stopped { terminated } + } + }; + let _ = stop_events.send(()); + let _ = forwarder.await; + let _ = write.lock().await.shutdown().await; + served } + + /// Stop with no connection open (P1R-C01): terminate every process and wait for them, as + /// `Stop` does. How many were still running. + pub async fn stop(&self) -> u32 { + self.stop.request(); + self.end_processes().await + } + + /// Terminate every process, those a request started after the stop request included, and + /// wait until each is reaped and its output has ended, at most [`STOP_DRAIN_BOUND`]: a pipe + /// process's own children may keep its output open, as a pipe process has no process group. + /// How many processes the stop terminated. + async fn end_processes(&self) -> u32 { + let terminated = self.stop.terminate(); + let deadline = tokio::time::Instant::now() + STOP_DRAIN_BOUND; + while self.runner.occupied_slots() > 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(20)).await; + } + terminated + } +} + +/// Why an independent Runner's connection stopped reading. +enum Ending { + Detach(String), + Lost, + /// With the request's id when `Stop` asked for it. + Stop(Option), } pub async fn write_frame( @@ -598,16 +1023,11 @@ async fn dispatch(runner: &InProcessRunner, op: RunnerOp) -> Result Ok(RunnerOpResult::Hello { - protocol: WIRE_PROTOCOL, - #[cfg(feature = "devguard")] - registration: true, - #[cfg(feature = "devguard")] - admission: true, - #[cfg(feature = "devguard")] - launch: false, - }), + RunnerOp::Hello => unreachable!("hello is answered before dispatch"), RunnerOp::Replay { .. } => unreachable!("replay is handled before dispatch"), + RunnerOp::Detach | RunnerOp::Stop => { + unreachable!("detach and stop are handled before dispatch") + } #[cfg(feature = "devguard")] RunnerOp::Registration => unreachable!("registration is handled before dispatch"), #[cfg(feature = "devguard")] @@ -629,8 +1049,8 @@ mod tests { } #[test] - fn wire_protocol_is_v7() { - assert_eq!(WIRE_PROTOCOL, 7); + fn wire_protocol_is_v8() { + assert_eq!(WIRE_PROTOCOL, 8); } #[tokio::test] @@ -868,12 +1288,14 @@ mod tests { match response(&mut read).await.result { Some(RunnerOpResult::Hello { protocol, + lifecycle, registration, admission, launch, + .. }) => assert_eq!( - (protocol, registration, admission, launch), - (WIRE_PROTOCOL, true, true, false) + (protocol, lifecycle, registration, admission, launch), + (WIRE_PROTOCOL, RunnerLifecycle::Managed, true, true, false) ), other => panic!("unexpected {other:?}"), } @@ -958,6 +1380,10 @@ mod tests { assert!(matches!( old, RunnerOpResult::Hello { + lifecycle: RunnerLifecycle::Managed, + identity: None, + previous_controller: false, + inherited_processes: 0, registration: false, admission: false, launch: false, @@ -1210,6 +1636,87 @@ mod tests { stop.store(true, Ordering::Relaxed); assert!(registering.await.unwrap() > 0); } + + /// An independent Runner keeps its registration across its connections and states it in + /// each `Hello`, and ends a connection only once its `Registration` requests were + /// answered, so the next connection starts after them (P1R-C01). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_runner_answers_registration_before_the_next_connection() { + let dir = private_directory(); + let credential = dir.path().join("consumer.secret"); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&credential) + .unwrap() + .write_all("ef".repeat(32).as_bytes()) + .unwrap(); + // An authority that accepts each session and answers nothing, so a registration + // session lasts until DevGuard's frame deadline. + let socket = dir.path().join("silent.sock"); + let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let stream = stream.unwrap(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(2)); + drop(stream); + }); + } + }); + let owner = Owner::new( + OwnerSettings { + socket, + consumer: "codespace".into(), + generation: "g1".into(), + }, + OwnerCredential::File(credential), + ); + let (worker, events) = host_worker(); + let mut independent = IndependentRunner::new(worker, events).with_registration( + Some(OwnerRegistration::new(owner, ResourceOwner::Worker)), + None, + ); + for connection in 0..2 { + let (client, server) = tokio::net::UnixStream::pair().unwrap(); + let serving = tokio::spawn(async move { + let served = independent.serve(server).await; + (served, independent) + }); + let (mut read, mut write) = client.into_split(); + request(&mut write, "rrpc-hello", RunnerOp::Hello).await; + match response(&mut read).await.result { + Some(RunnerOpResult::Hello { + lifecycle, + registration, + admission, + launch, + .. + }) => assert_eq!( + (lifecycle, registration, admission, launch), + (RunnerLifecycle::Independent, true, true, false), + "connection {connection}" + ), + other => panic!("unexpected {other:?}"), + } + request(&mut write, "rrpc-reg", RunnerOp::Registration).await; + request(&mut write, "rrpc-detach", RunnerOp::Detach).await; + // Both are answered before the connection ends. + let mut answered = Vec::new(); + while let Some(frame) = read_frame(&mut read).await.unwrap() { + let envelope: WireEnvelope = serde_json::from_slice(&frame).unwrap(); + if envelope.kind == WireKind::Response { + answered.push(envelope.request_id.unwrap()); + } + } + answered.sort(); + assert_eq!(answered, ["rrpc-detach", "rrpc-reg"]); + let (served, returned) = serving.await.unwrap(); + assert_eq!(served, Served::Detached); + independent = returned; + } + } } #[tokio::test] @@ -1236,4 +1743,583 @@ mod tests { other => panic!("unexpected {other:?}"), } } + + /// A managed worker refuses `Detach` and `Stop` and keeps serving; its `Hello` states its + /// lifecycle and identity (P1R-C01). + #[tokio::test] + async fn a_managed_worker_refuses_detach_and_stop() { + let (client, server) = tokio::net::UnixStream::pair().unwrap(); + let (worker, events) = host_worker(); + tokio::spawn(async move { + serve_runner_connection(server, worker, events) + .await + .expect("serve"); + }); + let (mut read, mut write) = client.into_split(); + for (id, op) in [ + ("rrpc-detach", RunnerOp::Detach), + ("rrpc-stop", RunnerOp::Stop), + ("rrpc-hello", RunnerOp::Hello), + ] { + write_frame(&mut write, &WireEnvelope::request(id.into(), op)) + .await + .unwrap(); + let reply = read_frame(&mut read).await.unwrap().unwrap(); + let reply: WireEnvelope = serde_json::from_slice(&reply).unwrap(); + assert_eq!(reply.request_id.as_deref(), Some(id)); + match reply.result { + Some(RunnerOpResult::Hello { + lifecycle, + identity, + previous_controller, + inherited_processes, + .. + }) => { + assert_eq!(lifecycle, RunnerLifecycle::Managed); + assert_eq!(identity, Some(RunnerIdentity::this_process())); + assert!(!previous_controller); + assert_eq!(inherited_processes, 0); + } + None => assert_eq!(reply.error.unwrap().code, ErrorCode::InvalidCommand), + other => panic!("unexpected {other:?}"), + } + } + } + + /// An independent Runner across its connections, through the gateway's client (P1R-C01). + mod independent { + use super::*; + use crate::{RunnerApplyPatchRequest, RunnerReadProcess, RunnerWriteStdin, UdsRunner}; + use codespace_domain::{ProcessState, ProcessTermination, Profile, WorkspaceId}; + use std::path::Path; + use std::time::Instant; + + /// An independent Runner that serves the streams sent to it in turn, as the worker binary + /// serves each connection it accepts. Dropping it stops the Runner. + struct Host { + streams: mpsc::UnboundedSender, + served: mpsc::UnboundedReceiver, + stop: StopHandle, + runner: InProcessRunner, + } + + impl Host { + fn start() -> Self { + let (runner, events) = host_worker(); + let independent = IndependentRunner::new(runner.clone(), events); + let stop = independent.stop_handle(); + let (streams, incoming) = mpsc::unbounded_channel::(); + let incoming = Arc::new(Mutex::new(incoming)); + let (report, served) = mpsc::unbounded_channel(); + tokio::spawn(independent.run( + move || { + let incoming = incoming.clone(); + async move { + incoming + .lock() + .await + .recv() + .await + .ok_or_else(|| std::io::Error::other("no more connections")) + } + }, + move |ending| { + if let Ok(ending) = ending { + let _ = report.send(ending); + } + }, + )); + Self { + streams, + served, + stop, + runner, + } + } + + /// A new gateway's connection, after its handshake. + async fn connect(&self) -> UdsRunner { + self.connect_reporting_exits(Arc::new(|_: &str| {})).await + } + + async fn connect_reporting_exits(&self, on_exit: ShellRelease) -> UdsRunner { + let (client, server) = tokio::net::UnixStream::pair().unwrap(); + self.streams.send(server).unwrap(); + let gateway = UdsRunner::from_stream(client, on_exit); + gateway.handshake().await.unwrap(); + gateway + } + + /// A connection the gateway's client does not speak for. + fn connect_raw(&self) -> tokio::net::UnixStream { + let (client, server) = tokio::net::UnixStream::pair().unwrap(); + self.streams.send(server).unwrap(); + client + } + + /// How the connection being served ended. + async fn served(&mut self) -> Served { + tokio::time::timeout(Duration::from_secs(20), self.served.recv()) + .await + .expect("the connection ends") + .expect("the Runner reports it") + } + } + + impl Drop for Host { + fn drop(&mut self) { + self.stop.request(); + } + } + + fn workspace(root: &Path) -> Workspace { + Workspace::new( + WorkspaceId("demo".into()), + root.to_path_buf(), + Profile::WorkspaceWrite, + ) + } + + fn command( + argv: &[&str], + process_id: &str, + timeout_ms: u64, + tty: bool, + ) -> RunnerExecRequest { + let mut request = RunnerExecRequest::for_host( + argv.iter().map(|arg| arg.to_string()).collect(), + ProcessId(process_id.into()), + Profile::WorkspaceWrite, + ); + request.timeout_ms = timeout_ms; + request.tty = tty; + request + } + + fn code(err: RunnerError) -> ErrorCode { + match err.as_execution() { + Some(body) => body.code, + None => panic!("{err:?}"), + } + } + + /// The process's output from its start, once it contains `expected`. + async fn output_with(gateway: &UdsRunner, process_id: &ProcessId, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let read = gateway + .read_process(RunnerReadProcess { + process_id: process_id.clone(), + cursor: 0, + }) + .await + .unwrap(); + if read.chunk.contains(expected) { + return; + } + assert!(Instant::now() < deadline, "{process_id:?}: {read:?}"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + async fn exited(runner: &R, process_id: &ProcessId) -> RunnerProcessStatus { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let status = runner.process_status(process_id).await.unwrap(); + if status.state == ProcessState::Exited { + return status; + } + assert!(Instant::now() < deadline, "{status:?}"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + async fn next_exit(exited_ids: &mut mpsc::UnboundedReceiver) -> Option { + tokio::time::timeout(Duration::from_secs(10), exited_ids.recv()) + .await + .expect("an exit event") + } + + async fn disconnected(gateway: &UdsRunner, process_id: &ProcessId) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match gateway.process_status(process_id).await { + Err(RunnerError::Execution(body)) + if body.code == ErrorCode::RunnerDisconnected => + { + return + } + other => assert!(Instant::now() < deadline, "{other:?}"), + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + /// After a detach the next gateway reads the output and status of the same process_id, + /// writes to it and resizes it, and the process times out at its original deadline. + /// That gateway starts nothing new, also once the inherited processes ended. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn the_next_gateway_inherits_the_processes_but_starts_nothing() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let first = host.connect().await; + let hello = first.hello(); + assert_eq!(hello.lifecycle, RunnerLifecycle::Independent); + assert_eq!(hello.identity, Some(RunnerIdentity::this_process())); + assert!(!hello.previous_controller); + assert_eq!(hello.inherited_processes, 0); + assert!(!first.refuses_new_work()); + let pipe = ProcessId("proc-pipe".into()); + let tty = ProcessId("proc-tty".into()); + let started = Instant::now(); + first + .exec( + &ws, + command( + &[ + "/bin/sh", + "-c", + "echo ready; read line; echo got $line; exec sleep 30", + ], + &pipe.0, + 3_000, + false, + ), + ) + .await + .unwrap(); + first + .exec( + &ws, + command( + &["/bin/sh", "-c", "echo tty-ready; exec sleep 30"], + &tty.0, + 60_000, + true, + ), + ) + .await + .unwrap(); + output_with(&first, &pipe, "ready").await; + first.detach().await.unwrap(); + assert_eq!(host.served().await, Served::Detached); + // The gateway that detached reaches the Runner no more. + assert_eq!( + code(first.process_status(&pipe).await.unwrap_err()), + ErrorCode::RunnerDisconnected + ); + + let second = host.connect().await; + let inherited = second.hello(); + assert_eq!(inherited.identity, hello.identity); + assert!(inherited.previous_controller); + assert_eq!(inherited.inherited_processes, 2); + assert!(second.refuses_new_work()); + output_with(&second, &pipe, "ready").await; + second + .write_stdin(RunnerWriteStdin { + process_id: pipe.clone(), + data: "hello\n".into(), + }) + .await + .unwrap(); + output_with(&second, &pipe, "got hello").await; + output_with(&second, &tty, "tty-ready").await; + assert_eq!( + second.resize(&tty, 40, 100).await.unwrap(), + RunnerResizeResult { + rows: 40, + cols: 100 + } + ); + let status = exited(&second, &pipe).await; + let elapsed = started.elapsed(); + assert_eq!( + status.termination, + Some(ProcessTermination::Timeout), + "{status:?}" + ); + assert!( + elapsed >= Duration::from_millis(3_000) && elapsed < Duration::from_secs(10), + "{elapsed:?}" + ); + // Nothing new starts, and the workspace is not changed. + let marker = root.path().join("started"); + let touch = command( + &["/usr/bin/touch", &marker.display().to_string()], + "proc-new", + 5_000, + false, + ); + assert_eq!( + code(second.exec(&ws, touch.clone()).await.unwrap_err()), + ErrorCode::RecoveryPending + ); + let patch = RunnerApplyPatchRequest { + patch: "*** Begin Patch\n*** Add File: added.txt\n+added\n*** End Patch\n".into(), + expected_versions: Default::default(), + check_only: false, + }; + assert_eq!( + code(second.apply_patch(&ws, patch.clone()).await.unwrap_err()), + ErrorCode::RecoveryPending + ); + // An inherited process still ends on request. + second.terminate(&tty).await.unwrap(); + assert_eq!( + exited(&second, &tty).await.termination, + Some(ProcessTermination::Terminated) + ); + // The barrier stays once the inherited processes ended. + second.detach().await.unwrap(); + assert_eq!(host.served().await, Served::Detached); + let third = host.connect().await; + assert!(third.hello().previous_controller); + assert_eq!(third.hello().inherited_processes, 0); + assert!(third.refuses_new_work()); + assert_eq!( + code(third.exec(&ws, touch).await.unwrap_err()), + ErrorCode::RecoveryPending + ); + assert_eq!( + code(third.apply_patch(&ws, patch).await.unwrap_err()), + ErrorCode::RecoveryPending + ); + assert!(!marker.exists()); + assert!(!root.path().join("added.txt").exists()); + assert_eq!(third.stop_runner().await.unwrap(), 0); + assert_eq!(host.served().await, Served::Stopped { terminated: 0 }); + } + + /// Only a connection that starts work becomes the controller: one that only read leaves + /// the next gateway free to start work. + #[tokio::test] + async fn a_gateway_that_starts_nothing_leaves_no_controller() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let reader = host.connect().await; + reader.find(&ws, None, None, None).await.unwrap(); + reader.detach().await.unwrap(); + assert_eq!(host.served().await, Served::Detached); + let worker = host.connect().await; + assert!(!worker.hello().previous_controller); + assert!(!worker.refuses_new_work()); + worker + .exec( + &ws, + command(&["/bin/sh", "-c", "exit 0"], "proc-ok", 5_000, false), + ) + .await + .unwrap(); + worker.close().await; + assert_eq!(host.served().await, Served::Lost); + assert!(host.connect().await.hello().previous_controller); + } + + /// An explicit stop ends the processes, a tty's process group included, and the Runner. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_stop_ends_the_processes_and_the_runner() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let gateway = host.connect().await; + let pipe = ProcessId("proc-pipe".into()); + let tty = ProcessId("proc-tty".into()); + gateway + .exec(&ws, command(&["/bin/sleep", "30"], &pipe.0, 60_000, false)) + .await + .unwrap(); + gateway + .exec( + &ws, + command( + &["/bin/sh", "-c", "echo tty-ready; exec sleep 30"], + &tty.0, + 60_000, + true, + ), + ) + .await + .unwrap(); + output_with(&gateway, &tty, "tty-ready").await; + let started = Instant::now(); + assert_eq!(gateway.stop_runner().await.unwrap(), 2); + assert!( + started.elapsed() < STOP_DRAIN_BOUND, + "{:?}", + started.elapsed() + ); + assert_eq!(host.served().await, Served::Stopped { terminated: 2 }); + assert_eq!(host.runner.occupied_slots(), 0); + for process_id in [&pipe, &tty] { + let status = exited(&host.runner, process_id).await; + assert_eq!( + status.termination, + Some(ProcessTermination::Terminated), + "{status:?}" + ); + } + assert_eq!( + code(gateway.process_status(&pipe).await.unwrap_err()), + ErrorCode::RunnerDisconnected + ); + } + + /// A stop request, as the Runner's own SIGTERM or SIGINT makes, ends the connection being + /// served and the processes; the gateway's later requests are refused. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_stop_request_ends_the_connection_and_the_processes() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let gateway = host.connect().await; + let pipe = ProcessId("proc-pipe".into()); + gateway + .exec(&ws, command(&["/bin/sleep", "30"], &pipe.0, 60_000, false)) + .await + .unwrap(); + host.stop.request(); + assert_eq!(host.served().await, Served::Stopped { terminated: 1 }); + assert_eq!( + exited(&host.runner, &pipe).await.termination, + Some(ProcessTermination::Terminated) + ); + disconnected(&gateway, &pipe).await; + } + + /// A connection that ends without a word is a loss: the processes keep running, and an + /// exit while no gateway is connected reaches the next one. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_lost_connection_keeps_the_processes_and_their_exit_events() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let first = host.connect().await; + let quick = ProcessId("proc-quick".into()); + let long = ProcessId("proc-long".into()); + first + .exec(&ws, command(&["/bin/sleep", "2"], &quick.0, 60_000, false)) + .await + .unwrap(); + first + .exec(&ws, command(&["/bin/sleep", "30"], &long.0, 60_000, false)) + .await + .unwrap(); + first.close().await; + assert_eq!(host.served().await, Served::Lost); + assert_eq!( + exited(&host.runner, &quick).await.termination, + Some(ProcessTermination::Exited) + ); + let (exits, mut exited_ids) = mpsc::unbounded_channel(); + let second = host + .connect_reporting_exits(Arc::new(move |process_id: &str| { + let _ = exits.send(process_id.to_string()); + })) + .await; + assert!(second.hello().previous_controller); + assert_eq!(second.hello().inherited_processes, 1); + assert_eq!( + next_exit(&mut exited_ids).await.as_deref(), + Some("proc-quick") + ); + assert_eq!( + second.process_status(&long).await.unwrap().state, + ProcessState::Running + ); + second.terminate(&long).await.unwrap(); + assert_eq!( + next_exit(&mut exited_ids).await.as_deref(), + Some("proc-long") + ); + } + + /// A gateway whose connection ends right after its `Detach` or `Stop` still detaches or + /// stops: the Runner acts on what it read, though its answer is lost. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_detach_or_stop_whose_answer_is_lost_still_holds() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let gateway = host.connect().await; + let long = ProcessId("proc-long".into()); + gateway + .exec(&ws, command(&["/bin/sleep", "30"], &long.0, 60_000, false)) + .await + .unwrap(); + gateway.close().await; + assert_eq!(host.served().await, Served::Lost); + for (op, ending) in [ + (RunnerOp::Detach, Served::Detached), + (RunnerOp::Stop, Served::Stopped { terminated: 1 }), + ] { + let (_, mut write) = host.connect_raw().into_split(); + write_frame(&mut write, &WireEnvelope::request("rrpc-end".into(), op)) + .await + .unwrap(); + drop(write); + assert_eq!(host.served().await, ending); + assert_eq!( + host.runner.process_status(&long).await.unwrap().state == ProcessState::Running, + ending == Served::Detached + ); + } + assert_eq!( + exited(&host.runner, &long).await.termination, + Some(ProcessTermination::Terminated) + ); + } + + /// When a call outlives the gateway's deadline the gateway closes the connection: its + /// later requests are refused, and the Runner treats the closed connection as a loss and + /// keeps the processes. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_call_past_its_deadline_closes_the_connection_and_keeps_the_processes() { + let mut host = Host::start(); + let root = tempfile::tempdir().unwrap(); + let ws = workspace(root.path()); + let gateway = host + .connect() + .await + .with_call_deadline(Duration::from_millis(500)); + // A process that never reads its stdin: a large write to it waits for its timeout. + let stuck = ProcessId("proc-stuck".into()); + let long = ProcessId("proc-long".into()); + gateway + .exec(&ws, command(&["/bin/sleep", "30"], &stuck.0, 2_000, false)) + .await + .unwrap(); + gateway + .exec(&ws, command(&["/bin/sleep", "30"], &long.0, 60_000, false)) + .await + .unwrap(); + let err = gateway + .write_stdin(RunnerWriteStdin { + process_id: stuck.clone(), + data: "x".repeat(4 * 1024 * 1024), + }) + .await + .unwrap_err(); + assert!( + matches!(err, RunnerError::TransportAmbiguous { .. }), + "{err:?}" + ); + assert_eq!( + code(gateway.process_status(&long).await.unwrap_err()), + ErrorCode::RunnerDisconnected + ); + assert_eq!(host.served().await, Served::Lost); + let next = host.connect().await; + assert_eq!( + next.process_status(&long).await.unwrap().state, + ProcessState::Running + ); + assert_eq!( + exited(&next, &stuck).await.termination, + Some(ProcessTermination::Timeout) + ); + } + } } diff --git a/crates/runner/tests/uds_runner.rs b/crates/runner/tests/uds_runner.rs index e0a30c0..910a043 100644 --- a/crates/runner/tests/uds_runner.rs +++ b/crates/runner/tests/uds_runner.rs @@ -121,7 +121,7 @@ async fn uds_process_status_echo_and_protocol_6_hello() { .expect("serve runner"); }); let runner = UdsRunner::from_stream(client, Arc::new(|_| {})); - runner.handshake().await.expect("hello protocol 7"); + runner.handshake().await.expect("hello protocol 8"); let dir = tempdir().unwrap(); let ws = workspace(dir.path()); let process_id = ProcessId("proc-uds-status".into()); diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index b9af3d7..dda4fa3 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -16,6 +16,38 @@ pub enum RunnerMode { Uds, } +/// How a UDS Runner's processes relate to the gateway (P1R-C01). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum RunnerLifecycleMode { + /// The worker belongs to the gateway, which starts it or connects to it through + /// `--runner-socket`; its processes end with the gateway's connection. + #[default] + Managed, + /// An independent Runner that an operator or a harness started: the gateway only connects + /// to it through `--runner-socket`, and its processes outlive the gateway. Not a supported + /// mode yet. + Independent, +} + +/// What a gateway asks of its independent Runner when it shuts down normally (P1R-C01). +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum RunnerExitIntent { + /// Leave the Runner, whose processes keep running. + Detach, + /// Stop the Runner: it ends its processes and exits. + Stop, +} + +/// The gateway's Runner lifecycle once its settings were checked together (P1R-C01). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum GatewayLifecycle { + /// The in-process runner, or a worker that belongs to the gateway. + #[default] + Managed, + /// An independent Runner, and what a normal shutdown asks of it. + Independent { exit_intent: RunnerExitIntent }, +} + #[derive(Debug, Clone, Parser)] #[command( name = "codespace-mcp", @@ -69,6 +101,18 @@ pub struct Cli { #[arg(long, env = "CODESPACE_RUNTIME_BIN")] pub runtime_bin: Option, + /// The UDS Runner's lifecycle: `managed` (default), or `independent` for a Runner that an + /// operator or a harness started, which the gateway only connects to through + /// `--runner-socket` (P1R-C01). `independent` is not a supported mode yet. + #[arg(long, value_enum, default_value_t = RunnerLifecycleMode::Managed)] + pub runner_lifecycle: RunnerLifecycleMode, + + /// Required with `--runner-lifecycle independent`, and only with it: what a normal + /// shutdown (end of stdin, SIGTERM, SIGINT) asks of the Runner, `detach` or `stop`. SIGUSR1 + /// ends the gateway for a restart and always detaches. + #[arg(long, value_enum)] + pub runner_exit_intent: Option, + /// Opt-in DevGuard connection: status (CSRG-U1) or the execution owner's registration /// (CSRG-U2). #[cfg(feature = "devguard")] @@ -77,6 +121,41 @@ pub struct Cli { } impl Cli { + /// The gateway's Runner lifecycle, or why its settings do not fit together (P1R-C01). + pub fn lifecycle(&self) -> Result { + let exit_intent = match (self.runner_lifecycle, self.runner_exit_intent) { + (RunnerLifecycleMode::Managed, None) => return Ok(GatewayLifecycle::Managed), + (RunnerLifecycleMode::Managed, Some(_)) => { + return Err( + "--runner-exit-intent is only for --runner-lifecycle independent".into(), + ) + } + (RunnerLifecycleMode::Independent, None) => { + return Err( + "--runner-lifecycle independent needs --runner-exit-intent detach \ + or stop" + .into(), + ) + } + (RunnerLifecycleMode::Independent, Some(exit_intent)) => exit_intent, + }; + if self.runner != RunnerMode::Uds { + return Err("--runner-lifecycle independent needs --runner uds".into()); + } + if self.runtime_bin.is_some() || self.runner_dir.is_some() { + return Err( + "--runner-lifecycle independent connects to a Runner that an operator \ + or a harness started; --runtime-bin and --runner-dir are for a worker \ + the gateway starts" + .into(), + ); + } + if self.runner_socket.is_none() { + return Err("--runner-lifecycle independent needs --runner-socket".into()); + } + Ok(GatewayLifecycle::Independent { exit_intent }) + } + pub fn mode(&self) -> TransportMode { if self.http { TransportMode::Http @@ -126,6 +205,83 @@ mod tests { assert_eq!(host.runner, RunnerMode::InProcess); } + /// An independent Runner is only connected to, through `--runner-socket`, and a normal + /// shutdown's intent toward it is always stated; a managed one takes no intent (P1R-C01). + #[test] + fn only_an_independent_runner_with_an_exit_intent_is_accepted() { + let lifecycle = |args: &[&str]| { + let mut argv = vec!["codespace-mcp"]; + argv.extend_from_slice(args); + let mut cli = Cli::try_parse_from(argv).expect("parses"); + // Runner settings that the environment of the test run may give. + if !args.contains(&"--runner") { + cli.runner = RunnerMode::InProcess; + } + if !args.contains(&"--runner-socket") { + cli.runner_socket = None; + } + if !args.contains(&"--runner-dir") { + cli.runner_dir = None; + } + if !args.contains(&"--runtime-bin") { + cli.runtime_bin = None; + } + cli.lifecycle() + }; + let socket = ["--runner", "uds", "--runner-socket", "/run/cs/runner.sock"]; + let independent = ["--runner-lifecycle", "independent"]; + assert_eq!(lifecycle(&[]), Ok(GatewayLifecycle::Managed)); + assert_eq!(lifecycle(&socket), Ok(GatewayLifecycle::Managed)); + for (intent, exit_intent) in [ + ("detach", RunnerExitIntent::Detach), + ("stop", RunnerExitIntent::Stop), + ] { + assert_eq!( + lifecycle(&[&socket[..], &independent, &["--runner-exit-intent", intent]].concat()), + Ok(GatewayLifecycle::Independent { exit_intent }) + ); + } + let detach = ["--runner-exit-intent", "detach"]; + for (args, problem) in [ + ([&socket[..], &detach].concat(), "only for"), + ( + [&socket[..], &independent].concat(), + "needs --runner-exit-intent", + ), + ([&independent[..], &detach].concat(), "needs --runner uds"), + ( + [&["--runner", "uds"][..], &independent, &detach].concat(), + "needs --runner-socket", + ), + ( + [ + &socket[..], + &independent, + &detach, + &["--runtime-bin", "/usr/local/bin/codespace-codex-runtime"], + ] + .concat(), + "--runtime-bin", + ), + ( + [ + &socket[..], + &independent, + &detach, + &["--runner-dir", "/run/cs"], + ] + .concat(), + "--runner-dir", + ), + ] { + let err = lifecycle(&args).expect_err(problem); + assert!(err.contains(problem), "{args:?}: {err}"); + } + let err = Cli::try_parse_from(["codespace-mcp", "--runner-lifecycle", "detached"]) + .expect_err("an unknown lifecycle"); + assert!(err.to_string().contains("detached"), "{err}"); + } + /// Without the `devguard` feature the gateway has no DevGuard setting at all. #[cfg(not(feature = "devguard"))] #[test] diff --git a/crates/server/src/devguard.rs b/crates/server/src/devguard.rs index b0b4d11..2f7eca8 100644 --- a/crates/server/src/devguard.rs +++ b/crates/server/src/devguard.rs @@ -33,7 +33,7 @@ use codespace_runner::launch::{LaunchReport, Launcher}; use codespace_runner::registration::{ handoff, status_info, Owner, OwnerCredential, OwnerRegistration, OwnerSettings, }; -use codespace_runner::RuntimeBackend; +use codespace_runner::{RunnerLifecycle, RuntimeBackend}; use tokio::sync::Mutex; use crate::config::RunnerMode; @@ -217,7 +217,8 @@ pub(crate) enum ExecutionOwner { /// The gateway runs executions in its own process and registers itself; with a launch /// helper it launches governed executions (CSRG-U4). InProcess(Arc, Option), - /// The worker the gateway started owns them, registers itself and reports to the gateway. + /// The worker the gateway started owns them, registers itself and reports to the gateway; + /// so does an independent Runner, which reads the consumer secret itself (P1R-C01). Worker, /// A worker the gateway did not start: nothing can hand it the consumer secret. UnstartedWorker, @@ -244,14 +245,16 @@ pub struct Registration { impl Registration { /// The registration for the gateway's runner mode. In InProcess mode the gateway reads /// the consumer secret from its file for each session, as a status probe does, and - /// launches through `launcher`; a worker the gateway starts gets its own. + /// launches through `launcher`; a worker the gateway starts gets its own, and an + /// independent Runner reads its own (P1R-C01): `worker_registers` says the UDS worker is + /// one of those. pub fn new( settings: Settings, runner: RunnerMode, - starts_worker: bool, + worker_registers: bool, launcher: Option, ) -> Arc { - let owner = match (runner, starts_worker) { + let owner = match (runner, worker_registers) { (RunnerMode::InProcess, _) => ExecutionOwner::InProcess( OwnerRegistration::new( Owner::new( @@ -282,11 +285,16 @@ impl Registration { /// Whether this gateway's execution owner can prepare governed executions with `runner` /// (CSRG-U3): the gateway itself in InProcess mode, or the worker it started when that - /// worker states it. + /// worker states it. An independent Runner prepares them only when it also states that it + /// launches them (P1R-C01). pub(crate) fn prepares(&self, runner: &RuntimeBackend) -> bool { match (&self.owner, runner) { (ExecutionOwner::InProcess(..), RuntimeBackend::InProcess(_)) => true, - (ExecutionOwner::Worker, RuntimeBackend::Uds(worker)) => worker.states_admission(), + (ExecutionOwner::Worker, RuntimeBackend::Uds(worker)) => { + worker.states_admission() + && (worker.hello().lifecycle == RunnerLifecycle::Managed + || worker.states_launch()) + } _ => false, } } @@ -1894,7 +1902,8 @@ pub(crate) mod tests { #[tokio::test] async fn a_worker_that_does_not_state_registration_is_refused() { use codespace_runner::{ - read_frame, write_frame, RunnerOpResult, UdsRunner, WireEnvelope, WIRE_PROTOCOL, + read_frame, write_frame, RunnerLifecycle, RunnerOpResult, UdsRunner, WireEnvelope, + WIRE_PROTOCOL, }; for registration in [false, true] { let (client, mut server) = tokio::net::UnixStream::pair().unwrap(); @@ -1903,6 +1912,10 @@ pub(crate) mod tests { let request: WireEnvelope = serde_json::from_slice(&frame).unwrap(); let hello = RunnerOpResult::Hello { protocol: WIRE_PROTOCOL, + lifecycle: RunnerLifecycle::Managed, + identity: None, + previous_controller: false, + inherited_processes: 0, registration, admission: registration, launch: false, diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 2b214e8..1c663af 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -1,10 +1,13 @@ +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use anyhow::Result; use clap::Parser; use codespace_policy::Registry; use codespace_runner::{RuntimeBackend, ShellRelease}; -use codespace_server::config::{Cli, RunnerMode, TransportMode}; +use codespace_server::config::{ + Cli, GatewayLifecycle, RunnerExitIntent, RunnerMode, TransportMode, +}; use codespace_server::http::serve_http_handler; use codespace_server::logging; use codespace_server::runtime::RuntimeProcess; @@ -16,65 +19,183 @@ fn main() -> Result<()> { // First, while this is the only thread: no later fork can then copy libnotify's // initialization in progress (see `prepare_fork_spawns`). codespace_runner::prepare_fork_spawns(); - tokio::runtime::Builder::new_multi_thread() + let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() - .expect("Failed building the Runtime") - .block_on(serve()) -} - -async fn serve() -> Result<()> { + .expect("Failed building the Runtime"); logging::init(); let cli = Cli::parse(); + let lifecycle = cli.lifecycle().map_err(anyhow::Error::msg)?; + let result = runtime.block_on(serve(cli, lifecycle)); + if let GatewayLifecycle::Independent { .. } = lifecycle { + // A signal can end the gateway while a read of its stdin waits on one of the runtime's + // blocking threads for as long as the client keeps stdin open: the gateway exits without + // waiting for it (P1R-C01). + runtime.shutdown_background(); + } + result +} + +async fn serve(cli: Cli, lifecycle: GatewayLifecycle) -> Result<()> { let registry = load_registry(&cli)?; let store = load_store(&cli)?; - let started = start_runner(&cli, store.clone()).await?; + let started = start_runner(&cli, lifecycle, store.clone()).await?; let handler = CodeSpace::from_cli(&cli, registry, store, started.runner.clone()); #[cfg(feature = "devguard")] handler.register_owner_now(); - let result = match cli.mode() { - TransportMode::Stdio => stdio::serve_handler(handler).await, - TransportMode::Http => { - let (bound, _cancel) = serve_http_handler(cli.http_config(), handler).await?; - tracing::info!(%bound, "http ready"); - tokio::signal::ctrl_c().await?; - Ok(()) + let (result, ending) = match lifecycle { + GatewayLifecycle::Managed => { + let result = match cli.mode() { + TransportMode::Stdio => stdio::serve_handler(handler).await, + TransportMode::Http => { + let (bound, _cancel) = serve_http_handler(cli.http_config(), handler).await?; + tracing::info!(%bound, "http ready"); + tokio::signal::ctrl_c().await?; + Ok(()) + } + }; + (result, Ending::Shutdown) } + GatewayLifecycle::Independent { .. } => serve_until_signal(&cli, handler).await?, }; - started.stop().await; - result + let stopped = started.stop(ending).await; + result.and(stopped) +} + +/// How the gateway ends (P1R-C01). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ending { + /// A normal shutdown: the end of stdin, SIGTERM or SIGINT. + Shutdown, + /// SIGUSR1: the gateway ends to be restarted, so it always detaches from an independent + /// Runner. + Restart, +} + +/// Serve until the transport ends or a signal ends the gateway (P1R-C01): SIGTERM and SIGINT +/// shut it down as the end of stdin does, SIGUSR1 ends it for a restart. +async fn serve_until_signal(cli: &Cli, handler: CodeSpace) -> Result<(Result<()>, Ending)> { + use tokio::signal::unix::{signal, SignalKind}; + let mut terminate = signal(SignalKind::terminate())?; + let mut interrupt = signal(SignalKind::interrupt())?; + let mut restart = signal(SignalKind::user_defined1())?; + let serving = async { + match cli.mode() { + TransportMode::Stdio => stdio::serve_handler(handler).await, + TransportMode::Http => { + let (bound, _cancel) = serve_http_handler(cli.http_config(), handler).await?; + tracing::info!(%bound, "http ready"); + std::future::pending::>().await + } + } + }; + Ok(tokio::select! { + result = serving => (result, Ending::Shutdown), + _ = terminate.recv() => (Ok(()), Ending::Shutdown), + _ = interrupt.recv() => (Ok(()), Ending::Shutdown), + _ = restart.recv() => (Ok(()), Ending::Restart), + }) } struct StartedRunner { runner: RuntimeBackend, process: Option, + lifecycle: GatewayLifecycle, + /// Set before the gateway ends its connection to an independent Runner itself. + leaving: Arc, } impl StartedRunner { /// Gateway shutdown ends a worker this gateway started, and with it the worker's processes, - /// before the gateway exits. - async fn stop(self) { - let Self { runner, process } = self; + /// before the gateway exits. An independent Runner is left or stopped as the gateway's exit + /// intent says, and always left for a restart (P1R-C01). + async fn stop(self, ending: Ending) -> Result<()> { + let Self { + runner, + process, + lifecycle, + leaving, + } = self; + if let (GatewayLifecycle::Independent { exit_intent }, RuntimeBackend::Uds(runner)) = + (lifecycle, &runner) + { + leaving.store(true, Ordering::SeqCst); + let intent = match ending { + Ending::Restart => RunnerExitIntent::Detach, + Ending::Shutdown => exit_intent, + }; + match intent { + RunnerExitIntent::Detach => match runner.detach().await { + Ok(()) => tracing::info!( + "detached from the independent Runner; its processes keep running" + ), + Err(err) => tracing::warn!( + error = %err, + "the independent Runner did not confirm the detach; its processes keep \ + running" + ), + }, + RunnerExitIntent::Stop => match runner.stop_runner().await { + Ok(terminated) => { + tracing::info!(terminated, "stopped the independent Runner") + } + Err(err) => { + return Err(anyhow::anyhow!( + "the independent Runner did not confirm its stop ({err}); its \ + processes may keep running, so stop it with SIGTERM" + )) + } + }, + } + return Ok(()); + } drop(runner); if let Some(mut process) = process { process.wait_exit().await; } + Ok(()) } } -async fn start_runner(cli: &Cli, store: Arc) -> Result { +async fn start_runner( + cli: &Cli, + lifecycle: GatewayLifecycle, + store: Arc, +) -> Result { let on_release: ShellRelease = { let store_for_lease = store.clone(); Arc::new(move |process_id: &str| { store_for_lease.release_process(process_id); }) }; + let leaving = Arc::new(AtomicBool::new(false)); match cli.runner { RunnerMode::InProcess => Ok(StartedRunner { runner: RuntimeBackend::in_process(on_release), process: None, + lifecycle, + leaving, }), RunnerMode::Uds => { + if let (GatewayLifecycle::Independent { .. }, Some(socket)) = + (lifecycle, &cli.runner_socket) + { + // The gateway only connects; the Runner registers itself, from its own + // credential file, and nothing is handed to it (P1R-C01). + let runner = + RuntimeProcess::connect_independent(socket, on_release, leaving.clone()) + .await?; + #[cfg(feature = "devguard")] + if cli.devguard.worker().is_some() { + codespace_server::runtime::require_registration(&runner, socket)?; + } + return Ok(StartedRunner { + runner: RuntimeBackend::Uds(runner), + process: None, + lifecycle, + leaving, + }); + } if let Some(bin) = &cli.runtime_bin { // The worker owns the executions, so it registers itself (CSRG-U2). #[cfg(feature = "devguard")] @@ -90,6 +211,8 @@ async fn start_runner(cli: &Cli, store: Arc) -> Result { return Ok(StartedRunner { runner: RuntimeBackend::Uds(runner), process: Some(process), + lifecycle, + leaving, }); } let (process, runner) = @@ -98,6 +221,8 @@ async fn start_runner(cli: &Cli, store: Arc) -> Result { return Ok(StartedRunner { runner: RuntimeBackend::Uds(runner), process: Some(process), + lifecycle, + leaving, }); } let socket = cli.runner_socket.as_ref().ok_or_else(|| { @@ -109,6 +234,8 @@ async fn start_runner(cli: &Cli, store: Arc) -> Result { Ok(StartedRunner { runner: RuntimeBackend::Uds(runner), process: None, + lifecycle, + leaving, }) } } diff --git a/crates/server/src/mcp.rs b/crates/server/src/mcp.rs index 3cd4514..66adc08 100644 --- a/crates/server/src/mcp.rs +++ b/crates/server/src/mcp.rs @@ -50,6 +50,8 @@ pub struct CodeSpace { /// The execution owner's registration `workspace_info` reports (CSRG-U2). #[cfg(feature = "devguard")] pub(crate) registration: Option>, + /// Whether the runner is an independent Runner, and what a shutdown asks of it (P1R-C01). + pub(crate) lifecycle: crate::config::GatewayLifecycle, } fn err_json(err: ErrorBody) -> String { @@ -143,6 +145,62 @@ tty_size is not an exec_command argument. Claim user intents only at major checkpoints and before work_finish."; +/// The lifetime paragraph of [`MCP_INSTRUCTIONS`]: the in-process runner's, or a worker's that +/// belongs to the gateway. +const MANAGED_LIFETIME: &str = "\ +Managed process lifetime is owned by the runner instance, not the MCP \ +session. Client or HTTP disconnect keeps the process running. Losing the \ +UDS worker connection or shutting down the gateway terminates the owned \ +subtree and drops handles. Restart does not recover process_id. If you lose \ +the spawn response before receiving process_id, do not search for the lost \ +handle and do not start a duplicate command."; + +/// The lifetime paragraph for an independent Runner (P1R-C01), after what the gateway's +/// shutdown does. It states only what the Runner guarantees: a pipe process has no process +/// group of its own, so terminating one signals that process alone. +const INDEPENDENT_LIFETIME: &str = "\ +Managed process lifetime is owned by an independent Runner that this \ +gateway did not start, not by the MCP session or the gateway. Client or \ +HTTP disconnect keeps the process running. {shutdown} If this gateway \ +loses its Runner connection, the processes keep running, but this gateway \ +refuses every later request that needs the Runner with RUNNER_DISCONNECTED \ +until it is restarted. A restarted gateway can read the output and status \ +of, write to, resize and terminate the processes started before, with the \ +same process_id, but it refuses new exec_command, apply_patch and \ +operation_resume with RECOVERY_PENDING until the Runner is stopped and a \ +new one is started. Terminating a pipe process signals that process only, \ +not its process group, so processes it started may outlive it. If you lose \ +the spawn response before receiving process_id, do not search for the lost \ +handle and do not start a duplicate command."; + +const DETACH_SHUTDOWN: &str = "\ +When this gateway shuts down it detaches: the Runner keeps the processes, \ +their output and their timeouts."; + +const STOP_SHUTDOWN: &str = "\ +When this gateway shuts down it stops the Runner, which terminates every \ +process, a pipe process alone and a tty process with its process group, \ +and exits, so their handles are lost. A gateway ended to be restarted \ +detaches instead and leaves them running."; + +/// The instructions for the gateway's runner (P1R-C01). +fn instructions(lifecycle: crate::config::GatewayLifecycle) -> String { + use crate::config::{GatewayLifecycle, RunnerExitIntent}; + match lifecycle { + GatewayLifecycle::Managed => MCP_INSTRUCTIONS.to_string(), + GatewayLifecycle::Independent { exit_intent } => { + let shutdown = match exit_intent { + RunnerExitIntent::Detach => DETACH_SHUTDOWN, + RunnerExitIntent::Stop => STOP_SHUTDOWN, + }; + MCP_INSTRUCTIONS.replace( + MANAGED_LIFETIME, + &INDEPENDENT_LIFETIME.replace("{shutdown}", shutdown), + ) + } + } +} + #[derive(Debug, Clone, Serialize, JsonSchema)] struct OkBody { ok: bool, @@ -181,7 +239,47 @@ impl CodeSpace { resource_authority: None, #[cfg(feature = "devguard")] registration: None, + lifecycle: crate::config::GatewayLifecycle::Managed, + } + } + + /// The handler of a gateway connected to an independent Runner (P1R-C01): its instructions + /// and `workspace_info` state that Runner's process lifetime. + pub fn with_lifecycle(mut self, lifecycle: crate::config::GatewayLifecycle) -> Self { + self.lifecycle = lifecycle; + self + } + + /// New work is refused when the runner is an independent Runner that an earlier gateway + /// controlled (P1R-C01), before any approval, lease or claim: what that gateway left is not + /// recovered yet. + fn refuse_new_work(&self) -> Result<(), ErrorBody> { + if self.runner.refuses_new_work() { + return Err(codespace_runner::recovery_pending()); + } + Ok(()) + } + + /// `info` with the process lifetime of the gateway's runner (P1R-C01). + fn with_lifetime(&self, mut info: WorkspaceInfo) -> WorkspaceInfo { + use crate::config::{GatewayLifecycle, RunnerExitIntent}; + use codespace_domain::ProcessDisconnectAction; + let GatewayLifecycle::Independent { exit_intent } = self.lifecycle else { + return info; + }; + if let Some(lifetime) = info + .execution + .as_mut() + .and_then(|execution| execution.process.capabilities.as_mut()) + .map(|capabilities| &mut capabilities.lifetime) + { + lifetime.runner_disconnect = ProcessDisconnectAction::KeepRunning; + lifetime.gateway_shutdown = match exit_intent { + RunnerExitIntent::Detach => ProcessDisconnectAction::KeepRunning, + RunnerExitIntent::Stop => ProcessDisconnectAction::Terminate, + }; } + info } #[tool( @@ -193,6 +291,7 @@ impl CodeSpace { Parameters(params): Parameters, ) -> Result, String> { let info = lookup(&self.registry, params.workspace_id).map_err(err_json)?; + let info = self.with_lifetime(info); #[cfg(feature = "devguard")] let info = self.with_resource_authority_status(info).await; Ok(Json(info)) @@ -256,6 +355,7 @@ impl CodeSpace { .map_err(err_json)?; allow(ws, Action::Write, &ClientClaims::default()).map_err(err_json)?; ws.require_file_write().map_err(err_json)?; + self.refuse_new_work().map_err(err_json)?; if let Err(err) = self.maybe_hold_patch(ws, ¶ms) { return Err(err_json(err)); } @@ -296,6 +396,7 @@ impl CodeSpace { if params.command.is_empty() || params.command[0].is_empty() { return Err(err_json(invalid_argv())); } + self.refuse_new_work().map_err(err_json)?; // A workspace this gateway cannot govern is refused before any approval or lease. self.participation(ws).await.map_err(err_json)?; if let Err(err) = self.maybe_hold_exec(ws, ¶ms) { @@ -711,6 +812,8 @@ impl CodeSpace { &self, params: OperationResumeParams, ) -> Result { + // Before the claim, so the approval stays as it is (P1R-C01). + self.refuse_new_work()?; match self.store.claim_resume(¶ms.approval_id)? { ResumeClaim::ReplaySuccess(result) => Ok(result), ResumeClaim::ReplayError(err) => Err(err), @@ -1202,7 +1305,9 @@ impl CodeSpace { store: Arc, runner: RuntimeBackend, ) -> Self { - let handler = Self::with_store_and_runner(registry, store, runner); + let lifecycle = cli.lifecycle().unwrap_or_default(); + let handler = + Self::with_store_and_runner(registry, store, runner).with_lifecycle(lifecycle); #[cfg(feature = "devguard")] if let Some(settings) = cli.devguard.settings() { if cli.devguard.mode == crate::devguard::DevGuardMode::Register { @@ -1217,10 +1322,16 @@ impl CodeSpace { ); } } + // A worker the gateway starts registers itself, and so does an independent + // Runner (P1R-C01). let registration = crate::devguard::Registration::new( settings, cli.runner, - cli.runtime_bin.is_some(), + cli.runtime_bin.is_some() + || matches!( + lifecycle, + crate::config::GatewayLifecycle::Independent { .. } + ), launcher, ); tracing::info!( @@ -1296,7 +1407,7 @@ impl ServerHandler for CodeSpace { codespace_domain::SERVER_NAME, codespace_domain::SERVER_VERSION, )) - .with_instructions(MCP_INSTRUCTIONS.to_string()) + .with_instructions(instructions(self.lifecycle)) } fn supported_protocol_versions(&self) -> Cow<'static, [ProtocolVersion]> { @@ -2784,8 +2895,8 @@ mod devguard_tests { #[tokio::test] async fn a_preparation_lost_with_the_worker_is_unknown() { use codespace_runner::{ - read_frame, write_frame, RunnerOp, RunnerOpResult, UdsRunner, WireEnvelope, - WIRE_PROTOCOL, + read_frame, write_frame, RunnerLifecycle, RunnerOp, RunnerOpResult, UdsRunner, + WireEnvelope, WIRE_PROTOCOL, }; let dir = short_directory(); std::fs::write(dir.path().join("a.txt"), "kept").unwrap(); @@ -2799,6 +2910,10 @@ mod devguard_tests { Some(RunnerOp::Hello) => { let hello = RunnerOpResult::Hello { protocol: WIRE_PROTOCOL, + lifecycle: RunnerLifecycle::Managed, + identity: None, + previous_controller: false, + inherited_processes: 0, registration: true, admission: true, launch: false, @@ -2881,8 +2996,8 @@ mod devguard_tests { #[tokio::test] async fn a_launch_lost_with_the_worker_is_unknown_and_never_sent_again() { use codespace_runner::{ - read_frame, write_frame, RunnerOp, RunnerOpResult, UdsRunner, WireEnvelope, - WIRE_PROTOCOL, + read_frame, write_frame, RunnerLifecycle, RunnerOp, RunnerOpResult, UdsRunner, + WireEnvelope, WIRE_PROTOCOL, }; use std::sync::atomic::AtomicUsize; let dir = short_directory(); @@ -2896,6 +3011,10 @@ mod devguard_tests { Some(RunnerOp::Hello) => { let hello = RunnerOpResult::Hello { protocol: WIRE_PROTOCOL, + lifecycle: RunnerLifecycle::Managed, + identity: None, + previous_controller: false, + inherited_processes: 0, registration: true, admission: true, launch: true, @@ -4164,3 +4283,302 @@ mod devguard_tests { drop(control); } } + +/// A gateway connected to an independent Runner (P1R-C01). +#[cfg(test)] +mod independent_tests { + use super::tests::{exec_params, participation_registry}; + use super::*; + use crate::config::{GatewayLifecycle, RunnerExitIntent}; + use codespace_domain::{ + ApprovalDecision, ProcessDisconnectAction, ProcessState, ProcessStatusParams, + TerminateProcessParams, WorkspaceId, + }; + use codespace_runner::{host_worker, IndependentRunner, StopHandle, UdsRunner}; + use std::collections::BTreeMap; + use std::time::{Duration, Instant}; + use tokio::net::UnixStream; + use tokio::sync::mpsc; + + const DETACHING: GatewayLifecycle = GatewayLifecycle::Independent { + exit_intent: RunnerExitIntent::Detach, + }; + + /// An independent Runner in this process, serving the streams sent to it in turn. Dropping + /// it stops the Runner. + struct Runner { + streams: mpsc::UnboundedSender, + stop: StopHandle, + } + + impl Runner { + fn start() -> Self { + let (runner, events) = host_worker(); + let independent = IndependentRunner::new(runner, events); + let stop = independent.stop_handle(); + let (streams, incoming) = mpsc::unbounded_channel::(); + let incoming = Arc::new(tokio::sync::Mutex::new(incoming)); + tokio::spawn(independent.run( + move || { + let incoming = incoming.clone(); + async move { + incoming + .lock() + .await + .recv() + .await + .ok_or_else(|| std::io::Error::other("no more gateways")) + } + }, + |_| {}, + )); + Self { streams, stop } + } + + /// A gateway with its own handle on the operations database, connected to the Runner. + async fn gateway(&self, root: &std::path::Path, database: &std::path::Path) -> CodeSpace { + let store = Arc::new(Store::open(database).unwrap()); + let (client, server) = UnixStream::pair().unwrap(); + self.streams.send(server).unwrap(); + let releases = store.clone(); + let runner = UdsRunner::from_stream( + client, + Arc::new(move |process_id: &str| releases.release_process(process_id)), + ); + runner.handshake().await.unwrap(); + CodeSpace::with_store_and_runner( + participation_registry(root, "off"), + store, + RuntimeBackend::Uds(runner), + ) + .with_lifecycle(DETACHING) + } + } + + impl Drop for Runner { + fn drop(&mut self) { + self.stop.request(); + } + } + + async fn detach(gateway: &CodeSpace) { + match &gateway.runner { + RuntimeBackend::Uds(runner) => runner.detach().await.unwrap(), + RuntimeBackend::InProcess(_) => unreachable!("an independent Runner"), + } + } + + fn refused(result: Result) -> ErrorBody { + match result { + Ok(_) => panic!("not refused"), + Err(err) => serde_json::from_str(&err).unwrap(), + } + } + + /// After a detach the next gateway reads and controls the processes it inherits, but new + /// executions, patches and approval resumes are refused before anything is claimed, so a + /// granted approval stays granted. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_independent_gateway_after_a_detach_starts_nothing_and_keeps_the_approval() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let database = dir.path().join("operations.db"); + let runner = Runner::start(); + let first = runner.gateway(&root, &database).await; + assert!(!first.runner.refuses_new_work()); + let touch = format!("{}/resumed", root.display()); + let held = refused( + first + .exec_command(Parameters(exec_params( + "gov-confirm", + &["/usr/bin/touch", &touch], + false, + ))) + .await, + ); + assert_eq!(held.code, ErrorCode::ApprovalRequired, "{held:?}"); + let approval_id = ApprovalId(held.approval_id.unwrap()); + first + .approval_resolve(Parameters(ApprovalResolveParams { + approval_id: approval_id.clone(), + decision: ApprovalDecision::Grant, + })) + .await + .unwrap(); + let started = first + .exec_command(Parameters(exec_params( + "demo", + &["/bin/sh", "-c", "echo inherited; exec sleep 30"], + false, + ))) + .await + .unwrap() + .0; + detach(&first).await; + + let next = runner.gateway(&root, &database).await; + assert!(next.runner.refuses_new_work()); + let info = next + .workspace_info(Parameters(WorkspaceInfoParams { + workspace_id: Some("demo".into()), + })) + .await + .unwrap() + .0; + let lifetime = info + .execution + .unwrap() + .process + .capabilities + .unwrap() + .lifetime; + assert_eq!( + lifetime.runner_disconnect, + ProcessDisconnectAction::KeepRunning + ); + assert_eq!( + lifetime.gateway_shutdown, + ProcessDisconnectAction::KeepRunning + ); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let read = next + .read_process(Parameters(ReadProcessParams { + process_id: started.process_id.clone(), + cursor: 0, + })) + .await + .unwrap() + .0; + if read.chunk.contains("inherited") { + break; + } + assert!(Instant::now() < deadline, "{read:?}"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + let status = |gateway: &CodeSpace| { + let process_id = started.process_id.clone(); + let gateway = gateway.clone(); + async move { + gateway + .process_status(Parameters(ProcessStatusParams { process_id })) + .await + .unwrap() + .0 + .state + } + }; + assert_eq!(status(&next).await, ProcessState::Running); + let refusal = refused( + next.exec_command(Parameters(exec_params( + "demo", + &["/bin/echo", "new"], + false, + ))) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + let refusal = refused( + next.apply_patch(Parameters(ApplyPatchParams { + workspace_id: WorkspaceId("demo".into()), + patch: "*** Begin Patch\n*** Add File: added.txt\n+added\n*** End Patch\n".into(), + expected_versions: BTreeMap::new(), + operation_key: None, + check_only: false, + work_id: None, + })) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + let refusal = refused( + next.operation_resume(Parameters(OperationResumeParams { + approval_id: approval_id.clone(), + })) + .await, + ); + assert_eq!(refusal.code, ErrorCode::RecoveryPending, "{refusal:?}"); + assert_eq!( + next.store.inspect_approval(&approval_id).unwrap().0, + ApprovalState::Granted + ); + next.terminate_process(Parameters(TerminateProcessParams { + process_id: started.process_id.clone(), + })) + .await + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while status(&next).await != ProcessState::Exited { + assert!(Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!root.join("resumed").exists()); + assert!(!root.join("added.txt").exists()); + } + + /// The first gateway of a fresh independent Runner works as a managed one does. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn the_first_independent_gateway_works_normally() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("ws"); + std::fs::create_dir(&root).unwrap(); + let runner = Runner::start(); + let gateway = runner + .gateway(&root, &dir.path().join("operations.db")) + .await; + // A patch reaches the Runner, which applies it as a managed worker does, given the + // patch helper. + match gateway + .apply_patch(Parameters(ApplyPatchParams { + workspace_id: WorkspaceId("demo".into()), + patch: "*** Begin Patch\n*** Add File: added.txt\n+added\n*** End Patch\n".into(), + expected_versions: BTreeMap::new(), + operation_key: None, + check_only: false, + work_id: None, + })) + .await + { + Ok(applied) => assert_eq!(applied.0.status, PatchStatus::Applied), + Err(err) => assert_ne!(refused::<()>(Err(err)).code, ErrorCode::RecoveryPending), + } + let started = gateway + .exec_command(Parameters(exec_params("demo", &["/bin/echo", "ok"], true))) + .await + .unwrap() + .0; + assert_eq!(started.dispatch_status, ExecDispatchStatus::Confirmed); + } + + /// Each independent mode states only what its Runner guarantees, and the rest of the + /// instructions stay as they are. + #[test] + fn independent_instructions_replace_only_the_lifetime_paragraph() { + assert_eq!(MCP_INSTRUCTIONS.matches(MANAGED_LIFETIME).count(), 1); + assert_eq!(instructions(GatewayLifecycle::Managed), MCP_INSTRUCTIONS); + for (exit_intent, shutdown, says) in [ + (RunnerExitIntent::Detach, DETACH_SHUTDOWN, "it detaches"), + (RunnerExitIntent::Stop, STOP_SHUTDOWN, "it stops the Runner"), + ] { + let text = instructions(GatewayLifecycle::Independent { exit_intent }); + let lifetime = INDEPENDENT_LIFETIME.replace("{shutdown}", shutdown); + assert_eq!(text.replace(&lifetime, MANAGED_LIFETIME), MCP_INSTRUCTIONS); + for expected in [ + "owned by an independent Runner", + "Client or HTTP disconnect keeps the process running", + "RUNNER_DISCONNECTED", + "RECOVERY_PENDING", + "same process_id", + "signals that process only", + says, + ] { + assert!(text.contains(expected), "{expected}: {text}"); + } + assert!(!text.contains("subtree"), "{text}"); + assert!( + !text.contains("Restart does not recover process_id"), + "{text}" + ); + } + } +} diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index 80d6ee9..d2f783e 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, Context, Result}; -use codespace_runner::{allocate_private_runner_dir, runner_socket_path, ShellRelease, UdsRunner}; +use codespace_runner::{ + allocate_private_runner_dir, runner_socket_path, RunnerLifecycle, ShellRelease, UdsRunner, +}; use codespace_store::Store; use tokio::net::UnixStream; use tokio::process::Command; @@ -125,6 +127,69 @@ impl RuntimeProcess { .handshake() .await .map_err(|err| anyhow!("runner hello: {err}"))?; + // Its processes would outlive a connection that this gateway treats as theirs (P1R-C01). + if runner.hello().lifecycle == RunnerLifecycle::Independent { + runner.close().await; + return Err(anyhow!( + "the worker at {} is an independent Runner; connect to it with \ + --runner-lifecycle independent", + socket.display() + )); + } + Ok(runner) + } + + /// Connect to an independent Runner that an operator or a harness started (P1R-C01). Its + /// processes outlive this connection: a lost connection releases nothing, and every later + /// request is refused until the gateway is restarted. A worker that is not an independent + /// Runner is refused. `leaving` is set before the gateway itself ends the connection. + pub async fn connect_independent( + socket: &Path, + on_process_exit: ShellRelease, + leaving: Arc, + ) -> Result { + let stream = UnixStream::connect(socket) + .await + .with_context(|| format!("connect {}", socket.display()))?; + let runner = UdsRunner::from_stream_with_disconnect( + stream, + on_process_exit, + Arc::new(move || { + if !leaving.load(std::sync::atomic::Ordering::SeqCst) { + tracing::error!( + "lost the connection to the independent Runner: its processes keep \ + running, and this gateway refuses every request that needs the Runner \ + until it is restarted" + ); + } + }), + ); + runner + .handshake() + .await + .map_err(|err| anyhow!("runner hello: {err}"))?; + let hello = runner.hello(); + if hello.lifecycle != RunnerLifecycle::Independent { + runner.close().await; + return Err(anyhow!( + "--runner-lifecycle independent: the worker at {} is {}, not an independent Runner", + socket.display(), + hello.lifecycle.as_str() + )); + } + tracing::info!( + runner_pid = hello.identity.as_ref().map(|identity| identity.pid), + previous_controller = hello.previous_controller, + inherited_processes = hello.inherited_processes, + "connected to an independent Runner; it is not a supported mode yet" + ); + if hello.previous_controller { + tracing::warn!( + "an earlier gateway controlled this independent Runner: its processes can be \ + queried, read, written to, resized and terminated, but new executions, patches \ + and approval resumes are refused with RECOVERY_PENDING" + ); + } Ok(runner) } @@ -164,7 +229,7 @@ impl Drop for RuntimeProcess { /// A worker whose `Hello` did not state that it registers was built without DevGuard. #[cfg(feature = "devguard")] -pub(crate) fn require_registration(runner: &UdsRunner, bin: &Path) -> Result<()> { +pub fn require_registration(runner: &UdsRunner, bin: &Path) -> Result<()> { if runner.states_registration() { return Ok(()); } diff --git a/crates/server/tests/independent_runner.rs b/crates/server/tests/independent_runner.rs new file mode 100644 index 0000000..81d82c7 --- /dev/null +++ b/crates/server/tests/independent_runner.rs @@ -0,0 +1,446 @@ +//! The gateway with an independent Runner (P1R-C01), each started as an operator starts it: the +//! Runner outlives a gateway, the next gateway inherits its processes but starts nothing, and a +//! gateway that ends leaves or stops the Runner as its exit intent says. These tests need the +//! worker binary (`CODESPACE_RUNTIME_BIN`) and return early without it, as the gateway's other +//! worker tests do. + +use std::path::{Path, PathBuf}; +use std::process::{ExitStatus, Stdio}; +use std::time::Duration; + +use codespace_domain::{TOOL_EXEC_COMMAND, TOOL_PROCESS_STATUS, TOOL_READ_PROCESS}; +use rmcp::model::{CallToolRequestParams, CallToolResult}; +use rmcp::service::{RunningService, ServiceError}; +use rmcp::{object, RoleClient, ServiceExt}; +use tokio::process::{Child, Command}; +use tokio::time::{sleep, timeout, Instant}; + +/// The process timeout that every gateway here gives its executions. +const TIMEOUT: Duration = Duration::from_secs(5); + +fn runtime_bin() -> Option { + std::env::var_os("CODESPACE_RUNTIME_BIN").map(PathBuf::from) +} + +fn payload(result: &CallToolResult) -> serde_json::Value { + result.structured_content.clone().unwrap_or_else(|| { + serde_json::from_str(&result.content[0].as_text().unwrap().text).unwrap() + }) +} + +fn error_body(result: &Result) -> serde_json::Value { + let text = match result { + Ok(r) => r + .content + .iter() + .filter_map(|c| c.as_text().map(|t| t.text.clone())) + .collect::>() + .join(""), + Err(err) => err.to_string(), + }; + if let Ok(value) = serde_json::from_str::(&text) { + return value; + } + let start = text.find('{').unwrap_or_else(|| panic!("{text}")); + let end = text.rfind('}').unwrap_or_else(|| panic!("{text}")); + serde_json::from_str(&text[start..=end]).unwrap_or_else(|_| panic!("{text}")) +} + +/// A workspace, the gateways' operations database, and where the Runner's socket goes. +struct Setup { + root: tempfile::TempDir, + cfg: PathBuf, + ws: PathBuf, + socket: PathBuf, +} + +impl Setup { + fn new() -> Self { + let root = tempfile::tempdir().unwrap(); + let ws = root.path().join("ws"); + std::fs::create_dir(&ws).unwrap(); + let cfg = root.path().join("workspaces.json"); + std::fs::write( + &cfg, + serde_json::json!({ + "workspaces": { "demo": { "root": ws, "profile": "workspace-write" } } + }) + .to_string(), + ) + .unwrap(); + // The Runner creates its socket's directory itself. + let socket = root.path().join("runner").join("runner.sock"); + Self { + root, + cfg, + ws, + socket, + } + } + + /// An independent Runner, started as an operator starts it, once it listens. + async fn start_runner(&self, bin: &Path) -> Child { + let runner = Command::new(bin) + .arg(&self.socket) + .args(["--lifecycle", "independent"]) + .kill_on_drop(true) + .spawn() + .expect("spawn the Runner"); + // A connection that says nothing leaves the Runner as it was. + let deadline = Instant::now() + Duration::from_secs(15); + while std::os::unix::net::UnixStream::connect(&self.socket).is_err() { + assert!(Instant::now() < deadline, "the Runner does not listen"); + sleep(Duration::from_millis(20)).await; + } + runner + } + + /// The gateway binary with these settings alone: the environment of the test run may name a + /// worker for the gateway to start, which an independent Runner's gateway refuses. + fn gateway_command(&self, args: &[&str]) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_codespace-mcp")); + for variable in [ + "CODESPACE_RUNNER", + "CODESPACE_RUNNER_SOCKET", + "CODESPACE_RUNNER_DIR", + "CODESPACE_RUNTIME_BIN", + ] { + command.env_remove(variable); + } + command + .env("CODESPACE_CONFIG", &self.cfg) + .env( + "CODESPACE_OPERATIONS_DB", + self.root.path().join("ops.sqlite"), + ) + .env( + "CODESPACE_PROCESS_TIMEOUT_SECS", + TIMEOUT.as_secs().to_string(), + ) + .args(args) + .kill_on_drop(true); + command + } + + /// A gateway connected to the independent Runner, whose normal shutdown asks `exit_intent` + /// of it. + async fn gateway(&self, exit_intent: &str) -> Gateway { + let socket = self.socket.display().to_string(); + let mut process = self + .gateway_command(&[ + "--runner", + "uds", + "--runner-socket", + &socket, + "--runner-lifecycle", + "independent", + "--runner-exit-intent", + exit_intent, + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn the gateway"); + let stdout = process.stdout.take().unwrap(); + let stdin = process.stdin.take().unwrap(); + let client = ().serve((stdout, stdin)).await.expect("initialize"); + Gateway { client, process } + } + + /// The error of a gateway that does not start with `args`. + async fn refused(&self, args: &[&str]) -> String { + let output = timeout( + Duration::from_secs(30), + self.gateway_command(args).stdin(Stdio::null()).output(), + ) + .await + .expect("the gateway ends") + .expect("run the gateway"); + assert!(!output.status.success(), "{args:?}: {output:?}"); + String::from_utf8_lossy(&output.stderr).into_owned() + } +} + +struct Gateway { + client: RunningService, + process: Child, +} + +impl Gateway { + async fn exec(&self, command: &[&str]) -> Result { + self.client + .call_tool( + CallToolRequestParams::new(TOOL_EXEC_COMMAND).with_arguments(object!({ + "workspace_id": "demo", + "command": command + })), + ) + .await + } + + /// Start `command` and return its process id. + async fn started(&self, command: &[&str]) -> String { + let started = self.exec(command).await.expect("exec"); + let body = payload(&started); + assert_eq!(body["dispatch_status"], "confirmed", "{body}"); + body["process_id"].as_str().unwrap().to_string() + } + + async fn status(&self, process_id: &str) -> serde_json::Value { + let status = self + .client + .call_tool( + CallToolRequestParams::new(TOOL_PROCESS_STATUS) + .with_arguments(object!({ "process_id": process_id })), + ) + .await + .expect("process_status"); + payload(&status) + } + + /// The process's output once it contains `expected`. + async fn output(&self, process_id: &str, expected: &str) -> String { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let read = self + .client + .call_tool( + CallToolRequestParams::new(TOOL_READ_PROCESS) + .with_arguments(object!({ "process_id": process_id, "cursor": 0 })), + ) + .await + .expect("read_process"); + let chunk = payload(&read)["chunk"].as_str().unwrap_or("").to_string(); + if chunk.contains(expected) { + return chunk; + } + assert!(Instant::now() < deadline, "missing {expected:?}: {chunk:?}"); + sleep(Duration::from_millis(40)).await; + } + } + + /// End the gateway's stdin, as a client that ends does, and wait for it to exit. + async fn shut_down(self) -> ExitStatus { + let Self { + client, + mut process, + } = self; + client.cancel().await.expect("close the gateway's stdin"); + exited(&mut process).await + } + + /// Send the gateway `signal` and wait for it to exit. + async fn signal(self, signal: &str) -> ExitStatus { + let Self { + client, + mut process, + } = self; + let pid = process.id().unwrap().to_string(); + let sent = std::process::Command::new("kill") + .args([signal, &pid]) + .status() + .unwrap(); + assert!(sent.success()); + let status = exited(&mut process).await; + drop(client); + status + } + + /// SIGKILL: the gateway says nothing to its Runner. + async fn kill(self) { + let Self { + client, + mut process, + } = self; + process.kill().await.expect("SIGKILL the gateway"); + drop(client); + } +} + +async fn exited(process: &mut Child) -> ExitStatus { + timeout(Duration::from_secs(15), process.wait()) + .await + .expect("the process exits") + .expect("wait for the process") +} + +/// Whether the count in `beat` still goes up. +async fn still_beating(beat: &Path) -> bool { + let read = || std::fs::read_to_string(beat).unwrap_or_default(); + let deadline = Instant::now() + Duration::from_secs(10); + while read().trim().is_empty() && Instant::now() < deadline { + sleep(Duration::from_millis(20)).await; + } + let before = read(); + sleep(Duration::from_millis(600)).await; + before != read() +} + +/// A gateway killed while its process runs leaves the process to the Runner: the next gateway +/// reads it and sees it end at its original timeout, not at one restarted when that gateway +/// connected, but starts nothing itself, and its normal shutdown with the intent `stop` then +/// stops the Runner (P1R-C01). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_independent_runner_keeps_a_killed_gateways_process_to_its_original_timeout() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let first = setup.gateway("detach").await; + let started = Instant::now(); + let process_id = first + .started(&["/bin/sh", "-c", "echo started; exec sleep 60"]) + .await; + first.output(&process_id, "started").await; + first.kill().await; + // Long enough that a timeout restarted by the next gateway would come visibly later. + sleep(Duration::from_secs(3)).await; + let reconnected = Instant::now(); + let next = setup.gateway("stop").await; + assert_eq!(next.status(&process_id).await["state"], "running"); + next.output(&process_id, "started").await; + let refused = next.exec(&["/bin/echo", "new"]).await; + assert_eq!(error_body(&refused)["code"], "RECOVERY_PENDING"); + let status = loop { + let status = next.status(&process_id).await; + if status["state"] == "exited" { + break status; + } + assert!( + reconnected.elapsed() < TIMEOUT, + "the process outlived its timeout: {status}" + ); + sleep(Duration::from_millis(50)).await; + }; + assert_eq!(status["termination"], "timeout", "{status}"); + assert!(started.elapsed() >= TIMEOUT); + let ended = next.shut_down().await; + assert!(ended.success(), "{ended:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); + assert!(!setup.socket.exists()); +} + +/// A gateway's normal shutdown, the end of stdin or SIGTERM, leaves or stops its independent +/// Runner as the gateway's exit intent says, and SIGUSR1, which ends the gateway for a restart, +/// always leaves it (P1R-C01). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_gateway_leaves_or_stops_its_independent_runner_as_its_exit_intent_says() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let mut runner = setup.start_runner(&bin).await; + let beat = setup.ws.join("beat"); + let first = setup.gateway("detach").await; + first + .started(&[ + "/bin/sh", + "-c", + "i=0; while :; do i=$((i+1)); echo $i > beat; sleep 0.1; done", + ]) + .await; + assert!(still_beating(&beat).await); + let ended = first.shut_down().await; + assert!(ended.success(), "{ended:?}"); + assert!(still_beating(&beat).await); + assert!(runner.try_wait().unwrap().is_none()); + + let restarted = setup.gateway("stop").await; + let ended = restarted.signal("-USR1").await; + assert!(ended.success(), "{ended:?}"); + assert!(still_beating(&beat).await); + assert!(runner.try_wait().unwrap().is_none()); + + let last = setup.gateway("stop").await; + let ended = last.signal("-TERM").await; + assert!(ended.success(), "{ended:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); + assert!(!still_beating(&beat).await); + assert!(!setup.socket.exists()); +} + +/// A gateway does not start with lifecycle settings that do not fit together, nor on a Runner of +/// the other lifecycle, and a gateway refused by an independent Runner leaves it as it was: the +/// next gateway starts work normally (P1R-C01). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_gateway_refuses_independent_settings_or_a_runner_that_do_not_fit() { + let Some(bin) = runtime_bin() else { + return; + }; + let setup = Setup::new(); + let socket = setup.socket.display().to_string(); + let uds = ["--runner", "uds", "--runner-socket", &socket]; + for (args, problem) in [ + ( + [&uds[..], &["--runner-lifecycle", "independent"]].concat(), + "needs --runner-exit-intent", + ), + ( + [&uds[..], &["--runner-exit-intent", "stop"]].concat(), + "only for --runner-lifecycle independent", + ), + ( + vec![ + "--runner-lifecycle", + "independent", + "--runner-exit-intent", + "stop", + ], + "needs --runner uds", + ), + ] { + let refused = setup.refused(&args).await; + assert!(refused.contains(problem), "{args:?}: {refused}"); + } + + let mut runner = setup.start_runner(&bin).await; + let refused = setup.refused(&uds).await; + assert!( + refused.contains("is an independent Runner"), + "a managed gateway: {refused}" + ); + let next = setup.gateway("stop").await; + let process_id = next.started(&["/bin/echo", "after"]).await; + next.output(&process_id, "after").await; + let ended = next.shut_down().await; + assert!(ended.success(), "{ended:?}"); + let status = exited(&mut runner).await; + assert!(status.success(), "{status:?}"); + + // A worker that belongs to its gateway is not an independent Runner. + let managed_socket = setup.root.path().join("managed.sock"); + let mut managed = Command::new(&bin) + .arg(&managed_socket) + .kill_on_drop(true) + .spawn() + .expect("spawn a managed worker"); + let deadline = Instant::now() + Duration::from_secs(15); + while !managed_socket.exists() { + assert!( + Instant::now() < deadline, + "the managed worker does not listen" + ); + sleep(Duration::from_millis(20)).await; + } + let managed_socket = managed_socket.display().to_string(); + let refused = setup + .refused(&[ + "--runner", + "uds", + "--runner-socket", + &managed_socket, + "--runner-lifecycle", + "independent", + "--runner-exit-intent", + "detach", + ]) + .await; + assert!( + refused.contains("not an independent Runner"), + "an independent gateway: {refused}" + ); + exited(&mut managed).await; +} diff --git a/docs/agent-integration.md b/docs/agent-integration.md index de9bcde..ca8ca41 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -150,7 +150,7 @@ Interactive input and cancellation use the same handle: A live command occupies the workspace. Wait for it to end or terminate it before applying a patch or starting another command. Reads and searches remain available. Long-lived development servers therefore require a workflow that stops them before edits. -`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. +`workspace_info.execution.process.capabilities.lifetime` advertises the owner: the runner instance, not the MCP session. Streamable HTTP or MCP client disconnect keeps the process running; reconnect with the saved `process_id`. Losing the UDS worker connection or shutting down the gateway (including stdio EOF) terminates the owned subtree and drops handles. Restart does not restore `process_id`. With an [independent Runner](runner-isolation.md#independent-runner), which is not a supported mode yet, `runner_disconnect` is `keep_running`, and `gateway_shutdown` is `keep_running` when the gateway detaches and `terminate` when it stops the Runner; read `initialize.instructions` for what that gateway does. If the spawn response is lost before you receive `process_id`, do not search with `process_status` and do not start a duplicate command. ## Confirm a held mutation diff --git a/docs/architecture.md b/docs/architecture.md index 4df63a7..e905fbd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,7 +51,7 @@ If helper application fails, the Runner attempts per-file snapshot restoration. ## Process lifetime -MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. +MCP request completion does not end a managed process. Clients continue with its `process_id`. Server restart loses those handles. In UDS mode the gateway owns the worker: internal disconnect/shutdown ends the worker and its children, with no reconnect. An [independent Runner](runner-isolation.md#independent-runner), not a supported mode yet, outlives its gateways instead. See [runner isolation](runner-isolation.md) for the distinct transport and isolation boundaries. ## Planned execution coordination diff --git a/docs/codex-reuse.md b/docs/codex-reuse.md index 433eb79..63a1605 100644 --- a/docs/codex-reuse.md +++ b/docs/codex-reuse.md @@ -37,7 +37,7 @@ Agent → CodeSpace MCP/policy/store → Runner contract Core crates have no direct Codex dependencies. Adapter crates are separate Cargo workspaces to accommodate the pinned upstream workspace dependencies. The filesystem and PTY adapters are library dependencies of the Runner; patch and Linux sandbox operations use helper processes. An isolated Cargo workspace alone does not create a process or security boundary. -The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 7 and sandbox-helper protocol version 1 are separate contracts. +The Linux sandbox helper is binary-only. `codespace-linux-sandbox-protocol` contains its CodeSpace-owned handshake data and no Codex types. Worker UDS protocol version 8 and sandbox-helper protocol version 1 are separate contracts. diff --git a/docs/devguard-integration.md b/docs/devguard-integration.md index 79cabca..1f5d958 100644 --- a/docs/devguard-integration.md +++ b/docs/devguard-integration.md @@ -2,13 +2,13 @@ # DevGuard integration roadmap -> **Status: suspended as an implementation directive.** The CS-RG work order, the planned consumer boundary and the design revision 1 references on this page must not be implemented as described; current status and prerequisites are unaffected. This is pending the CS-RG integration-boundary revalidation, an owner-directed review of the CodeSpace integration plan; it is not a work unit. No replacement architecture has been approved; the owner decides after reviewing its results. This notice suspends directives only and relaxes no safety requirement. The text below is retained unchanged for historical traceability, except the current status, the [status-only connection](#devguard-status-connection) of CSRG-U1, which the owner approved on 2026-10-02 outside this suspension, the [execution-owner registration](#devguard-owner-registration) of CSRG-U2, which the owner directed on 2026-10-03, the [admission and pre-spawn preparation](#devguard-admission) of CSRG-U3, which the owner directed on 2026-10-07, the [managed launch](#devguard-launch) of CSRG-U4, which the owner directed on 2026-10-08, the [lifecycle after launch](#devguard-lifecycle) of CSRG-U5, which the owner directed on 2026-10-09, and the [qualification and adoption](#devguard-qualification) of CSRG-U6, which the owner directed on 2026-10-12. +> **Status: suspended as an implementation directive.** The CS-RG work order, the planned consumer boundary and the design revision 1 references on this page must not be implemented as described; current status and prerequisites are unaffected. This is pending the CS-RG integration-boundary revalidation, an owner-directed review of the CodeSpace integration plan; it is not a work unit. No replacement architecture has been approved; the owner decides after reviewing its results. This notice suspends directives only and relaxes no safety requirement. The text below is retained unchanged for historical traceability, except the current status, the [status-only connection](#devguard-status-connection) of CSRG-U1, which the owner approved on 2026-10-02 outside this suspension, the [execution-owner registration](#devguard-owner-registration) of CSRG-U2, which the owner directed on 2026-10-03, the [admission and pre-spawn preparation](#devguard-admission) of CSRG-U3, which the owner directed on 2026-10-07, the [managed launch](#devguard-launch) of CSRG-U4, which the owner directed on 2026-10-08, the [lifecycle after launch](#devguard-lifecycle) of CSRG-U5, which the owner directed on 2026-10-09, the [qualification and adoption](#devguard-qualification) of CSRG-U6, which the owner directed on 2026-10-12, and the [Gateway recovery scope](#resource-gateway-recovery) of P1-RECOVERY, which the owner directed on 2026-10-14. [English](devguard-integration.md) | [한국어](ko/devguard-integration.md) [DevGuard](https://github.com/novelKR/DevGuard) is an independent resource authority for development workloads, licensed under Apache-2.0 like CodeSpace. Its approved integration path adds a shared admission and accounting layer while CodeSpace keeps process ownership, PTY, input/output, permissions, approval holds and workspace coordination. -**Current status:** DevGuard has completed DG-1 in its independent repository ([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [milestone ledger](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). On top of the DG-0 contracts it provides native macOS host evidence, an authenticated daemon installed as the current user's LaunchAgent, the fenced launch helper and reconciliation, the `devguard` command-line owner with Cargo adapters, parent leases for candidate tests, and upgrade and repair. Its macOS SLO is qualified for release `0.1.0-6e7e065-94d85365` (a release ID, not a Git tag), built from the client pin's commit and measured on 2026-10-13 on the measured host under the policy that includes CodeSpace's `codespace` consumer ([DevGuard #34](https://github.com/novelKR/DevGuard/pull/34)); Linux enforcement is not qualified. CodeSpace consumption (CS-RG) is qualified on macOS for one exact combination. CSRG-U1 added an opt-in, status-only connection: a gateway built with the `devguard` feature and started with `--devguard status` reports DevGuard's status in `workspace_info`, through DevGuard's client pinned at [`6e7e065`](upstream-lock.md#devguard-client-pin). With CSRG-U2, the process that owns CodeSpace's executions registers itself with DevGuard (`--devguard register`), and a workspace's `resources` setting can require resource participation. With CSRG-U3, that owner takes a process slot and has DevGuard admit each such execution before anything is created. With CSRG-U4 and a configured launch helper (`--devguard-launch-helper`), it then launches the execution once through DevGuard's launch helper, which DevGuard authorizes after binding and policing its scope; without a helper it cancels the unstarted attempt and refuses the execution. Nothing runs without DevGuard. With CSRG-U5, each process has one reaper; DevGuard observes a governed execution's scope before its root is reaped, on a pipe through CodeSpace's own reaper and on a PTY through the pinned Codex PTY's waiter with the [recorded patch](upstream-lock.md#codex-downstream-patch); killing a governed process also terminates its scope; and an approval whose governed execution provably never started can be resumed again. CSRG-U6 qualified one exact combination under the [CS-RG qualification protocol](cs-rg-qualification.md), in the runner modes `in-process` and `uds` with `--runtime-bin`, each with pipe and PTY executions, as the [qualification report](cs-rg-qualification-report.md) records ([qualification and adoption](#devguard-qualification)); Linux `required` stays unsupported until DG-LINUX. The qualified release remains a candidate for execution, not a selected pin; the client pin is a source pin of the client crates. +**Current status:** DevGuard has completed DG-1 in its independent repository ([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [milestone ledger](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). On top of the DG-0 contracts it provides native macOS host evidence, an authenticated daemon installed as the current user's LaunchAgent, the fenced launch helper and reconciliation, the `devguard` command-line owner with Cargo adapters, parent leases for candidate tests, and upgrade and repair. Its macOS SLO is qualified for release `0.1.0-6e7e065-94d85365` (a release ID, not a Git tag), built from the client pin's commit and measured on 2026-10-13 on the measured host under the policy that includes CodeSpace's `codespace` consumer ([DevGuard #34](https://github.com/novelKR/DevGuard/pull/34)); Linux enforcement is not qualified. CodeSpace consumption (CS-RG) is qualified on macOS for one exact combination. CSRG-U1 added an opt-in, status-only connection: a gateway built with the `devguard` feature and started with `--devguard status` reports DevGuard's status in `workspace_info`, through DevGuard's client pinned at [`6e7e065`](upstream-lock.md#devguard-client-pin). With CSRG-U2, the process that owns CodeSpace's executions registers itself with DevGuard (`--devguard register`), and a workspace's `resources` setting can require resource participation. With CSRG-U3, that owner takes a process slot and has DevGuard admit each such execution before anything is created. With CSRG-U4 and a configured launch helper (`--devguard-launch-helper`), it then launches the execution once through DevGuard's launch helper, which DevGuard authorizes after binding and policing its scope; without a helper it cancels the unstarted attempt and refuses the execution. Nothing runs without DevGuard. With CSRG-U5, each process has one reaper; DevGuard observes a governed execution's scope before its root is reaped, on a pipe through CodeSpace's own reaper and on a PTY through the pinned Codex PTY's waiter with the [recorded patch](upstream-lock.md#codex-downstream-patch); killing a governed process also terminates its scope; and an approval whose governed execution provably never started can be resumed again. CSRG-U6 qualified one exact combination under the [CS-RG qualification protocol](cs-rg-qualification.md), in the runner modes `in-process` and `uds` with `--runtime-bin`, each with pipe and PTY executions, as the [qualification report](cs-rg-qualification-report.md) records ([qualification and adoption](#devguard-qualification)); Linux `required` stays unsupported until DG-LINUX. The qualified release remains a candidate for execution, not a selected pin; the client pin is a source pin of the client crates. P1-RECOVERY started on the owner's direction of 2026-10-14 and is in progress, not qualified ([Gateway recovery scope](#resource-gateway-recovery)). DevGuard [design revision 1](https://github.com/novelKR/DevGuard/blob/d4981b4c241cff42687f5c2c681b583c7847776e/docs/design-revision-1.md), merged in [DevGuard PR #8](https://github.com/novelKR/DevGuard/pull/8) as `d4981b4c241cff42687f5c2c681b583c7847776e`, revised the CS-RG execution layer and work order. It changes plans and design decisions that bind later implementation. It does not change current CodeSpace behavior or any implementation or qualification status. @@ -82,6 +82,7 @@ CSRG-U2 lets the process that owns CodeSpace's executions establish and prove a | `in-process` | the gateway, which runs the executions | read from the credential file for each session, as status reads it | | `uds` with `--runtime-bin` | the worker the gateway starts, which owns the execution handles | read by the gateway once and handed to that worker alone through one private descriptor | | `uds` with `--runner-socket` | nothing: `unsupported_mode` | none: the gateway did not start that worker and cannot hand it the secret | +| `uds` with `--runner-socket` and `--runner-lifecycle independent` | the [independent Runner](runner-isolation.md#independent-runner), which owns the execution handles and outlives each gateway ([P1-RECOVERY](#resource-gateway-recovery)) | read by the Runner itself from the file given with its own `--devguard-credential-file`; the gateway hands it nothing | DevGuard takes the registered process from the session's peer, so a process can only register itself, and the gateway never registers on its worker's behalf. The owner also checks that the identity DevGuard registered is its own process and stays the same across sessions. @@ -105,7 +106,7 @@ When the owner cannot be asked (`owner_unreachable`, `unsupported_mode`), the au "resources": {"participation": "required"}}}} ``` -`off`, the default, and a missing setting leave execution as before. With `required` and `--devguard register`, every new execution goes to the owner's [preparation](#devguard-admission) (CSRG-U3). Without the feature, without `--devguard register`, or with an owner that cannot prepare (a worker the gateway did not start), every new execution, from `exec_command` or a resumed approval, is refused with `RESOURCE_POLICY_UNSUPPORTED` before an approval, a lease or a process exists; the message says whether the execution owner is registered. `required` never falls back to running without the authority. An unknown `resources` setting fails the registry load. Reads, finds, patches and the control of running processes are unaffected. +`off`, the default, and a missing setting leave execution as before. With `required` and `--devguard register`, every new execution goes to the owner's [preparation](#devguard-admission) (CSRG-U3). Without the feature, without `--devguard register`, or with an owner that cannot prepare (a worker the gateway did not start, except an independent Runner that states that it registers and launches), every new execution, from `exec_command` or a resumed approval, is refused with `RESOURCE_POLICY_UNSUPPORTED` before an approval, a lease or a process exists; the message says whether the execution owner is registered. `required` never falls back to running without the authority. An unknown `resources` setting fails the registry load. Reads, finds, patches and the control of running processes are unaffected. **Process control.** `process_status`, `read_process`, `write_stdin`, `process_resize`, `terminate_process` and timeouts never ask DevGuard. In UDS mode the worker answers registration requests beside the others, so a registration session never delays them. @@ -263,7 +264,7 @@ CSRG-U6 qualified CS-RG under the [CS-RG qualification protocol](cs-rg-qualifica **Adoption.** The claim holds for the qualified combination only: a different release, policy, host, toolchain or code tree is a new candidate that needs its own runs. To use it: 1. Run the qualified DevGuard release as the current user's service, with a consumer for CodeSpace provisioned as a control service: role `control_service`, a generation, a private credential file, one instance and a static control reservation, qualified at 150 mCPU, 512 MiB and 40 tasks. DevGuard reads its policy only when it starts, and a policy change needs DG-1's measurement again. -2. Build `codespace-mcp` with `-p codespace-server --features devguard`, and for `uds` the worker with `--features devguard`, from a commit with the qualified code tree. +2. Build `codespace-mcp` with `-p codespace-server --features devguard`, and for `uds` the worker with `--features devguard`, from a commit with the qualified code tree, such as `ca306ee` itself: P1-RECOVERY's changes give later commits of main another code tree. 3. Start the gateway with `--devguard register`, the consumer settings of the [status-only connection](#devguard-status-connection) and `--devguard-launch-helper` set to the release's `devguard-launch`, in `in-process` mode or in `uds` mode with `--runtime-bin`. 4. Give each workspace whose executions must be governed `"resources": {"participation": "required"}` and a `request` that fits the work capacity. `workspace_info.resource_authority` reports the owner `registered` and `governs_execution: true` once the owner is registered and has a launch helper. 5. Start or restart DevGuard's service while memory pressure is normal: a service that starts under memory warning admits nothing until memory has been normal for 30 seconds ([DevGuard operations](https://github.com/novelKR/DevGuard/blob/c077c747972cebca828a903968868f5713ce5664/docs/operations.md#available-commands)). @@ -337,6 +338,8 @@ Default participation remains `off`; `required` is an explicit operator choice i ## Initial Gateway recovery scope +**Status, P1-RECOVERY (owner directive 2026-10-14):** P1-RECOVERY is implemented one pull request per unit, as re-baselined in DevGuard's [P1-RECOVERY plan](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/milestones/P1-RECOVERY.md#re-baseline-2026-10-14). Its first unit, P1R-C01, adds the [independent Runner](runner-isolation.md#independent-runner), an opt-in mode that is not supported yet: an operator or a harness starts the Runner, and its processes, output and timeouts outlive each gateway, which detaches from it or stops it as its exit intent says. A gateway does not reconnect yet, and a gateway that connects after another one started work refuses new executions, patches and approval resumes with `RECOVERY_PENDING` until the Runner is stopped and a new one is started. Reconnection, process records, reconciliation and Runner loss come in the later units, and the mode is supported only once the last unit qualifies it. In this mode the Runner is DevGuard's execution owner and reads the consumer secret from its own credential file, as DevGuard's [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/planning/decisions.md#adr-001--one-registration-by-the-execution-owner) now provides; its gateway admits `required` workspaces only when the Runner states that it registers and launches through a launch helper. The rest of this section is the scope as planned. + P1-RECOVERY introduces an operator-selected independent Runner mode and explicit capability. Existing modes keep their current shutdown/disconnect contracts. InProcess shares the Gateway PID and is excluded from live Gateway restart recovery. The new mode distinguishes normal shutdown, explicit service stop, restart detach and unexpected connection loss. A surviving Runner retains handles, PTY/pipe ownership, bounded output and the original timeout during detach or Gateway loss. Reconnection authenticates the new Gateway, fences stale controller epochs and reconciles workspace occupancy, approvals and resource leases before mutations resume. Reconnecting to an existing execution does not require a new workload budget. diff --git a/docs/error-codes.md b/docs/error-codes.md index 95e0d36..305a888 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -54,6 +54,8 @@ Errors use uppercase identifiers and a message. An `operation_id` may be present | `APPROVAL_NOT_FOUND` | Unknown confirmation-hold id | | `APPROVAL_CONFLICT` | Hold is still pending, already decided, or a resume is already in progress | | `APPROVAL_AMBIGUOUS` | Resume was interrupted and the terminal result is not on disk. Includes `approval_id`; patch cases may also include `operation_id` | +| `RECOVERY_PENDING` | An earlier gateway started work on this gateway's [independent Runner](runner-isolation.md#independent-runner). The processes it left can be queried, read, written to, resized and terminated, but no execution, patch or approval resume starts until the Runner is stopped and a new one is started. The approval stays as it was. Not occupancy; retrying does not help | +| `RUNNER_DISCONNECTED` | The gateway lost its connection to its [independent Runner](runner-isolation.md#independent-runner) and does not reconnect. The request was not sent, and the Runner's processes keep running; restart the gateway to reach them | | `INTERNAL` | Scheduler or store invariant failed. Not occupancy; do not retry as `WORKSPACE_BUSY` | ## Dispatch and completion diff --git a/docs/ko/agent-integration.md b/docs/ko/agent-integration.md index f91cadc..4aac13e 100644 --- a/docs/ko/agent-integration.md +++ b/docs/ko/agent-integration.md @@ -150,7 +150,7 @@ MCP 클라이언트 SDK로 stdio 또는 Streamable HTTP를 초기화하고, 초 실행 중인 명령은 작업 공간을 점유합니다. 패치를 적용하거나 다른 명령을 시작하려면 기존 명령이 끝날 때까지 기다리거나 종료하세요. 읽기와 검색은 계속 가능합니다. 개발 서버를 오래 실행하는 작업 흐름이라면 수정 전에 서버를 멈추는 절차가 필요합니다. -`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. +`workspace_info.execution.process.capabilities.lifetime`은 소유자가 MCP 세션이 아니라 러너 인스턴스임을 알립니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지하므로, 저장한 `process_id`로 다시 연결하면 됩니다. UDS worker 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 소유 서브트리가 종료되고 핸들이 사라집니다. 재시작은 `process_id`를 복구하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)에서는 `runner_disconnect`가 `keep_running`이고, `gateway_shutdown`은 게이트웨이가 detach하면 `keep_running`, Runner를 중지하면 `terminate`입니다. 그 게이트웨이가 하는 일은 `initialize.instructions`에서 확인하세요. spawn 응답을 받기 전에 `process_id`를 잃으면 `process_status`로 찾지 말고 명령을 중복 시작하지 마세요. ## 보류된 변경 확인 diff --git a/docs/ko/architecture.md b/docs/ko/architecture.md index 8b8096d..ef8a41a 100644 --- a/docs/ko/architecture.md +++ b/docs/ko/architecture.md @@ -58,7 +58,7 @@ UDS worker와 기본 러너는 모두 같은 호스트에서 실행합니다. ## 프로세스 수명 -MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. +MCP 요청이 끝나도 관리 중인 프로세스는 유지됩니다. 클라이언트는 `process_id`로 후속 호출을 수행합니다. 서버 재시작 후에는 핸들이 사라집니다. UDS 모드에서는 게이트웨이가 worker를 관리하므로 내부 연결 종료나 서버 종료 시 worker와 자식 프로세스가 종료되며 재접속은 지원하지 않습니다. 아직 지원하는 모드가 아닌 [독립 Runner](runner-isolation.md#independent-runner)는 이와 달리 게이트웨이보다 오래 삽니다. [러너 격리](runner-isolation.md)에서 전송 경계와 격리 경계를 구분해 설명합니다. ## 계획된 실행 조정 구조 diff --git a/docs/ko/codex-reuse.md b/docs/ko/codex-reuse.md index 6617ad0..89c4480 100644 --- a/docs/ko/codex-reuse.md +++ b/docs/ko/codex-reuse.md @@ -42,7 +42,7 @@ CodeSpace는 특정 버전에 고정한 Codex 소스의 실행 라이브러리 핵심 crate에는 직접적인 Codex 의존성이 없습니다. 어댑터는 고정된 업스트림의 workspace 의존성을 수용하기 위해 별도의 Cargo workspace로 구성합니다. 파일 시스템·PTY 어댑터는 Runner의 라이브러리 의존성이며, 패치와 Linux 샌드박스는 도우미 프로세스를 사용합니다. Cargo workspace를 분리하는 것만으로 프로세스나 보안 경계가 생기지는 않습니다. -Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 7과 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. +Linux 샌드박스 도우미는 실행 파일만 제공합니다. `codespace-linux-sandbox-protocol`에는 CodeSpace가 정의한 핸드셰이크 데이터만 있고 Codex 타입은 없습니다. worker의 UDS 프로토콜 버전 8과 샌드박스 도우미 프로토콜 버전 1은 별개의 계약입니다. diff --git a/docs/ko/devguard-integration.md b/docs/ko/devguard-integration.md index 24041c1..363daea 100644 --- a/docs/ko/devguard-integration.md +++ b/docs/ko/devguard-integration.md @@ -2,13 +2,13 @@ # DevGuard 결합 로드맵 -> **상태: 구현 지시로서 효력 중지.** 이 페이지의 ‘CS-RG 작업 순서’, ‘소비 경계’, ‘설계 개정 1 참조’ 절은 적힌 대로 구현하지 않습니다. 현재 상태와 선행 조건은 영향이 없습니다. CodeSpace 통합 계획에 대해 소유자가 지시한 검토인 CS-RG 통합 경계 재검증(작업 단위가 아닙니다)이 끝날 때까지 적용됩니다. 대체 구조는 승인되지 않았으며, 소유자가 재검증 결과를 검토한 뒤 결정합니다. 이 표기는 지시의 효력만 중지하며 어떤 안전 요구도 완화하지 않습니다. 아래 내용은 이력 추적을 위해 바꾸지 않고 남깁니다. 다만 현재 상태, 소유자가 2026-10-02에 이 효력 중지와 별도로 승인한 CSRG-U1의 [상태 전용 연결](#devguard-status-connection), 소유자가 2026-10-03에 지시한 CSRG-U2의 [실행 소유자 등록](#devguard-owner-registration), 소유자가 2026-10-07에 지시한 CSRG-U3의 [실행 허용과 spawn 전 준비](#devguard-admission), 소유자가 2026-10-08에 지시한 CSRG-U4의 [관리형 실행](#devguard-launch), 소유자가 2026-10-09에 지시한 CSRG-U5의 [실행 뒤 생명주기](#devguard-lifecycle), 소유자가 2026-10-12에 지시한 CSRG-U6의 [qualification과 도입](#devguard-qualification)은 예외입니다. +> **상태: 구현 지시로서 효력 중지.** 이 페이지의 ‘CS-RG 작업 순서’, ‘소비 경계’, ‘설계 개정 1 참조’ 절은 적힌 대로 구현하지 않습니다. 현재 상태와 선행 조건은 영향이 없습니다. CodeSpace 통합 계획에 대해 소유자가 지시한 검토인 CS-RG 통합 경계 재검증(작업 단위가 아닙니다)이 끝날 때까지 적용됩니다. 대체 구조는 승인되지 않았으며, 소유자가 재검증 결과를 검토한 뒤 결정합니다. 이 표기는 지시의 효력만 중지하며 어떤 안전 요구도 완화하지 않습니다. 아래 내용은 이력 추적을 위해 바꾸지 않고 남깁니다. 다만 현재 상태, 소유자가 2026-10-02에 이 효력 중지와 별도로 승인한 CSRG-U1의 [상태 전용 연결](#devguard-status-connection), 소유자가 2026-10-03에 지시한 CSRG-U2의 [실행 소유자 등록](#devguard-owner-registration), 소유자가 2026-10-07에 지시한 CSRG-U3의 [실행 허용과 spawn 전 준비](#devguard-admission), 소유자가 2026-10-08에 지시한 CSRG-U4의 [관리형 실행](#devguard-launch), 소유자가 2026-10-09에 지시한 CSRG-U5의 [실행 뒤 생명주기](#devguard-lifecycle), 소유자가 2026-10-12에 지시한 CSRG-U6의 [qualification과 도입](#devguard-qualification), 소유자가 2026-10-14에 지시한 P1-RECOVERY의 [Gateway 복구 범위](#resource-gateway-recovery)는 예외입니다. [English](../devguard-integration.md) | [한국어](devguard-integration.md) [DevGuard](https://github.com/novelKR/DevGuard)는 개발 작업의 자원을 중앙에서 관리하는 독립 시스템이며 CodeSpace와 동일한 Apache-2.0 라이선스를 적용한다. 승인된 결합 경로에 따라 실행 허용과 자원 회계를 공통 계층에 맡기고, CodeSpace는 프로세스 소유권, PTY, 입출력, 권한, 승인 hold와 workspace 조정을 계속 담당한다. -**현재 상태:** DevGuard는 독립 저장소에서 DG-1을 완료했다([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [마일스톤 원장](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). DG-0 계약 위에 macOS 실제 호스트 증거, 현재 사용자의 LaunchAgent로 설치하는 인증 daemon, fenced launch helper와 대조, Cargo adapter를 갖춘 `devguard` 명령행 owner, 후보 시험용 parent lease, upgrade와 repair를 제공한다. macOS SLO qualification은 release `0.1.0-6e7e065-94d85365`(Git tag가 아닌 release ID)에 대해 마쳤다. 이 release는 client pin의 commit에서 빌드했고, 2026-10-13에 측정한 호스트에서 CodeSpace의 `codespace` 소비자를 포함한 정책으로 측정했다([DevGuard #34](https://github.com/novelKR/DevGuard/pull/34)). Linux 강제 보호는 qualification하지 않았다. CodeSpace 소비(CS-RG)는 정확히 한 조합에 대해 macOS에서 qualification을 마쳤다. CSRG-U1은 opt-in 상태 전용 연결을 추가했다. `devguard` feature로 빌드하고 `--devguard status`로 시작한 gateway는 [`6e7e065`](upstream-lock.md#devguard-client-pin)에 고정한 DevGuard client로 DevGuard의 상태를 `workspace_info`에 보고한다. CSRG-U2부터는 CodeSpace의 실행을 소유한 프로세스가 스스로를 DevGuard에 등록하며(`--devguard register`), workspace의 `resources` 설정으로 자원 참여를 필수로 지정할 수 있다. CSRG-U3부터 그 소유자는 이런 실행마다 무엇을 만들기 전에 프로세스 slot을 잡고 DevGuard의 실행 허용을 받는다. CSRG-U4부터는 launch helper를 설정했으면(`--devguard-launch-helper`) 그 실행을 DevGuard의 launch helper로 한 번 실행한다. DevGuard는 helper의 scope를 묶고 정책을 적용한 뒤에 실행을 승인한다. helper가 없으면 시작하지 않은 attempt를 취소하고 실행을 거부한다. DevGuard 없이 실행하는 것은 없다. CSRG-U5부터 프로세스마다 reaper는 하나다. DevGuard는 관리 대상 실행의 scope를 root가 reap되기 전에 관찰한다. pipe에서는 CodeSpace 자신의 reaper가, PTY에서는 [기록된 패치](upstream-lock.md#codex-downstream-patch)를 적용한 고정 Codex PTY의 waiter가 그 기회를 준다. 관리 대상 프로세스를 죽이면 그 scope도 종료하며, 시작하지 않았음이 증명된 관리 대상 실행의 승인은 다시 재개할 수 있다. CSRG-U6은 [CS-RG qualification 절차](cs-rg-qualification.md)에 따라 정확히 한 조합을 qualification했다. runner 모드는 `in-process`와 `--runtime-bin`을 쓴 `uds`이고, 각각 pipe와 PTY 실행을 포함한다. 그 결과는 [qualification 보고서](cs-rg-qualification-report.md)에 있다([qualification과 도입](#devguard-qualification)). Linux `required`는 DG-LINUX 전까지 지원하지 않는다. qualification을 마친 release는 실행에 쓸 후보로 남으며 선택된 pin이 아니다. client pin은 client crate의 소스 pin이다. +**현재 상태:** DevGuard는 독립 저장소에서 DG-1을 완료했다([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [마일스톤 원장](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). DG-0 계약 위에 macOS 실제 호스트 증거, 현재 사용자의 LaunchAgent로 설치하는 인증 daemon, fenced launch helper와 대조, Cargo adapter를 갖춘 `devguard` 명령행 owner, 후보 시험용 parent lease, upgrade와 repair를 제공한다. macOS SLO qualification은 release `0.1.0-6e7e065-94d85365`(Git tag가 아닌 release ID)에 대해 마쳤다. 이 release는 client pin의 commit에서 빌드했고, 2026-10-13에 측정한 호스트에서 CodeSpace의 `codespace` 소비자를 포함한 정책으로 측정했다([DevGuard #34](https://github.com/novelKR/DevGuard/pull/34)). Linux 강제 보호는 qualification하지 않았다. CodeSpace 소비(CS-RG)는 정확히 한 조합에 대해 macOS에서 qualification을 마쳤다. CSRG-U1은 opt-in 상태 전용 연결을 추가했다. `devguard` feature로 빌드하고 `--devguard status`로 시작한 gateway는 [`6e7e065`](upstream-lock.md#devguard-client-pin)에 고정한 DevGuard client로 DevGuard의 상태를 `workspace_info`에 보고한다. CSRG-U2부터는 CodeSpace의 실행을 소유한 프로세스가 스스로를 DevGuard에 등록하며(`--devguard register`), workspace의 `resources` 설정으로 자원 참여를 필수로 지정할 수 있다. CSRG-U3부터 그 소유자는 이런 실행마다 무엇을 만들기 전에 프로세스 slot을 잡고 DevGuard의 실행 허용을 받는다. CSRG-U4부터는 launch helper를 설정했으면(`--devguard-launch-helper`) 그 실행을 DevGuard의 launch helper로 한 번 실행한다. DevGuard는 helper의 scope를 묶고 정책을 적용한 뒤에 실행을 승인한다. helper가 없으면 시작하지 않은 attempt를 취소하고 실행을 거부한다. DevGuard 없이 실행하는 것은 없다. CSRG-U5부터 프로세스마다 reaper는 하나다. DevGuard는 관리 대상 실행의 scope를 root가 reap되기 전에 관찰한다. pipe에서는 CodeSpace 자신의 reaper가, PTY에서는 [기록된 패치](upstream-lock.md#codex-downstream-patch)를 적용한 고정 Codex PTY의 waiter가 그 기회를 준다. 관리 대상 프로세스를 죽이면 그 scope도 종료하며, 시작하지 않았음이 증명된 관리 대상 실행의 승인은 다시 재개할 수 있다. CSRG-U6은 [CS-RG qualification 절차](cs-rg-qualification.md)에 따라 정확히 한 조합을 qualification했다. runner 모드는 `in-process`와 `--runtime-bin`을 쓴 `uds`이고, 각각 pipe와 PTY 실행을 포함한다. 그 결과는 [qualification 보고서](cs-rg-qualification-report.md)에 있다([qualification과 도입](#devguard-qualification)). Linux `required`는 DG-LINUX 전까지 지원하지 않는다. qualification을 마친 release는 실행에 쓸 후보로 남으며 선택된 pin이 아니다. client pin은 client crate의 소스 pin이다. P1-RECOVERY는 소유자 지시로 2026-10-14에 시작해 진행 중이며 qualification하지 않았다([Gateway 복구 범위](#resource-gateway-recovery)). DevGuard [설계 개정 1](https://github.com/novelKR/DevGuard/blob/d4981b4c241cff42687f5c2c681b583c7847776e/docs/ko/design-revision-1.md)은 [DevGuard PR #8](https://github.com/novelKR/DevGuard/pull/8)의 병합 `d4981b4c241cff42687f5c2c681b583c7847776e`로 반영되었으며 CS-RG의 실행 계층과 작업 순서를 개정했다. 이 개정은 계획과 이후 구현을 구속하는 설계 결정을 바꾼다. 현재 CodeSpace 동작이나 구현·qualification 상태는 바꾸지 않는다. @@ -82,6 +82,7 @@ CSRG-U2는 CodeSpace의 실행을 소유한 프로세스가 DevGuard에 등록 | `in-process` | 실행을 직접 수행하는 gateway | 상태 연결처럼 세션마다 credential 파일에서 읽음 | | `uds`와 `--runtime-bin` | gateway가 시작하고 실행 handle을 소유하는 worker | gateway가 한 번 읽어 비공개 descriptor 하나로 그 worker에게만 넘김 | | `uds`와 `--runner-socket` | 없음: `unsupported_mode` | 없음: gateway가 시작하지 않은 worker에는 secret을 넘길 수 없음 | +| `uds`, `--runner-socket`, `--runner-lifecycle independent` | 실행 handle을 소유하고 모든 gateway보다 오래 사는 [독립 Runner](runner-isolation.md#independent-runner)([P1-RECOVERY](#resource-gateway-recovery)) | Runner가 자기 `--devguard-credential-file`로 받은 파일에서 직접 읽음. gateway는 아무것도 넘기지 않음 | DevGuard는 등록할 프로세스를 세션의 peer에서 얻으므로 프로세스는 자신만 등록할 수 있고, gateway가 worker를 대신해 등록하는 일은 없다. 소유자는 DevGuard가 등록한 정체성이 자기 프로세스이며 세션마다 같은지도 확인한다. @@ -105,7 +106,7 @@ DevGuard는 등록할 프로세스를 세션의 peer에서 얻으므로 프로 "resources": {"participation": "required"}}}} ``` -기본값 `off`와 설정이 없는 경우는 실행을 이전과 같게 둔다. `required`이고 `--devguard register`로 시작했으면 새 실행은 모두 소유자의 [준비](#devguard-admission)로 간다(CSRG-U3). feature 없이 빌드했거나 `--devguard register` 없이 시작했거나 소유자가 준비할 수 없으면(gateway가 시작하지 않은 worker) `exec_command`나 재개한 승인으로 들어온 새 실행을 승인, lease, 프로세스가 생기기 전에 `RESOURCE_POLICY_UNSUPPORTED`로 모두 거부한다. 메시지는 실행 소유자가 등록되었는지 알려 준다. `required`가 자원 관리 없이 실행하는 쪽으로 물러서는 일은 없다. 알 수 없는 `resources` 설정은 registry 로드를 실패시킨다. 읽기, 찾기, 패치와 실행 중인 프로세스의 제어는 영향을 받지 않는다. +기본값 `off`와 설정이 없는 경우는 실행을 이전과 같게 둔다. `required`이고 `--devguard register`로 시작했으면 새 실행은 모두 소유자의 [준비](#devguard-admission)로 간다(CSRG-U3). feature 없이 빌드했거나 `--devguard register` 없이 시작했거나 소유자가 준비할 수 없으면(gateway가 시작하지 않은 worker. 단, 등록하고 launch한다고 알리는 독립 Runner는 제외) `exec_command`나 재개한 승인으로 들어온 새 실행을 승인, lease, 프로세스가 생기기 전에 `RESOURCE_POLICY_UNSUPPORTED`로 모두 거부한다. 메시지는 실행 소유자가 등록되었는지 알려 준다. `required`가 자원 관리 없이 실행하는 쪽으로 물러서는 일은 없다. 알 수 없는 `resources` 설정은 registry 로드를 실패시킨다. 읽기, 찾기, 패치와 실행 중인 프로세스의 제어는 영향을 받지 않는다. **프로세스 제어.** `process_status`, `read_process`, `write_stdin`, `process_resize`, `terminate_process`와 timeout은 DevGuard에 묻지 않는다. UDS 모드의 worker는 등록 요청을 다른 요청과 나란히 처리하므로 등록 세션이 이들을 지연시키지 않는다. @@ -263,7 +264,7 @@ CSRG-U6은 [CS-RG qualification 절차](cs-rg-qualification.md)에 따라 qualif **도입.** 이 주장은 qualification한 조합에만 해당한다. release, 정책, 호스트, 도구, code tree 가운데 하나라도 다르면 새 후보이며 따로 실행해야 한다. 쓰는 방법은 다음과 같다. 1. qualification한 DevGuard release를 현재 사용자의 서비스로 실행하고, CodeSpace용 소비자를 control service로 준비한다. role은 `control_service`이고 generation, 비공개 credential 파일, 인스턴스 하나, 정적 control 예약이 필요하다. 예약은 150 mCPU, 512 MiB, 40 tasks로 qualification했다. DevGuard는 시작할 때만 정책을 읽으며, 정책을 바꾸면 DG-1 측정을 다시 해야 한다. -2. `codespace-mcp`는 `-p codespace-server --features devguard`로, `uds`의 worker는 `--features devguard`로, qualification한 code tree를 가진 commit에서 빌드한다. +2. `codespace-mcp`는 `-p codespace-server --features devguard`로, `uds`의 worker는 `--features devguard`로, qualification한 code tree를 가진 commit(예: `ca306ee` 자체)에서 빌드한다. P1-RECOVERY의 변경 때문에 이후 main의 commit은 code tree가 다르다. 3. gateway를 `--devguard register`, [상태 전용 연결](#devguard-status-connection)의 소비자 설정, release의 `devguard-launch`를 가리키는 `--devguard-launch-helper`로 시작한다. 모드는 `in-process`이거나 `--runtime-bin`을 쓴 `uds`다. 4. 실행을 관리해야 하는 workspace마다 `"resources": {"participation": "required"}`와 work capacity에 들어가는 `request`를 준다. 소유자가 등록되고 launch helper가 있으면 `workspace_info.resource_authority`는 소유자를 `registered`로, `governs_execution`을 `true`로 보고한다. 5. DevGuard 서비스는 메모리 pressure가 정상일 때 시작하거나 재시작한다. 메모리 경고 중에 시작한 서비스는 메모리가 30초 동안 정상이 될 때까지 아무것도 받아들이지 않는다([DevGuard 운영 문서](https://github.com/novelKR/DevGuard/blob/c077c747972cebca828a903968868f5713ce5664/docs/operations.md#available-commands)). @@ -337,6 +338,8 @@ contract/client pin, daemon/helper artifact와 CodeSpace Runner wire는 각각 ## 최초 Gateway 복구 범위 +**상태, P1-RECOVERY(소유자 지시 2026-10-14):** P1-RECOVERY는 DevGuard의 [P1-RECOVERY 계획](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/milestones/P1-RECOVERY.md#재기준-2026-10-14)에서 다시 잡은 기준대로 단위마다 PR 하나로 구현한다. 첫 단위 P1R-C01은 아직 지원하지 않는 opt-in 모드인 [독립 Runner](runner-isolation.md#independent-runner)를 더한다. 운영자나 하네스가 Runner를 시작하고, Runner의 프로세스, 출력, timeout은 모든 gateway보다 오래 산다. gateway는 종료 의도대로 Runner에서 detach하거나 Runner를 중지한다. gateway는 아직 다시 연결하지 않으며, 다른 gateway가 작업을 시작한 뒤에 연결한 gateway는 Runner를 중지하고 새로 시작할 때까지 새 실행, 패치, 승인 재개를 `RECOVERY_PENDING`으로 거부한다. 재연결, process 기록, 대조, Runner 손실 처리는 이후 단위에서 더하며, 마지막 단위가 qualification한 뒤에야 이 모드를 지원한다. 이 모드에서는 Runner가 DevGuard의 실행 소유자이며, DevGuard [ADR-001](https://github.com/novelKR/DevGuard/blob/165cb84a991fbc941b822681b47ef7dd07363878/docs/ko/planning/decisions.md#adr-001--실행-소유자인-runner의-단일-등록)의 보완대로 소비자 secret을 자기 자격 파일에서 직접 읽는다. 그 gateway는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 `required` workspace를 받아들인다. 이 절의 나머지는 계획한 범위다. + P1-RECOVERY는 운영자가 선택하는 독립 Runner 모드와 명시적인 capability를 도입한다. 기존 모드의 종료·연결 소실 계약은 유지한다. InProcess는 Gateway와 PID를 공유하므로 Gateway 재시작 중 live 복구 대상에서 제외한다. 새 모드는 정상 종료, 명시적인 서비스 중지, 재시작용 detach, 예상치 못한 연결 소실을 구분한다. 살아 있는 Runner는 detach나 Gateway 소실 중에도 handle, PTY·pipe 소유권, 상한이 있는 출력과 원래 timeout을 유지한다. 새 Gateway는 인증하고 오래된 제어자 epoch의 mutation을 차단하며, workspace 점유·승인·자원 lease를 대조한 뒤 변경 작업을 재개한다. 기존 실행에 재연결할 때 새로운 작업 예산을 요구하지 않는다. diff --git a/docs/ko/error-codes.md b/docs/ko/error-codes.md index 35de1f4..0e45668 100644 --- a/docs/ko/error-codes.md +++ b/docs/ko/error-codes.md @@ -57,6 +57,8 @@ | `APPROVAL_NOT_FOUND` | 알 수 없는 확인 홀드 ID | | `APPROVAL_CONFLICT` | 홀드가 아직 대기 중이거나 이미 결정되었거나, 재개가 이미 진행 중임 | | `APPROVAL_AMBIGUOUS` | 재개가 중단되어 단말 결과가 디스크에 없음. `approval_id` 포함. 패치는 `operation_id`도 있을 수 있음 | +| `RECOVERY_PENDING` | 이 게이트웨이의 [독립 Runner](runner-isolation.md#independent-runner)에서 이전 게이트웨이가 작업을 시작함. 남은 프로세스는 조회, 출력 읽기, stdin 쓰기, 크기 변경, 종료가 가능하지만, Runner를 중지하고 새로 시작하기 전에는 실행, 패치, 승인 재개를 시작하지 않음. 승인은 그대로 남음. 점유가 아니므로 재시도해도 소용없음 | +| `RUNNER_DISCONNECTED` | 게이트웨이가 [독립 Runner](runner-isolation.md#independent-runner)와의 연결을 잃었고 다시 연결하지 않음. 요청은 보내지 않았고 Runner의 프로세스는 계속 실행됨. 그 프로세스에 닿으려면 게이트웨이를 재시작할 것 | | `INTERNAL` | 스케줄러·스토어 불변조건 실패. 점유가 아니므로 `WORKSPACE_BUSY`로 재시도하지 말 것 | ## 실행 요청과 완료의 구분 diff --git a/docs/ko/operations.md b/docs/ko/operations.md index 46de268..1df8474 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -115,7 +115,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. +worker는 같은 호스트에서 실행하는 별도 프로세스이며 컨테이너가 아닙니다. 게이트웨이가 전용 소켓 디렉터리를 만들고 자식 프로세스를 관리합니다. worker 연결이 끊기거나 게이트웨이가 종료되면 해당 worker의 프로세스도 종료됩니다. 재접속과 프로세스 복구는 지원하지 않습니다. MCP/HTTP 클라이언트 끊김은 그 프로세스를 죽이지 않습니다. 기본값은 `in-process`입니다. 프로세스가 게이트웨이보다 오래 사는 [독립 Runner](runner-isolation.md#independent-runner)는 아직 지원하는 모드가 아닙니다. diff --git a/docs/ko/runner-isolation.md b/docs/ko/runner-isolation.md index f76f334..c293036 100644 --- a/docs/ko/runner-isolation.md +++ b/docs/ko/runner-isolation.md @@ -13,12 +13,47 @@ `in-process`는 `codespace-mcp` 안에서 프로세스를 관리합니다. `uds`(Unix domain socket, Unix 도메인 소켓)는 같은 호스트의 `codespace-codex-runtime` 안에서 같은 관리 코드를 실행합니다. worker는 시작 시 Codex의 프로세스 보호 설정을 적용하고 전용 Unix 소켓을 엽니다. worker 자체를 보호하는 것과 실행할 명령에 샌드박스를 적용하는 것은 별개입니다. -게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 7 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. +게이트웨이는 임시 디렉터리 또는 `CODESPACE_RUNNER_DIR` 아래에 권한 0700의 고유 디렉터리를 만듭니다. 내부 통신은 u32 길이 접두부, 버전 8 핸드셰이크, 요청 ID, 프로세스 종료 이벤트를 사용하는 CodeSpace JSON입니다. Codex App Server RPC가 아닙니다. 같은 연결에서의 요청 재처리는 재접속 복구를 뜻하지 않습니다. -관리 프로세스의 수명은 MCP 연결이 아니라 러너 인스턴스가 소유합니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지합니다. 게이트웨이와 worker의 UDS 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 관리 중인 worker와 자식 프로세스가 종료되고 핸들이 사라집니다. 종료할 때 게이트웨이는 worker와의 연결을 닫으므로 worker는 종료하면서 자기 프로세스를 끝내고, 5초 뒤에도 실행 중인 worker는 강제로 종료됩니다. 재시작은 `process_id`를 복구하지 않습니다. +관리 프로세스의 수명은 MCP 연결이 아니라 러너 인스턴스가 소유합니다. Streamable HTTP나 MCP 클라이언트 끊김은 프로세스를 유지합니다. 게이트웨이와 worker의 UDS 연결이 끊기거나 게이트웨이가 종료되면(stdio EOF 포함) 관리 중인 worker와 자식 프로세스가 종료되고 핸들이 사라집니다. 종료할 때 게이트웨이는 worker와의 연결을 닫으므로 worker는 종료하면서 자기 프로세스를 끝내고, 5초 뒤에도 실행 중인 worker는 강제로 종료됩니다. 재시작은 `process_id`를 복구하지 않습니다. [독립 Runner](#independent-runner)는 예외로, 그 프로세스는 게이트웨이보다 오래 삽니다. **자식 프로세스를 시작하는 방식.** 파이프 명령, `tty: true` 명령, 패치 도우미, Linux 샌드박스 도우미, worker는 fork와 exec로 시작합니다. 모두 exec 전에 자식 프로세스 안에서 실행할 단계가 있기 때문입니다. 파이프, 패치 도우미, 샌드박스 도우미, worker 자식은 표준 디스크립터만 남기고([#79](https://github.com/novelKR/CodeSpace/issues/79)), PTY 자식은 새 세션의 리더가 됩니다. macOS에서는 같은 프로세스의 다른 스레드가 시스템 알림 라이브러리(libnotify)를 초기화하는 도중에 fork한 자식이 명령을 실행하기도 전에 `fork` 안에서 신호를 받아 종료됩니다. `process_status`는 이렇게 종료된 파이프 명령을 `signaled`로, `tty: true` 명령을 종료 코드 1로 끝난 것으로 보고합니다([Agent Loop 연동](agent-integration.md)). 그래서 `codespace-mcp`와 `codespace-codex-runtime`은 스레드를 시작하기 전에 이 초기화를 마치고, 위의 spawn은 모두 fork하기 전에 초기화를 마칩니다. 이 조치는 이 원인 하나를 없앨 뿐이며, 다중 스레드 프로세스에서 fork하는 것을 일반적으로 안전하게 만들지는 않습니다. + + +## 독립 Runner + +> **아직 지원하는 모드가 아닙니다.** 독립 Runner는 게이트웨이 복구의 첫 단계입니다. 게이트웨이는 독립 Runner에 다시 연결하지 않으며, 다른 게이트웨이 뒤에 연결한 게이트웨이는 새 작업을 시작할 수 없습니다. 재연결과 상태 대조는 다음 단계에서 더하며, 이 모드는 qualification을 마친 뒤에야 지원합니다. + +독립 Runner는 자신에게 연결하는 게이트웨이보다 오래 삽니다. 게이트웨이가 종료되거나 재시작되거나 강제로 종료되어도 프로세스와 그 출력, timeout은 계속됩니다. 운영자나 하네스가 독립 Runner를 시작하고, 게이트웨이는 연결만 합니다. + +```bash +codespace-codex-runtime "$RUNNER_DIR/runner.sock" --lifecycle independent +codespace-mcp --runner uds --runner-socket "$RUNNER_DIR/runner.sock" \ + --runner-lifecycle independent --runner-exit-intent detach +``` + +Runner는 소켓 디렉터리를 권한 0700으로 직접 만듭니다. 이미 있는 디렉터리는 같은 사용자의 디렉터리이고 권한이 0700이며 심볼릭 링크가 아닐 때만 받아들입니다. 이미 있는 디렉터리를 바꾸지 않으며 `/`, `/tmp`, `/var/tmp`, `$HOME`은 거부합니다. 게이트웨이 연결은 한 번에 하나씩 받고, 앞 연결의 요청이 모두 끝난 뒤에 다음 연결을 받습니다. 중지할 때는 소켓을 지우고, 디렉터리도 직접 만든 것이면 지웁니다. + +게이트웨이의 `--runner-lifecycle`이 Runner의 수명 모드와 다르거나, `--runner-lifecycle independent`에 `--runner uds`, `--runner-socket`, `--runner-exit-intent` 중 하나가 없거나, `--runtime-bin`이나 `--runner-dir`가 함께 있으면 게이트웨이는 시작하지 않습니다. `--runner-exit-intent`는 `--runner-lifecycle independent`와 함께일 때만 받습니다. + +| 사건 | 관리 worker(기본값) | 독립 Runner | +| --- | --- | --- | +| 게이트웨이 종료: stdin이 끝나거나 SIGTERM이나 SIGINT를 받음 | worker가 프로세스를 끝내고 종료합니다 | `--runner-exit-intent detach`면 게이트웨이가 detach하고 프로세스는 계속됩니다. `stop`이면 Runner가 프로세스를 끝내고 종료합니다 | +| 재시작하려고 게이트웨이에 SIGUSR1을 보냄 | 처리하지 않으므로 게이트웨이가 강제 종료된 경우와 같습니다 | 종료 의도와 관계없이 게이트웨이가 detach하고 프로세스는 계속됩니다 | +| 게이트웨이가 강제 종료되거나, detach나 stop 없이 연결이 끝남 | worker가 프로세스를 끝내고 종료합니다 | 중지가 아니라 연결 손실입니다. 프로세스는 계속되고 Runner는 다음 게이트웨이를 기다립니다 | +| Runner 호출이 120초 deadline을 넘김 | 게이트웨이가 연결을 닫고, worker는 프로세스를 끝내고 종료합니다 | 게이트웨이가 연결을 닫고, 프로세스는 계속됩니다 | +| 게이트웨이가 Runner에 중지를 요청하거나, Runner가 SIGTERM이나 SIGINT를 받음 | 관리 worker는 요청을 거부하며 이 신호를 처리하지 않습니다 | Runner가 프로세스를 끝내고 최대 5초 동안 기다린 뒤 종료합니다 | +| 프로세스가 timeout에 도달함 | 프로세스를 강제로 종료합니다 | 게이트웨이 연결 여부와 관계없이 원래 deadline에 프로세스를 강제로 종료합니다 | + +프로세스를 끝낼 때 tty 프로세스는 그 프로세스 그룹에 신호를 보내지만, 파이프 프로세스는 그 프로세스에만 보냅니다. 파이프 프로세스에는 자기 프로세스 그룹이 없으므로, 그 프로세스가 시작한 프로세스는 중지할 때를 포함해 더 오래 남을 수 있습니다. + +**연결을 잃었을 때.** 게이트웨이는 다시 연결하지 않습니다. Runner와의 연결이 끝나면 Runner가 필요한 요청을 모두 `RUNNER_DISCONNECTED`로 거부하며, Runner의 프로세스에 다시 닿으려면 게이트웨이를 재시작해야 합니다. 연결이 끝날 때 이미 보낸 요청은 지금처럼 불확실한 결과를 받습니다. 실행이라면 `dispatch_status: unknown`입니다. + +**이전 게이트웨이가 있었을 때.** 새로 시작한 Runner에 처음 연결한 게이트웨이는 관리 worker와 같이 동작합니다. 한 게이트웨이가 그 Runner에서 작업(실행, 패치, DevGuard를 쓰면 준비나 launch)을 시작한 뒤에는, 다음 게이트웨이들이 Runner의 프로세스를 넘겨받습니다. 그 게이트웨이는 `process_id`로 프로세스를 조회하고, 출력을 읽고, stdin에 쓰고, 크기를 바꾸고, 종료할 수 있지만, 새 실행, 패치, 승인 재개는 `RECOVERY_PENDING`으로 거부하고 승인은 그대로 둡니다. 넘겨받은 프로세스가 모두 끝난 뒤에도 마찬가지이므로, 작업을 다시 시작하려면 Runner를 중지하고 새로 시작합니다. 게이트웨이는 Runner의 식별 정보(프로세스 ID, 부팅과 시작 시각), 이전에 다른 게이트웨이가 제어했는지, 넘겨받은 프로세스 수를 로그에 남깁니다. + +**DevGuard.** `devguard` feature로 빌드한 독립 Runner는 worker의 DevGuard 설정으로 스스로 등록하며, 소비자 secret은 `--devguard-credential-file`로 지정한 비공개 파일에서 직접 읽습니다. `--devguard register`로 시작한 게이트웨이는 Runner에 secret을 넘기지 않습니다. 자원 참여를 요구하는 workspace는 Runner가 등록하고 launch helper로 실행한다고 알릴 때만 실행하며, 그렇지 않으면 `RESOURCE_POLICY_UNSUPPORTED`로 거부합니다. [DevGuard 결합](devguard-integration.md#resource-gateway-recovery)을 참고하세요. + diff --git a/docs/operations.md b/docs/operations.md index c5ac1f3..0eb8ec7 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -109,7 +109,7 @@ export CODESPACE_RUNNER=uds export CODESPACE_RUNTIME_BIN="$PWD/dist/codespace-codex-runtime" ``` -The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. +The worker runs on the same host and is not a container. The gateway creates a private socket directory and owns the child. Worker connection loss or gateway shutdown ends that worker's processes; reconnect and process recovery are not supported. MCP/HTTP client disconnect does not kill those processes. The default remains `in-process`. An [independent Runner](runner-isolation.md#independent-runner), whose processes outlive the gateway, is not a supported mode yet. diff --git a/docs/runner-isolation.md b/docs/runner-isolation.md index 1b80daf..0ba8417 100644 --- a/docs/runner-isolation.md +++ b/docs/runner-isolation.md @@ -10,12 +10,47 @@ There are three separate questions: which process runs the work, whether command `in-process` runs the supervisor inside `codespace-mcp`. `uds` (Unix domain socket) runs that supervisor inside `codespace-codex-runtime` on the same host. The worker starts with Codex process hardening and binds a private Unix socket. Hardening the worker does not sandbox its commands. -The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 7, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. +The gateway creates a unique 0700 directory beneath its temporary directory or `CODESPACE_RUNNER_DIR`. The internal protocol is CodeSpace JSON with a u32 length prefix, handshake version 8, request IDs, and process-exit events. It is not Codex App Server RPC. Same-connection replay is not reconnect recovery. -Managed process lifetime is owned by the runner instance, not the MCP connection. Streamable HTTP or MCP client disconnect keeps processes running. Gateway/worker UDS disconnect or gateway shutdown (including stdio EOF) ends the owned worker and its children; process handles are lost. At shutdown the gateway closes the worker's connection, so the worker ends its processes as it exits; a worker still running 5 seconds later is killed. Restart does not restore `process_id`. +Managed process lifetime is owned by the runner instance, not the MCP connection. Streamable HTTP or MCP client disconnect keeps processes running. Gateway/worker UDS disconnect or gateway shutdown (including stdio EOF) ends the owned worker and its children; process handles are lost. At shutdown the gateway closes the worker's connection, so the worker ends its processes as it exits; a worker still running 5 seconds later is killed. Restart does not restore `process_id`. An [independent Runner](#independent-runner) is the exception: its processes outlive the gateway. **How children start.** Pipe commands, `tty: true` commands, the patch helper, the Linux sandbox helper and the worker are started with fork and exec, because each runs a step in the child before exec: pipe, patch-helper, sandbox-helper and worker children keep only their standard descriptors ([#79](https://github.com/novelKR/CodeSpace/issues/79)), and a PTY child becomes the leader of a new session. On macOS, a child forked while another thread of the same process is initializing the system's notification library (libnotify) is killed by a signal inside `fork`, before the command runs. `process_status` reports a pipe command killed this way as `signaled`, and a `tty: true` command as an exit with code 1 ([Agent Loop integration](agent-integration.md)). So `codespace-mcp` and `codespace-codex-runtime` complete that initialization before they start any thread, and every one of these spawns completes it before it forks. This removes that one cause; it does not make forking from a multithreaded process safe in general. + + +## Independent Runner + +> **Not a supported mode yet.** The independent Runner is the first step of Gateway recovery. A gateway does not reconnect to it, and a gateway that connects after another one cannot start new work. Later steps add reconnection and reconciliation, and the mode is supported only once it is qualified. + +An independent Runner outlives the gateways that connect to it: its processes, their output and their timeouts carry on when a gateway exits, restarts or is killed. An operator or a harness starts it, and a gateway only connects to it: + +```bash +codespace-codex-runtime "$RUNNER_DIR/runner.sock" --lifecycle independent +codespace-mcp --runner uds --runner-socket "$RUNNER_DIR/runner.sock" \ + --runner-lifecycle independent --runner-exit-intent detach +``` + +The Runner creates its socket's directory with mode 0700, or accepts an existing one only when it is a directory of the same user with mode 0700 and not a symbolic link. It never changes an existing directory, and it refuses `/`, `/tmp`, `/var/tmp` and `$HOME`. It serves one gateway connection at a time, and accepts the next only after every request of the previous one has ended. When it stops, it removes its socket, and the directory if it created it. + +A gateway does not start when its `--runner-lifecycle` differs from the Runner's lifecycle, or when `--runner-lifecycle independent` comes without `--runner uds`, `--runner-socket` or `--runner-exit-intent`, or with `--runtime-bin` or `--runner-dir`. `--runner-exit-intent` is accepted only with `--runner-lifecycle independent`. + +| Event | Managed worker (default) | Independent Runner | +| --- | --- | --- | +| The gateway shuts down: its stdin ends, or it receives SIGTERM or SIGINT | The worker ends its processes and exits | With `--runner-exit-intent detach` the gateway detaches, and the processes carry on. With `stop` the Runner ends its processes and exits | +| The gateway receives SIGUSR1 to be restarted | Not handled: as when the gateway is killed | The gateway detaches whatever its exit intent, and the processes carry on | +| The gateway is killed, or its connection ends without a detach or a stop | The worker ends its processes and exits | A loss, never a stop: the processes carry on, and the Runner waits for the next gateway | +| A call to the Runner passes its 120-second deadline | The gateway closes the connection, and the worker ends its processes and exits | The gateway closes the connection, and the processes carry on | +| A gateway asks the Runner to stop, or the Runner receives SIGTERM or SIGINT | A managed worker refuses the request and does not handle these signals | The Runner ends its processes, waits up to 5 seconds for them, and exits | +| A process reaches its timeout | The process is killed | The process is killed at its original deadline, whether or not a gateway is connected | + +Ending a process signals a tty process's process group, but a pipe process alone: a pipe process has no process group of its own, so the processes it started may outlive it, a stop included. + +**A lost connection.** A gateway does not reconnect. Once its connection to the Runner has ended, it refuses every request that needs the Runner with `RUNNER_DISCONNECTED`, and it has to be restarted to reach the Runner's processes again. A request that was already sent when the connection ended keeps the uncertain result it has today, such as `dispatch_status: unknown` for an execution. + +**After an earlier gateway.** The first gateway that connects to a newly started Runner works as it does with a managed worker. Once a gateway has started work there (an execution, a patch, or with DevGuard a preparation or a launch), every later gateway inherits the Runner's processes: it can query, read, write to, resize and terminate them with their `process_id`, but it refuses new executions, patches and approval resumes with `RECOVERY_PENDING`, and leaves the approvals as they are. This holds after the inherited processes have ended too; to start work again, stop the Runner and start a new one. The gateway logs the Runner's identity (its process ID, boot and start time), whether a gateway controlled it before and how many processes it inherited. + +**DevGuard.** With the `devguard` feature, the independent Runner registers itself with the worker's DevGuard settings and reads the consumer secret from the private file given with `--devguard-credential-file`; its gateway, started with `--devguard register`, hands it no secret. Workspaces that require resource participation run only when the Runner states that it registers and launches through a launch helper, and are refused with `RESOURCE_POLICY_UNSUPPORTED` otherwise. See [DevGuard integration](devguard-integration.md#resource-gateway-recovery). + ## Linux command sandbox diff --git a/docs/translations.json b/docs/translations.json index 63e5303..716660e 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -78,8 +78,8 @@ "최소-도입-조건", "최초-gateway-복구-범위" ], - "source_sha256": "3158dd7bc4040281e0666a63aa6675bd34df702c20d9a84a039db6fcd25efb6d", - "translation_sha256": "845f9a126b8e34948fd21d51ac168d9beb0f7d66c0577e6ed7d4919e58bb95f1" + "source_sha256": "2baf08aad540a8065010299f5f48d2614492938e2df400133dfaa08cb10bfb2c", + "translation_sha256": "9d0ee4193edbc99e725622d78b6178850c7658a7354a9f40184c36c127ec7f09" }, { "id": "cs-rg-qualification", @@ -257,8 +257,8 @@ "재시도와-복구", "파일-읽기와-패치" ], - "source_sha256": "eb38ede0d2c1f2457e238c92ee554c7dc292ba50841835f0c658431f6c9a6f33", - "translation_sha256": "a71c333f14a70d9848e14fe1633e34de3d9616fca1ab5e432d0ed1387aa3c218" + "source_sha256": "a0ac8251ae13700936d1f2011aeb709100b1c3ea5cfd60ec9053575f463c4909", + "translation_sha256": "ea48623f485e9b683065e990a1d3cca15f05b83f315384fbb8792116a8df7e80" }, { "id": "operations", @@ -309,8 +309,8 @@ "작업-공간-등록", "첫-연결-확인" ], - "source_sha256": "de7bbd350892090e5fb352a06e47ce8e95f96b0c2b2dc6128a4edef5f1d9af2b", - "translation_sha256": "aed949328aff2698dbc138d4335ceb97f5c42f0fb1ed8499eb6c0e6469fdc90d" + "source_sha256": "99d45a3addf7a209b087b3663af59dddf73b3a0043c0f0e9194e73fd24732c07", + "translation_sha256": "972fd7b98c226bb7607b17e6cbd872ae584cd189c6017ae49a8c9b6a3f8bd385" }, { "id": "chatgpt-connector", @@ -399,8 +399,8 @@ "현재-실행-구조", "확장-시-유지할-경계" ], - "source_sha256": "df2bd3224b5f47d28038f11bf065ec48ed08bc8a4bc30c3163adcec65b0eecba", - "translation_sha256": "300c857fa22a155c01070a7c4f7867e43085d8c15773acd19e60d745917cd2d2" + "source_sha256": "ff67a5d9dcfa1943b2a54eb27c41e3ca3d2e24f532c12f4b06091f0f5e31821c", + "translation_sha256": "6ea5c1e57e1dafea7e2d2806fca18574584b9308b411e810e9ae704a7c6ef751" }, { "id": "execution-substrate", @@ -590,6 +590,7 @@ "compose-픽스처가-하는-일", "container-fixture-and-verification", "host-and-worker-execution", + "independent-runner", "later-process-split", "linux-command-sandbox", "linux-명령-샌드박스", @@ -601,13 +602,14 @@ "what-the-compose-fixture-does", "이후-프로세스-분리", "네트워크와-파일-접근-범위", + "독립-runner", "러너-격리", "이후-프로세스-분리", "컨테이너-실험-구성과-검증", "호스트와-worker-실행" ], - "source_sha256": "50fe627ce573f1b77b4a35a71a83d897d06ad62f17077425d36d01b32f8f51e4", - "translation_sha256": "69ac617af227ade40277d11c48dccda71d287caee9a0e0a6357eeddaa5da6c26" + "source_sha256": "513bdd237464fa077354b3279b336dae6d9276403b13ca3bf3465dea9898d08c", + "translation_sha256": "c49af49c145ae3e13c07ee7eb58da5179685720aa039fd66c6dc05680a9061f1" }, { "id": "error-codes", @@ -635,8 +637,8 @@ "전송-실패", "전송-실패-작업-없음" ], - "source_sha256": "04e3006bc1d7eed2b8a94f44b71dbaa417fceb23f168d0ed4b480825af0c8629", - "translation_sha256": "f286fa26fa1055491de9245c537716ada252177a0d699b7dda37ecd20eee485c" + "source_sha256": "d37b93d073a059ff993b2309efdedc98d51a80f3f32b5802f66b62d0e9e1338a", + "translation_sha256": "9e100facbb0cc372e235733e6c093eb1a4a3e4d9903343bf6b098df8e745e294" }, { "id": "codex-reuse", @@ -717,8 +719,8 @@ "핀-6b9826e의-후보", "핵심-대-어댑터" ], - "source_sha256": "121d4af8ffeecbe2cf1cf34c23046664637e6d906a38f7cfd2f7a7a0d5ebe656", - "translation_sha256": "fd1b6ffcc4f23f07caad0421546ee59fcae3b0ad6926a0b1900df16e8ac9c911" + "source_sha256": "cfacce5889ed00ed9a7338f98b80c20f05947ee0af5e913fb29e4b64869a403b", + "translation_sha256": "bef8eb0b952cf5c47e6df069076dca6dfbae97dcb07f78609f52f977172a79cb" }, { "id": "upstream-lock", diff --git a/scripts/validate-upstream.py b/scripts/validate-upstream.py index 9126d41..752e1f8 100644 --- a/scripts/validate-upstream.py +++ b/scripts/validate-upstream.py @@ -80,12 +80,18 @@ def stages(): # launch helper, and the DevGuard tests of the runner (registration, its wire, the # preparation of governed executions, CSRG-U3, and their launch, CSRG-U4), the worker # and the gateway where socket creation is not atomically close-on-exec, the gateway's - # against the worker, the fixture authority and the helper built here. + # against the worker, the fixture authority and the helper built here. With them, the + # independent Runner (P1R-C01): its identity, socket directory and wire, the gateway's + # settings and refusals, then the worker binary and the gateway binary against it, + # with the default worker built again over the DevGuard one. + devguard_binaries() + [cargo('test', 'devguard'), cargo('test', 'codex-runtime', '--features', 'devguard', '--bin', 'codespace-codex-runtime'), - cargo('test', 'root', '-p', 'codespace-runner', '--features', 'devguard', '--lib', '--', 'registration', 'wire', 'admission', 'launch'), - cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--lib', '--', 'devguard')], + cargo('test', 'root', '-p', 'codespace-runner', '--features', 'devguard', '--lib', '--', 'registration', 'wire', 'admission', 'launch', 'independent', 'lifecycle'), + cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--lib', '--', 'devguard', 'independent'), + cargo('build', 'codex-runtime', '--bin', 'codespace-codex-runtime'), + cargo('test', 'codex-runtime', '--test', 'runtime_binary', '--', 'independent'), + cargo('test', 'root', '-p', 'codespace-server', '--features', 'devguard', '--test', 'independent_runner')], # Children killed inside fork on macOS: fresh-process trials of reads concurrent with # pipe spawns (crates/runner/tests/fork_race.rs); any child without an exit code fails. 'macos-fork-race': [cargo('test', 'root', '-p', 'codespace-runner', '--test', 'fork_race', '--',