diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a4222e00..d0fd40e6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -66,6 +66,7 @@ Each subject has one doc in `docs/architecture/`. `pnpm run check:docs` enforces | [Relay](docs/architecture/relay.md) | `crates/relay`: storage on SQLite or Postgres, files, housekeeping, quotas, metrics, rate limits, auth, tables and migrations, the blob, sync socket, and push APIs, deploys | | [Computer display](docs/architecture/computer-display.md) | Proposed authenticated display/input transport, Device-bound sessions, lifecycle fences, client feasibility and measurement gaps | | [Protocols](docs/architecture/protocols.md) | The app ↔ CLI websocket and the CLI ↔ relay requests and blobs | +| [Remote requests](docs/architecture/remote-requests.md) | Device-to-Runner request signatures, authority, envelopes and deadlines | | [Computer ownership](docs/architecture/computer-ownership.md) | Proposed encrypted ownership, async lifecycle, capability compatibility and old-client fixture | | [Computer ownership compatibility](docs/architecture/computer-ownership-compatibility.md) | Source-extracted fixture, hashes, replay and evidence limits | | [Managed policy receipts and replay](docs/architecture/managed-policy-replay.md) | Staged opaque policy/roster commits, durable receipt retention, authenticated finite replay cuts and teardown boundaries | diff --git a/crates/cli/src/app.rs b/crates/cli/src/app.rs index 4686ffdd..617211d1 100644 --- a/crates/cli/src/app.rs +++ b/crates/cli/src/app.rs @@ -1245,10 +1245,12 @@ impl App { /// Every turn in flight this Device knows of, by job id: what the other Devices list while /// the relay last saw them online, then this Device's own jobs, including one it sent to a - /// Runner that has not listed it yet. + /// Runner that has not listed it yet. A job sent to a Runner the relay's last machine list + /// shows offline is left out, as that Runner's own list is: its wait goes on, so a result + /// that comes later still counts. fn known_turns(&self) -> std::collections::BTreeMap { let mut turns = std::collections::BTreeMap::new(); - { + let offline_runners: std::collections::HashSet = { let state = self.state.lock().unwrap(); for (device_id, listed) in &state.device_turns { if state.turns_online.contains(device_id) { @@ -1257,8 +1259,12 @@ impl App { } } } - } + state.listed_machines.keys().filter(|id| !state.turns_online.contains(*id)).cloned().collect() + }; for (id, job) in self.running_jobs.lock().unwrap().iter() { + if job.runner_id.as_ref().is_some_and(|runner| offline_runners.contains(runner)) { + continue; + } turns.entry(id.clone()).or_insert_with(|| job.turn(id)); } turns @@ -1273,11 +1279,7 @@ impl App { } let mut announced = self.announced_turns.lock().unwrap(); let known = self.known_turns(); - for (id, turn) in announced.iter() { - if !known.contains_key(id) { - self.emit(Event::JobFinished { chat_id: turn.chat_id.clone(), bot_id: turn.bot_id.clone(), job_id: id.clone(), routine_id: turn.routine_id.clone() }); - } - } + self.announce_ended(&announced, &known); for (id, turn) in &known { let before = announced.get(id); if before.is_none() { @@ -1292,6 +1294,28 @@ impl App { *announced = known; } + /// Tells the local app at once which of the turns it was told of this Device no longer + /// knows, and announces none: a Device the relay now shows offline lists nothing, and its + /// turns end here without waiting for the rest of the sync round, which a relay that drops + /// the session may never finish. What starts waits for `turns_changed` after the pull. + pub fn turns_ended(&self) { + if self.bulk_sync.load(Ordering::Relaxed) { + return; + } + let mut announced = self.announced_turns.lock().unwrap(); + let known = self.known_turns(); + self.announce_ended(&announced, &known); + announced.retain(|id, _| known.contains_key(id)); + } + + fn announce_ended(&self, announced: &std::collections::BTreeMap, known: &std::collections::BTreeMap) { + for (id, turn) in announced.iter() { + if !known.contains_key(id) { + self.emit(Event::JobFinished { chat_id: turn.chat_id.clone(), bot_id: turn.bot_id.clone(), job_id: id.clone(), routine_id: turn.routine_id.clone() }); + } + } + } + /// A turn here started, ended, or changed what it is doing: the other Devices hear it /// through this Device's machine blob, and the local app through `turns_changed`. pub fn local_turns_changed(&self) { diff --git a/crates/cli/src/sync.rs b/crates/cli/src/sync.rs index 8cd2d72c..8337f903 100644 --- a/crates/cli/src/sync.rs +++ b/crates/cli/src/sync.rs @@ -841,6 +841,7 @@ async fn refresh_presence(app: &Arc, url: &str, token: &str) -> Result<(), tracing::error!(%error, "dropping another Device's turns"); } } + app.turns_ended(); Ok(()) } @@ -1799,6 +1800,125 @@ mod tests { } } + /// A Runner that leaves the relay mid-turn ends its turns here as soon as the presence + /// refresh hears it, without waiting for the rest of the sync round: the turn it lists, and + /// the job this Device sent it and still waits on. Its eventual result reaches the waiter + /// while offline, and a reconnect advertises a still-tracked turn once. + #[cfg(feature = "server")] + #[tokio::test] + async fn a_runner_leaving_the_relay_ends_its_turns_here_at_once() { + use axum::{routing::get, Json, Router}; + use std::sync::atomic::AtomicBool; + let scratch = scratch_app(); + let app = &scratch.0; + let online = Arc::new(AtomicBool::new(true)); + let listed = online.clone(); + let relay = Router::new().route("/v1/machines", get(move || { + let online = listed.load(Ordering::Relaxed); + async move { Json(serde_json::json!({ "machines": [{ "machine_pubkey": "mac", "box_pubkey": "box", "last_seen": 1, "created_at": 1, "online": online }], "now": crate::config::now_unix() })) } + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { axum::serve(listener, relay).await.unwrap() }); + let turn = |job_id: &str| LiveTurn { job_id: job_id.into(), chat_id: "chat".into(), bot_id: "bot".into(), routine_id: None, activity: None }; + let mut events = app.events.subscribe(); + let mut heard = |pick: fn(Event) -> Option| -> Vec { + let mut ids: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()).filter_map(pick).collect(); + ids.sort(); + ids + }; + let started = |event: Event| match event { Event::JobStarted { job_id, .. } => Some(job_id), _ => None }; + let finished = |event: Event| match event { Event::JobFinished { job_id, .. } => Some(job_id), _ => None }; + let ids = |ids: &[&str]| { + let mut ids: Vec<_> = ids.iter().map(|id| id.to_string()).collect(); + ids.sort(); + ids + }; + + app.running_jobs.lock().unwrap().insert( + "sent-job".into(), + crate::app::RunningJob { chat_id: "chat".into(), bot_id: "bot".into(), routine_id: None, runner_id: Some("mac".into()), cancel: Default::default(), activity: None }, + ); + refresh_presence(app, &url, "test").await.unwrap(); + assert!(app.device_is_online("mac"), "the fixture establishes the Runner online before it leaves"); + app.set_device_turns("mac", vec![turn("mac-job"), turn("sent-job")]); + assert_eq!(heard(started), ids(&["mac-job", "sent-job"])); + + let (tx, mut result) = tokio::sync::oneshot::channel(); + app.pending_results.lock().unwrap().insert("sent-job".into(), tx); + let cancel = app.running_jobs.lock().unwrap()["sent-job"].cancel.clone(); + online.store(false, Ordering::Relaxed); + app.bulk_sync.store(true, Ordering::Relaxed); + refresh_presence(app, &url, "test").await.unwrap(); + assert!(heard(finished).is_empty(), "presence refresh respects bulk batching"); + app.bulk_sync.store(false, Ordering::Relaxed); + refresh_presence(app, &url, "test").await.unwrap(); + assert_eq!(heard(finished), ids(&["mac-job", "sent-job"]), "the refresh alone tells the app"); + assert!(app.running_turns().is_empty()); + assert!(app.running_jobs.lock().unwrap().contains_key("sent-job"), "the wait for the result goes on"); + + assert!(app.pending_results.lock().unwrap().contains_key("sent-job")); + assert!(!cancel.is_cancelled()); + assert!(matches!(result.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty))); + crate::runtime::deliver_job_result(app, JobResult { + job_id: "sent-job".into(), chat_id: "chat".into(), bot_id: "bot".into(), outcome: "sent".into(), + }); + assert_eq!(result.await.unwrap(), "sent", "an offline Runner's eventual result is accepted"); + assert!(!app.pending_results.lock().unwrap().contains_key("sent-job")); + + online.store(true, Ordering::Relaxed); + refresh_presence(app, &url, "test").await.unwrap(); + assert!(heard(started).is_empty(), "presence alone must not announce new starts"); + app.set_device_turns("mac", vec![turn("sent-job")]); + assert_eq!(heard(started), ids(&["sent-job"])); + app.turns_changed(); + assert!(heard(started).is_empty(), "reconnect advertises once"); + app.set_device_turns("mac", Vec::new()); + app.running_jobs.lock().unwrap().remove("sent-job"); + app.turns_changed(); + assert_eq!(heard(finished), ids(&["sent-job"])); + server.abort(); + } + + #[tokio::test] + async fn offline_turn_visibility_preserves_unknown_local_and_bulk_jobs() { + let scratch = scratch_app(); + let app = &scratch.0; + for (id, runner) in [("local", None), ("unknown", Some("unlisted")), ("online", Some("online")), ("offline", Some("offline"))] { + app.running_jobs.lock().unwrap().insert(id.into(), crate::app::RunningJob { + chat_id: "chat".into(), bot_id: "bot".into(), routine_id: None, + runner_id: runner.map(str::to_string), cancel: Default::default(), activity: None, + }); + } + let mut events = app.events.subscribe(); + app.turns_changed(); + let starts: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()) + .filter(|e| matches!(e, Event::JobStarted { .. })).collect(); + assert_eq!(starts.len(), 4, "initial presence is unknown, not offline"); + { + let mut state = app.state.lock().unwrap(); + state.listed_machines.insert("offline".into(), 1); + state.listed_machines.insert("online".into(), 1); + state.turns_online.insert("online".into()); + } + app.bulk_sync.store(true, Ordering::Relaxed); + app.turns_ended(); + app.turns_changed(); + assert!(events.try_recv().is_err(), "bulk mode defers turn events"); + app.bulk_sync.store(false, Ordering::Relaxed); + app.emit(Event::Snapshot(app.snapshot())); + app.turns_changed(); + assert!(matches!(events.try_recv().unwrap(), Event::Snapshot(_))); + assert!(matches!(events.try_recv().unwrap(), Event::JobFinished { job_id, .. } if job_id == "offline")); + assert!(events.try_recv().is_err()); + let visible: std::collections::BTreeSet<_> = app.running_turns().iter() + .map(|t| t["job_id"].as_str().unwrap().to_string()).collect(); + assert_eq!(visible, ["local", "unknown", "online"].into_iter().map(str::to_string).collect()); + assert_eq!(app.running_jobs.lock().unwrap().len(), 4); + app.turns_ended(); + assert!(events.try_recv().is_err(), "ended visibility emits once"); + } + fn scratch_app() -> ScratchApp { let home = std::env::temp_dir().join(format!("beans-sync-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&home).unwrap(); diff --git a/docs/architecture/protocols.md b/docs/architecture/protocols.md index cb5d96ea..c8cf6983 100644 --- a/docs/architecture/protocols.md +++ b/docs/architecture/protocols.md @@ -22,10 +22,11 @@ Memory-service configuration RPCs use `memory.connections.*`, `memory.embeddings Guided custom-provider `providers.list_models`, `providers.connect_custom` and `providers.refresh` accept optional `runner_id`; refresh optionally limits work to `kind`. Their selected-Runner sealed verbs are `providers.custom.preview/save/refresh`, admitted through the existing request mechanism. Preview is non-mutating. Save/refresh return revision receipts while authoritative credentials still travel only in the encrypted `credentials` blob. Remote setup waits 60 seconds for the Runner and up to 30 seconds for those revisions to sync. No Runner selection preserves existing non-guided CLI behavior. -All sealed remote request verbs wrap their body in `{ payload, signature }`. The Ed25519 signature binds a domain, request ID, verb, requester signing public key, target Runner, creation time and payload. The three guided provider verbs retain their existing `beans-provider-request-v1` domain and wire format; other verbs use `beans-request-v1`. The shared Runner request dispatcher checks current local paired membership and verifies the signature before executing any verb. A relay-listed Device with missing metadata remains eligible; a claimed `requested_by` inside an anonymous sealed box alone is not authenticated authority. Removed requesters, unsigned bodies and forged signatures reject before dispatch. Previously queued unsigned requests are not re-signed or executed; paired clients must upgrade together for remote requests. This request-body cutover does not change protocol 5, the account format or stored account data. Membership is admission-time authority, not a promise to roll back already-started asynchronous effects. Guided provider membership remains checked through the post-discovery credential commit. Guided clients retain a UUID `custom:setup-*` kind for add retries; exact committed replay returns the same kind and stamp without another catalog request. +[Remote requests](remote-requests.md#request-authentication) describes sealed request signatures, admission-time authority and the coordinated paired-client request-body cutover. Custom credentials and provider statuses add optional `capabilities { context_window?, images?, tools? }`; status deserialization defaults absent declarations. Positive integer context windows and boolean/null support values are validated before HTTP. Omitted saves keep declarations, null clears them, objects replace them. Model statuses carry effective declarations ahead of discovery; declaration status separately lets sheets reopen Unknown/Yes/No choices. This is an additive protocol-5 field, without storage-format or home migration. +A presence refresh immediately emits `job.finished` for advertised turns whose Runner the relay now lists offline, including locally tracked remote jobs awaiting a result. It announces new starts only after the sync pull. Initial or unlisted presence does not hide those waiting jobs. Visibility ending preserves the running job, cancellation token and result waiter; eventual results still complete the wait. Reconnection re-advertises turns through the next machine blob and sync notification. Bulk sync defers turn events until after its snapshot. Account alias merge and recovery: [Device names](device-names.md). ## CLI ↔ relay @@ -39,7 +40,7 @@ Account alias merge and recovery: [Device names](device-names.md). - PUT/GET non-file blobs as base64url ciphertext in JSON, and PUT/GET files as binary ciphertext. `roster` is a whole-roster snapshot of bots, chats, and routines; `policy` is an append-only encrypted explicit Pause, bot capability, or bot removal action with a logical version and Device tie-break. Devices reconcile those actions with the latest roster before publishing queued snapshots, so a stale Device cannot undo a newer policy action even when its roster upload races another Device. The relay cannot read policy values. `chat` is one upsert or removal of a message; `machine` is a Device’s metadata and the turns in flight on it; `credentials` is the account's provider credentials; `key` is the DEK sealed to the content key. - Individual `DELETE /v1/blobs/{id}` protects `roster`, `policy` and `memory_config` for every protocol-admitted client. SQLite and Postgres apply the kind restriction in the atomic DELETE itself, without a lookup/check/delete race. Protected ids return `404` and leave rows, sequence and usage unchanged; file cleanup and consumed-envelope deletion remain available. - Jobs: `kind=job` with `recipient_machine_pubkey`, sealed to that machine’s box key, deleted by the Runner after the turn. -- Questions: `kind=request` sealed to one Runner (`crates/cli/src/requests.rs`: `{ id, verb, requested_by, body }`), answered with a `kind=response` sealed to the Device that asked (`{ request_id, body, error? }`); each side deletes the blob it consumed, and the asker gives up after 20 s, or 75 s for the start of a plugin sign-in, whose discovery and registration the Runner cuts off at 45 s, or 150 s for `mcp.reconnect`, which answers once a server has connected or failed. The verbs are `memory.read` and `memory.write`, so a bot's memory can be shown and edited from a Device that is not its Runner; `plugins.install` / `plugins.uninstall` / `plugins.variables` / `plugins.connect` / `plugins.sign_out` / `plugins.detail`, so plugins on a Runner are managed from any Device, and `plugins.sign_in.finish` / `plugins.sign_in.cancel`, with which the Device that asked for a sign-in (its `redirect_uri` on `plugins.connect` or `permission.answer`) hands back where its browser landed (see Sign-in in [Codemode and Plugins](plugins.md#plugins)); the `mcp.*` methods but `mcp.parse`, so a Runner's own MCP servers are managed from any Device; `permission.answer`, so a permission card is answered from any Device; `bash.stdin`, `bash.stop`, and `bash.background`, so a command a bot runs is answered, stopped, or sent to the background from any Device; `chats.send_now`, so a message the bot's turn holds is read at once from any Device. A request is refused up front when the Runner is unknown or offline. +- [Remote requests](remote-requests.md#delivery-and-timeouts) describes sealed request/response envelopes, supported verbs, consumption and deadlines. - Slots: a `roster`, `chat`, `machine`, or `credentials` blob goes up with a `slot` (`Slot` in `app.rs`, `ChatBlob::slot`), and the relay deletes the blobs of that slot it supersedes, so the log holds a message once and not once per streamed chunk. A message's versions and its removal share the slot named by the message id. An upsert sets `keep_first`, which spares the slot's oldest blob: its seq holds the message's place in the log, so a Device that replays from the start gets the transcript in order, first the early version and later the final one. A tool row is the exception: its running version goes up without arguments or results, for the status line on other Devices, and the finished row supersedes it outright, so the relay keeps one version of each call, at the finished row's place. A row that shows a terminal keeps its first place as a message does, since the transcript shows it. A removal supersedes every version. The SQLite outbox does the same before upload: a queued version updates its row without moving its queue position. `App::upsert_message` holds one lock from the SQLite write that gives a message its `position` to the enqueue, so the sending Device's transcript order is the log's order. - Groups: a `chat` blob and a message's `file` blob go up with `group`, the chat's id (`ChatBlob::group`). Deleting a chat queues its id in `State::group_deletes`, drops what the outbox still held for it, and the sync cycle calls `DELETE /v1/groups/{group}` until it lands: the relay deletes the group's rows and the attachments' objects in one transaction, gives the bytes back to the quota, and remembers the group in `deleted_groups`, so a blob that names it later (a Runner finishing a turn in a chat another Device deleted) gets `409` and the outbox drops it. The other Devices learn of the deletion from the roster. A bot's avatar is a `file` with no group: the Device that replaces or removes it, or deletes its bot, removes its local copy and queues the attachment id in `State::blob_deletes`, and the sync cycle calls `DELETE /v1/blobs/{id}` until it lands, which removes the row and the object. - `GET /v1/groups/{group}/blobs?before=&limit=` reads a chat backwards a message at a time: `{ slots: [{ place, blobs }], has_more }`, oldest first, where a slot is a message (its first and latest blob, or its removal) or the chat's read mark, and `place` is the slot's lowest seq. A message other than a tool row keeps the place its first version took however late its last one landed, so pages read backwards give the transcript in the log's order. A page is up to 500 slots and 8 MiB, and leaves out the group's `file` blobs. A relay without the route answers `404`, and the Device replays the log. diff --git a/docs/architecture/remote-requests.md b/docs/architecture/remote-requests.md new file mode 100644 index 00000000..6a534d19 --- /dev/null +++ b/docs/architecture/remote-requests.md @@ -0,0 +1,9 @@ +# Remote requests + +## Request authentication + +All sealed remote request verbs wrap their body in `{ payload, signature }`. The Ed25519 signature binds a domain, request ID, verb, requester signing public key, target Runner, creation time and payload. The three guided provider verbs retain their existing `beans-provider-request-v1` domain and wire format; other verbs use `beans-request-v1`. The shared Runner request dispatcher checks current local paired membership and verifies the signature before executing any verb. A relay-listed Device with missing metadata remains eligible; a claimed `requested_by` inside an anonymous sealed box alone is not authenticated authority. Removed requesters, unsigned bodies and forged signatures reject before dispatch. Previously queued unsigned requests are not re-signed or executed; paired clients must upgrade together for remote requests. This request-body cutover does not change protocol 5, the account format or stored account data. Membership is admission-time authority, not a promise to roll back already-started asynchronous effects. Guided provider membership remains checked through the post-discovery credential commit. Guided clients retain a UUID `custom:setup-*` kind for add retries; exact committed replay returns the same kind and stamp without another catalog request. + +## Delivery and timeouts + +`kind=request` sealed to one Runner (`crates/cli/src/requests.rs`: `{ id, verb, requested_by, body }`), answered with a `kind=response` sealed to the Device that asked (`{ request_id, body, error? }`); each side deletes the blob it consumed, and the asker gives up after 20 s, or 75 s for the start of a plugin sign-in, whose discovery and registration the Runner cuts off at 45 s, or 150 s for `mcp.reconnect`, which answers once a server has connected or failed. The verbs are `memory.read` and `memory.write`, so a bot's memory can be shown and edited from a Device that is not its Runner; `plugins.install` / `plugins.uninstall` / `plugins.variables` / `plugins.connect` / `plugins.sign_out` / `plugins.detail`, so plugins on a Runner are managed from any Device, and `plugins.sign_in.finish` / `plugins.sign_in.cancel`, with which the Device that asked for a sign-in (its `redirect_uri` on `plugins.connect` or `permission.answer`) hands back where its browser landed (see Sign-in in [Codemode and Plugins](plugins.md#plugins)); the `mcp.*` methods but `mcp.parse`, so a Runner's own MCP servers are managed from any Device; `permission.answer`, so a permission card is answered from any Device; `bash.stdin`, `bash.stop`, and `bash.background`, so a command a bot runs is answered, stopped, or sent to the background from any Device; `chats.send_now`, so a message the bot's turn holds is read at once from any Device. A request is refused up front when the Runner is unknown or offline.