Skip to content
1 change: 1 addition & 0 deletions ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ Each subject has one doc in `docs/architecture/`. `pnpm run check:docs` enforces
| [Relay](docs/architecture/relay.md) | `crates/relay`: storage on SQLite or Postgres, files, housekeeping, quotas, metrics, rate limits, auth, tables and migrations, the blob, sync socket, and push APIs, deploys |
| [Computer display](docs/architecture/computer-display.md) | Proposed authenticated display/input transport, Device-bound sessions, lifecycle fences, client feasibility and measurement gaps |
| [Protocols](docs/architecture/protocols.md) | The app ↔ CLI websocket and the CLI ↔ relay requests and blobs |
| [Remote requests](docs/architecture/remote-requests.md) | Device-to-Runner request signatures, authority, envelopes and deadlines |
| [Computer ownership](docs/architecture/computer-ownership.md) | Proposed encrypted ownership, async lifecycle, capability compatibility and old-client fixture |
| [Computer ownership compatibility](docs/architecture/computer-ownership-compatibility.md) | Source-extracted fixture, hashes, replay and evidence limits |
| [Managed policy receipts and replay](docs/architecture/managed-policy-replay.md) | Staged opaque policy/roster commits, durable receipt retention, authenticated finite replay cuts and teardown boundaries |
Expand Down
40 changes: 32 additions & 8 deletions crates/cli/src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1245,10 +1245,12 @@ impl App {

/// Every turn in flight this Device knows of, by job id: what the other Devices list while
/// the relay last saw them online, then this Device's own jobs, including one it sent to a
/// Runner that has not listed it yet.
/// Runner that has not listed it yet. A job sent to a Runner the relay's last machine list
/// shows offline is left out, as that Runner's own list is: its wait goes on, so a result
/// that comes later still counts.
fn known_turns(&self) -> std::collections::BTreeMap<String, LiveTurn> {
let mut turns = std::collections::BTreeMap::new();
{
let offline_runners: std::collections::HashSet<String> = {
let state = self.state.lock().unwrap();
for (device_id, listed) in &state.device_turns {
if state.turns_online.contains(device_id) {
Expand All @@ -1257,8 +1259,12 @@ impl App {
}
}
}
}
state.listed_machines.keys().filter(|id| !state.turns_online.contains(*id)).cloned().collect()
};
for (id, job) in self.running_jobs.lock().unwrap().iter() {
if job.runner_id.as_ref().is_some_and(|runner| offline_runners.contains(runner)) {
continue;
}
turns.entry(id.clone()).or_insert_with(|| job.turn(id));
}
turns
Expand All @@ -1273,11 +1279,7 @@ impl App {
}
let mut announced = self.announced_turns.lock().unwrap();
let known = self.known_turns();
for (id, turn) in announced.iter() {
if !known.contains_key(id) {
self.emit(Event::JobFinished { chat_id: turn.chat_id.clone(), bot_id: turn.bot_id.clone(), job_id: id.clone(), routine_id: turn.routine_id.clone() });
}
}
self.announce_ended(&announced, &known);
for (id, turn) in &known {
let before = announced.get(id);
if before.is_none() {
Expand All @@ -1292,6 +1294,28 @@ impl App {
*announced = known;
}

/// Tells the local app at once which of the turns it was told of this Device no longer
/// knows, and announces none: a Device the relay now shows offline lists nothing, and its
/// turns end here without waiting for the rest of the sync round, which a relay that drops
/// the session may never finish. What starts waits for `turns_changed` after the pull.
pub fn turns_ended(&self) {
if self.bulk_sync.load(Ordering::Relaxed) {
return;
}
let mut announced = self.announced_turns.lock().unwrap();
let known = self.known_turns();
self.announce_ended(&announced, &known);
announced.retain(|id, _| known.contains_key(id));
}

fn announce_ended(&self, announced: &std::collections::BTreeMap<String, LiveTurn>, known: &std::collections::BTreeMap<String, LiveTurn>) {
for (id, turn) in announced.iter() {
if !known.contains_key(id) {
self.emit(Event::JobFinished { chat_id: turn.chat_id.clone(), bot_id: turn.bot_id.clone(), job_id: id.clone(), routine_id: turn.routine_id.clone() });
}
}
}

/// A turn here started, ended, or changed what it is doing: the other Devices hear it
/// through this Device's machine blob, and the local app through `turns_changed`.
pub fn local_turns_changed(&self) {
Expand Down
120 changes: 120 additions & 0 deletions crates/cli/src/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,7 @@ async fn refresh_presence(app: &Arc<App>, url: &str, token: &str) -> Result<(),
tracing::error!(%error, "dropping another Device's turns");
}
}
app.turns_ended();
Ok(())
}

Expand Down Expand Up @@ -1799,6 +1800,125 @@ mod tests {
}
}

/// A Runner that leaves the relay mid-turn ends its turns here as soon as the presence
/// refresh hears it, without waiting for the rest of the sync round: the turn it lists, and
/// the job this Device sent it and still waits on. Its eventual result reaches the waiter
/// while offline, and a reconnect advertises a still-tracked turn once.
#[cfg(feature = "server")]
#[tokio::test]
async fn a_runner_leaving_the_relay_ends_its_turns_here_at_once() {
use axum::{routing::get, Json, Router};
use std::sync::atomic::AtomicBool;
let scratch = scratch_app();
let app = &scratch.0;
let online = Arc::new(AtomicBool::new(true));
let listed = online.clone();
let relay = Router::new().route("/v1/machines", get(move || {
let online = listed.load(Ordering::Relaxed);
async move { Json(serde_json::json!({ "machines": [{ "machine_pubkey": "mac", "box_pubkey": "box", "last_seen": 1, "created_at": 1, "online": online }], "now": crate::config::now_unix() })) }
}));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let server = tokio::spawn(async move { axum::serve(listener, relay).await.unwrap() });
let turn = |job_id: &str| LiveTurn { job_id: job_id.into(), chat_id: "chat".into(), bot_id: "bot".into(), routine_id: None, activity: None };
let mut events = app.events.subscribe();
let mut heard = |pick: fn(Event) -> Option<String>| -> Vec<String> {
let mut ids: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()).filter_map(pick).collect();
ids.sort();
ids
};
let started = |event: Event| match event { Event::JobStarted { job_id, .. } => Some(job_id), _ => None };
let finished = |event: Event| match event { Event::JobFinished { job_id, .. } => Some(job_id), _ => None };
let ids = |ids: &[&str]| {
let mut ids: Vec<_> = ids.iter().map(|id| id.to_string()).collect();
ids.sort();
ids
};

app.running_jobs.lock().unwrap().insert(
"sent-job".into(),
crate::app::RunningJob { chat_id: "chat".into(), bot_id: "bot".into(), routine_id: None, runner_id: Some("mac".into()), cancel: Default::default(), activity: None },
);
refresh_presence(app, &url, "test").await.unwrap();
assert!(app.device_is_online("mac"), "the fixture establishes the Runner online before it leaves");
app.set_device_turns("mac", vec![turn("mac-job"), turn("sent-job")]);
assert_eq!(heard(started), ids(&["mac-job", "sent-job"]));

let (tx, mut result) = tokio::sync::oneshot::channel();
app.pending_results.lock().unwrap().insert("sent-job".into(), tx);
let cancel = app.running_jobs.lock().unwrap()["sent-job"].cancel.clone();
online.store(false, Ordering::Relaxed);
app.bulk_sync.store(true, Ordering::Relaxed);
refresh_presence(app, &url, "test").await.unwrap();
assert!(heard(finished).is_empty(), "presence refresh respects bulk batching");
app.bulk_sync.store(false, Ordering::Relaxed);
refresh_presence(app, &url, "test").await.unwrap();
assert_eq!(heard(finished), ids(&["mac-job", "sent-job"]), "the refresh alone tells the app");
assert!(app.running_turns().is_empty());
assert!(app.running_jobs.lock().unwrap().contains_key("sent-job"), "the wait for the result goes on");

assert!(app.pending_results.lock().unwrap().contains_key("sent-job"));
assert!(!cancel.is_cancelled());
assert!(matches!(result.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)));
crate::runtime::deliver_job_result(app, JobResult {
job_id: "sent-job".into(), chat_id: "chat".into(), bot_id: "bot".into(), outcome: "sent".into(),
});
assert_eq!(result.await.unwrap(), "sent", "an offline Runner's eventual result is accepted");
assert!(!app.pending_results.lock().unwrap().contains_key("sent-job"));

online.store(true, Ordering::Relaxed);
refresh_presence(app, &url, "test").await.unwrap();
assert!(heard(started).is_empty(), "presence alone must not announce new starts");
app.set_device_turns("mac", vec![turn("sent-job")]);
assert_eq!(heard(started), ids(&["sent-job"]));
app.turns_changed();
assert!(heard(started).is_empty(), "reconnect advertises once");
app.set_device_turns("mac", Vec::new());
app.running_jobs.lock().unwrap().remove("sent-job");
app.turns_changed();
assert_eq!(heard(finished), ids(&["sent-job"]));
server.abort();
}

#[tokio::test]
async fn offline_turn_visibility_preserves_unknown_local_and_bulk_jobs() {
let scratch = scratch_app();
let app = &scratch.0;
for (id, runner) in [("local", None), ("unknown", Some("unlisted")), ("online", Some("online")), ("offline", Some("offline"))] {
app.running_jobs.lock().unwrap().insert(id.into(), crate::app::RunningJob {
chat_id: "chat".into(), bot_id: "bot".into(), routine_id: None,
runner_id: runner.map(str::to_string), cancel: Default::default(), activity: None,
});
}
let mut events = app.events.subscribe();
app.turns_changed();
let starts: Vec<_> = std::iter::from_fn(|| events.try_recv().ok())
.filter(|e| matches!(e, Event::JobStarted { .. })).collect();
assert_eq!(starts.len(), 4, "initial presence is unknown, not offline");
{
let mut state = app.state.lock().unwrap();
state.listed_machines.insert("offline".into(), 1);
state.listed_machines.insert("online".into(), 1);
state.turns_online.insert("online".into());
}
app.bulk_sync.store(true, Ordering::Relaxed);
app.turns_ended();
app.turns_changed();
assert!(events.try_recv().is_err(), "bulk mode defers turn events");
app.bulk_sync.store(false, Ordering::Relaxed);
app.emit(Event::Snapshot(app.snapshot()));
app.turns_changed();
assert!(matches!(events.try_recv().unwrap(), Event::Snapshot(_)));
assert!(matches!(events.try_recv().unwrap(), Event::JobFinished { job_id, .. } if job_id == "offline"));
assert!(events.try_recv().is_err());
let visible: std::collections::BTreeSet<_> = app.running_turns().iter()
.map(|t| t["job_id"].as_str().unwrap().to_string()).collect();
assert_eq!(visible, ["local", "unknown", "online"].into_iter().map(str::to_string).collect());
assert_eq!(app.running_jobs.lock().unwrap().len(), 4);
app.turns_ended();
assert!(events.try_recv().is_err(), "ended visibility emits once");
}

fn scratch_app() -> ScratchApp {
let home = std::env::temp_dir().join(format!("beans-sync-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&home).unwrap();
Expand Down
Loading
Loading