diff --git a/AGENTS.md b/AGENTS.md index 07ed301..a0d3a8e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,11 +26,14 @@ The design lives in `docs/design/sync.md`; the live protocols (the server's live ```text crates/ - proto craft-cloud-proto wire types: names, IDs, hashes, manifests, commit ops, API bodies, limits, join codes and join strings (`join.rs`) + api craft-cloud-api what apps need of the protocol, with no postcard: IDs, names, limits, the agent's local API (`local.rs`, `/v1/blobs` too), error codes and the account's devices (`api.rs`) + proto craft-cloud-proto wire types: hashes, manifests, commit ops, API bodies, join codes and join strings (`join.rs`), the live protocol (`live.rs`); re-exports `craft-cloud-api` under the same paths chunker craft-cloud-chunker FastCDC `fastcdc-v1` + BLAKE3, frozen by golden vectors store craft-cloud-store blob store: disk, S3, memory; checks every blob in and out meta craft-cloud-meta metadata on SQLite or Postgres; the batch commit; the change feed; invites and pairings (`join.rs`); live documents' logs and snapshots (`live.rs`) sync craft-cloud-sync the sync engine over `Fs`/`Transport` traits; `MemFs`; scenario tests and the simulation + kit craft-cloud-kit the app kit (`docs/kit.md`): an app's model live through the agent; the `LiveDocument` trait and `Session` (`session.rs`: open, the `local_change` hook, `poll`, undo through Loro's undo manager, reconnect and the 64 MiB queue, presence, saves), the change writer that cuts commits under 512 KiB (`writer.rs`), the mappings (`tree.rs`, `text.rs`, `table.rs`, `timeline.rs`, `opaque.rs`, `list.rs`), values, chunk references and blob stores (`value.rs`) + egui craft-cloud-egui Craft Cloud's piece of an app's window: the account line, the shared recents, a live document's badge, and `CloudWatch` that keeps them current app craft-cloud-app what apps link: a blocking client of the agent's local API (status, sign-in, joining, computers, pause, autostart, recents, live documents at the level of bytes: `live.rs`) apps/ craft-cloud-server HTTP API (`/api/v1`), the live service (`/api/v1/live`, `live.rs`: logs, documents by import, snapshots and compaction, presence), join codes (`join.rs`: pairings, join, the `/join/{code}` page), auth, admin CLI (`init`, `serve`, `user` with `invite`, `device`, `reset-epoch`, `live`) @@ -41,14 +44,17 @@ apps/ craft-cloud-desktop the Craft Cloud app (`craft-cloud`, eframe/egui): the pure model (`model.rs`), the agent behind a trait and calls on threads (`agent.rs`), starting the agent (`start_agent.rs`), one app per user (`single_instance.rs`), the window (`ui/`, `app.rs`), the tray (`tray/`: tray-icon in the window's process on Windows and macOS; on Linux a StatusNotifierItem over zbus in a process of its own, `craft-cloud --tray`, `tray/process.rs`, since Wayland can't hide a window), the app's own start at login (`autostart.rs`), its icons (`assets/`, drawn by `cargo xtask icons`) tests/snapshots/ every screen as kittest draws it (`UPDATE_SNAPSHOTS=1` redraws them) craft-cloud-setup the Windows setup (`craft-cloud-setup.exe`): install, upgrade, move from a hand-made install, `--uninstall`; the exes built in by `cargo xtask package` + craft-cloud-kit-demo the app kit's demo (never packaged or installed): a story and a layer tree, live through the kit; its model and adapter (`model.rs`), its window (`ui.rs`); tests drive two windows on one fake hub, and time the kit on a big document packaging/linux/ the Linux tarball's `install.sh` and `uninstall.sh` (into `~/.local`) xtask/ cargo xtask ci | licenses | icons | package | version docs/ design documents and the self-hosting guide ``` -Layering: `proto` and `chunker` depend on nothing else in the workspace. `store` and `meta` depend on `proto`. +Layering: `api` and `chunker` depend on nothing else in the workspace; `proto` depends on `api` only. What apps link +(`app`, `kit`, `egui`) depends on `api`, never on `proto`, so it brings no crate a Craft app doesn't already lock but +Loro's own (`cargo xtask suite-deps ` checks). `store` and `meta` depend on `proto`. The server depends on all three; nothing depends on the server outside tests. `sync` depends on `proto` and `chunker` -(its tests also use `meta` and `store` as an in-process server). `app` depends on `proto` only and has no async +(its tests also use `meta` and `store` as an in-process server). `app` depends on `api` only and has no async runtime. The agent depends on `proto`, `sync` and `app` (its `recents` commands are clients like an app); its tests also run the server on a loopback port, and `tests/network/` runs the server's router and agents over `turmoil` (a dev-dependency: the shipped binaries never link it). That test target includes sync's test harness diff --git a/Cargo.lock b/Cargo.lock index 8873682..7b4cbe7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "craft-cloud-agent" -version = "0.0.1" +version = "0.1.0" dependencies = [ "async-trait", "axum", @@ -1119,6 +1119,7 @@ dependencies = [ "clap", "craft-cloud-app", "craft-cloud-chunker", + "craft-cloud-kit", "craft-cloud-meta", "craft-cloud-proto", "craft-cloud-server", @@ -1153,11 +1154,24 @@ dependencies = [ "winresource", ] +[[package]] +name = "craft-cloud-api" +version = "0.1.0" +dependencies = [ + "caseless", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", + "unicode-normalization", + "uuid", +] + [[package]] name = "craft-cloud-app" -version = "0.0.1" +version = "0.1.0" dependencies = [ - "craft-cloud-proto", + "craft-cloud-api", "httparse", "serde", "serde_json", @@ -1169,7 +1183,7 @@ dependencies = [ [[package]] name = "craft-cloud-chunker" -version = "0.0.1" +version = "0.1.0" dependencies = [ "blake3", "proptest", @@ -1178,7 +1192,7 @@ dependencies = [ [[package]] name = "craft-cloud-desktop" -version = "0.0.1" +version = "0.1.0" dependencies = [ "craft-cloud-app", "craft-cloud-proto", @@ -1197,12 +1211,51 @@ dependencies = [ "zbus", ] +[[package]] +name = "craft-cloud-egui" +version = "0.1.0" +dependencies = [ + "craft-cloud-app", + "craft-cloud-kit", + "egui", + "egui_kittest", +] + +[[package]] +name = "craft-cloud-kit" +version = "0.1.0" +dependencies = [ + "blake3", + "craft-cloud-api", + "craft-cloud-app", + "getrandom 0.4.3", + "loro", + "proptest", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-cloud-kit-demo" +version = "0.1.0" +dependencies = [ + "craft-cloud-app", + "craft-cloud-egui", + "craft-cloud-kit", + "eframe", + "egui", + "egui_kittest", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "craft-cloud-meta" -version = "0.0.1" +version = "0.1.0" dependencies = [ "blake3", "craft-cloud-proto", + "serde_json", "sqlx", "tempfile", "thiserror 2.0.21", @@ -1212,10 +1265,10 @@ dependencies = [ [[package]] name = "craft-cloud-proto" -version = "0.0.1" +version = "0.1.0" dependencies = [ "blake3", - "caseless", + "craft-cloud-api", "getrandom 0.4.3", "postcard", "proptest", @@ -1228,7 +1281,7 @@ dependencies = [ [[package]] name = "craft-cloud-server" -version = "0.0.1" +version = "0.1.0" dependencies = [ "argon2", "axum", @@ -1257,7 +1310,7 @@ dependencies = [ [[package]] name = "craft-cloud-setup" -version = "0.0.1" +version = "0.1.0" dependencies = [ "rfd", "tempfile", @@ -1267,7 +1320,7 @@ dependencies = [ [[package]] name = "craft-cloud-store" -version = "0.0.1" +version = "0.1.0" dependencies = [ "async-trait", "blake3", @@ -1282,7 +1335,7 @@ dependencies = [ [[package]] name = "craft-cloud-sync" -version = "0.0.1" +version = "0.1.0" dependencies = [ "async-trait", "blake3", @@ -7167,7 +7220,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" [[package]] name = "xtask" -version = "0.0.1" +version = "0.1.0" dependencies = [ "serde_json", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index e359df9..3c65eb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/*", "apps/*", "xtask"] [workspace.package] -version = "0.0.1" +version = "0.1.0" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.94" @@ -17,6 +17,7 @@ dbg_macro = "warn" todo = "warn" [workspace.dependencies] +craft-cloud-api = { path = "crates/api" } craft-cloud-proto = { path = "crates/proto" } craft-cloud-chunker = { path = "crates/chunker" } craft-cloud-store = { path = "crates/store" } @@ -25,6 +26,8 @@ craft-cloud-server = { path = "apps/craft-cloud-server" } craft-cloud-sync = { path = "crates/sync" } craft-cloud-agent = { path = "apps/craft-cloud-agent" } craft-cloud-app = { path = "crates/app" } +craft-cloud-kit = { path = "crates/kit" } +craft-cloud-egui = { path = "crates/egui" } argon2 = "0.6" async-trait = "0.1.89" # `ws`: the server's live service and the agent's local API are WebSockets (tokio-tungstenite, MIT). diff --git a/README.md b/README.md index 7e23ee7..48d6196 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ all your machines, and the folder `~/Craft Cloud` stays in sync, with version hi There is no hosted service and no account with anyone but you. Nothing phones home. > **Status: early development.** File sync, shared recent files and the Craft Cloud app work. Live -> documents (an edit shows on your other machines as you type) work in the agent and the server; the apps take -> them up next. +> documents (an edit shows on your other machines as you type) work in the agent and the server, and the app kit +> lets a Craft app take them up with one adapter; the apps take them up next. - **Big files sync quickly.** Files are split into content-defined chunks, so a change to a 20 GB project uploads only what changed, and an interrupted upload resumes. @@ -38,6 +38,7 @@ that is signed in, and enter its code on the new one. - [Running the server](docs/server.md): reverse proxy, Postgres and S3, accounts, invites, backups - [The agent on the command line](docs/agent.md): `login`, `run`, `sync`, `join`, start at login - [Development](docs/development.md): how it fits together, building, testing, the sync engine's rules +- [The app kit](docs/kit.md): how a Craft app makes its documents live - Design: [`sync.md`](docs/design/sync.md) and [`live.md`](docs/design/live.md). Contributors and AI agents start with [`AGENTS.md`](AGENTS.md). diff --git a/apps/craft-cloud-agent/Cargo.toml b/apps/craft-cloud-agent/Cargo.toml index c67c82d..3c5008a 100644 --- a/apps/craft-cloud-agent/Cargo.toml +++ b/apps/craft-cloud-agent/Cargo.toml @@ -59,6 +59,7 @@ path = "src/main.rs" winresource.workspace = true [dev-dependencies] +craft-cloud-kit.workspace = true craft-cloud-meta = { workspace = true, features = ["test-support"] } craft-cloud-server = { workspace = true, features = ["test-support"] } craft-cloud-store.workspace = true diff --git a/apps/craft-cloud-agent/src/cli.rs b/apps/craft-cloud-agent/src/cli.rs index e1ae35a..194d471 100644 --- a/apps/craft-cloud-agent/src/cli.rs +++ b/apps/craft-cloud-agent/src/cli.rs @@ -922,6 +922,9 @@ fn show_time(ms: i64) -> String { } } +/// How soon a pass that left a Craft save waiting is followed by another. +const SAVE_WAIT_RECHECK: std::time::Duration = std::time::Duration::from_secs(2); + pub(crate) fn now_ms() -> i64 { Timestamp::now().as_millisecond() } @@ -1435,6 +1438,8 @@ impl Session { let transport = HttpTransport::new(account.server.clone(), &account.token)?; let config = Config::new(account.device_name.clone(), account.device); let mut engine = Engine::new(fs, transport, std::mem::take(&mut saved.state), config, conflict_stamp)?; + // What Craft apps saved from live documents (their local API's `saved`): see `live::saves`. + engine.set_save_records(std::sync::Arc::new(crate::live::saves::SaveBook::new(data.path(), account.space))); // `state.json` is also written at the pass's checkpoints, so a crash or a kill loses at most what the pass did // since the last one (the last pass's outcome stays as it was until this one ends). let last_pass = std::sync::Arc::new(std::sync::Mutex::new(saved.last_pass)); @@ -1496,6 +1501,15 @@ impl Session { (Ok(_), Some(stability), Some(started)) => stability.pass_finished(started), _ => None, }; + // A Craft save waiting for its record, or two whose order isn't known yet: looked at again soon (a record that + // lands asks for a pass by itself; a wait that runs out doesn't). + let waits = |why: &String| why.starts_with(craft_cloud_sync::SAVE_RECORD_WAITS) || why.starts_with(craft_cloud_sync::SAVE_ORDER_WAITS); + if let Ok(report) = &result + && report.skipped.iter().any(|(_, why)| waits(why)) + { + let soon = SAVE_WAIT_RECHECK; + self.recheck_in = Some(self.recheck_in.map_or(soon, |at| at.min(soon))); + } let (outcome, summary) = match &result { Ok(report) => { let summary = PassSummary::from_report(report); diff --git a/apps/craft-cloud-agent/src/daemon.rs b/apps/craft-cloud-agent/src/daemon.rs index ccc4315..0841eea 100644 --- a/apps/craft-cloud-agent/src/daemon.rs +++ b/apps/craft-cloud-agent/src/daemon.rs @@ -71,6 +71,15 @@ pub struct RunOptions { /// Where the local API's `autostart` puts the agent's start-at-login entry (and a sign-out removes it from): /// this user's, or nowhere / a temporary place in tests, which must not touch the machine's own. pub autostart: crate::autostart::Target, + /// Tests: how long the live hub holds every app's save record back before taking it (a slow agent, as under load). + #[doc(hidden)] + pub record_delay: Duration, + /// How long after an app session had a document open an unannounced change of its file is still waited for as a + /// Craft save on its way (`live::saves::SESSION_GRACE`, 60 s; tests shorten it). + pub session_grace: Duration, + /// Tests: how late the live hub reads every app's save announcement (a hub busy elsewhere). + #[doc(hidden)] + pub announce_delay: Duration, } impl Default for RunOptions { @@ -85,6 +94,9 @@ impl Default for RunOptions { poll_busy: Duration::from_secs(5), fail_watch: false, autostart: crate::autostart::Target::User, + record_delay: Duration::ZERO, + session_grace: crate::live::saves::SESSION_GRACE, + announce_delay: Duration::ZERO, } } } @@ -745,6 +757,9 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou token: account.token.clone(), machine: account.device_name.clone(), folder: account.folder.clone(), + record_delay: options.record_delay, + session_grace: options.session_grace, + announce_delay: options.announce_delay, }; let hub = match Hub::start(hub_config, control.clone(), user_paused) { Ok(hub) => Some(hub), diff --git a/apps/craft-cloud-agent/src/doc.rs b/apps/craft-cloud-agent/src/doc.rs index c3cb907..601ef33 100644 --- a/apps/craft-cloud-agent/src/doc.rs +++ b/apps/craft-cloud-agent/src/doc.rs @@ -105,6 +105,10 @@ fn session( }; let mirror = LoroDoc::new(); mirror.import(document.state()).map_err(|e| usage(format!("the document's state doesn't read: {e}")))?; + for update in document.state_updates() { + // (An update this copy can't take is the other side's; the agent has it, and so will a fresh open.) + let _ = mirror.import(update); + } let text = |mirror: &LoroDoc| mirror.get_text(TEXT).to_string(); // What came right after the state (updates still waiting for their history), before anything else. let take = |mirror: &LoroDoc, event: &Event| { @@ -128,7 +132,7 @@ fn session( let left = until.saturating_duration_since(Instant::now()); match document.recv_timeout(left) { Some(Event::Acked(acked)) if acked == seq => break, - Some(Event::Refused { seq: refused, message }) if refused == seq => { + Some(Event::Refused { seq: refused, message, .. }) if refused == seq => { return Err(usage(format!("the agent refused the change: {message}"))); } Some(Event::Closed(why)) => return Err(usage(format!("the connection to the agent ended: {why}"))), @@ -157,7 +161,7 @@ fn session( say(format!("the connection to the agent ended: {why}")); return Ok(Exit::Failed); } - Some(Event::Presence(_) | Event::Acked(_) | Event::Refused { .. }) | None => {} + Some(Event::Presence(_) | Event::Acked(_) | Event::Refused { .. } | Event::File(_)) | None => {} } } } diff --git a/apps/craft-cloud-agent/src/http.rs b/apps/craft-cloud-agent/src/http.rs index ddb7d82..16efff9 100644 --- a/apps/craft-cloud-agent/src/http.rs +++ b/apps/craft-cloud-agent/src/http.rs @@ -40,7 +40,7 @@ use craft_cloud_proto::api::{ }; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_CHUNK_BYTES, MAX_MANIFEST_BYTES, MAX_PAGE_NODES}; use craft_cloud_proto::local::MAX_MACHINE_BYTES; -use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, ManifestId, Op, PageCursor}; +use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, ManifestId, NodeId, Op, PageCursor, SaveRecord, VersionId}; use craft_cloud_sync::{Transport, TransportError}; use reqwest::header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE, HeaderMap, HeaderValue, RETRY_AFTER, USER_AGENT}; use reqwest::{Client, Method, StatusCode, Url}; @@ -749,6 +749,13 @@ impl Transport for HttpTransport { let req = self.request(Method::POST, "commit", MAX_SMALL_JSON_BYTES).json(&CommitRequest { ops: ops.to_vec() })?; json(self.send(req).await?, MAX_SMALL_JSON_BYTES).await } + + async fn attach_save(&self, node: NodeId, version: VersionId, save: &SaveRecord) -> Result<(), TransportError> { + let body = craft_cloud_proto::api::SaveAttach { node, version, save: save.clone() }; + let req = self.request(Method::POST, "saves", MAX_SMALL_JSON_BYTES).json(&body)?; + let _: CommitResponse = json(self.send(req).await?, MAX_SMALL_JSON_BYTES).await?; + Ok(()) + } } #[cfg(test)] diff --git a/apps/craft-cloud-agent/src/live/app_blobs.rs b/apps/craft-cloud-agent/src/live/app_blobs.rs new file mode 100644 index 0000000..ec788b9 --- /dev/null +++ b/apps/craft-cloud-agent/src/live/app_blobs.rs @@ -0,0 +1,205 @@ +//! Apps' blobs (`/v1/blobs`, [`craft_cloud_proto::local::blobs`]): binary content an app keeps out of a live +//! document, named by the manifest it cuts into and kept in `/live//blobs/`. +//! +//! - **Put**: the bytes are cut as a file's content is ([`super::blobs::cut_content`]), a `.pending` mark is +//! written, then the bytes (both atomically), and the app is answered. The upload (the chunks the server lacks, then +//! the manifest, through the chunk API) follows at once when online; the mark goes once it is done. Offline, or when +//! it fails, the mark stays and [`upload_pending`] tries again later (the local API runs it every +//! [`RETRY_EVERY`]). A mark whose bytes never got written (a crash between the two) is dropped. +//! - **Get**: from the disk, checked against its name on every read; else downloaded (each chunk checked, the whole +//! against the manifest), kept, and answered. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use craft_cloud_proto::limits::MAX_BLOB_BYTES; +use craft_cloud_proto::local::blobs::{Stored, hex, parse_hex}; +use craft_cloud_proto::{ManifestId, SpaceId}; +use craft_cloud_sync::transport::{Transport, TransportError}; + +use super::blobs::{cut_content, download_content, upload_content}; +use crate::account::write_atomic; + +/// How often the local API tries again to upload blobs put while offline. +pub const RETRY_EVERY: Duration = Duration::from_secs(30); + +/// Why a blob call failed. +#[derive(Debug, thiserror::Error)] +pub enum BlobError { + #[error("a blob is 1 byte to {MAX_BLOB_BYTES} bytes")] + Size, + #[error("not a blob's name")] + BadName, + #[error("no such blob")] + NotFound, + #[error("the server can't be reached to fetch the blob: {0}")] + Offline(String), + #[error("{0}")] + Failed(String), +} + +/// Where a space's blobs are kept. +pub fn dir(data: &Path, space: SpaceId) -> PathBuf { + data.join("live").join(space.to_string()).join("blobs") +} + +fn content_path(dir: &Path, hash: &[u8; 32]) -> PathBuf { + dir.join(hex(hash)) +} + +fn pending_path(dir: &Path, hash: &[u8; 32]) -> PathBuf { + dir.join(format!("{}.pending", hex(hash))) +} + +fn io(e: std::io::Error) -> BlobError { + BlobError::Failed(e.to_string()) +} + +/// The name `bytes` have: the ID of the manifest they cut into. +pub fn name(bytes: &[u8]) -> Result<[u8; 32], BlobError> { + if bytes.is_empty() || bytes.len() > MAX_BLOB_BYTES { + return Err(BlobError::Size); + } + let (manifest, _) = cut_content(bytes).map_err(BlobError::Failed)?; + Ok(*manifest.id().as_bytes()) +} + +/// Keeps `bytes` in `dir`, marked for upload (blocking: run it off the async threads). +pub fn put(dir: &Path, bytes: &[u8]) -> Result { + let hash = name(bytes)?; + let stored = Stored { hash, len: bytes.len() as u64 }; + std::fs::create_dir_all(dir).map_err(io)?; + let content = content_path(dir, &hash); + if content.exists() && !pending_path(dir, &hash).exists() { + // Kept already (uploaded, or downloaded): nothing to send. + return Ok(stored); + } + write_atomic(&pending_path(dir, &hash), b"", true).map_err(io)?; + if !content.exists() { + write_atomic(&content, bytes, true).map_err(io)?; + } + Ok(stored) +} + +/// The bytes named `hash` if this machine has them, checked against the name (blocking). +pub fn read_local(dir: &Path, hash: &[u8; 32]) -> Result>, BlobError> { + let bytes = match std::fs::read(content_path(dir, hash)) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(io(e)), + }; + if name(&bytes).ok().as_ref() != Some(hash) { + // A damaged copy is never served: it is fetched again. + let _ = std::fs::remove_file(content_path(dir, hash)); + return Ok(None); + } + Ok(Some(bytes)) +} + +/// Keeps a downloaded blob (no upload mark: the server has it). +pub fn keep(dir: &Path, hash: &[u8; 32], bytes: &[u8]) -> Result<(), BlobError> { + std::fs::create_dir_all(dir).map_err(io)?; + write_atomic(&content_path(dir, hash), bytes, true).map_err(io) +} + +/// The bytes named `hash`: from `dir`, else downloaded through `transport` (if any: `None` is offline) and kept. +pub async fn get(dir: PathBuf, hash: [u8; 32], transport: Option<&T>) -> Result, BlobError> { + let local_dir = dir.clone(); + let local = tokio::task::spawn_blocking(move || read_local(&local_dir, &hash)).await.map_err(|e| BlobError::Failed(e.to_string()))??; + if let Some(bytes) = local { + return Ok(bytes); + } + let Some(transport) = transport else { return Err(BlobError::Offline("not signed in".into())) }; + let bytes = match download_content(transport, &ManifestId::from_bytes(hash), MAX_BLOB_BYTES as u64).await { + Ok(bytes) => bytes, + Err(TransportError::NotFound) => return Err(BlobError::NotFound), + Err(TransportError::Unavailable(e)) => return Err(BlobError::Offline(e)), + Err(e) => return Err(BlobError::Failed(e.to_string())), + }; + let (keep_dir, kept) = (dir, bytes.clone()); + // Failing to keep it only means fetching it again next time. + let _ = tokio::task::spawn_blocking(move || keep(&keep_dir, &hash, &kept)).await; + Ok(bytes) +} + +/// The blobs in `dir` still marked for upload (a mark without bytes is dropped). +fn pending(dir: &Path) -> Vec<[u8; 32]> { + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + let mut out = Vec::new(); + for entry in entries.flatten() { + let file = entry.file_name(); + let Some(hash) = file.to_str().and_then(|n| n.strip_suffix(".pending")).and_then(parse_hex) else { continue }; + if content_path(dir, &hash).exists() { + out.push(hash); + } else { + let _ = std::fs::remove_file(entry.path()); + } + } + out +} + +/// Uploads one kept blob and drops its mark. +pub async fn upload_one(dir: &Path, hash: [u8; 32], transport: &T) -> Result<(), BlobError> { + let path = content_path(dir, &hash); + let cut = tokio::task::spawn_blocking(move || -> Result<_, BlobError> { + let bytes = std::fs::read(&path).map_err(io)?; + cut_content(&bytes).map_err(BlobError::Failed) + }) + .await + .map_err(|e| BlobError::Failed(e.to_string()))??; + let (manifest, chunks) = cut; + if *manifest.id().as_bytes() != hash { + let _ = std::fs::remove_file(content_path(dir, &hash)); + let _ = std::fs::remove_file(pending_path(dir, &hash)); + return Err(BlobError::Failed("a kept blob was damaged on disk".into())); + } + match upload_content(transport, &manifest, &chunks).await { + Ok(()) => { + let _ = std::fs::remove_file(pending_path(dir, &hash)); + Ok(()) + } + Err(TransportError::Unavailable(e)) => Err(BlobError::Offline(e)), + Err(e) => Err(BlobError::Failed(e.to_string())), + } +} + +/// Uploads every blob in `dir` still marked; returns how many went up. Stops at the first that can't (offline). +pub async fn upload_pending(dir: &Path, transport: &T) -> usize { + let scan = dir.to_owned(); + let hashes = tokio::task::spawn_blocking(move || pending(&scan)).await.unwrap_or_default(); + let mut done = 0; + for hash in hashes { + match upload_one(dir, hash, transport).await { + Ok(()) => done += 1, + Err(BlobError::Offline(_)) => break, + Err(e) => crate::runlog::note(format_args!("a blob didn't upload: {e}")), + } + } + done +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A put is kept with its mark (the mark first); the same bytes have the same name; a damaged copy is never + /// served; a mark without bytes is dropped. + #[test] + fn blobs_are_kept_named_and_checked() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path().join("blobs"); + let bytes: Vec = (0..3_000_000u32).map(|i| (i.wrapping_mul(2_654_435_761) >> 11) as u8).collect(); + let stored = put(&dir, &bytes).unwrap(); + assert_eq!(stored.len, bytes.len() as u64); + assert_eq!(put(&dir, &bytes).unwrap(), stored, "the same name every time"); + assert!(pending_path(&dir, &stored.hash).exists()); + assert_eq!(pending(&dir), vec![stored.hash]); + assert_eq!(read_local(&dir, &stored.hash).unwrap(), Some(bytes.clone())); + std::fs::write(content_path(&dir, &stored.hash), b"damaged").unwrap(); + assert_eq!(read_local(&dir, &stored.hash).unwrap(), None, "a damaged copy isn't served"); + assert!(pending(&dir).is_empty(), "its mark goes with it"); + assert!(matches!(put(&dir, &[]), Err(BlobError::Size))); + assert!(matches!(put(&dir, &vec![0; MAX_BLOB_BYTES + 1]), Err(BlobError::Size))); + assert!(put(&dir, &vec![1; MAX_BLOB_BYTES]).is_ok(), "the limit itself is fine"); + } +} diff --git a/apps/craft-cloud-agent/src/live/blobs.rs b/apps/craft-cloud-agent/src/live/blobs.rs index 822840f..b53f800 100644 --- a/apps/craft-cloud-agent/src/live/blobs.rs +++ b/apps/craft-cloud-agent/src/live/blobs.rs @@ -12,6 +12,9 @@ use craft_cloud_proto::live::MAX_SNAPSHOT_BYTES; use craft_cloud_proto::{ChunkId, ChunkRef, ChunkerParamsId, DocId, FileHash, Manifest, ManifestId}; use craft_cloud_sync::transport::{Transport, TransportError}; +/// Content's chunks, each once, with their IDs. +pub type Chunks = Vec<(ChunkId, Vec)>; + /// How many chunk IDs one "which are missing" question names. const MISSING_QUERY_IDS: usize = 1000; @@ -52,6 +55,13 @@ pub fn cut(doc: DocId, bytes: &[u8]) -> Result { if bytes.is_empty() || bytes.len() > MAX_SNAPSHOT_BYTES { return Err(format!("a snapshot is 1 byte to {MAX_SNAPSHOT_BYTES} bytes; this one is {}", bytes.len())); } + let (manifest, chunks) = cut_content(bytes)?; + Ok(Upload { doc, manifest, chunks }) +} + +/// `bytes` (not empty) cut into chunks as a file's content is (`fastcdc-v1`), each chunk once, with the manifest that +/// names them: the same bytes always give the same manifest. Snapshots and apps' blobs both go up this way. +pub fn cut_content(bytes: &[u8]) -> Result<(Manifest, Chunks), String> { let chunker = Chunker::new(FASTCDC_V1).map_err(|e| e.to_string())?; let mut chunks: Vec<(ChunkId, Vec)> = Vec::new(); let mut seen = BTreeSet::new(); @@ -66,34 +76,39 @@ pub fn cut(doc: DocId, bytes: &[u8]) -> Result { let refs = summary.chunks.iter().map(|c| ChunkRef { id: ChunkId::from_bytes(c.hash), len: c.len }).collect(); let params = ChunkerParamsId::new(FASTCDC_V1.name).map_err(|e| e.to_string())?; let manifest = Manifest::new(params, summary.size, FileHash::from_bytes(summary.file_hash), refs).map_err(|e| e.to_string())?; - Ok(Upload { doc, manifest, chunks }) + Ok((manifest, chunks)) } /// Uploads a snapshot: the chunks the server lacks, then the manifest (and, if the server says it lacks some of them /// after all, those once more). pub async fn upload(transport: &T, up: &Upload) -> Result<(), TransportError> { - let ids: Vec = up.chunks.iter().map(|(id, _)| *id).collect(); + upload_content(transport, &up.manifest, &up.chunks).await +} + +/// Uploads content cut by [`cut_content`]: the chunks the server lacks, then the manifest. +pub async fn upload_content(transport: &T, manifest: &Manifest, chunks: &[(ChunkId, Vec)]) -> Result<(), TransportError> { + let id = manifest.id(); + let ids: Vec = chunks.iter().map(|(id, _)| *id).collect(); let mut missing = BTreeSet::new(); for batch in ids.chunks(MISSING_QUERY_IDS) { missing.extend(transport.missing_chunks(batch).await?); } - for (id, data) in &up.chunks { + for (id, data) in chunks { if missing.contains(id) { transport.put_chunk(id, Bytes::from(data.clone())).await?; } } - let encoded = Bytes::from(up.manifest.encode()); - match transport.put_manifest(&up.id(), encoded.clone()).await { + let encoded = Bytes::from(manifest.encode()); + match transport.put_manifest(&id, encoded.clone()).await { Err(TransportError::MissingChunks(lost)) => { - for id in lost { - let (_, data) = up - .chunks + for lost_id in lost { + let (_, data) = chunks .iter() - .find(|(c, _)| *c == id) - .ok_or_else(|| TransportError::Protocol(format!("the server asked for chunk {id}, which the snapshot doesn't have")))?; - transport.put_chunk(&id, Bytes::from(data.clone())).await?; + .find(|(c, _)| *c == lost_id) + .ok_or_else(|| TransportError::Protocol(format!("the server asked for chunk {lost_id}, which the content doesn't have")))?; + transport.put_chunk(&lost_id, Bytes::from(data.clone())).await?; } - transport.put_manifest(&up.id(), encoded).await + transport.put_manifest(&id, encoded).await } other => other, } @@ -102,13 +117,18 @@ pub async fn upload(transport: &T, up: &Upload) -> Result /// Downloads the snapshot `id` names: its manifest (checked against `id`), then each chunk (checked against its ID and /// length), the whole checked against the manifest's size (at most [`MAX_SNAPSHOT_BYTES`]) and hash. pub async fn download(transport: &T, id: &ManifestId) -> Result, TransportError> { - let bad = |what: &str| TransportError::Protocol(format!("the snapshot {id} doesn't read: {what}")); + download_content(transport, id, MAX_SNAPSHOT_BYTES as u64).await +} + +/// Downloads the content `id` names, as [`download`], of at most `max` bytes. +pub async fn download_content(transport: &T, id: &ManifestId, max: u64) -> Result, TransportError> { + let bad = |what: &str| TransportError::Protocol(format!("the content {id} doesn't read: {what}")); let manifest = Manifest::decode(&transport.get_manifest(id).await?).map_err(|e| bad(&e.to_string()))?; if manifest.id() != *id { return Err(bad("its manifest isn't the one asked for")); } - if manifest.size() == 0 || manifest.size() > MAX_SNAPSHOT_BYTES as u64 { - return Err(bad("its size is no snapshot's")); + if manifest.size() == 0 || manifest.size() > max { + return Err(bad("its size is over the limit")); } let mut out = Vec::with_capacity(usize::try_from(manifest.size()).unwrap_or(0)); for chunk in manifest.chunks() { diff --git a/apps/craft-cloud-agent/src/live/copy.rs b/apps/craft-cloud-agent/src/live/copy.rs index 9970b0f..cd8cc72 100644 --- a/apps/craft-cloud-agent/src/live/copy.rs +++ b/apps/craft-cloud-agent/src/live/copy.rs @@ -39,6 +39,11 @@ pub type Spans = BTreeMap>; /// A version: how far each peer's operations reach. pub type Version = BTreeMap; +/// Loro frontiers of `(peer, counter)` operation IDs (`None` for a counter past Loro's). +pub fn frontiers(ids: &[(u64, u32)]) -> Option { + ids.iter().map(|(peer, counter)| Some(loro::ID::new(*peer, i32::try_from(*counter).ok()?))).collect::>>().map(loro::Frontiers::from) +} + /// The ranges of operations an update (or snapshot) holds, read from its header without importing it; `None` for /// bytes Loro can't read. pub fn spans_of(bytes: &[u8]) -> Option> { @@ -87,6 +92,27 @@ fn usable_peer(peer: u64) -> u64 { if peer == u64::MAX { peer - 1 } else { peer } } +/// The most bytes of updates one of Loro's batch imports takes ([`LiveCopy::import_batch`]). +const BATCH_GROUP_BYTES: usize = 16 * 1024; + +/// `updates` in consecutive groups of at most `max` bytes (an update bigger than that alone). +fn groups(updates: &[Vec], max: usize) -> Vec<&[Vec]> { + let mut out = Vec::new(); + let (mut start, mut size) = (0, 0usize); + for (i, u) in updates.iter().enumerate() { + if i > start && size.saturating_add(u.len()) > max { + out.extend(updates.get(start..i)); + start = i; + size = 0; + } + size = size.saturating_add(u.len()); + } + if start < updates.len() { + out.extend(updates.get(start..)); + } + out +} + /// One live document's copy on this machine. pub struct LiveCopy { pub(crate) doc: LoroDoc, @@ -186,7 +212,12 @@ impl LiveCopy { let _ = self.guard(|doc| { #[cfg(test)] Self::failing(&updates.iter().map(Vec::as_slice).collect::>()); - doc.import_batch(updates).map_err(|e| UpdateError::Malformed(e.to_string())) + // In groups of at most `BATCH_GROUP_BYTES`: Loro's batch import of much text at once takes time growing + // with its square (a 1 MB paste in pieces: seconds), while small groups keep the batch's speed. + for group in groups(updates, BATCH_GROUP_BYTES) { + doc.import_batch(group).map_err(|e| UpdateError::Malformed(e.to_string()))?; + } + Ok(()) }); Ok(self.version()? != before) } @@ -211,6 +242,24 @@ impl LiveCopy { .unwrap_or(false) } + /// Whether every operation `ids` names (`(peer, counter)`: a save's frontiers) is in this copy. + pub fn holds(&self, ids: &[(u64, u32)]) -> bool { + self.guard(|doc| { + let vv = doc.oplog_vv(); + Ok(ids.iter().all(|(peer, counter)| i32::try_from(*counter).is_ok_and(|c| vv.includes_id(loro::ID::new(*peer, c))))) + }) + .unwrap_or(false) + } + + /// Whether the document as frontiers `a` name it holds every operation of frontiers `b`; `None` when this copy + /// lacks one of their operations (it can't tell). + pub fn includes(&self, a: &[(u64, u32)], b: &[(u64, u32)]) -> Option { + let (fa, fb) = (frontiers(a)?, frontiers(b)?); + self.guard(|doc| Ok(doc.cmp_frontiers(&fa, &fb).ok().map(|o| matches!(o, Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal))))) + .ok() + .flatten() + } + /// How far each peer's operations reach in this copy. pub fn version(&self) -> Result { self.guard(|doc| Ok(doc.oplog_vv().iter().map(|(peer, end)| (*peer, *end)).collect())) @@ -472,6 +521,64 @@ mod tests { doc.export(ExportMode::updates(&before)).unwrap() } + fn heads(doc: &LoroDoc) -> Vec<(u64, u32)> { + doc.oplog_frontiers().iter().map(|id| (id.peer, id.counter as u32)).collect() + } + + /// Which of two saves is newer, from the copy's operations: one made after the other includes it, two made apart + /// include neither, and frontiers whose operations aren't here yet can't be told (none) until they are. + #[test] + fn saves_are_ordered_by_the_copys_operations() { + let (x, y) = (LoroDoc::new(), LoroDoc::new()); + x.set_peer_id(1).unwrap(); + y.set_peer_id(2).unwrap(); + let base = text_update(&x, 0, "base"); + y.import(&base).unwrap(); + let first = heads(&x); + let ux = text_update(&x, 4, " x"); + let uy = text_update(&y, 4, " y"); + let (on_x, on_y) = (heads(&x), heads(&y)); + let copy = LiveCopy::new(9); + copy.import(&base).unwrap(); + assert!(copy.holds(&first) && !copy.holds(&on_x)); + assert_eq!(copy.includes(&on_x, &first), None, "x's save isn't here yet"); + copy.import(&ux).unwrap(); + copy.import(&uy).unwrap(); + assert_eq!(copy.includes(&on_x, &first), Some(true)); + assert_eq!(copy.includes(&first, &on_x), Some(false)); + assert_eq!(copy.includes(&on_x, &on_y), Some(false), "made apart"); + assert_eq!(copy.includes(&on_y, &on_x), Some(false)); + assert_eq!(copy.includes(&on_x, &on_x), Some(true)); + let both = heads(&{ + x.import(&uy).unwrap(); + x + }); + assert_eq!(both.len(), 2, "two heads"); + assert_eq!(copy.includes(&both, &on_x), Some(true)); + assert_eq!(copy.includes(&both, &on_y), Some(true)); + } + + /// A batch is imported in small groups: a 2.5 MB paste in pieces (which Loro's batch import takes minutes for in + /// one go) comes in at once, and the groups never pass the limit but for one big update alone. + #[test] + fn a_big_paste_in_a_batch_is_imported_in_small_groups() { + let sizes = [10usize, 20_000, 3, 9_000, 9_000, 1]; + let updates: Vec> = sizes.iter().map(|n| vec![0; *n]).collect(); + let g = groups(&updates, BATCH_GROUP_BYTES); + assert_eq!(g.iter().map(|g| g.len()).collect::>(), vec![1, 1, 2, 2]); + assert!(groups(&[], BATCH_GROUP_BYTES).is_empty()); + let other = LoroDoc::new(); + other.set_peer_id(2).unwrap(); + let pieces: Vec> = (0..5).map(|i| text_update(&other, i * 500 * 1024, &"p".repeat(500 * 1024))).collect(); + let copy = LiveCopy::new(1); + let started = std::time::Instant::now(); + assert!(copy.import_batch(&pieces).unwrap()); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + let back = LoroDoc::new(); + back.import(©.snapshot().unwrap()).unwrap(); + assert_eq!(back.get_text("text").len_utf8(), 5 * 500 * 1024, "all of it"); + } + /// An update Loro refuses leaves the copy as it was (Ruling: validation without a scratch copy). #[test] fn a_refused_update_leaves_the_copy_as_it_was() { diff --git a/apps/craft-cloud-agent/src/live/docs.rs b/apps/craft-cloud-agent/src/live/docs.rs index 87e6f99..ecf67fc 100644 --- a/apps/craft-cloud-agent/src/live/docs.rs +++ b/apps/craft-cloud-agent/src/live/docs.rs @@ -125,8 +125,8 @@ async fn on_message(socket: &mut WebSocket, link: &mut Link, api: &ApiState, mes } }; match request { - DocsRequest::Open { id, path, app } => match link.hub(api).await { - Some(hub) => hub.docs(DocsCall::Open { conn: link.conn, id, path, app }).await.map(|()| true).map_err(|_| End::Hub), + DocsRequest::Open { id, path, app, file_hash } => match link.hub(api).await { + Some(hub) => hub.docs(DocsCall::Open { conn: link.conn, id, path, app, file_hash }).await.map(|()| true).map_err(|_| End::Hub), None => { let event = DocsEvent::NotLive { id, @@ -140,6 +140,16 @@ async fn on_message(socket: &mut WebSocket, link: &mut Link, api: &ApiState, mes Some(hub) => hub.docs(DocsCall::Close { conn: link.conn, doc }).await.map(|()| true).map_err(|_| End::Hub), None => Ok(true), }, + // In order with the connection's changes: the hub keeps it once the changes it covers are in. + DocsRequest::Saved { doc, hash, frontiers } => match link.hub.clone() { + Some(hub) => hub.docs(DocsCall::Saved { conn: link.conn, doc, hash, frontiers }).await.map(|()| true).map_err(|_| End::Hub), + None => Ok(true), + }, + // At once, before the app writes the file: the sync waits for that save's record. + DocsRequest::Saving { doc, hash } => match link.hub.clone() { + Some(hub) => hub.docs(DocsCall::Saving { conn: link.conn, doc, hash }).await.map(|()| true).map_err(|_| End::Hub), + None => Ok(true), + }, } } Message::Binary(bytes) => { @@ -180,8 +190,8 @@ async fn on_message(socket: &mut WebSocket, link: &mut Link, api: &ApiState, mes /// Sends one of the hub's messages to the app; false when the app is gone (or too slow). async fn send_out(socket: &mut WebSocket, out: DocOut) -> bool { match out { - DocOut::Opened { id, doc, state, snapshot, waiting } => { - send_json(socket, &DocsEvent::Opened { id, doc, state }).await && send_binary(socket, STATE, id, &snapshot).await && { + DocOut::Opened { id, doc, state, changed_outside, snapshot, waiting } => { + send_json(socket, &DocsEvent::Opened { id, doc, state, changed_outside }).await && send_binary(socket, STATE, id, &snapshot).await && { let mut ok = true; for update in &waiting { ok = ok && send_binary(socket, UPDATE, id, update).await; @@ -198,6 +208,7 @@ async fn send_out(socket: &mut WebSocket, out: DocOut) -> bool { DocOut::Status { doc, online, waiting, deleted, stopped } => { send_json(socket, &DocsEvent::Status { doc, online, waiting, deleted, stopped }).await } + DocOut::File { doc, frontiers } => send_json(socket, &DocsEvent::File { doc, frontiers }).await, } } diff --git a/apps/craft-cloud-agent/src/live/hub.rs b/apps/craft-cloud-agent/src/live/hub.rs index bb397f0..cd0e5b5 100644 --- a/apps/craft-cloud-agent/src/live/hub.rs +++ b/apps/craft-cloud-agent/src/live/hub.rs @@ -11,7 +11,7 @@ //! many changes wait, whether its file was deleted) goes to its apps when it changes. Nothing the hub sends an app ever //! waits for the app: a connection whose queue is full is dropped. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -45,6 +45,12 @@ pub struct HubConfig { pub machine: String, /// The synced folder (the paths apps open documents by are in it). pub folder: PathBuf, + /// Tests: how long every save record is held back before it is taken (a slow agent). + pub record_delay: Duration, + /// How long after a session an unannounced change is still waited for (`saves::SESSION_GRACE`). + pub session_grace: Duration, + /// Tests: how late the hub reads every save announcement (a hub busy elsewhere). + pub announce_delay: Duration, } /// How long an open of a file that isn't uploaded yet waits for its node (a sync pass is started for it). @@ -62,8 +68,9 @@ pub type ConnId = u64; /// What the hub sends a docs connection, in order. #[derive(Debug, Clone)] pub enum DocOut { - /// The document opened as `id` is live: its state (a snapshot, then updates still waiting for their history). - Opened { id: u64, doc: DocId, state: DocState, snapshot: Vec, waiting: Vec> }, + /// The document opened as `id` is live: its state (a snapshot, then updates still waiting for their history), and + /// whether the file the app read was edited outside Craft. + Opened { id: u64, doc: DocId, state: DocState, changed_outside: Option, snapshot: Vec, waiting: Vec> }, /// Send the file's content as an import. NeedsImport { id: u64 }, /// The file opens as a plain file. @@ -78,6 +85,8 @@ pub enum DocOut { Presence { id: u64, bytes: Arc> }, /// How the document stands now. Status { doc: DocId, online: bool, waiting: u64, deleted: bool, stopped: bool }, + /// The record of the document's file as it is on disk now (the live document's frontiers it was saved from). + File { doc: DocId, frontiers: Option> }, } /// What a docs connection asks the hub. @@ -93,6 +102,21 @@ pub enum DocsCall { id: u64, path: String, app: String, + /// The hash (hex) of the file's bytes as the app read them. + file_hash: Option, + }, + /// The app is about to write `doc`'s file with content `hash` (hex); its record follows. + Saving { + conn: ConnId, + doc: DocId, + hash: String, + }, + /// The app is about to write `doc`'s file: content `hash` (hex), from the document at `frontiers`. + Saved { + conn: ConnId, + doc: DocId, + hash: String, + frontiers: Vec<(u64, u32)>, }, Import { conn: ConnId, @@ -314,8 +338,41 @@ struct Docs { importer: BTreeMap, lookups: Vec, told: BTreeMap, + /// Opens not answered yet: the hash (hex) of the file each read. + opens: HashMap<(ConnId, u64), Option>, + /// Each open document's file as last looked at: its length and modification time then, its content's hash, and + /// the record its apps were last told. + files: HashMap, + /// Save records that came before the changes they cover: kept once those are in this machine's copy, dropped when + /// their connection closes the document first. + early: Vec, + /// Documents the sync asked about and this copy couldn't tell yet: a pass is asked for once operations come. + asked: BTreeSet, + /// Tests ([`HubConfig::announce_delay`]): announcements read late, and when. + late_announcements: Vec<(Instant, ConnId, DocId, String)>, } +/// A save record waiting for its operations ([`Docs::early`]). +struct EarlyRecord { + conn: ConnId, + doc: DocId, + hash: String, + frontiers: Vec<(u64, u32)>, + /// Not taken before then (tests: [`HubConfig::record_delay`]). + not_before: Instant, +} + +/// A document's file as last looked at ([`State::tell_files`]). +#[derive(Default)] +struct FileSeen { + stamp: Option<(u64, std::time::SystemTime)>, + hash: Option, + told: Option>>, +} + +/// How often the files of open documents are looked at (whether a save, here or synced from elsewhere, changed them). +const FILES_LOOK_EVERY: Duration = Duration::from_secs(1); + /// `text` cut to `max` bytes at a character boundary, without control characters. fn clean(text: &str, max: usize) -> String { let mut out = String::new(); @@ -403,6 +460,8 @@ struct State { /// Syncing is paused: no connection is dialed (see [`HubHandle::set_paused`]). paused: bool, docs: Docs, + /// What apps on this machine saved (for whether a file opened was edited outside Craft). + saves: super::saves::SaveBook, } /// A refusal sent to an app. @@ -434,6 +493,8 @@ impl State { } } let frames = self.doc_events(); + // Remote changes may be what a record waits for. + self.take_early(); self.tell_status(); for note in self.replica.take_notes() { self.say(note); @@ -483,6 +544,10 @@ impl State { } } DocEvent::Remote(doc, bytes) => { + // The sync waits to learn which of two saves is newer: ask it to look again now. + if self.docs.asked.remove(&doc) { + self.control.request_pass(); + } let bytes = Arc::new(bytes); for (conn, id) in self.handles_of(doc) { frames.extend(self.tell(conn, DocOut::Update { id, bytes: bytes.clone() })); @@ -519,12 +584,174 @@ impl State { let Some((snapshot, waiting)) = self.replica.document_state(doc) else { return self.tell(conn, DocOut::NotLive { id, reason: NotLiveReason::Offline, message: "the document can't be read here".into() }); }; + let file_hash = self.docs.opens.remove(&(conn, id)).flatten(); + let changed_outside = file_hash.and_then(|hash| self.changed_outside(doc, &hash)); let Some(c) = self.docs.conns.get_mut(&conn) else { return Vec::new() }; c.handles.insert(id, doc); self.replica.set_open(doc, true); + self.sessions_changed(&[doc]); let (online, waiting_n) = self.replica.document_status(doc).unwrap_or((false, 0)); - let state = DocState { online, waiting: waiting_n, bytes: snapshot.len() as u64 }; - self.tell(conn, DocOut::Opened { id, doc, state, snapshot, waiting }) + let state = DocState { online, waiting: waiting_n, bytes: snapshot.len() as u64, updates: waiting.len() as u64 }; + let mut frames = self.tell(conn, DocOut::Opened { id, doc, state, changed_outside, snapshot, waiting }); + // The file's record, to this open now (and to the others if it changed). + let frontiers = self.file_record(doc); + frames.extend(self.tell(conn, DocOut::File { doc, frontiers })); + self.tell_files(); + frames + } + + /// Whether the file content `hash` (hex) an app read for `doc` was edited outside Craft (see + /// [`super::saves::changed_outside`]); none when the sync doesn't know the file. + fn changed_outside(&mut self, doc: DocId, hash: &str) -> Option { + let node = self.replica.document_node(doc)?; + let saved_here = self.saves.find(doc, hash).is_some(); + let state = self.lookup.state()?; + let base = state.base.get(&node).and_then(|e| match &e.kind { + craft_cloud_sync::BaseKind::File { version, hash, .. } => Some((*version, *hash)), + craft_cloud_sync::BaseKind::Dir { .. } => None, + }); + let remote = state.remote.get(&node).and_then(|r| r.file.as_ref()); + super::saves::changed_outside(doc, hash, saved_here, base, remote) + } + + /// Tells the apps of each open document the record of its file as it is on disk now, when that changed (after + /// an open, every [`FILES_LOOK_EVERY`]). + fn tell_files(&mut self) { + let open: Vec = { + let mut docs: Vec = self.docs.conns.values().flat_map(|c| c.handles.values().copied()).collect(); + docs.sort_unstable(); + docs.dedup(); + docs + }; + self.docs.files.retain(|doc, _| open.contains(doc)); + for doc in open { + let version = self.file_record(doc); + let seen = self.docs.files.entry(doc).or_default(); + if seen.told.as_ref() == Some(&version) { + continue; + } + seen.told = Some(version.clone()); + let mut conns: Vec = self.handles_of(doc).into_iter().map(|(conn, _)| conn).collect(); + conns.dedup(); + for conn in conns { + let _ = self.tell(conn, DocOut::File { doc, frontiers: version.clone() }); + } + } + } + + /// The record of `doc`'s file as it is on disk: a save this machine's apps made of that content, or the server's + /// version's record when the file is that version. None when no record is known (or the file can't be read). + fn file_record(&mut self, doc: DocId) -> Option> { + let node = self.replica.document_node(doc)?; + let folder = self.config.folder.clone(); + let state = self.lookup.state()?; + let rel = state.base_path(node)?; + let (base, remote) = match (&state.base.get(&node)?.kind, state.remote.get(&node).and_then(|r| r.file.as_ref())) { + (craft_cloud_sync::BaseKind::File { version, hash, .. }, remote) => ((*version, *hash), remote.cloned()), + _ => return None, + }; + let path = rel.names().iter().fold(folder, |p, name| p.join(name.as_str())); + let meta = std::fs::metadata(&path).ok()?; + let stamp = Some((meta.len(), meta.modified().ok()?)); + let seen = self.docs.files.entry(doc).or_default(); + if seen.stamp != stamp || seen.hash.is_none() { + // Read again only when the file changed (an unchanged file of the agreed size is mostly the agreed + // content, but only reading it says so). + if meta.len() > MAX_SNAPSHOT_BYTES as u64 { + return None; + } + seen.hash = std::fs::read(&path).ok().map(|bytes| craft_cloud_proto::FileHash::from_bytes(*blake3::hash(&bytes).as_bytes())); + seen.stamp = stamp; + } + let hash = seen.hash?; + let hex: String = hash.as_bytes().iter().map(|b| format!("{b:02x}")).collect(); + if let Some(record) = self.saves.find(doc, &hex) { + return Some(record.frontiers); + } + let (version, agreed) = base; + let remote = remote?; + if hash != agreed || remote.version != version { + return None; + } + remote.save.filter(|s| s.doc == doc).map(|s| s.frontiers) + } + + /// An app is about to write `doc`'s file with content `hash` from the document at `frontiers`: kept in the save + /// book once every operation the frontiers name is in this machine's copy (the changes come first on the app's + /// stream; one that hasn't come yet holds the record back until it does). + fn saved(&mut self, conn: ConnId, doc: DocId, hash: String, frontiers: Vec<(u64, u32)>) -> Vec { + if let Err(message) = super::saves::SaveBook::check(&hash, &frontiers) { + return self.tell(conn, refusal(None, None, craft_cloud_proto::api::codes::INVALID, message)); + } + let not_before = Instant::now() + self.config.record_delay; + if self.config.record_delay.is_zero() && self.replica.document_holds(doc, &frontiers) { + self.keep_record(doc, &hash, frontiers); + } else { + self.docs.early.push(EarlyRecord { conn, doc, hash, frontiers, not_before }); + } + Vec::new() + } + + /// Writes a record to the save book, asks for a sync pass (a record that came after its upload is added to that + /// version), and has the file's record told again (the app took this save for the file's; a write that then + /// failed shows by the agent's next word). + fn keep_record(&mut self, doc: DocId, hash: &str, frontiers: Vec<(u64, u32)>) { + let at = crate::cli::now_ms(); + if let Err(e) = self.saves.add(doc, hash, frontiers, at) { + self.say(format!("a save record couldn't be kept: {e}")); + return; + } + self.control.request_pass(); + if let Some(seen) = self.docs.files.get_mut(&doc) { + seen.told = None; + } + } + + /// Whether app sessions have `docs` open now, told the save book and kept on disk (a restarted agent still waits + /// for saves its apps may have been making). + fn sessions_changed(&mut self, docs: &[DocId]) { + let now = std::time::Instant::now(); + for doc in docs { + let open = !self.handles_of(*doc).is_empty(); + self.saves.with_pending(|p| p.session(*doc, open, now)); + } + if let Err(e) = self.saves.keep_sessions() { + self.say(format!("which documents apps had open couldn't be written down: {e}")); + } + } + + /// An app connection announced a save: noted only of a document it has open, at most so many per connection + /// ([`super::saves::Pending`]). + fn announce(&mut self, conn: ConnId, doc: DocId, hash: &str) -> Vec { + let open = self.docs.conns.get(&conn).is_some_and(|c| c.handles.values().any(|d| *d == doc)); + if super::saves::SaveBook::check(hash, &[]).is_err() || !self.saves.with_pending(|p| p.announce(conn, doc, hash, open)) { + return self.tell(conn, refusal(None, None, craft_cloud_proto::api::codes::INVALID, "a save of a document not open here")); + } + Vec::new() + } + + /// Keeps the early records whose operations have come. + fn take_early(&mut self) { + let now = Instant::now(); + for (at, conn, doc, hash) in std::mem::take(&mut self.docs.late_announcements) { + if at <= now { + let _ = self.announce(conn, doc, &hash); + } else { + self.docs.late_announcements.push((at, conn, doc, hash)); + } + } + if self.docs.early.is_empty() { + return; + } + let early = std::mem::take(&mut self.docs.early); + let now = Instant::now(); + for r in early { + if r.not_before <= now && self.replica.document_holds(r.doc, &r.frontiers) { + self.keep_record(r.doc, &r.hash, r.frontiers); + } else { + self.docs.early.push(r); + } + } } /// A docs connection ended (or was dropped): its documents are closed, its waiting opens dropped, an import it was @@ -534,6 +761,13 @@ impl State { for doc in c.handles.values() { self.replica.set_open(*doc, false); } + // Its records still waiting for their changes: those never came (the version, if any, has no record). Its + // announcements, and its sessions, hold a grace more: it may be reconnecting. + self.docs.early.retain(|r| r.conn != conn); + self.saves.with_pending(|p| p.gone(conn, std::time::Instant::now())); + let docs: Vec = c.handles.values().copied().collect(); + self.sessions_changed(&docs); + self.docs.opens.retain(|(c, _), _| *c != conn); self.docs.lookups.retain(|l| l.conn != conn); for waiting in self.docs.waiting.values_mut() { waiting.retain(|(c, _)| *c != conn); @@ -559,7 +793,7 @@ impl State { } /// An app opens `path` as `id`. - fn open(&mut self, conn: ConnId, id: u64, path: &str, app: &str) -> Vec { + fn open(&mut self, conn: ConnId, id: u64, path: &str, app: &str, file_hash: Option) -> Vec { let not_live = |reason, message: &str| DocOut::NotLive { id, reason, message: message.to_owned() }; let Some(c) = self.docs.conns.get(&conn) else { return Vec::new() }; if c.handles.contains_key(&id) @@ -582,6 +816,7 @@ impl State { Ok(m) if m.is_file() => {} _ => return self.tell(conn, not_live(NotLiveReason::NotInFolder, "there is no such file in the synced folder")), } + self.docs.opens.insert((conn, id), file_hash); match self.lookup.file_node(&rel) { Some(node) => self.open_node(conn, id, node, &app), None => { @@ -664,7 +899,16 @@ impl State { self.docs.conns.insert(conn, DocsConn { out, handles: BTreeMap::new(), presence: HashMap::new() }); Vec::new() } - DocsCall::Open { conn, id, path, app } => self.open(conn, id, &path, &app), + DocsCall::Open { conn, id, path, app, file_hash } => self.open(conn, id, &path, &app, file_hash), + DocsCall::Saved { conn, doc, hash, frontiers } => self.saved(conn, doc, hash, frontiers), + DocsCall::Saving { conn, doc, hash } => { + if !self.config.announce_delay.is_zero() { + // Tests: a hub that reads the announcement late (busy elsewhere). + self.docs.late_announcements.push((Instant::now() + self.config.announce_delay, conn, doc, hash)); + return Vec::new(); + } + self.announce(conn, doc, &hash) + } DocsCall::Import { conn, id, bytes } => { let doc = self.docs.importer.iter().find(|(_, who)| **who == (conn, id)).map(|(doc, _)| *doc); let Some(doc) = doc else { @@ -689,6 +933,8 @@ impl State { match self.replica.local_update(doc, &bytes) { Ok(mut frames) => { frames.extend(self.tell(conn, DocOut::Ack { id, seq })); + // A record waiting for this change. + self.take_early(); // The document's other apps on this machine get it at once. let bytes = Arc::new(bytes); for (other, other_id) in self.handles_of(doc) { @@ -732,6 +978,11 @@ impl State { self.replica.set_open(doc, false); } } + // Closed before the changes its records cover came: those records are dropped, and its announcements + // with them (the session's grace still holds a while: `Pending`). + self.docs.early.retain(|r| !(r.conn == conn && r.doc == doc)); + self.saves.with_pending(|p| p.closed(conn, doc)); + self.sessions_changed(&[doc]); Vec::new() } DocsCall::Gone { conn } => self.gone(conn), @@ -928,7 +1179,14 @@ async fn run( let (blob_tx, mut blob_rx) = mpsc::unbounded_channel(); let transport = HttpTransport::new(config.server.clone(), &config.token).ok().map(Arc::new); let blobs = Blobs { transport, done: blob_tx, generation: 0, uploading: BTreeMap::new(), held: BTreeMap::new() }; + let saves = super::saves::SaveBook::new(config.data.path(), config.space); + // Documents apps had open before the agent (re)started: their files may hold saves whose records come on reconnect. + saves.with_pending(|p| p.set_grace(config.session_grace)); + saves.restore_sessions(); + // The sync engine's questions: which of two saves of a document is newer (this copy's operations say). + let (_orders, mut orders) = super::saves::Orders::serve(config.data.path(), config.space); let mut state = State { + saves, config, replica, blobs, @@ -949,6 +1207,7 @@ async fn run( let mut resend = tokio::time::interval(RESEND_EVERY); let mut look = tokio::time::interval(NODE_LOOK_EVERY); let mut write_documents = tokio::time::interval(DOCUMENTS_FLUSH_EVERY); + let mut look_files = tokio::time::interval(FILES_LOOK_EVERY); loop { if state.socket.is_none() && dialing.is_none() && !state.paused && Instant::now() >= state.retry_at { let (server, token) = (state.config.server.clone(), state.config.token.clone()); @@ -1007,6 +1266,22 @@ async fn run( Err(_) => state.back_off(None), } } + Some(order) = orders.recv() => { + let mut answer = state.replica.document_includes(order.doc, &order.a, &order.b); + if answer.is_none() { + // Not loaded, or put aside: loaded and followed (within the usual caps), asked again from what is + // on disk, and once the server's operations come a pass is asked for, to ask again soon. + let frames = state.replica.consult_document(order.doc); + answer = state.replica.document_includes(order.doc, &order.a, &order.b); + if answer.is_none() { + state.docs.asked.insert(order.doc); + } + let _ = order.reply.send(answer); + state.send(frames).await; + } else { + let _ = order.reply.send(answer); + } + } Some(done) = blob_rx.recv() => { let frames = state.blob_done(done); state.send(frames).await; @@ -1061,6 +1336,9 @@ async fn run( let frames = state.look_again(); state.send(frames).await; } + _ = look_files.tick(), if !state.docs.files.is_empty() || state.docs.conns.values().any(|c| !c.handles.is_empty()) => { + state.tell_files(); + } _ = write_documents.tick() => { if let Err(e) = state.replica.flush_documents_now_and_then() { state.say(format!("the live documents couldn't be written to disk: {e}")); diff --git a/apps/craft-cloud-agent/src/live/mod.rs b/apps/craft-cloud-agent/src/live/mod.rs index f786ee8..aff00c0 100644 --- a/apps/craft-cloud-agent/src/live/mod.rs +++ b/apps/craft-cloud-agent/src/live/mod.rs @@ -15,6 +15,7 @@ //! The local API's recents WebSocket ([`recents_socket`]) is a client of the hub like any other: an app subscribes //! to its list (the whole list comes now and after every change) and adds entries (answered once they are on disk). +pub mod app_blobs; pub mod blobs; pub mod copy; pub mod docs; @@ -22,6 +23,7 @@ pub mod hub; pub mod link; pub mod recents; pub mod replica; +pub mod saves; pub mod store; use std::sync::Arc; diff --git a/apps/craft-cloud-agent/src/live/replica.rs b/apps/craft-cloud-agent/src/live/replica.rs index af50c13..a2cfa70 100644 --- a/apps/craft-cloud-agent/src/live/replica.rs +++ b/apps/craft-cloud-agent/src/live/replica.rs @@ -1788,6 +1788,39 @@ impl Replica { Some((snapshot, after)) } + /// Whether this machine's copy of a document of a file holds every operation `ids` names (a save's frontiers). + pub fn document_holds(&self, id: DocId, ids: &[(u64, u32)]) -> bool { + self.docs.get(&id).filter(|d| d.kind.is_document()).is_some_and(|d| d.body.copy().holds(ids)) + } + + /// Whether a document of a file, as frontiers `a` name it, holds every operation of frontiers `b`; `None` when this + /// machine's copy can't tell (it lacks their operations, or isn't loaded). + pub fn document_includes(&self, id: DocId, a: &[(u64, u32)], b: &[(u64, u32)]) -> Option { + self.docs.get(&id).filter(|d| d.kind.is_document())?.body.copy().includes(a, b) + } + + /// A document of a file whose saves are asked about: loaded from disk if it isn't, and followed again if it was + /// put aside, within the same caps as an open (the least recently used make room), so its copy can tell which of + /// two saves is newer once the server's operations are in. Frames to send. + pub fn consult_document(&mut self, id: DocId) -> Vec { + self.uses += 1; + let (up, epoch, uses) = (self.link == Link::Up, self.epoch, self.uses); + let documents = self.protocol >= LIVE_DOCUMENTS_PROTOCOL; + self.load(id); + let Some(doc) = self.docs.get_mut(&id).filter(|d| d.joined && d.kind.is_document()) else { return Vec::new() }; + doc.last_used = uses; + let was_parked = std::mem::replace(&mut doc.parked, false); + let mut frames = self.unload(); + if was_parked { + frames.extend(self.evict()); + } + let Some(doc) = self.docs.get_mut(&id) else { return frames }; + if up && documents && doc.sub == Sub::No && doc.retry.is_none() { + frames.extend(subscribe(id, doc, epoch)); + } + frames + } + /// Whether a document being joined waits for an app's import (the server doesn't have it). pub fn needs_import(&self, id: DocId) -> bool { self.docs.get(&id).is_some_and(|d| d.join == Some(Join::NeedsImport)) @@ -1827,7 +1860,8 @@ impl Replica { /// returns, and sent when the link is up. Returns the frames to send. pub fn local_update(&mut self, id: DocId, bytes: &[u8]) -> Result, LocalError> { let Some(doc) = self.docs.get_mut(&id).filter(|d| d.joined && d.kind.is_document()) else { - return Err(LocalError::Invalid("the document isn't open here".into())); + // (After a disk error the copy may be being read back: for now, worth sending again.) + return Err(LocalError::Io(std::io::Error::other("the document isn't open here (being read back from disk?)"))); }; if bytes.is_empty() || bytes.len() > MAX_UPDATE_BYTES { return Err(LocalError::Invalid(format!("an update is 1 to {MAX_UPDATE_BYTES} bytes"))); diff --git a/apps/craft-cloud-agent/src/live/saves.rs b/apps/craft-cloud-agent/src/live/saves.rs new file mode 100644 index 0000000..383f119 --- /dev/null +++ b/apps/craft-cloud-agent/src/live/saves.rs @@ -0,0 +1,536 @@ +//! Save records: what Craft apps tell the agent when they save a live document's file (the content's hash and the +//! document's frontiers then, `DocsRequest::Saved`), kept in `/live//saves.json` (atomically, the +//! newest [`SAVES_PER_DOC`] of each document, [`MAX_SAVES`] in all) so a pass after a restart still has them. The sync +//! engine reads them ([`SaveRecords`]): a version it uploads with saved content carries the record, and two saved +//! versions of one document never make a conflict copy. Which of two records is newer only the live hub can say, from +//! its copy of the document ([`Orders`]). + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use craft_cloud_proto::live::node_doc; +use craft_cloud_proto::ops::MAX_SAVE_HEADS; +use craft_cloud_proto::{DocId, FileHash, NodeId, SaveRecord, SpaceId}; +use craft_cloud_sync::SaveRecords; +use serde::{Deserialize, Serialize}; + +use crate::account::write_atomic; +use crate::cli::now_ms; + +/// How many saves the book keeps of one document (newest first)… +pub const SAVES_PER_DOC: usize = 32; +/// …and in all. +pub const MAX_SAVES: usize = 4096; +/// How often the book looks whether its file changed (the engine asks for every file of every pass). +const LOOK_EVERY: Duration = Duration::from_millis(500); +/// How long the sync waits for the live hub to say which of two saves is newer. +const ORDER_WAIT: Duration = Duration::from_secs(2); + +/// One save. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct Entry { + doc: DocId, + /// The content's BLAKE3 hash, hex. + hash: String, + frontiers: Vec<(u64, u32)>, + at_ms: i64, +} + +/// Adds from several app connections at once take turns. +static WRITING: Mutex<()> = Mutex::new(()); + +/// Each book's saves possibly on their way ([`Pending`]), by book file. +static PENDING: Mutex> = Mutex::new(BTreeMap::new()); +/// How long content is waited for in all, at most, from the first time it was: no announcement renews it. +pub const MAX_PENDING_WAIT: Duration = Duration::from_secs(10 * 60); +/// How long after an app session had a document open a change to its file still counts as a Craft save on its way +/// (the session may be reconnecting, or the agent restarting), and an app connection's announcements still hold after +/// it went. +pub const SESSION_GRACE: Duration = Duration::from_secs(60); +/// Most announcements one app connection has held at once (the kit keeps 16 records); past it, the oldest go. +pub const MAX_ANNOUNCED_PER_CONN: usize = 64; + +/// Whether an app session has a document open on this machine, or since when none has. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Seen { + Open, + ClosedAt(Instant), +} + +/// Saves a Craft app may be making whose records haven't come: the sync waits for them rather than take the content +/// for an edit outside Craft ([`SaveRecords::pending`]). +/// +/// Content of a document's file counts as such a save while an app session has the document open on this machine and +/// for [`SESSION_GRACE`] after (reconnecting, or the agent restarting: the agent keeps when sessions had documents open, +/// [`Pending::sessions`]), or while a connection's announcement of it holds (at most [`MAX_ANNOUNCED_PER_CONN`] per +/// connection, only of documents it has open, for [`SESSION_GRACE`] after it went). Content waits from the first time +/// it was waited for: an announced save at most [`MAX_PENDING_WAIT`], one only a session makes likely (never announced: +/// saved while the app had no agent) at most [`SESSION_GRACE`]; never again after, and announcing renews nothing. Past +/// that the content counts as made outside Craft, as without an app. +#[derive(Debug, Default)] +pub struct Pending { + by_conn: HashMap>, + /// Announcements of connections that went, and until when they hold. + gone: HashMap<(DocId, String), Instant>, + sessions: HashMap, + /// The first time each content was waited for. + first: HashMap<(DocId, String), Instant>, + /// The grace, when not [`SESSION_GRACE`] (tests). + grace: Option, +} + +impl Pending { + /// Another grace than [`SESSION_GRACE`] (tests: the run loop's `session_grace`). + pub fn set_grace(&mut self, grace: Duration) { + self.grace = Some(grace); + } + + fn grace(&self) -> Duration { + self.grace.unwrap_or(SESSION_GRACE) + } + + /// Connection `conn` announced a save of `doc` with content `hash`; refused (false) when it hasn't `doc` open. + pub fn announce(&mut self, conn: u64, doc: DocId, hash: &str, open: bool) -> bool { + if !open { + return false; + } + let mine = self.by_conn.entry(conn).or_default(); + if !mine.iter().any(|(d, h)| *d == doc && h == hash) { + mine.push_back((doc, hash.to_owned())); + while mine.len() > MAX_ANNOUNCED_PER_CONN { + mine.pop_front(); + } + } + true + } + + /// Connection `conn` closed `doc`: its announcements of it no longer hold. + pub fn closed(&mut self, conn: u64, doc: DocId) { + if let Some(mine) = self.by_conn.get_mut(&conn) { + mine.retain(|(d, _)| *d != doc); + } + } + + /// Connection `conn` went without closing: its announcements hold [`SESSION_GRACE`] more. + pub fn gone(&mut self, conn: u64, now: Instant) { + let grace = self.grace(); + for key in self.by_conn.remove(&conn).unwrap_or_default() { + self.gone.insert(key, now + grace); + } + } + + /// Whether an app session has `doc` open on this machine now. + pub fn session(&mut self, doc: DocId, open: bool, now: Instant) { + let seen = if open { Seen::Open } else { Seen::ClosedAt(now) }; + self.sessions.insert(doc, seen); + } + + /// The record of `doc`'s content `hash` came: nothing more to wait for. + pub fn recorded(&mut self, doc: DocId, hash: &str) { + let key = (doc, hash.to_owned()); + for mine in self.by_conn.values_mut() { + mine.retain(|k| *k != key); + } + self.gone.remove(&key); + self.first.remove(&key); + } + + /// Whether content `hash` of `doc`'s file is (possibly) a Craft save on its way, to be waited for now. + pub fn is_pending(&mut self, doc: DocId, hash: &str, now: Instant) -> bool { + let (key, grace) = ((doc, hash.to_owned()), self.grace()); + let session = match self.sessions.get(&doc) { + Some(Seen::Open) => true, + Some(Seen::ClosedAt(at)) => now.saturating_duration_since(*at) < grace, + None => false, + }; + let announced = self.by_conn.values().any(|mine| mine.contains(&key)) || self.gone.get(&key).is_some_and(|until| now < *until); + if !session && !announced { + return false; + } + // (Old firsts are forgotten after a day; one that waited its time stays done until then.) + self.first.retain(|_, at| now.saturating_duration_since(*at) < Duration::from_secs(24 * 3600)); + let first = *self.first.entry(key).or_insert(now); + now.saturating_duration_since(first) < if announced { MAX_PENDING_WAIT } else { grace } + } + + /// When sessions last had documents open, as kept on disk: by document, `None` while one has it open, else when it + /// closed (ms since the epoch). + pub fn sessions(&self, now: Instant, now_ms: i64) -> BTreeMap> { + self.sessions + .iter() + .filter_map(|(doc, seen)| match seen { + Seen::Open => Some((*doc, None)), + Seen::ClosedAt(at) => { + let ago = now.saturating_duration_since(*at); + (ago < self.grace()).then(|| (*doc, Some(now_ms - i64::try_from(ago.as_millis()).unwrap_or(i64::MAX)))) + } + }) + .collect() + } + + /// What [`Pending::sessions`] kept, read back at the agent's start: one open then went at the restart. + pub fn restore(&mut self, kept: &BTreeMap>, now: Instant, now_ms: i64) { + for (doc, closed) in kept { + let ago = closed.map_or(Duration::ZERO, |ms| Duration::from_millis(u64::try_from(now_ms.saturating_sub(ms)).unwrap_or(u64::MAX))); + if ago < self.grace() { + self.sessions.insert(*doc, Seen::ClosedAt(now.checked_sub(ago).unwrap_or(now))); + } + } + } +} + +/// The books' caches, by file (shared by every [`SaveBook`] of one file in the process). +type Cache = Arc>>; +static CACHES: Mutex> = Mutex::new(BTreeMap::new()); + +/// The book as last read: when it was looked at, the file's modification time and length then, and its entries by +/// document and content. +type Read = (Instant, Option<(std::time::SystemTime, u64)>, Arc>); + +/// A space's save records. +#[derive(Debug, Clone)] +pub struct SaveBook { + path: PathBuf, + space: SpaceId, + /// Read again after an add here, or when the file changed (looked at no more often than [`LOOK_EVERY`]). + cache: Cache, +} + +/// A question for the live hub: whether `doc` as frontiers `a` name it holds every operation of frontiers `b`. The +/// answer is `None` when its copy lacks the operations (or the document isn't here). +#[derive(Debug)] +pub struct Order { + pub doc: DocId, + pub a: Vec<(u64, u32)>, + pub b: Vec<(u64, u32)>, + pub reply: std::sync::mpsc::Sender>, +} + +/// Where each running hub takes [`Order`] questions, by its save book's file (one per data folder and space). +static ORDERS: Mutex>> = Mutex::new(BTreeMap::new()); + +/// The live hub's end of [`Order`] questions: registered while it lives. +pub struct Orders { + key: PathBuf, +} + +impl Orders { + /// Takes the questions about the save book of `data` and `space` from now on (until dropped). + pub fn serve(data: &Path, space: SpaceId) -> (Self, tokio::sync::mpsc::UnboundedReceiver) { + let key = SaveBook::new(data, space).path; + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + ORDERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner).insert(key.clone(), tx); + (Self { key }, rx) + } +} + +impl Drop for Orders { + fn drop(&mut self) { + ORDERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner).remove(&self.key); + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +impl SaveBook { + pub fn new(data: &Path, space: SpaceId) -> Self { + let path = data.join("live").join(space.to_string()).join("saves.json"); + // One cache for every book of this file in the process, so an add is seen at once by the sync engine's. + let cache = CACHES.lock().unwrap_or_else(std::sync::PoisonError::into_inner).entry(path.clone()).or_default().clone(); + Self { path, space, cache } + } + + /// The entries on disk; one that doesn't read (an earlier build's form, a damaged one) is left out, the rest kept. + fn entries(&self) -> Vec { + let all: Vec = std::fs::read(&self.path).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default(); + all.into_iter().filter_map(|e| serde_json::from_value(e).ok()).collect() + } + + /// The entries by document and content, from the cache unless the file changed. + fn by_key(&self) -> Arc> { + let mut cache = self.cache.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some((at, _, map)) = cache.as_ref() + && at.elapsed() < LOOK_EVERY + { + return map.clone(); + } + let stamp = std::fs::metadata(&self.path).ok().and_then(|m| Some((m.modified().ok()?, m.len()))); + if let Some((at, seen, map)) = cache.as_mut() + && *seen == stamp + && stamp.is_some() + { + *at = Instant::now(); + return map.clone(); + } + let mut map = HashMap::new(); + // Newest first in the file: the first of a key is kept. + for e in self.entries() { + map.entry((e.doc, e.hash.clone())).or_insert(e); + } + let map = Arc::new(map); + *cache = Some((Instant::now(), stamp, map.clone())); + map + } + + /// This book's saves possibly on their way ([`Pending`]), shared by every book of its file in the process. + pub fn with_pending(&self, f: impl FnOnce(&mut Pending) -> T) -> T { + let mut all = PENDING.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + f(all.entry(self.path.clone()).or_default()) + } + + /// Where [`Pending::sessions`] is kept: next to the book. + fn sessions_path(&self) -> PathBuf { + self.path.with_file_name("sessions.json") + } + + /// Writes when sessions last had documents open (atomically), so a restarted agent still waits for their saves. + pub fn keep_sessions(&self) -> Result<(), String> { + let kept = self.with_pending(|p| p.sessions(Instant::now(), now_ms())); + let json = serde_json::to_vec(&kept).map_err(|e| e.to_string())?; + if let Some(dir) = self.path.parent() { + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + } + write_atomic(&self.sessions_path(), &json, true).map_err(|e| e.to_string()) + } + + /// Reads back what [`SaveBook::keep_sessions`] wrote (at the agent's start). + pub fn restore_sessions(&self) { + let kept: BTreeMap> = + std::fs::read(self.sessions_path()).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default(); + self.with_pending(|p| p.restore(&kept, Instant::now(), now_ms())); + } + + /// The record of `doc`'s save with content `hash` (hex), if an app on this machine saved exactly that. + pub fn find(&self, doc: DocId, hash: &str) -> Option { + self.by_key().get(&(doc, hash.to_owned())).map(|e| SaveRecord { doc, frontiers: e.frontiers.clone() }) + } + + /// Whether `hash` and `frontiers` look like a save's: 64 lowercase hex digits, and at most [`MAX_SAVE_HEADS`] + /// heads (`Err` says why). + pub fn check(hash: &str, frontiers: &[(u64, u32)]) -> Result<(), String> { + if hash.len() != 64 || !hash.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err("a save's hash is 64 lowercase hex digits".into()); + } + if frontiers.len() > MAX_SAVE_HEADS { + return Err(format!("a save's frontiers have at most {MAX_SAVE_HEADS} heads")); + } + Ok(()) + } + + /// Notes a save of `doc` with content `hash` (64 hex digits) at `frontiers`. Refused (`Err` with why) when the + /// hash or the frontiers don't look like a save's ([`SaveBook::check`]). + pub fn add(&self, doc: DocId, hash: &str, frontiers: Vec<(u64, u32)>, at_ms: i64) -> Result<(), String> { + Self::check(hash, &frontiers)?; + // Its record came: no longer on its way. + self.with_pending(|p| p.recorded(doc, hash)); + let mut frontiers = frontiers; + frontiers.sort_unstable(); + frontiers.dedup(); + let _turn = WRITING.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut entries = self.entries(); + entries.retain(|e| !(e.doc == doc && e.hash == hash)); + entries.insert(0, Entry { doc, hash: hash.to_owned(), frontiers, at_ms }); + // The newest of each document, and of all. + let mut per_doc: HashMap = HashMap::new(); + entries.retain(|e| { + let n = per_doc.entry(e.doc).or_default(); + *n += 1; + *n <= SAVES_PER_DOC + }); + entries.truncate(MAX_SAVES); + if let Some(dir) = self.path.parent() { + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + } + let json = serde_json::to_vec(&entries).map_err(|e| e.to_string())?; + let written = write_atomic(&self.path, &json, true).map_err(|e| e.to_string()); + // (A write within the file clock's resolution could keep its stamp.) + *self.cache.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + written + } +} + +/// Whether the content `hash` (hex) an app read from `doc`'s file was edited outside Craft, decided once, at the open: +/// - an app on this machine saved exactly that ([`SaveBook::find`], `saved_here`): no; +/// - it is the content the sync agreed on (`base`: version and hash) and that version is the server's: no if the +/// version carries a record of this document (an app on another machine saved it), yes if none; +/// - it is the agreed content but the server has a newer version: the sync brings that one; can't tell (none); +/// - other content (an edit here the sync hasn't taken, with no record here): yes. +/// +/// No window of recent saves: every Craft save of the file is either this machine's record or its version's. +pub fn changed_outside( + doc: DocId, + hash: &str, + saved_here: bool, + base: Option<(craft_cloud_proto::VersionId, FileHash)>, + remote: Option<&craft_cloud_proto::FileState>, +) -> Option { + if saved_here { + return Some(false); + } + let (version, agreed) = base?; + if hex(agreed.as_bytes()) != hash { + return Some(true); + } + let remote = remote?; + if remote.version != version { + return None; + } + Some(!remote.save.as_ref().is_some_and(|s| s.doc == doc)) +} + +impl SaveRecords for SaveBook { + fn record(&self, node: NodeId, hash: &FileHash) -> Option { + self.find(node_doc(self.space, node), &hex(hash.as_bytes())) + } + + fn pending(&self, node: NodeId, hash: &FileHash) -> bool { + let doc = node_doc(self.space, node); + let hash = hex(hash.as_bytes()); + self.find(doc, &hash).is_none() && self.with_pending(|p| p.is_pending(doc, &hash, Instant::now())) + } + + /// Asked of the live hub, which holds the document's operations; `None` when no hub runs, it doesn't answer in + /// time, or its copy can't tell. + fn includes(&self, a: &SaveRecord, b: &SaveRecord) -> Option { + if a.doc != b.doc { + return Some(false); + } + let hub = ORDERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&self.path).cloned()?; + let (reply, answer) = std::sync::mpsc::channel(); + hub.send(Order { doc: a.doc, a: a.frontiers.clone(), b: b.frontiers.clone(), reply }).ok()?; + answer.recv_timeout(ORDER_WAIT).ok().flatten() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn saves_are_kept_by_document_and_content() { + let temp = tempfile::tempdir().unwrap(); + let space = SpaceId::generate(); + let book = SaveBook::new(temp.path(), space); + let node = NodeId::generate(); + let doc = node_doc(space, node); + let hash = FileHash::from_bytes([7; 32]); + assert_eq!(book.record(node, &hash), None); + book.add(doc, &hex(&[7; 32]), vec![(2, 1), (1, 9)], 1).unwrap(); + assert_eq!(book.record(node, &hash), Some(SaveRecord { doc, frontiers: vec![(1, 9), (2, 1)] }), "sorted"); + assert_eq!(book.record(NodeId::generate(), &hash), None, "another file's document"); + assert!(book.add(doc, "nope", vec![], 1).is_err()); + assert!(book.add(doc, &hex(&[1; 32]), vec![(0, 0); MAX_SAVE_HEADS + 1], 1).is_err()); + // Many saves of one document keep its newest; another document's saves stay. + let other = node_doc(space, NodeId::generate()); + book.add(other, &hex(&[9; 32]), vec![(5, 5)], 1).unwrap(); + for i in 0..SAVES_PER_DOC + 5 { + book.add(doc, &hex(&[(i % 250) as u8; 32]), vec![(1, i as u32)], i as i64).unwrap(); + } + assert_eq!(book.entries().iter().filter(|e| e.doc == doc).count(), SAVES_PER_DOC); + assert!(book.find(other, &hex(&[9; 32])).is_some(), "a busy document never pushes another's out"); + let reopened = SaveBook::new(temp.path(), space); + assert!(reopened.record(node, &FileHash::from_bytes([20; 32])).is_some(), "kept on disk"); + assert!(reopened.record(node, &FileHash::from_bytes([0; 32])).is_none(), "the oldest went"); + // An entry in an earlier build's form is left out; the others still read. + let mut on_disk: Vec = serde_json::from_slice(&std::fs::read(&book.path).unwrap()).unwrap(); + on_disk.insert(0, serde_json::json!({ "doc": doc, "hash": hex(&[200; 32]), "version": [[1, 5]], "at_ms": 1 })); + std::fs::write(&book.path, serde_json::to_vec(&on_disk).unwrap()).unwrap(); + let reread = SaveBook { path: book.path.clone(), space, cache: Arc::new(Mutex::new(None)) }; + assert!(reread.record(node, &FileHash::from_bytes([20; 32])).is_some(), "the others read"); + assert!(reread.record(node, &FileHash::from_bytes([200; 32])).is_none()); + // No hub: which is newer can't be told. + let (a, b) = (SaveRecord { doc, frontiers: vec![(1, 2)] }, SaveRecord { doc, frontiers: vec![(1, 1)] }); + assert_eq!(book.includes(&a, &b), None); + } + + /// The bounds on saves possibly on their way: a session's document waits while open and a grace after; an + /// announcement holds only for a document its connection has open, at most 64 per connection (the oldest go), + /// a grace after the connection went, and no longer once closed; each content waits 10 minutes in all, from its + /// first wait, and announcing again doesn't renew it; a record ends the wait; what is kept on disk comes back. + #[test] + fn pending_saves_are_bounded() { + let t0 = Instant::now(); + let (doc, other) = (DocId::generate(), DocId::generate()); + let mut p = Pending::default(); + assert!(!p.is_pending(doc, "h", t0), "nothing known"); + // Sessions. + p.session(doc, true, t0); + assert!(p.is_pending(doc, "h", t0), "while open"); + assert!(p.is_pending(doc, "o", t0)); + assert!(!p.is_pending(doc, "o", t0 + SESSION_GRACE), "never announced: a grace at most, even while open"); + p.session(doc, false, t0); + assert!(p.is_pending(doc, "h2", t0 + SESSION_GRACE - Duration::from_secs(1)), "a grace after"); + assert!(!p.is_pending(doc, "h3", t0 + SESSION_GRACE), "not past it"); + // Announcements: only of an open document. + assert!(!p.announce(1, other, "a", false)); + assert!(!p.is_pending(other, "a", t0)); + assert!(p.announce(1, other, "a", true)); + assert!(p.is_pending(other, "a", t0)); + // At most 64 per connection: the oldest go. + for i in 0..MAX_ANNOUNCED_PER_CONN { + assert!(p.announce(1, other, &format!("n{i}"), true)); + } + assert!(!p.is_pending(other, "a", t0), "the oldest went"); + assert!(p.is_pending(other, "n63", t0)); + // Closed: they no longer hold. Gone: a grace more. + p.closed(1, other); + assert!(!p.is_pending(other, "n63", t0)); + assert!(p.announce(2, other, "g", true)); + p.gone(2, t0); + assert!(p.is_pending(other, "g", t0 + SESSION_GRACE - Duration::from_secs(1))); + assert!(!p.is_pending(other, "g", t0 + SESSION_GRACE)); + // Ten minutes in all, never renewed. + assert!(p.announce(3, other, "long", true)); + assert!(p.is_pending(other, "long", t0)); + assert!(p.is_pending(other, "long", t0 + MAX_PENDING_WAIT - Duration::from_secs(1))); + assert!(!p.is_pending(other, "long", t0 + MAX_PENDING_WAIT)); + assert!(p.announce(4, other, "long", true)); + assert!(!p.is_pending(other, "long", t0 + MAX_PENDING_WAIT + Duration::from_secs(5)), "announcing again renews nothing"); + // A record ends it. + assert!(p.announce(3, other, "r", true)); + p.recorded(other, "r"); + p.closed(3, other); + assert!(!p.is_pending(other, "r", t0)); + // Kept on disk and read back: a document open at the restart waits a grace from it. + let mut q = Pending::default(); + q.session(doc, true, t0); + let kept = q.sessions(t0, 1_000_000); + let mut back = Pending::default(); + back.restore(&kept, t0, 1_000_000); + assert!(back.is_pending(doc, "x", t0 + Duration::from_secs(1))); + assert!(!back.is_pending(doc, "y", t0 + SESSION_GRACE)); + } + + /// Whether a file was edited outside Craft: by this machine's records, or the agreed version's record, whatever + /// number of saves came before. + #[test] + fn a_file_is_changed_outside_unless_a_craft_save_has_its_content() { + use craft_cloud_proto::{DeviceId, Epoch, FileState, ManifestId, VersionId}; + let doc = DocId::generate(); + let (v, h) = (VersionId::generate(), FileHash::from_bytes([5; 32])); + let file = |version: VersionId, save: Option| FileState { + version, + manifest: ManifestId::from_bytes([0; 32]), + size: 1, + mtime_ms: 0, + exec: false, + device: DeviceId::generate(), + epoch: Epoch::generate(), + save, + }; + let ours = Some(SaveRecord { doc, frontiers: vec![(1, 2)] }); + let theirs = Some(SaveRecord { doc: DocId::generate(), frontiers: vec![(1, 2)] }); + let agreed = hex(h.as_bytes()); + assert_eq!(changed_outside(doc, "ab", true, None, None), Some(false), "saved here"); + assert_eq!(changed_outside(doc, &agreed, false, Some((v, h)), Some(&file(v, ours.clone()))), Some(false), "saved elsewhere"); + assert_eq!(changed_outside(doc, &agreed, false, Some((v, h)), Some(&file(v, None))), Some(true), "a version without a record"); + assert_eq!(changed_outside(doc, &agreed, false, Some((v, h)), Some(&file(v, theirs))), Some(true), "another document's record"); + assert_eq!(changed_outside(doc, &agreed, false, Some((v, h)), Some(&file(VersionId::generate(), ours))), None, "a newer version comes"); + assert_eq!(changed_outside(doc, "other", false, Some((v, h)), None), Some(true), "edited here"); + assert_eq!(changed_outside(doc, &agreed, false, None, None), None, "not synced"); + } +} diff --git a/apps/craft-cloud-agent/src/local_api.rs b/apps/craft-cloud-agent/src/local_api.rs index f2af1c4..3e2f8bb 100644 --- a/apps/craft-cloud-agent/src/local_api.rs +++ b/apps/craft-cloud-agent/src/local_api.rs @@ -33,10 +33,14 @@ use craft_cloud_proto::local::{ }; use tokio::sync::{mpsc, oneshot}; +use craft_cloud_proto::limits::MAX_BLOB_BYTES; +use craft_cloud_proto::local::blobs::{Stored, parse_hex}; + use crate::account::{Account, DataDir, random_hex, write_atomic}; use crate::autostart::{self, Roots}; use crate::cli::{AgentError, Exit}; use crate::http::HttpTransport; +use crate::live::app_blobs::{self, BlobError}; /// What the local API asks the `run` loop to do, with where to send the outcome. pub enum Command { @@ -253,11 +257,20 @@ pub struct ApiState { data: PathBuf, } +impl ApiState { + /// The agent's data folder. + pub fn data(&self) -> &std::path::Path { + &self.data + } +} + /// The running local API. Dropping it stops the server and removes `local-api.json`. pub struct LocalApi { port: u16, file: PathBuf, task: tokio::task::JoinHandle<()>, + /// The task that uploads blobs put while offline. + retry: tokio::task::JoinHandle<()>, shutdown: Option>, } @@ -270,6 +283,7 @@ impl LocalApi { let listener = tokio::net::TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))).await.map_err(io)?; let port = listener.local_addr().map_err(io)?.port(); let token = random_hex(32).map_err(io)?; + let retry = tokio::spawn(retry_blobs(control.clone(), data.path().to_owned())); let state = Arc::new(ApiState { port, token: token.clone(), control, data: data.path().to_owned() }); let app = router(state); let (shutdown, shutting_down) = oneshot::channel::<()>(); @@ -290,9 +304,10 @@ impl LocalApi { let json = serde_json::to_vec_pretty(&record).map_err(|e| AgentError::Io { path: file.clone(), source: std::io::Error::other(e) })?; if let Err(e) = write_atomic(&file, &json, true) { task.abort(); + retry.abort(); return Err(io(e)); } - Ok(Self { port, file, task, shutdown: Some(shutdown) }) + Ok(Self { port, file, task, retry, shutdown: Some(shutdown) }) } pub fn port(&self) -> u16 { @@ -303,6 +318,7 @@ impl LocalApi { /// [`STOP_GRACE`], and removes `local-api.json`. pub async fn stop(mut self) { let _ = std::fs::remove_file(&self.file); + self.retry.abort(); if let Some(shutdown) = self.shutdown.take() { let _ = shutdown.send(()); } @@ -316,6 +332,7 @@ const STOP_GRACE: std::time::Duration = std::time::Duration::from_secs(2); impl Drop for LocalApi { fn drop(&mut self) { self.task.abort(); + self.retry.abort(); let _ = std::fs::remove_file(&self.file); } } @@ -338,6 +355,8 @@ fn router(state: Api) -> Router { .route(paths::PAUSE, post(pause)) .route(paths::RESUME, post(resume)) .route(paths::QUIT, post(quit)) + .route(paths::BLOBS, axum::routing::put(put_blob).layer(DefaultBodyLimit::max(MAX_BLOB_BYTES))) + .route(&format!("{}/{{hash}}", paths::BLOBS), get(get_blob)) .fallback(|| async { LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such call") }) .layer(DefaultBodyLimit::max(MAX_LOCAL_JSON_BYTES)) .layer(middleware::from_fn_with_state(state.clone(), guard)) @@ -394,6 +413,59 @@ async fn logout(State(state): State, body: Result, Json ask(&state, |reply| Command::Logout(request, reply)).await } +fn blob_error(e: BlobError) -> LocalError { + let message = e.to_string(); + match e { + BlobError::Size => LocalError::new(StatusCode::PAYLOAD_TOO_LARGE, api_codes::TOO_LARGE, message), + BlobError::BadName => LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, message), + BlobError::NotFound => LocalError::new(StatusCode::NOT_FOUND, codes::BLOB_NOT_FOUND, message), + BlobError::Offline(_) => LocalError::new(StatusCode::SERVICE_UNAVAILABLE, codes::OFFLINE, message), + BlobError::Failed(_) => LocalError::new(StatusCode::INTERNAL_SERVER_ERROR, api_codes::INTERNAL, message), + } +} + +/// `PUT /v1/blobs`: keeps the body as a blob, answers its name, and uploads it (now if it can, later otherwise). +async fn put_blob( + State(state): State, + body: Result, +) -> Result, LocalError> { + let body = body.map_err(|e| LocalError::new(e.status(), api_codes::TOO_LARGE, e.body_text()))?; + let account = state.control.account().ok_or(AgentError::NotSignedIn)?; + let dir = app_blobs::dir(&state.data, account.space); + let put_dir = dir.clone(); + let stored = tokio::task::spawn_blocking(move || app_blobs::put(&put_dir, &body)) + .await + .map_err(|e| LocalError::new(StatusCode::INTERNAL_SERVER_ERROR, api_codes::INTERNAL, e.to_string()))? + .map_err(blob_error)?; + if let Ok(transport) = HttpTransport::new(account.server.clone(), &account.token) { + tokio::spawn(async move { + // This one, and any put while offline before it. Offline, they wait for the next try. + app_blobs::upload_pending(&dir, &transport).await; + }); + } + Ok(Json(stored)) +} + +/// `GET /v1/blobs/{hash}`: the blob's bytes, downloaded first if this machine doesn't have them. +async fn get_blob(State(state): State, hash: Result, PathRejection>) -> Result { + let hash = parse_hex(&path_id(hash)?).ok_or_else(|| blob_error(BlobError::BadName))?; + let account = state.control.account().ok_or(AgentError::NotSignedIn)?; + let transport = HttpTransport::new(account.server.clone(), &account.token).ok(); + let bytes = app_blobs::get(app_blobs::dir(&state.data, account.space), hash, transport.as_ref()).await.map_err(blob_error)?; + Ok(([(axum::http::header::CONTENT_TYPE, "application/octet-stream")], bytes).into_response()) +} + +/// Uploads, every [`app_blobs::RETRY_EVERY`], the blobs put while the server couldn't be reached. +async fn retry_blobs(control: Control, data: PathBuf) { + let mut every = tokio::time::interval(app_blobs::RETRY_EVERY); + loop { + every.tick().await; + let Some(account) = control.account() else { continue }; + let Ok(transport) = HttpTransport::new(account.server.clone(), &account.token) else { continue }; + app_blobs::upload_pending(&app_blobs::dir(&data, account.space), &transport).await; + } +} + async fn join(State(state): State, body: Result, JsonRejection>) -> Result, LocalError> { let Json(request) = body?; if request.join.len() > MAX_JOIN_BYTES { diff --git a/apps/craft-cloud-agent/tests/docs_common/mod.rs b/apps/craft-cloud-agent/tests/docs_common/mod.rs index d1d015c..2f00063 100644 --- a/apps/craft-cloud-agent/tests/docs_common/mod.rs +++ b/apps/craft-cloud-agent/tests/docs_common/mod.rs @@ -129,7 +129,7 @@ impl World { .unwrap(); self.meta.add_manifest(self.space, &manifest, 100, 0).await.unwrap(); let node = NodeId::generate(); - let version = NewVersion { id: VersionId::generate(), manifest: manifest.id(), mtime_ms: 0, exec: false }; + let version = NewVersion { id: VersionId::generate(), manifest: manifest.id(), mtime_ms: 0, exec: false, save: None }; let device = self.machines.first().map_or_else(craft_cloud_proto::DeviceId::generate, |m| m.auth.device); self.meta.commit(self.space, device, &[Op::Create { node, parent: self.root, name: Name::new(&name).unwrap(), version }], 0).await.unwrap(); node diff --git a/apps/craft-cloud-agent/tests/live_autosave.rs b/apps/craft-cloud-agent/tests/live_autosave.rs new file mode 100644 index 0000000..80e010c --- /dev/null +++ b/apps/craft-cloud-agent/tests/live_autosave.rs @@ -0,0 +1,407 @@ +//! Two machines with one live document open, both autosaving the way the kit says (when dirty, or when the file is +//! behind the live document), against running agents and a real server: edits made at once on both, and saves made +//! while the server is away, end with the file holding every edit on both machines, no conflict copy, and then no +//! more saves (no loop). + +mod looping; + +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use craft_cloud_kit::{Agent as KitAgent, ElemId, KitError, LiveDocument, Mappings, Open, RemotePatch, Session, TableSource, Value}; +use looping::{Agent, Server, eventually}; + +/// The one column. +const TEXT: ElemId = ElemId(1); + +/// A list of lines, a line per row. +#[derive(Clone, Debug, PartialEq, Default)] +struct Lines { + rows: Arc>, +} + +impl Lines { + fn texts(&self) -> Vec { + self.rows.iter().map(|(_, t)| t.clone()).collect() + } + + fn with(&self, text: &str) -> Self { + let mut rows = (*self.rows).clone(); + rows.push((ElemId::new(), text.to_owned())); + Self { rows: Arc::new(rows) } + } + + fn to_file(&self) -> Vec { + self.texts().join("\n").into_bytes() + } + + fn from_file(bytes: &[u8]) -> Self { + let text = String::from_utf8_lossy(bytes); + Self { rows: Arc::new(text.lines().filter(|l| !l.is_empty()).map(|l| (ElemId::new(), l.to_owned())).collect()) } + } +} + +struct Rows<'a>(&'a Lines); + +impl TableSource for Rows<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(&self.0.rows) as usize) + } + + fn rows(&self) -> Vec { + self.0.rows.iter().map(|(id, _)| *id).collect() + } + + fn columns(&self) -> Vec { + vec![TEXT] + } + + fn cell(&self, row: ElemId, column: ElemId) -> Option { + (column == TEXT).then(|| self.0.rows.iter().find(|(id, _)| *id == row).map(|(_, t)| Value::from(t.as_str()))).flatten() + } + + fn cells(&self) -> Vec<(ElemId, ElemId, Value)> { + self.0.rows.iter().map(|(id, t)| (*id, TEXT, Value::from(t.as_str()))).collect() + } +} + +struct LinesApp; + +impl LiveDocument for LinesApp { + type Doc = Lines; + const APP: &'static str = "lines"; + const SCHEMA: u32 = 1; + + fn mappings(doc: &Lines) -> Mappings<'_> { + Mappings::new().table("lines", Rows(doc)) + } + + fn apply_remote(old: &Arc, patch: &RemotePatch<'_>) -> Result, KitError> { + let Some(table) = patch.table("lines") else { return Ok(old.clone()) }; + let rows = table + .rows() + .into_iter() + .map(|row| (row, table.cell(row, TEXT).and_then(|v| v.as_str().map(str::to_owned)).unwrap_or_default())) + .collect(); + Ok(Arc::new(Lines { rows: Arc::new(rows) })) + } + + fn normalise(_doc: &mut Lines) {} + + fn refresh_derived(_doc: &mut Lines) {} +} + +const FILE: &str = "n.lines"; +/// How late every save record reaches each agent's save book. +const RECORD_DELAY_MS: u64 = 1500; + +/// What the two windows share with the test. +#[derive(Default)] +struct Shared { + stop: AtomicBool, + saves: AtomicUsize, + /// When the last save was made, in ms since the test started. + last_save_ms: AtomicU64, + /// Windows done with their first half of edits, and with all. + half_done: AtomicUsize, + edits_done: AtomicUsize, + /// The second half may start (the server is away). + second_half: AtomicBool, +} + +/// One window: opens the file live, adds `edits` lines (one every 40 ms), and autosaves (at most every 100 ms) when +/// dirty or when the file is behind, in the kit's order (save point, record, then the write), until told to stop. +/// Its model and whether its file was behind at the end. +fn window(client: craft_cloud_app::Client, folder: PathBuf, name: &'static str, edits: usize, shared: Arc, t0: Instant) -> (Lines, bool) { + let path = folder.join(FILE); + let bytes = std::fs::read(&path).unwrap(); + let agent = KitAgent::from_fn(move || Ok(client.clone())); + let mut session = match Session::::open(&agent, FILE, Arc::new(Lines::from_file(&bytes)), Some(&bytes)).unwrap() { + Open::Live(s) => *s, + Open::NotLive(why) => panic!("{name}: not live: {why:?}"), + }; + let (mut made, mut dirty, mut next_edit, mut last_save) = (0, false, Instant::now(), Instant::now() - Duration::from_secs(1)); + let tmp = folder.parent().unwrap().join(format!(".{name}.tmp")); + while !shared.stop.load(Ordering::SeqCst) { + session.poll(); + let may = made < edits / 2 || (made < edits && shared.second_half.load(Ordering::SeqCst)); + if may && Instant::now() >= next_edit { + let old = session.model().clone(); + let new = Arc::new(old.with(&format!("{name} {made}"))); + if session.local_change(&old, &new).is_ok() { + made += 1; + dirty = true; + if made == edits / 2 { + shared.half_done.fetch_add(1, Ordering::SeqCst); + } + if made == edits { + shared.edits_done.fetch_add(1, Ordering::SeqCst); + } + } + next_edit = Instant::now() + Duration::from_millis(40); + } + if (dirty || session.file_behind()) && last_save.elapsed() >= Duration::from_millis(100) { + let point = session.save_point().unwrap(); + let bytes = point.model().to_file(); + session.record(&point, &bytes); + std::fs::write(&tmp, &bytes).unwrap(); + replace(&tmp, &path); + dirty = false; + last_save = Instant::now(); + shared.saves.fetch_add(1, Ordering::SeqCst); + shared.last_save_ms.store(t0.elapsed().as_millis() as u64, Ordering::SeqCst); + } + std::thread::sleep(Duration::from_millis(5)); + } + let behind = session.file_behind(); + (session.model().as_ref().clone(), behind) +} + +/// Puts `tmp` in `path`'s place, as an app's atomic save does. On Windows the agent may be reading the file just then +/// (its sync hashing it), which refuses the replace for a moment ("access denied"): tried again, as an app tries its +/// save again. +fn replace(tmp: &std::path::Path, path: &std::path::Path) { + let mut tries = 0; + loop { + match std::fs::rename(tmp, path) { + Ok(()) => return, + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied && tries < 100 => { + tries += 1; + std::thread::sleep(Duration::from_millis(20)); + } + Err(e) => panic!("the save couldn't replace {}: {e}", path.display()), + } + } +} + +/// Every file in `folder`, by name. +fn files(folder: &std::path::Path) -> Vec<(String, Vec)> { + let mut out: Vec<(String, Vec)> = std::fs::read_dir(folder) + .unwrap() + .flatten() + .filter(|e| e.path().is_file()) + .map(|e| (e.file_name().to_string_lossy().into_owned(), std::fs::read(e.path()).unwrap())) + .collect(); + out.sort(); + out +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn both_machines_autosaving_end_with_every_edit_in_the_file_and_stop() { + let server = Server::new().await; + let (pc, laptop) = (Agent::signed_in(&server, "pc").await, Agent::signed_in(&server, "laptop").await); + // Slow agents: every save record reaches the save book late, after the app wrote its file (as under load). + // And a hub too busy to read the announcement before the sync pass sees the file (the 200 ms stability window + // here): the Windows CI failure's path. + let slow = || craft_cloud_agent::daemon::RunOptions { + record_delay: Duration::from_millis(RECORD_DELAY_MS), + announce_delay: Duration::from_millis(1000), + ..looping::quick() + }; + let (run_pc, run_laptop) = (pc.run_with(slow()), laptop.run_with(slow())); + let (on_pc, on_laptop) = (pc.client().await, laptop.client().await); + std::fs::create_dir_all(&pc.folder).unwrap(); + std::fs::write(pc.folder.join(FILE), "start").unwrap(); + let there = laptop.folder.join(FILE); + eventually("the file reached the laptop", move || there.exists().then_some(())).await; + let shared = Arc::new(Shared::default()); + // More load on demand: `CRAFT_CLOUD_TEST_LOAD=8` adds eight threads that keep the CPUs busy throughout. + let burners: usize = std::env::var("CRAFT_CLOUD_TEST_LOAD").ok().and_then(|n| n.parse().ok()).unwrap_or(0); + for _ in 0..burners { + let s = shared.clone(); + std::thread::spawn(move || { + let mut x = 0u64; + while !s.stop.load(Ordering::Relaxed) { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1); + std::hint::black_box(x); + } + }); + } + let t0 = Instant::now(); + // The pc opens first (it imports the file), then the laptop. + let (s, folder, c) = (shared.clone(), pc.folder.clone(), on_pc.clone()); + let first = std::thread::spawn(move || window(c, folder, "pc", 12, s, t0)); + tokio::time::sleep(Duration::from_millis(500)).await; + let (s, folder, c) = (shared.clone(), laptop.folder.clone(), on_laptop.clone()); + let second = std::thread::spawn(move || window(c, folder, "laptop", 12, s, t0)); + // Halfway, the server goes away: both make the rest of their edits and save them; their saves go up once it is + // back, and the later upload wins without the other's last edits in it. Nothing is edited after that: only the + // file being behind makes the other window save again. + let s = shared.clone(); + eventually("both windows made half their edits", move || (s.half_done.load(Ordering::SeqCst) == 2).then_some(())).await; + server.gate.down(); + shared.second_half.store(true, Ordering::SeqCst); + let s = shared.clone(); + eventually("both windows made their edits", move || (s.edits_done.load(Ordering::SeqCst) == 2).then_some(())).await; + tokio::time::sleep(Duration::from_millis(500)).await; + server.gate.up(); + // Settled: both files have every line (the first, and 12 from each), alike, and no save for 3 s. + let (pc_folder, laptop_folder, s) = (pc.folder.clone(), laptop.folder.clone(), shared.clone()); + let deadline = Instant::now() + Duration::from_secs(60); + loop { + // A window that stopped early (its thread panicked) says why, rather than leave the files to never settle. + assert!(!first.is_finished() && !second.is_finished(), "a window stopped early (see its panic above)"); + let (on_pc, on_laptop) = (files(&pc_folder), files(&laptop_folder)); + let full = |f: &[(String, Vec)]| f.len() == 1 && Lines::from_file(&f[0].1).rows.len() == 25; + let quiet = t0.elapsed().as_millis() as u64 > s.last_save_ms.load(Ordering::SeqCst) + 3000; + if full(&on_pc) && on_pc == on_laptop && quiet { + break; + } + let show = |f: &[(String, Vec)]| f.iter().map(|(n, d)| format!("{n}: {}", String::from_utf8_lossy(d))).collect::>(); + assert!( + Instant::now() < deadline, + "never settled: pc {:?}, laptop {:?}, {} saves", + show(&on_pc), + show(&on_laptop), + s.saves.load(Ordering::SeqCst) + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + // Still quiet a while later: no save loop. + let saves = shared.saves.load(Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(3)).await; + assert_eq!(shared.saves.load(Ordering::SeqCst), saves, "no more saves once the file has everything"); + shared.stop.store(true, Ordering::SeqCst); + let (model_pc, behind_pc) = tokio::task::spawn_blocking(move || first.join().unwrap()).await.unwrap(); + let (model_laptop, behind_laptop) = tokio::task::spawn_blocking(move || second.join().unwrap()).await.unwrap(); + assert!(!behind_pc && !behind_laptop); + let mut want = model_pc.texts(); + want.sort(); + let mut got = model_laptop.texts(); + got.sort(); + assert_eq!(want, got); + assert_eq!(want.len(), 25, "every edit: {want:?}"); + let file = Lines::from_file(&files(&pc.folder)[0].1).texts(); + assert_eq!(file, model_pc.texts(), "the file is the live document"); + // Bounded: every save is an edit's, or one to catch up after another machine's save won. + assert!(saves <= 24 + 12, "{saves} saves"); + run_pc.stop().await.0.unwrap(); + run_laptop.stop().await.0.unwrap(); +} + +/// Saves the session's model in the kit's order (save point, record, write); the bytes. +fn save_now(session: &mut Session, path: &std::path::Path) -> Vec { + let point = session.save_point().unwrap(); + let bytes = point.model().to_file(); + session.record(&point, &bytes); + let tmp = path.with_extension("tmp-save"); + std::fs::write(&tmp, &bytes).unwrap(); + replace(&tmp, path); + bytes +} + +/// A save made while the app has no agent (the agent restarts) can't be announced: the agent, back, still waits for +/// it, since an app had the document open moments ago (kept on disk), while the laptop's own Craft save of the same +/// document has gone up meanwhile. The app reconnects within its backoff and records the save: no conflict copy, and +/// the version on the server carries the record. (Before: the agent's first pass made a conflict copy.) +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_save_made_while_the_agent_was_away_is_waited_for_and_never_conflicts() { + let server = Server::new().await; + let (pc, laptop) = (Agent::signed_in(&server, "pc").await, Agent::signed_in(&server, "laptop").await); + let (run_pc, run_laptop) = (pc.run(), laptop.run()); + let (_on_pc, on_laptop) = (pc.client().await, laptop.client().await); + std::fs::create_dir_all(&pc.folder).unwrap(); + std::fs::write(pc.folder.join(FILE), "start").unwrap(); + let there = laptop.folder.join(FILE); + eventually("the file reached the laptop", move || there.exists().then_some(())).await; + // The pc's app finds its agent afresh each time (it restarts below). + let pc_data = pc.data.path().to_owned(); + let pc_agent = KitAgent::from_fn(move || craft_cloud_app::Client::connect_in(&pc_data)); + let (pc_folder, laptop_folder) = (pc.folder.clone(), laptop.folder.clone()); + let open_on = |agent: KitAgent, folder: PathBuf| { + let bytes = std::fs::read(folder.join(FILE)).unwrap(); + match Session::::open(&agent, FILE, Arc::new(Lines::from_file(&bytes)), Some(&bytes)).unwrap() { + Open::Live(s) => *s, + Open::NotLive(why) => panic!("not live: {why:?}"), + } + }; + let (a_agent, a_folder) = (pc_agent.clone(), pc_folder.clone()); + let mut a = tokio::task::spawn_blocking(move || open_on(a_agent, a_folder)).await.unwrap(); + let laptop_client = on_laptop.clone(); + let b_folder = laptop_folder.clone(); + let mut b = tokio::task::spawn_blocking(move || open_on(KitAgent::from_fn(move || Ok(laptop_client.clone())), b_folder)).await.unwrap(); + // The pc's agent goes away; the app saves meanwhile, unannounced. + run_pc.stop().await.0.unwrap(); + let (mut a, pc_saved) = tokio::task::spawn_blocking(move || { + let deadline = Instant::now() + Duration::from_secs(10); + while a.poll().status.connected { + assert!(Instant::now() < deadline, "the app saw its agent go"); + std::thread::sleep(Duration::from_millis(20)); + } + let old = a.model().clone(); + a.local_change(&old, &Arc::new(old.with("pc, while its agent was away"))).unwrap(); + let saved = save_now(&mut a, &pc_folder.join(FILE)); + (a, saved) + }) + .await + .unwrap(); + // The laptop saves its own edit, and it goes up with its record. + let (b_back, laptop_saved) = tokio::task::spawn_blocking(move || { + let old = b.model().clone(); + b.local_change(&old, &Arc::new(old.with("the laptop's"))).unwrap(); + let saved = save_now(&mut b, &laptop_folder.join(FILE)); + (b, saved) + }) + .await + .unwrap(); + let mut b = b_back; + let m = server.meta.clone(); + let space = server.meta.user_by_name("ana").await.unwrap().unwrap().space; + let up = move || { + let m = m.clone(); + async move { + let tree = m.tree(space, None, 5000).await.unwrap(); + tree.nodes.into_iter().find_map(|n| match n.kind { + craft_cloud_proto::NodeKind::File(f) if n.name.as_ref().is_some_and(|x| x.as_str() == FILE) => Some(f), + _ => None, + }) + } + }; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if let Some(f) = up().await + && f.save.is_some() + && f.size == laptop_saved.len() as u64 + { + break; + } + assert!(Instant::now() < deadline, "the laptop's save went up"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + // The pc's agent is back; its app reconnects within its backoff and records the save it made. + let run_pc = pc.run(); + let _back = pc.client().await; + let (pc_dir, laptop_dir) = (pc.folder.clone(), laptop.folder.clone()); + let deadline = Instant::now() + Duration::from_secs(40); + loop { + a = tokio::task::spawn_blocking(move || { + a.poll(); + a + }) + .await + .unwrap(); + b = tokio::task::spawn_blocking(move || { + b.poll(); + b + }) + .await + .unwrap(); + let (on_pc, on_laptop) = (files(&pc_dir), files(&laptop_dir)); + let conflict = on_pc.iter().chain(&on_laptop).any(|(n, _)| n.contains("conflict")); + assert!(!conflict, "a conflict copy of two Craft saves: pc {on_pc:?}, laptop {on_laptop:?}"); + if let Some(f) = up().await + && f.save.is_some() + && f.size == pc_saved.len() as u64 + && on_pc == on_laptop + { + break; + } + assert!(Instant::now() < deadline, "never settled: pc {on_pc:?}, laptop {on_laptop:?}"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + drop((a, b)); + run_pc.stop().await.0.unwrap(); + run_laptop.stop().await.0.unwrap(); +} diff --git a/apps/craft-cloud-agent/tests/live_documents.rs b/apps/craft-cloud-agent/tests/live_documents.rs index dc752f1..5a38074 100644 --- a/apps/craft-cloud-agent/tests/live_documents.rs +++ b/apps/craft-cloud-agent/tests/live_documents.rs @@ -33,6 +33,9 @@ impl AppDoc { let loro = LoroDoc::new(); loro.set_peer_id(peer).unwrap(); loro.import(doc.state()).unwrap(); + for u in doc.state_updates() { + loro.import(u).unwrap(); + } Self { loro, doc } } @@ -40,6 +43,13 @@ impl AppDoc { self.loro.get_text("text").to_string() } + /// The document's heads, as a save record names them. + fn heads(&self) -> Vec<(u64, u32)> { + let mut heads: Vec<(u64, u32)> = self.loro.oplog_frontiers().iter().map(|id| (id.peer, id.counter as u32)).collect(); + heads.sort_unstable(); + heads + } + /// Applies what came; the other events. fn pump(&self) -> Vec { let mut others = Vec::new(); @@ -146,6 +156,153 @@ async fn a_file_opened_on_one_machine_is_live_on_the_other() { run_laptop.stop().await.0.unwrap(); } +/// The records the agent's save book holds, by content hash (hex), as the sync reads them. +fn book(data: &std::path::Path) -> Vec<(String, Vec<(u64, u32)>)> { + let Ok(spaces) = std::fs::read_dir(data.join("live")) else { return Vec::new() }; + let mut out = Vec::new(); + for space in spaces.flatten() { + let Ok(bytes) = std::fs::read(space.path().join("saves.json")) else { continue }; + let entries: Vec = serde_json::from_slice(&bytes).unwrap_or_default(); + for e in entries { + let frontiers = serde_json::from_value(e["frontiers"].clone()).unwrap_or_default(); + out.push((e["hash"].as_str().unwrap_or_default().to_owned(), frontiers)); + } + } + out +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Asks `agent`'s save book, as the sync does, whether the document at frontiers `a` holds `b`'s. +async fn ask(agent: &Agent, doc: craft_cloud_proto::DocId, a: &[(u64, u32)], b: &[(u64, u32)]) -> Option { + use craft_cloud_sync::SaveRecords; + let data = agent.data.path().to_owned(); + let space = agent.data.load_account().unwrap().unwrap().space; + let (a, b) = (craft_cloud_proto::SaveRecord { doc, frontiers: a.to_vec() }, craft_cloud_proto::SaveRecord { doc, frontiers: b.to_vec() }); + off(move || craft_cloud_agent::live::saves::SaveBook::new(&data, space).includes(&a, &b)).await +} + +/// The real save-order path (the save book asks the live hub, which asks its copy): a later save includes an earlier +/// one and not the other way round, and operations it hasn't got can't be told. After the agent restarts with no +/// window open, the document isn't loaded: the question loads it (and follows it again) and is answered from it, +/// soon, not left unknown until the 10-minute fallback (the reviewer's probe answered unknown for good). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn save_order_questions_load_the_document_to_answer() { + let server = Server::new().await; + let pc = Agent::signed_in(&server, "pc").await; + let run_pc = pc.run(); + let on_pc = pc.client().await; + std::fs::create_dir_all(&pc.folder).unwrap(); + std::fs::write(pc.folder.join("o.txt"), "start").unwrap(); + let c = on_pc.clone(); + eventually("synced", move || { + let docs = c.documents().ok()?; + matches!(docs.open("o.txt", "wordcraft").ok()?, Opened::NeedsImport(_)).then_some(()) + }) + .await; + let c = on_pc.clone(); + let (docs, app) = off(move || { + let docs = c.documents().unwrap(); + let doc = open(&docs, "o.txt", "start"); + (docs, AppDoc::new(doc, 61)) + }) + .await; + let doc = app.doc.id(); + let acked = |app: &AppDoc, seq: u64| { + let (deadline, mut seen) = (std::time::Instant::now() + Duration::from_secs(10), false); + while !seen { + seen = app.pump().contains(&Event::Acked(seq)); + assert!(std::time::Instant::now() < deadline, "acked"); + std::thread::sleep(Duration::from_millis(20)); + } + }; + let seq = app.append(" one"); + acked(&app, seq); + let h1 = app.heads(); + let mut last = 0; + for i in 0..5 { + last = app.append(&format!(" n{i}")); + } + acked(&app, last); + let h2 = app.heads(); + assert_eq!(ask(&pc, doc, &h2, &h1).await, Some(true)); + assert_eq!(ask(&pc, doc, &h1, &h2).await, Some(false)); + assert_eq!(ask(&pc, doc, &h2, &[(999, 0)]).await, None, "operations it hasn't got"); + drop((app, docs)); + tokio::time::sleep(Duration::from_millis(300)).await; + run_pc.stop().await.0.unwrap(); + let run_pc = pc.run(); + let _on_pc = pc.client().await; + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + if ask(&pc, doc, &h2, &h1).await == Some(true) { + break; + } + assert!(std::time::Instant::now() < deadline, "never answered after the restart"); + tokio::time::sleep(Duration::from_millis(200)).await; + } + run_pc.stop().await.0.unwrap(); +} + +/// A save record that comes before the changes it covers waits for them: kept once they are in the agent's copy, +/// never before; one whose app closes the document before its changes come is dropped, and its changes arriving +/// later on another connection don't bring it back. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_save_record_waits_for_the_changes_it_covers() { + let server = Server::new().await; + let pc = Agent::signed_in(&server, "pc").await; + let run_pc = pc.run(); + let on_pc = pc.client().await; + std::fs::create_dir_all(&pc.folder).unwrap(); + std::fs::write(pc.folder.join("w.txt"), "waits").unwrap(); + let c = on_pc.clone(); + eventually("the file is synced", move || { + let docs = c.documents().ok()?; + matches!(docs.open("w.txt", "wordcraft").ok()?, Opened::NeedsImport(_)).then_some(()) + }) + .await; + let c = on_pc.clone(); + let (docs, app) = off(move || { + let docs = c.documents().unwrap(); + let doc = open(&docs, "w.txt", "waits"); + (docs, AppDoc::new(doc, 51)) + }) + .await; + // A change made, its record sent first, the change after. + let t = app.loro.get_text("text"); + let before = app.loro.oplog_vv(); + t.insert(t.len_unicode(), " and saved").unwrap(); + app.loro.commit(); + let heads = app.heads(); + app.doc.saved(*blake3::hash(b"first").as_bytes(), &heads).unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + let data = pc.data.path().to_owned(); + assert!(!book(&data).iter().any(|(h, _)| *h == hex(blake3::hash(b"first").as_bytes())), "not before its change"); + app.doc.send(&app.loro.export(ExportMode::updates(&before)).unwrap()).unwrap(); + let (d, want) = (data.clone(), (hex(blake3::hash(b"first").as_bytes()), heads.clone())); + eventually("kept once its change is in", move || book(&d).contains(&want).then_some(())).await; + // Another, whose change never comes on its connection: the app closes the document first. + let before = app.loro.oplog_vv(); + t.insert(t.len_unicode(), "!").unwrap(); + app.loro.commit(); + let update = app.loro.export(ExportMode::updates(&before)).unwrap(); + app.doc.saved(*blake3::hash(b"second").as_bytes(), &app.heads()).unwrap(); + drop((app, docs)); + let c = on_pc.clone(); + let _later = off(move || { + let docs = c.documents().unwrap(); + let doc = open(&docs, "w.txt", "waits"); + doc.send(&update).unwrap(); + std::thread::sleep(Duration::from_millis(500)); + (docs, doc) + }) + .await; + assert!(!book(&data).iter().any(|(h, _)| *h == hex(blake3::hash(b"second").as_bytes())), "dropped with its session"); + run_pc.stop().await.0.unwrap(); +} + /// Snapshots go through the chunk API: the import goes up before `create` names it, a compaction's snapshot before /// `snapshot` names it, and a machine that never had the document downloads the server's snapshot before the log /// above it. The live connection never carries one. @@ -395,14 +552,14 @@ mod raw { } } - /// The next control message, skipping binary ones and `status` events (which come whenever the document's state - /// changes, between any two answers). + /// The next control message, skipping binary ones and `status` and `file` events (which come whenever the + /// document's state or its file changes, between any two answers). pub async fn next_json(ws: &mut Ws) -> Option { loop { match next(ws).await? { Message::Text(text) => { let value: serde_json::Value = serde_json::from_str(text.as_str()).ok()?; - if value["type"] != "status" { + if value["type"] != "status" && value["type"] != "file" { return Some(value); } } @@ -595,3 +752,159 @@ async fn signed_out_every_file_is_a_plain_file() { assert!(matches!(opened, Opened::NotLive(NotLiveReason::NotSignedIn)), "{opened:?}"); running.stop().await.0.unwrap(); } + +/// Apps' blobs: one put on the pc (bigger than a chunk) is got whole on the laptop, which downloads it; the same bytes +/// have the same name everywhere; one put while the server can't be reached stays on the pc's disk and goes up with +/// the next put once it can; an unknown name is `blob_not_found`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn blobs_put_on_one_machine_are_got_on_the_other() { + let server = Server::new().await; + let pc = Agent::signed_in(&server, "pc").await; + let laptop = Agent::signed_in(&server, "laptop").await; + let (run_pc, run_laptop) = (pc.run(), laptop.run()); + let (on_pc, on_laptop) = (pc.client().await, laptop.client().await); + let big: Vec = (0..9_000_000u32).map(|i| (i.wrapping_mul(2_654_435_761) >> 9) as u8).collect(); + let (c, b) = (on_pc.clone(), big.clone()); + let stored = off(move || c.blobs().put(&b).unwrap()).await; + assert_eq!(stored.len, big.len() as u64); + let c = on_laptop.clone(); + let got = eventually("the laptop gets the pc's blob", move || c.blobs().get(&stored.hash, stored.len).ok()).await; + assert!(got == big, "the bytes came whole"); + let (c, b) = (on_laptop.clone(), big.clone()); + assert_eq!(off(move || c.blobs().put(&b).unwrap()).await, stored, "the same name on every machine"); + let c = on_laptop.clone(); + let unknown = off(move || c.blobs().get(&[7; 32], 1)).await; + assert!(matches!(&unknown, Err(craft_cloud_app::Error::Refused { code, .. }) if code == "blob_not_found"), "{unknown:?}"); + + server.gate.down(); + let c = on_pc.clone(); + let offline = off(move || c.blobs().put(b"put while offline").unwrap()).await; + server.gate.up(); + let c = on_pc.clone(); + off(move || c.blobs().put(b"the next put, online").unwrap()).await; + let c = on_laptop.clone(); + let got = eventually("the offline put went up with the next", move || c.blobs().get(&offline.hash, offline.len).ok()).await; + assert_eq!(got, b"put while offline"); + run_pc.stop().await.0.unwrap(); + run_laptop.stop().await.0.unwrap(); +} + +/// An app's save of a live document: it tells the agent (`saved`), writes the file, and the version the agent uploads +/// for that content carries the save record; a later edit by another program goes up without one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_craft_save_goes_up_with_its_record() { + let server = Server::new().await; + let pc = Agent::signed_in(&server, "pc").await; + let run_pc = pc.run(); + let on_pc = pc.client().await; + std::fs::create_dir_all(&pc.folder).unwrap(); + std::fs::write(pc.folder.join("story.txt"), "start").unwrap(); + let c = on_pc.clone(); + eventually("the file is synced", move || { + let docs = c.documents().ok()?; + matches!(docs.open("story.txt", "wordcraft").ok()?, Opened::NeedsImport(_)).then_some(()) + }) + .await; + let c = on_pc.clone(); + let (docs, app) = off(move || { + let docs = c.documents().unwrap(); + let doc = open(&docs, "story.txt", "start"); + (docs, AppDoc::new(doc, 31)) + }) + .await; + let saved = b"start, then saved by the app".to_vec(); + // The app's change, then its record (after the change, on the same stream). + app.append(", then saved by the app"); + let heads = app.heads(); + app.doc.saved(*blake3::hash(&saved).as_bytes(), &heads).unwrap(); + // The agent writes the record down before the app writes the file (the app's call comes first). + let book = pc.data.path().join("live"); + eventually("the record is kept", move || { + std::fs::read_dir(&book).ok()?.flatten().any(|space| space.path().join("saves.json").exists()).then_some(()) + }) + .await; + std::fs::write(pc.folder.join("story.txt"), &saved).unwrap(); + let node = server_node(&server, "story.txt").await.unwrap(); + let space = server.meta.user_by_name("ana").await.unwrap().unwrap().space; + let current = |meta: craft_cloud_meta::Meta| async move { + let tree = meta.tree(space, None, 5000).await.unwrap(); + tree.nodes.into_iter().find(|n| n.id == node).and_then(|n| match n.kind { + craft_cloud_proto::NodeKind::File(f) => Some(f), + craft_cloud_proto::NodeKind::Dir => None, + }) + }; + let mut record = None; + for _ in 0..600 { + if let Some(f) = current(server.meta.clone()).await + && f.save.is_some() + { + record = f.save; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + let record = record.expect("the save went up with its record"); + assert_eq!(record.doc, app.doc.id()); + assert_eq!(record.frontiers, heads); + // Another program edits it: no record. + std::fs::write(pc.folder.join("story.txt"), "edited in a text editor").unwrap(); + let mut plain = false; + for _ in 0..600 { + if let Some(f) = current(server.meta.clone()).await + && f.save.is_none() + { + plain = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(plain, "an edit outside Craft goes up without a record"); + // An open says whether the bytes the app read were a Craft save: the app's save yes, the editor's no. + let c = on_pc.clone(); + let (outside, ours) = off(move || { + let docs = c.documents().unwrap(); + let ask = |bytes: &[u8]| { + let options = craft_cloud_app::live::OpenOptions { file_hash: Some(*blake3::hash(bytes).as_bytes()) }; + match docs.open_with("story.txt", "wordcraft", &options).unwrap() { + Opened::Live(d) => d.changed_outside(), + _ => panic!("not live"), + } + }; + (ask(b"edited in a text editor"), ask(b"start, then saved by the app")) + }) + .await; + assert_eq!((outside, ours), (Some(true), Some(false))); + // A save whose record comes after its upload (made while the agent was away): added to that version. + let late = b"saved while the agent was away".to_vec(); + std::fs::write(pc.folder.join("story.txt"), &late).unwrap(); + let mut up = None; + for _ in 0..600 { + if let Some(f) = current(server.meta.clone()).await + && f.size == late.len() as u64 + { + up = Some(f); + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + let up = up.expect("the save went up"); + assert_eq!(up.save, None); + app.append(" (later)"); + let late_heads = app.heads(); + app.doc.saved(*blake3::hash(&late).as_bytes(), &late_heads).unwrap(); + let mut added = None; + for _ in 0..600 { + if let Some(f) = current(server.meta.clone()).await + && f.save.is_some() + { + added = Some(f); + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + let added = added.expect("the late record was added"); + assert_eq!(added.version, up.version, "to the version that went up"); + assert_eq!(added.save.map(|s| s.frontiers), Some(late_heads)); + drop((docs, app)); + run_pc.stop().await.0.unwrap(); +} diff --git a/apps/craft-cloud-agent/tests/live_speed.rs b/apps/craft-cloud-agent/tests/live_speed.rs index 58fb88e..5d2257e 100644 --- a/apps/craft-cloud-agent/tests/live_speed.rs +++ b/apps/craft-cloud-agent/tests/live_speed.rs @@ -100,6 +100,11 @@ async fn time_keystrokes(scan: bool) -> Vec { mine.import(doc_pc.state()).unwrap(); let theirs = LoroDoc::new(); theirs.import(doc_laptop.state()).unwrap(); + for (doc, updates) in [(&mine, doc_pc.state_updates()), (&theirs, doc_laptop.state_updates())] { + for u in updates { + doc.import(u).unwrap(); + } + } let (pc_side, laptop_side) = (Arc::new(Mutex::new((mine, doc_pc))), Arc::new(Mutex::new((theirs, doc_laptop)))); // Both online: the laptop's document followed, and the pc's caught up with its own import (the server sends it // back once it is kept). Until then the pc's keystrokes wait in its journal, and the first ones would time the diff --git a/apps/craft-cloud-agent/tests/local_api.rs b/apps/craft-cloud-agent/tests/local_api.rs index 3da573e..2839d70 100644 --- a/apps/craft-cloud-agent/tests/local_api.rs +++ b/apps/craft-cloud-agent/tests/local_api.rs @@ -60,6 +60,9 @@ fn quick() -> RunOptions { poll_min_spacing: Duration::from_millis(100), poll_busy: Duration::from_millis(200), fail_watch: false, + record_delay: Duration::ZERO, + session_grace: Duration::from_secs(10), + announce_delay: Duration::ZERO, // Never this machine's own start-at-login entry. autostart: craft_cloud_agent::autostart::Target::Off, } diff --git a/apps/craft-cloud-agent/tests/looping/mod.rs b/apps/craft-cloud-agent/tests/looping/mod.rs index 1c9bfa5..8c7969b 100644 --- a/apps/craft-cloud-agent/tests/looping/mod.rs +++ b/apps/craft-cloud-agent/tests/looping/mod.rs @@ -185,12 +185,17 @@ impl Agent { } pub fn run(&self) -> Running { + self.run_with(quick()) + } + + /// Runs with other options than [`quick`]'s. + pub fn run_with(&self, options: RunOptions) -> Running { let (stop, stops) = mpsc::unbounded_channel(); let data = self.data.clone(); let thread = std::thread::spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); let mut out = Vec::new(); - let exit = runtime.block_on(run_loop(&data, quick(), stops, &mut out)); + let exit = runtime.block_on(run_loop(&data, options, stops, &mut out)); (exit, String::from_utf8_lossy(&out).into_owned()) }); Running { stop, thread: Some(thread) } @@ -232,6 +237,10 @@ pub fn quick() -> RunOptions { poll_min_spacing: Duration::from_millis(100), poll_busy: Duration::from_millis(200), fail_watch: false, + record_delay: Duration::ZERO, + // An unannounced change of a file an app had open is waited for this long (60 s on a real machine). + session_grace: Duration::from_secs(10), + announce_delay: Duration::ZERO, // Never this machine's own start-at-login entry. autostart: craft_cloud_agent::autostart::Target::Off, } diff --git a/apps/craft-cloud-agent/tests/network/live.rs b/apps/craft-cloud-agent/tests/network/live.rs index a253bd7..ee290b1 100644 --- a/apps/craft-cloud-agent/tests/network/live.rs +++ b/apps/craft-cloud-agent/tests/network/live.rs @@ -630,7 +630,13 @@ pub fn run_seed(seed: u64) -> Vec { meta.add_manifest(user.space, &manifest, 100, 0).await.unwrap(); let root = meta.space_info(user.space).await.unwrap().root; let node = NodeId::from_uuid(uuid::Uuid::from_u64_pair(seed, 7)); - let version = NewVersion { id: VersionId::from_uuid(uuid::Uuid::from_u64_pair(seed, 8)), manifest: manifest.id(), mtime_ms: 0, exec: false }; + let version = NewVersion { + id: VersionId::from_uuid(uuid::Uuid::from_u64_pair(seed, 8)), + manifest: manifest.id(), + mtime_ms: 0, + exec: false, + save: None, + }; meta.commit(user.space, auths[0].device, &[Op::Create { node, parent: root, name: Name::new("doc.txt").unwrap(), version }], 0) .await .unwrap(); diff --git a/apps/craft-cloud-agent/tests/network/main.rs b/apps/craft-cloud-agent/tests/network/main.rs index 049b2f7..cc0ecb5 100644 --- a/apps/craft-cloud-agent/tests/network/main.rs +++ b/apps/craft-cloud-agent/tests/network/main.rs @@ -87,6 +87,122 @@ struct Driver { at_crash: Vec>, seen: BTreeSet, destroyed: BTreeSet, + /// Craft saves of the live-linked file [`LIVE_FILE`] (see [`Driver::craft_save`]). + craft: CraftSaves, +} + +/// The live-linked file Craft apps save on every machine. +const LIVE_FILE: &str = "live.kitdemo"; + +/// The simulation's Craft apps: each machine's save book (what its apps told its agent they saved), and what each +/// machine's live document knows of every machine's edits (`knows[i][j]`: machine `i` has machine `j`'s edits up to +/// that count). A save writes the file with a record of the document's version on that machine. +struct CraftSaves { + rng: Rng, + doc: craft_cloud_proto::DocId, + books: Vec, + knows: Vec>, + saves: u32, + /// Records on their way to their save books ([`Driver::deliver_late`]). + late: Vec, +} + +/// A machine's save records. Their frontiers stand for version vectors here, compared as vectors (the agent asks its +/// copy of the document). The first time a pair is asked, the answer is "can't tell yet", as an agent that lacks the +/// operations says: the files wait a pass. +/// The third part: saves announced whose records are late (a slow agent), waited for by the sync. +#[derive(Default, Clone)] +struct CraftBook( + Arc>>, + Arc>>, + Arc>>, +); + +/// Two records' frontiers asked about. +type AskedPair = (Vec<(u64, u32)>, Vec<(u64, u32)>); + +impl craft_cloud_sync::SaveRecords for CraftBook { + fn record(&self, node: NodeId, hash: &FileHash) -> Option { + self.0.lock().unwrap().get(&(node, *hash)).cloned() + } + + fn includes(&self, a: &craft_cloud_proto::SaveRecord, b: &craft_cloud_proto::SaveRecord) -> Option { + if self.1.lock().unwrap().insert((a.frontiers.clone(), b.frontiers.clone())) { + return None; + } + Some(b.frontiers.iter().all(|(peer, end)| a.frontiers.iter().any(|(p, e)| p == peer && e >= end))) + } + + fn pending(&self, node: NodeId, hash: &FileHash) -> bool { + self.2.lock().unwrap().contains(&(node, *hash)) + } +} + +/// A save record on its way (machine, file and content, the record, steps until it reaches the save book). +type LateRecord = (usize, (NodeId, FileHash), craft_cloud_proto::SaveRecord, usize); + +impl Driver { + /// Late save records arrive: those whose steps ran out, or all (`all`, before the network heals). + fn deliver_late(&mut self, all: bool) { + let c = &mut self.craft; + for late in std::mem::take(&mut c.late) { + let (i, key, record, left) = late; + if all || left <= 1 { + if let Some(book) = c.books.get(i) { + book.2.lock().unwrap().remove(&key); + book.0.lock().unwrap().insert(key, record); + } + } else { + c.late.push((i, key, record, left - 1)); + } + } + } + + /// A Craft app on machine `i` saves [`LIVE_FILE`] (once the machine tracks it): its live document first takes + /// another machine's edits now and then (live updates arriving), then gets one of its own, and the save records + /// that version. The file's earlier content and this save's may both vanish (the live document holds them, and a + /// save that includes them, or a newer upload, replaces them), so both count as overwritten. + fn craft_save(&mut self, i: usize) { + let Some(m) = self.slots.get_mut(i).and_then(Option::as_mut) else { return }; + let target = common::path(LIVE_FILE); + let Some(node) = m.state().base.keys().copied().find(|id| m.state().base_path(*id).as_ref() == Some(&target)) else { return }; + let c = &mut self.craft; + let from = c.rng.below(MACHINES); + if c.rng.below(2) == 0 + && from != i + && let Some(theirs) = c.knows.get(from).cloned() + && let Some(mine) = c.knows.get_mut(i) + { + for (k, n) in mine.iter_mut().zip(theirs) { + *k = (*k).max(n); + } + } + let Some(mine) = c.knows.get_mut(i) else { return }; + if let Some(own) = mine.get_mut(i) { + *own += 1; + } + let version: Vec<(u64, u32)> = mine.iter().enumerate().filter(|(_, n)| **n > 0).map(|(j, n)| (j as u64 + 1, *n)).collect(); + c.saves += 1; + let data = format!("craft save {} by m{i}: {version:?}", c.saves).into_bytes(); + let before = m.fs().files().get(&target).map(|d| FileHash::from_bytes(*blake3::hash(d).as_bytes())); + if m.fs_mut().write(&target, &data, false, 0).is_err() { + return; + } + let hash = FileHash::from_bytes(*blake3::hash(&data).as_bytes()); + self.destroyed.extend(before); + self.destroyed.insert(hash); + let record = craft_cloud_proto::SaveRecord { doc: c.doc, frontiers: version }; + if let Some(book) = c.books.get(i) { + // Half the records are slow to reach the agent (announced at once, the record some steps later). + if c.rng.below(2) == 0 { + book.2.lock().unwrap().insert((node, hash)); + c.late.push((i, (node, hash), record, 1 + c.rng.below(6))); + } else { + book.0.lock().unwrap().insert((node, hash), record); + } + } + self.net.event(format!("{} m{i} craft save {}", stamp(), c.saves)); + } } /// What a pass did, without node IDs (they are random), for the trace. @@ -172,10 +288,18 @@ impl Driver { let mut writes = Rng::new(seed ^ 0x57AB_1E5E_ED00_0001); let mut extra = Rng::new(seed ^ 0xC04B_0ED0_0000_0002); let mut moments = Rng::new(seed ^ 0x7177_0000_0000_0005); + // The live-linked file starts on the first machine. + if let Some(m) = self.slots.first_mut().and_then(Option::as_mut) { + let _ = m.fs_mut().write(&common::path(LIVE_FILE), b"live document, first save", false, 0); + } for step in 0..steps { tokio::time::sleep(Duration::from_millis(moments.below(2_000) as u64)).await; let i = rng.below(MACHINES); self.join(i).await; + self.deliver_late(false); + if self.craft.rng.below(4) == 0 { + self.craft_save(i); + } let Some(m) = self.slots.get_mut(i).and_then(Option::as_mut) else { continue }; if writes.below(10) == 0 && let Some(f) = writes.pick(&files_of(m)).cloned() @@ -237,6 +361,16 @@ impl Driver { settle_and_check(self.seed, &self.net.server, &mut ms, out).await; for (i, m) in ms.iter().enumerate() { self.net.event(format!("m{i} ends with {}", listing(m))); + // Craft saves of one live document never make a conflict copy, however they crossed. (The first content + // is a plain file, written before the document was live: it may, as any file may.) + let copies: Vec = m + .fs() + .files() + .into_iter() + .filter(|(p, data)| p.to_string().starts_with("live (conflict") && data.starts_with(b"craft save")) + .map(|(p, _)| p.to_string()) + .collect(); + assert!(copies.is_empty(), "seed {}: m{i} has conflict copies of Craft saves of the live file: {copies:?}", self.seed); } } } @@ -249,7 +383,19 @@ fn run_seed(seed: u64, steps: usize) -> Vec { let latency = [2, 20, 100, 400][choose.below(4)]; let stormy = choose.below(4) != 0; let (lose, cut) = ([0, 6, 25][choose.below(3)], [0, 6, 25][choose.below(3)]); - let (net, machines) = world(seed, &Setup { page, latency, lose, cut, password: None }); + let (net, mut machines) = world(seed, &Setup { page, latency, lose, cut, password: None }); + let books: Vec = (0..MACHINES).map(|_| CraftBook::default()).collect(); + for (m, book) in machines.iter_mut().zip(&books) { + m.set_save_records(Arc::new(book.clone())); + } + let craft = CraftSaves { + rng: Rng::new(seed ^ 0xC2AF_75A7_E000_0009), + doc: craft_cloud_proto::DocId::from_uuid(uuid::Uuid::from_u64_pair(seed, 0x11fe)), + books, + knows: vec![vec![0; MACHINES]; MACHINES], + saves: 0, + late: Vec::new(), + }; net.event(format!("seed {seed}: pages of {page}, latency up to {latency} ms, stormy {stormy}, lose 1/{lose}, cut 1/{cut}")); let driving = net.clone(); simulate(seed, latency, &net, async move { @@ -275,8 +421,11 @@ fn run_seed(seed: u64, steps: usize) -> Vec { at_crash: vec![BTreeSet::new(); MACHINES], seen: BTreeSet::new(), destroyed: BTreeSet::new(), + craft, }; driver.run(steps).await; + // Every late record arrives. + driver.deliver_late(true); // The network heals and the server's faults stop. calm.store(true, Ordering::Relaxed); for i in 0..MACHINES { diff --git a/apps/craft-cloud-agent/tests/old_agent_blobs.rs b/apps/craft-cloud-agent/tests/old_agent_blobs.rs new file mode 100644 index 0000000..0aab215 --- /dev/null +++ b/apps/craft-cloud-agent/tests/old_agent_blobs.rs @@ -0,0 +1,56 @@ +//! An agent from before `/v1/blobs` (as it serves: axum, a 404 fallback that doesn't read the body, the 1 MiB body +//! limit), and the app client putting blobs of every size: always `agent_too_old`, never a broken connection. +//! (The review's probe, kept as a test.) + +use axum::Router; +use axum::extract::DefaultBodyLimit; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::routing::get; +use craft_cloud_proto::local::{LOCAL_API_FILE, LOCAL_API_MAX, LocalApiFile, RunState, Status}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_old_agent_is_too_old_for_blobs_of_any_size() { + let status = Status { + api: LOCAL_API_MAX, + signed_in: true, + account: Some("a".into()), + server: Some("https://x.invalid".into()), + folder: None, + machine: Some("m".into()), + online: true, + run_state: RunState::Idle, + autostart: None, + last_sync_ms: None, + removed: false, + }; + let app = Router::new() + .route("/v1/status", get(move || async move { axum::Json(status.clone()) })) + .fallback(|| async { + (StatusCode::NOT_FOUND, axum::Json(serde_json::json!({ "error": "not_found", "message": "no such call" }))).into_response() + }) + .layer(DefaultBodyLimit::max(1024 * 1024)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let dir = tempfile::tempdir().unwrap(); + let file = LocalApiFile { schema: craft_cloud_proto::local::LOCAL_API_SCHEMA, port, token: "t".into(), pid: 1 }; + std::fs::write(dir.path().join(LOCAL_API_FILE), serde_json::to_vec(&file).unwrap()).unwrap(); + let path = dir.path().to_owned(); + let results = tokio::task::spawn_blocking(move || { + let client = craft_cloud_app::Client::connect_in(&path).unwrap(); + let mut out = Vec::new(); + for len in [10usize, 512 * 1024, 5 * 1024 * 1024, 32 * 1024 * 1024] { + // A fresh client each time: the first question isn't remembered from the last. + let fresh = craft_cloud_app::Client::connect_in(&path).unwrap(); + out.push((len, fresh.blobs().put(&vec![7u8; len]))); + } + out.push((0, client.blobs().get(&[1; 32], 3).map(|_| craft_cloud_app::blobs::Stored { hash: [0; 32], len: 0 }))); + out + }) + .await + .unwrap(); + for (len, r) in results { + assert!(matches!(&r, Err(craft_cloud_app::Error::Refused { code, .. }) if code == "agent_too_old"), "{len} bytes: {r:?}"); + } +} diff --git a/apps/craft-cloud-agent/tests/run_loop.rs b/apps/craft-cloud-agent/tests/run_loop.rs index b5279d7..4f33ca9 100644 --- a/apps/craft-cloud-agent/tests/run_loop.rs +++ b/apps/craft-cloud-agent/tests/run_loop.rs @@ -232,6 +232,9 @@ fn quick() -> RunOptions { poll_min_spacing: Duration::from_millis(100), poll_busy: Duration::from_millis(200), fail_watch: false, + record_delay: Duration::ZERO, + session_grace: Duration::from_secs(10), + announce_delay: Duration::ZERO, // Never this machine's own start-at-login entry. autostart: craft_cloud_agent::autostart::Target::Off, } diff --git a/apps/craft-cloud-kit-demo/Cargo.toml b/apps/craft-cloud-kit-demo/Cargo.toml new file mode 100644 index 0000000..28f2331 --- /dev/null +++ b/apps/craft-cloud-kit-demo/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "craft-cloud-kit-demo" +description = "The app kit's demo: a rich-text story and a layer tree beside it, live through Craft Cloud (not packaged or installed)" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +craft-cloud-app.workspace = true +craft-cloud-egui.workspace = true +craft-cloud-kit.workspace = true +egui.workspace = true +# The window, as the Craft apps have it (wgpu; Wayland and X11 on Linux). +eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11"] } +# The `.kitdemo` file (JSON). +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +craft-cloud-app = { workspace = true, features = ["fake"] } +egui_kittest.workspace = true +tempfile.workspace = true + +[lints] +workspace = true diff --git a/apps/craft-cloud-kit-demo/src/lib.rs b/apps/craft-cloud-kit-demo/src/lib.rs new file mode 100644 index 0000000..c860803 --- /dev/null +++ b/apps/craft-cloud-kit-demo/src/lib.rs @@ -0,0 +1,9 @@ +//! The app kit's demo: a story and a layer tree in one window, live through Craft Cloud, saved to a `.kitdemo` file +//! (JSON). It shows what an app adds for live documents, and nothing more: its adapter for the kit +//! ([`model::DemoApp`]), the hook at the end of each command ([`ui::Demo::apply`]), a poll each frame, undo through +//! the session while live, and the save record when it saves. Not packaged or installed: `cargo run -p +//! craft-cloud-kit-demo -- `. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod model; +pub mod ui; diff --git a/apps/craft-cloud-kit-demo/src/main.rs b/apps/craft-cloud-kit-demo/src/main.rs new file mode 100644 index 0000000..7d656b3 --- /dev/null +++ b/apps/craft-cloud-kit-demo/src/main.rs @@ -0,0 +1,70 @@ +//! `craft-cloud-kit-demo `: opens `` (relative to the synced folder, or absolute inside it; a new file if +//! there is none) in the demo's window, live through the Craft Cloud agent running on this machine. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use craft_cloud_kit::Agent; +use craft_cloud_kit_demo::ui::Demo; + +struct App(Demo); + +impl eframe::App for App { + fn ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { + self.0.show(ui); + } + + fn on_exit(&mut self) { + // What wasn't saved yet goes to the file (the live document has it either way). + self.0.save(); + } +} + +fn main() -> ExitCode { + let Some(arg) = std::env::args().nth(1) else { + eprintln!("usage: craft-cloud-kit-demo (relative to the Craft Cloud folder, e.g. Notes/demo.kitdemo)"); + return ExitCode::from(2); + }; + let client = craft_cloud_app::Client::connect(); + let status = client.as_ref().ok().and_then(|c| c.status().ok()); + let folder: Option = status.as_ref().and_then(|s| s.folder.clone()).map(PathBuf::from); + let machine = status.as_ref().and_then(|s| s.machine.clone()).unwrap_or_else(|| "this computer".into()); + // The file as the agent knows it: relative to the synced folder, `/`-separated. + let (folder, rel) = match &folder { + Some(f) => match Path::new(&arg).strip_prefix(f) { + Ok(inside) => (f.clone(), inside.to_string_lossy().replace('\\', "/")), + Err(_) if Path::new(&arg).is_absolute() => { + eprintln!("{arg} isn't inside the Craft Cloud folder {}: it opens as a plain file", f.display()); + (Path::new(&arg).parent().map(Path::to_path_buf).unwrap_or_default(), file_name(&arg)) + } + Err(_) => (f.clone(), arg.replace('\\', "/")), + }, + None => { + let path = PathBuf::from(&arg); + (path.parent().map(Path::to_path_buf).unwrap_or_default(), file_name(&arg)) + } + }; + let agent = client.is_ok().then(Agent::local); + let options = eframe::NativeOptions { viewport: egui::ViewportBuilder::default().with_inner_size([1000.0, 700.0]), ..Default::default() }; + let title = format!("Craft Cloud kit demo: {rel}"); + let result = eframe::run_native( + &title, + options, + Box::new(move |cc| { + let ctx = cc.egui_ctx.clone(); + Ok(Box::new(App(Demo::open(agent, &folder, &rel, &machine, move || ctx.request_repaint())))) + }), + ); + match result { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("the window failed: {e}"); + ExitCode::FAILURE + } + } +} + +fn file_name(arg: &str) -> String { + Path::new(arg).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| arg.to_owned()) +} diff --git a/apps/craft-cloud-kit-demo/src/model.rs b/apps/craft-cloud-kit-demo/src/model.rs new file mode 100644 index 0000000..05a8385 --- /dev/null +++ b/apps/craft-cloud-kit-demo/src/model.rs @@ -0,0 +1,563 @@ +//! The demo's document: a story (rich text: bold runs, headings) and a tree of layers beside it, held copy-on-write +//! behind `Arc`s as the Craft apps hold theirs; its adapter for the kit ([`DemoApp`]); its edits; and its file (JSON). + +use std::borrow::Cow; +use std::cell::OnceCell; +use std::collections::HashMap; +use std::ops::Range; +use std::sync::Arc; + +use craft_cloud_kit::{Attrs, ElemId, KitError, LiveDocument, Mappings, Props, RemotePatch, RichText, TextSource, TreeSource, Value}; +use serde::{Deserialize, Serialize}; + +/// The character attribute for bold, and the paragraph attribute for a heading. +pub const BOLD: &str = "b"; +pub const HEADING: &str = "h"; + +#[derive(Debug, Clone, PartialEq)] +pub struct Layer { + pub id: ElemId, + pub name: String, + pub children: Vec>, +} + +pub type Layers = Arc>>; + +#[derive(Debug, Clone, PartialEq)] +pub struct DemoDoc { + pub story: Arc, + pub layers: Layers, + /// Derived (never synced): how many paragraphs the story has. (Counting words would scan the whole story after + /// every remote keystroke: about a millisecond for 1 MB. An app derives what it can cheaply, or incrementally.) + pub paragraphs: usize, +} + +impl Default for DemoDoc { + fn default() -> Self { + Self { story: Arc::new(RichText::plain("")), layers: Arc::new(Vec::new()), paragraphs: 1 } + } +} + +// The adapter. + +/// The demo's mapping: the tree `layers` (each layer's name) and the rich text `story`. +pub struct DemoApp; + +/// The layer tree as a tree source; its index is made only if the tree is compared (a keystroke in the story skips it). +pub struct LayersSrc<'a> { + roots: &'a Layers, + index: OnceCell>>, +} + +impl<'a> LayersSrc<'a> { + pub fn new(roots: &'a Layers) -> Self { + Self { roots, index: OnceCell::new() } + } + + fn index(&self) -> &HashMap> { + self.index.get_or_init(|| { + let mut index = HashMap::new(); + let mut stack: Vec<&'a Arc> = self.roots.iter().collect(); + while let Some(n) = stack.pop() { + index.insert(n.id, n); + stack.extend(n.children.iter()); + } + index + }) + } +} + +impl TreeSource for LayersSrc<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(self.roots) as usize) + } + fn roots(&self) -> Vec { + self.roots.iter().map(|n| n.id).collect() + } + fn children(&self, id: ElemId) -> Vec { + self.index().get(&id).map(|n| n.children.iter().map(|c| c.id).collect()).unwrap_or_default() + } + fn subtree_key(&self, id: ElemId) -> Option { + self.index().get(&id).map(|n| Arc::as_ptr(n) as usize) + } + fn write(&self, id: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError> { + if let Some(n) = self.index().get(&id) { + props.set("name", n.name.as_str())?; + } + Ok(()) + } +} + +/// The story as a text source (keyed by its `Arc`: unchanged, it is skipped). +pub struct StorySrc<'a>(pub &'a Arc); + +impl TextSource for StorySrc<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(self.0) as usize) + } + fn text(&self) -> &str { + &self.0.text + } + fn runs(&self) -> Cow<'_, [(Range, Attrs)]> { + Cow::Borrowed(&self.0.runs) + } + fn paragraphs(&self) -> Cow<'_, [Attrs]> { + Cow::Borrowed(&self.0.paragraphs) + } +} + +/// A layer from the tree as stored (its children too). +fn build(tree: &craft_cloud_kit::TreeView<'_>, ids: Vec) -> Vec> { + ids.into_iter() + .map(|id| { + let name = tree.props(id).and_then(|p| p.str("name")).unwrap_or_default(); + Arc::new(Layer { id, name, children: build(tree, tree.children(id)) }) + }) + .collect() +} + +impl LiveDocument for DemoApp { + type Doc = DemoDoc; + const APP: &'static str = "kitdemo"; + const SCHEMA: u32 = 1; + + fn mappings(doc: &DemoDoc) -> Mappings<'_> { + Mappings::new().tree("layers", LayersSrc::new(&doc.layers)).text("story", StorySrc(&doc.story)) + } + + fn apply_remote(old: &Arc, patch: &RemotePatch<'_>) -> Result, KitError> { + let mut new = (**old).clone(); + if let Some(tree) = patch.tree("layers") { + if tree.touch().structure || tree.touch().all { + new.layers = Arc::new(build(&tree, tree.roots())); + } else { + // Only names changed: just those layers (and the path to them) are new. + let mut layers = (*new.layers).clone(); + for id in &tree.touch().nodes { + let name = tree.props(*id).and_then(|p| p.str("name")).unwrap_or_default(); + layers = rename(&layers, *id, &name); + } + new.layers = Arc::new(layers); + } + } + if let Some(story) = patch.text("story") { + new.story = Arc::new(story); + } + Ok(Arc::new(new)) + } + + fn normalise(doc: &mut DemoDoc) { + // A layer with no name (two machines cleared it, say) gets one: what the app itself does. + let mut unnamed = Vec::new(); + let mut stack: Vec<&Arc> = doc.layers.iter().collect(); + while let Some(n) = stack.pop() { + if n.name.trim().is_empty() { + unnamed.push(n.id); + } + stack.extend(n.children.iter()); + } + if !unnamed.is_empty() { + let mut layers = (*doc.layers).clone(); + for id in unnamed { + layers = rename(&layers, id, "Untitled layer"); + } + doc.layers = Arc::new(layers); + } + } + + fn refresh_derived(doc: &mut DemoDoc) { + doc.paragraphs = doc.story.paragraphs.len(); + } +} + +// The layer edits, copy-on-write: what didn't change stays shared. + +fn contains(node: &Layer, id: ElemId) -> bool { + node.id == id || node.children.iter().any(|c| contains(c, id)) +} + +pub fn find(roots: &[Arc], id: ElemId) -> Option<&Arc> { + for n in roots { + if n.id == id { + return Some(n); + } + if let Some(found) = find(&n.children, id) { + return Some(found); + } + } + None +} + +/// Where a layer is: its parent (`None` at the top) and its place there. +pub fn place(roots: &[Arc], id: ElemId) -> Option<(Option, usize)> { + if let Some(i) = roots.iter().position(|n| n.id == id) { + return Some((None, i)); + } + for n in roots { + if let Some(i) = n.children.iter().position(|c| c.id == id) { + return Some((Some(n.id), i)); + } + if let Some(found) = place(&n.children, id) { + return Some(found); + } + } + None +} + +pub fn remove(roots: &[Arc], id: ElemId) -> (Vec>, Option>) { + let mut taken = None; + let mut out = Vec::with_capacity(roots.len()); + for n in roots { + if n.id == id { + taken = Some(n.clone()); + } else if taken.is_none() && contains(n, id) { + let (children, t) = remove(&n.children, id); + taken = t; + out.push(Arc::new(Layer { id: n.id, name: n.name.clone(), children })); + } else { + out.push(n.clone()); + } + } + (out, taken) +} + +pub fn insert(roots: &[Arc], parent: Option, at: usize, node: Arc) -> Vec> { + match parent { + None => { + let mut out = roots.to_vec(); + out.insert(at.min(out.len()), node); + out + } + Some(p) => roots + .iter() + .map(|n| { + if n.id == p { + let mut children = n.children.clone(); + children.insert(at.min(children.len()), node.clone()); + Arc::new(Layer { id: n.id, name: n.name.clone(), children }) + } else if contains(n, p) { + Arc::new(Layer { id: n.id, name: n.name.clone(), children: insert(&n.children, parent, at, node.clone()) }) + } else { + n.clone() + } + }) + .collect(), + } +} + +pub fn rename(roots: &[Arc], id: ElemId, name: &str) -> Vec> { + roots + .iter() + .map(|n| { + if n.id == id { + Arc::new(Layer { id, name: name.to_owned(), children: n.children.clone() }) + } else if contains(n, id) { + Arc::new(Layer { id: n.id, name: n.name.clone(), children: rename(&n.children, id, name) }) + } else { + n.clone() + } + }) + .collect() +} + +/// A layer moved up (`-1`) or down (`1`) among its siblings. +pub fn shift(roots: &[Arc], id: ElemId, by: isize) -> Vec> { + let Some((parent, at)) = place(roots, id) else { return roots.to_vec() }; + let to = at.saturating_add_signed(by); + let (rest, taken) = remove(roots, id); + match taken { + Some(n) => insert(&rest, parent, to, n), + None => roots.to_vec(), + } +} + +/// A layer made the last child of the sibling before it. +pub fn indent(roots: &[Arc], id: ElemId) -> Vec> { + let Some((parent, at)) = place(roots, id) else { return roots.to_vec() }; + if at == 0 { + return roots.to_vec(); + } + let siblings: Vec = match parent { + None => roots.iter().map(|n| n.id).collect(), + Some(p) => find(roots, p).map(|n| n.children.iter().map(|c| c.id).collect()).unwrap_or_default(), + }; + let Some(&before) = siblings.get(at - 1) else { return roots.to_vec() }; + let (rest, taken) = remove(roots, id); + match taken { + Some(n) => insert(&rest, Some(before), usize::MAX, n), + None => roots.to_vec(), + } +} + +/// A layer moved out of its parent, right after it. +pub fn outdent(roots: &[Arc], id: ElemId) -> Vec> { + let Some((Some(parent), _)) = place(roots, id) else { return roots.to_vec() }; + let Some((grand, at)) = place(roots, parent) else { return roots.to_vec() }; + let (rest, taken) = remove(roots, id); + match taken { + Some(n) => insert(&rest, grand, at + 1, n), + None => roots.to_vec(), + } +} + +/// Every layer, depth first, with its depth. +pub fn flatten(roots: &[Arc]) -> Vec<(usize, Arc)> { + let mut out = Vec::new(); + let mut stack: Vec<(usize, &Arc)> = roots.iter().rev().map(|n| (0, n)).collect(); + while let Some((depth, n)) = stack.pop() { + out.push((depth, n.clone())); + stack.extend(n.children.iter().rev().map(|c| (depth + 1, c))); + } + out +} + +// The story edits. + +/// Merges adjacent runs with the same attributes and drops empty ones. +fn tidy(runs: Vec<(Range, Attrs)>) -> Vec<(Range, Attrs)> { + let mut out: Vec<(Range, Attrs)> = Vec::with_capacity(runs.len()); + for (r, a) in runs { + if r.start >= r.end || a.is_empty() { + continue; + } + match out.last_mut() { + Some((last, la)) if last.end == r.start && *la == a => last.end = r.end, + _ => out.push((r, a)), + } + } + out +} + +fn prefix(a: &str, b: &str) -> usize { + let mut i = a.as_bytes().iter().zip(b.as_bytes()).take_while(|(x, y)| x == y).count(); + while i > 0 && !a.is_char_boundary(i) { + i -= 1; + } + i +} + +fn suffix(a: &str, b: &str, max: usize) -> usize { + let mut n = a.as_bytes().iter().rev().zip(b.as_bytes().iter().rev()).take(max).take_while(|(x, y)| x == y).count(); + while n > 0 && !a.is_char_boundary(a.len() - n) { + n -= 1; + } + n +} + +/// The story with its text replaced by `text` (as the text box edited it): typed text takes the attributes of the +/// character before it, a split paragraph's new part takes its attributes, and joined paragraphs keep the first's. +pub fn edit_text(old: &RichText, text: &str) -> RichText { + let p = prefix(&old.text, text); + let s = suffix(&old.text, text, old.text.len().min(text.len()) - p); + let (old_end, new_end) = (old.text.len() - s, text.len() - s); + let inserted = new_end - p; + let mut runs = Vec::with_capacity(old.runs.len() + 1); + let mut inherited = Attrs::new(); + for (r, a) in &old.runs { + if r.start < p && p <= r.end { + inherited = a.clone(); + } + if r.start < p { + runs.push((r.start..r.end.min(p), a.clone())); + } + if r.end > old_end { + runs.push((r.start.max(old_end) - old_end + new_end..r.end - old_end + new_end, a.clone())); + } + } + if inserted > 0 && !inherited.is_empty() { + runs.push((p..new_end, inherited)); + } + runs.sort_by_key(|(r, _)| r.start); + let before = old.text.get(..p).map_or(0, |t| t.matches('\n').count()); + let joined = old.text.get(p..old_end).map_or(0, |t| t.matches('\n').count()); + let split = text.get(p..new_end).map_or(0, |t| t.matches('\n').count()); + let mut paragraphs = old.paragraphs.clone(); + paragraphs.resize(old.text.matches('\n').count() + 1, Attrs::new()); + let keep = paragraphs.get(before).cloned().unwrap_or_default(); + let end = (before + 1 + joined).min(paragraphs.len()); + paragraphs.drain((before + 1).min(end)..end); + for _ in 0..split { + paragraphs.insert(before, keep.clone()); + } + RichText { text: text.to_owned(), runs: tidy(runs), paragraphs } +} + +/// The story with attribute `key` set (or removed) on bytes `range`. +pub fn format(old: &RichText, range: Range, key: &str, value: Option) -> RichText { + let (a, b) = (range.start.min(old.text.len()), range.end.min(old.text.len())); + let mut runs = Vec::with_capacity(old.runs.len() + 2); + let mut at = 0; + let mut pieces: Vec<(Range, Attrs)> = Vec::new(); + for (r, attrs) in &old.runs { + if r.start > at { + pieces.push((at..r.start, Attrs::new())); + } + pieces.push((r.clone(), attrs.clone())); + at = r.end; + } + if at < old.text.len() { + pieces.push((at..old.text.len(), Attrs::new())); + } + for (r, attrs) in pieces { + for (x, y, inside) in [(r.start, r.end.min(a), false), (r.start.max(a), r.end.min(b), true), (r.start.max(b), r.end, false)] { + if x >= y { + continue; + } + let mut attrs = attrs.clone(); + if inside { + match &value { + Some(v) => attrs.insert(key.to_owned(), v.clone()), + None => attrs.remove(key), + }; + } + runs.push((x..y, attrs)); + } + } + RichText { text: old.text.clone(), runs: tidy(runs), paragraphs: old.paragraphs.clone() } +} + +/// Whether every byte of `range` has attribute `key`. +pub fn has(story: &RichText, range: Range, key: &str) -> bool { + !range.is_empty() + && story + .runs + .iter() + .filter(|(_, a)| a.contains_key(key)) + .map(|(r, _)| r.end.min(range.end).saturating_sub(r.start.max(range.start))) + .sum::() + >= range.len() +} + +/// The paragraph that byte `at` is in. +pub fn paragraph_at(story: &RichText, at: usize) -> usize { + story.text.get(..at.min(story.text.len())).map_or(0, |t| t.matches('\n').count()) +} + +/// The story with paragraph `para`'s heading set (or removed). +pub fn heading(old: &RichText, para: usize, level: Option) -> RichText { + let mut paragraphs = old.paragraphs.clone(); + paragraphs.resize(old.text.matches('\n').count() + 1, Attrs::new()); + if let Some(p) = paragraphs.get_mut(para) { + match level { + Some(l) => p.insert(HEADING.to_owned(), Value::I64(l)), + None => p.remove(HEADING), + }; + } + RichText { text: old.text.clone(), runs: old.runs.clone(), paragraphs } +} + +// The file: `.kitdemo`, JSON. + +/// The file format's name and version. +pub const FORMAT: &str = "craft-cloud-kit-demo"; +pub const FORMAT_VERSION: u32 = 1; + +#[derive(Debug, Serialize, Deserialize)] +struct FileDoc { + format: String, + version: u32, + story: FileStory, + layers: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct FileStory { + text: String, + /// Bold byte ranges. + bold: Vec<[usize; 2]>, + /// Paragraphs that are headings: index and level. + headings: Vec<[i64; 2]>, +} + +#[derive(Debug, Serialize, Deserialize)] +struct FileLayer { + id: String, + name: String, + #[serde(default)] + children: Vec, +} + +/// The document as the file's bytes. +pub fn to_file(doc: &DemoDoc) -> Vec { + fn layer(l: &Layer) -> FileLayer { + FileLayer { id: l.id.hex(), name: l.name.clone(), children: l.children.iter().map(|c| layer(c)).collect() } + } + let story = FileStory { + text: doc.story.text.clone(), + bold: doc.story.runs.iter().filter(|(_, a)| a.contains_key(BOLD)).map(|(r, _)| [r.start, r.end]).collect(), + headings: doc + .story + .paragraphs + .iter() + .enumerate() + .filter_map(|(i, p)| p.get(HEADING).and_then(Value::as_i64).map(|l| [i as i64, l])) + .collect(), + }; + let file = FileDoc { format: FORMAT.into(), version: FORMAT_VERSION, story, layers: doc.layers.iter().map(|l| layer(l)).collect() }; + serde_json::to_vec_pretty(&file).unwrap_or_default() +} + +/// A document from a file's bytes; an error says why it doesn't read. +pub fn from_file(bytes: &[u8]) -> Result { + let file: FileDoc = serde_json::from_slice(bytes).map_err(|e| format!("not a demo file: {e}"))?; + if file.format != FORMAT || file.version > FORMAT_VERSION { + return Err(format!("a {} file of version {}: this demo reads {FORMAT} {FORMAT_VERSION}", file.format, file.version)); + } + fn layer(f: FileLayer) -> Result, String> { + let id = ElemId::from_hex(&f.id).ok_or_else(|| format!("{:?} isn't a layer ID", f.id))?; + Ok(Arc::new(Layer { id, name: f.name, children: f.children.into_iter().map(layer).collect::>()? })) + } + let mut story = RichText::plain(&file.story.text); + for [a, b] in file.story.bold { + if story.text.is_char_boundary(a.min(story.text.len())) && story.text.is_char_boundary(b.min(story.text.len())) { + story = format(&story, a..b, BOLD, Some(Value::Bool(true))); + } + } + for [para, level] in file.story.headings { + story = heading(&story, usize::try_from(para).unwrap_or(usize::MAX), Some(level)); + } + let mut doc = DemoDoc { story: Arc::new(story), layers: Arc::new(file.layers.into_iter().map(layer).collect::>()?), paragraphs: 0 }; + DemoApp::refresh_derived(&mut doc); + Ok(doc) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn typing_splitting_and_joining_keep_the_attributes() { + let s = RichText::plain("Hello world"); + let s = format(&s, 0..5, BOLD, Some(Value::Bool(true))); + let s = heading(&s, 0, Some(1)); + let typed = edit_text(&s, "Hellooo world"); + assert!(has(&typed, 0..7, BOLD) && !has(&typed, 7..8, BOLD), "{:?}", typed.runs); + let split = edit_text(&typed, "Hel\nlooo world"); + assert_eq!(split.paragraphs.len(), 2); + assert!(split.paragraphs.iter().all(|p| p.get(HEADING) == Some(&Value::I64(1)))); + let joined = edit_text(&split, "Hellooo world"); + assert_eq!(joined.paragraphs.len(), 1); + let unbold = format(&joined, 0..3, BOLD, None); + assert!(!has(&unbold, 0..3, BOLD) && has(&unbold, 3..7, BOLD)); + assert_eq!(paragraph_at(&split, 5), 1); + } + + #[test] + fn layers_move_and_the_file_reads_back() { + let (a, b, c) = (ElemId(1), ElemId(2), ElemId(3)); + let l = |id, name: &str| Arc::new(Layer { id, name: name.into(), children: Vec::new() }); + let roots = vec![l(a, "a"), l(b, "b"), l(c, "c")]; + let roots = indent(&roots, b); + assert_eq!(place(&roots, b), Some((Some(a), 0))); + let roots = outdent(&roots, b); + assert_eq!(place(&roots, b), Some((None, 1))); + let roots = shift(&roots, c, -1); + assert_eq!(flatten(&roots).iter().map(|(_, n)| n.id).collect::>(), vec![a, c, b]); + let story = heading(&format(&RichText::plain("Title\nbody"), 0..5, BOLD, Some(Value::Bool(true))), 0, Some(1)); + let mut doc = DemoDoc { story: Arc::new(story), layers: Arc::new(rename(&roots, c, "renamed")), paragraphs: 0 }; + DemoApp::refresh_derived(&mut doc); + let back = from_file(&to_file(&doc)).unwrap(); + assert_eq!(back, doc); + assert!(from_file(b"{}").is_err()); + } +} diff --git a/apps/craft-cloud-kit-demo/src/ui.rs b/apps/craft-cloud-kit-demo/src/ui.rs new file mode 100644 index 0000000..872148b --- /dev/null +++ b/apps/craft-cloud-kit-demo/src/ui.rs @@ -0,0 +1,627 @@ +//! The demo's window: the account line and its recents, the document's badge, the story (a text box with bold and +//! headings, other machines' cursors marked) and the layer tree beside it (add, rename, move, delete). +//! +//! The kit's hook is [`Demo::apply`]: every command ends there with the model before and after it. Each frame +//! [`Demo::show`] polls the session first. While the document isn't live (opening, or Craft Cloud isn't there) the +//! demo keeps its own snapshot undo; once live, undo goes through the session. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::mpsc::{Receiver, TryRecvError}; +use std::time::{Duration, Instant}; + +use craft_cloud_kit::{Agent, ElemId, NotLive, Open, Session, UndoKind, Value}; +use egui::text::LayoutJob; +use egui::{Color32, FontId, TextFormat, Ui}; + +use crate::model::{self, BOLD, DemoApp, DemoDoc, HEADING, Layer}; + +/// How long after a change the demo saves by itself. +pub const AUTOSAVE_AFTER: Duration = Duration::from_secs(5); +/// How often, and how many times, opening is tried while the file isn't synced yet. +const OPEN_RETRY: Duration = Duration::from_secs(2); +const OPEN_TRIES: u32 = 60; + +type Opened = Result, String>; + +/// Where the document stands. +enum Mode { + /// Being opened live, off the UI thread. + Opening(Receiver), + Live(Box>), + /// A plain file (the reason): the demo's own undo. + Local(String), +} + +/// What the window shows of other machines' cursors. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Cursor { + pub machine: String, + pub at: usize, +} + +/// The demo. +pub struct Demo { + /// The file, absolute, and as the agent knows it (relative to the synced folder). + file: PathBuf, + rel: String, + machine: String, + mode: Mode, + model: Arc, + /// The demo's own undo, while not live. + undo: Vec>, + redo: Vec>, + selected: Option, + rename: String, + cursor: Option<(usize, usize)>, + dirty_since: Option, + /// Since when the file has lacked edits the live document has (another machine's, or this one's after another + /// machine's save won over it): autosaved like a change made here. + behind_since: Option, + pub autosave: bool, + /// The last message for the status line. + pub message: String, + watch: Option, + /// The file's model and bytes as read at open (to bring a file changed outside Craft in). + read: (Arc, Vec), + composing: bool, + /// The user chose what to do with a file changed outside Craft. + outside_handled: bool, +} + +/// Reads the file at `path` (an empty document if there is none yet); its bytes too. +pub fn read_file(path: &Path) -> Result<(DemoDoc, Vec), String> { + match std::fs::read(path) { + Ok(bytes) => Ok((model::from_file(&bytes)?, bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok((DemoDoc::default(), Vec::new())), + Err(e) => Err(format!("{}: {e}", path.display())), + } +} + +/// Writes `bytes` to `path` atomically (a temporary file beside it, then renamed). +pub fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let dir = path.parent().unwrap_or(Path::new(".")); + std::fs::create_dir_all(dir)?; + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let tmp = dir.join(format!(".{name}.{}.tmp", std::process::id())); + let mut f = std::fs::File::create(&tmp)?; + f.write_all(bytes)?; + f.sync_all()?; + drop(f); + std::fs::rename(&tmp, path) +} + +impl Demo { + /// Opens `rel` (relative to the synced folder `folder`) as `machine`, live through `agent` (opened off this + /// thread; `wake` is called when it is open or something comes). + pub fn open(agent: Option, folder: &Path, rel: &str, machine: &str, wake: impl Fn() + Send + Sync + Clone + 'static) -> Self { + let file = folder.join(rel); + let (doc, bytes, message) = match read_file(&file) { + Ok((doc, bytes)) => (doc, bytes, String::new()), + Err(e) => (DemoDoc::default(), Vec::new(), e), + }; + let mut doc = doc; + ::refresh_derived(&mut doc); + let model = Arc::new(doc); + // A new file is written once, so the file sync has something to make live. + if bytes.is_empty() && !file.exists() { + let _ = write_atomic(&file, &model::to_file(&model)); + } + let bytes = std::fs::read(&file).unwrap_or(bytes); + let mode = match agent.clone() { + None => Mode::Local("Craft Cloud isn't running on this computer".into()), + Some(agent) => { + let (tx, rx) = std::sync::mpsc::channel(); + let (rel, file_model, file_bytes, w) = (rel.to_owned(), model.clone(), bytes.clone(), wake.clone()); + let _ = std::thread::Builder::new().name("kit-demo-open".into()).spawn(move || { + let mut tries = 0; + let result = loop { + tries += 1; + match Session::::open(&agent, &rel, file_model.clone(), Some(&file_bytes)) { + Ok(Open::NotLive(NotLive::Reason(craft_cloud_app::live::NotLiveReason::NotSynced))) if tries < OPEN_TRIES => { + std::thread::sleep(OPEN_RETRY); + } + other => break other.map_err(|e| e.to_string()), + } + }; + let _ = tx.send(result); + w(); + }); + Mode::Opening(rx) + } + }; + let watch = agent.map(|a| { + let w = wake.clone(); + craft_cloud_egui::CloudWatch::start("kitdemo", move || a.client(), w) + }); + Self { + file, + rel: rel.to_owned(), + machine: machine.to_owned(), + mode, + read: (model.clone(), bytes), + model, + undo: Vec::new(), + redo: Vec::new(), + selected: None, + rename: String::new(), + cursor: None, + dirty_since: None, + behind_since: None, + autosave: true, + message, + watch, + composing: false, + outside_handled: false, + } + } + + pub fn model(&self) -> &Arc { + &self.model + } + + pub fn is_live(&self) -> bool { + matches!(self.mode, Mode::Live(_)) + } + + pub fn session(&self) -> Option<&Session> { + match &self.mode { + Mode::Live(s) => Some(s), + _ => None, + } + } + + /// Takes what came (the open, remote changes) without drawing: what each frame does first. + pub fn poll(&mut self) { + if let Mode::Opening(rx) = &self.mode { + match rx.try_recv() { + Ok(Ok(Open::Live(session))) => { + self.model = session.model().clone(); + self.undo.clear(); + self.redo.clear(); + if let Some(w) = &self.watch { + w.add_recent(&self.rel, false); + } + self.mode = Mode::Live(session); + } + Ok(Ok(Open::NotLive(why))) => self.mode = Mode::Local(format!("not live: {why:?}")), + Ok(Err(e)) => self.mode = Mode::Local(e), + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => self.mode = Mode::Local("the open stopped".into()), + } + } + if let Mode::Live(session) = &mut self.mode { + session.hold_remote(self.composing); + if session.poll().changed { + self.model = session.model().clone(); + if self.selected.is_some_and(|id| model::find(&self.model.layers, id).is_none()) { + self.selected = None; + } + } + } + } + + /// The hook: a command made `new` from the current model. Live, the session writes the difference (one undo + /// step) and sends it; otherwise the demo keeps the old model for its own undo. + pub fn apply(&mut self, mut new: DemoDoc) { + ::refresh_derived(&mut new); + let new = Arc::new(new); + let old = self.model.clone(); + if *new == *old { + return; + } + match &mut self.mode { + Mode::Live(session) => match session.local_change(&old, &new) { + Ok(()) => self.model = new, + Err(e) => { + self.message = e.to_string(); + // (Read-only, stalled or out of step: the session's model stands.) + self.model = session.model().clone(); + } + }, + _ => { + self.undo.push(old); + self.redo.clear(); + self.model = new; + } + } + self.dirty_since.get_or_insert_with(Instant::now); + } + + pub fn undo(&mut self) { + match &mut self.mode { + Mode::Live(session) => { + if let Err(e) = session.undo() { + self.message = e.to_string(); + } + self.model = session.model().clone(); + } + _ => { + if let Some(prev) = self.undo.pop() { + self.redo.push(std::mem::replace(&mut self.model, prev)); + } + } + } + self.dirty_since.get_or_insert_with(Instant::now); + } + + pub fn redo(&mut self) { + match &mut self.mode { + Mode::Live(session) => { + if let Err(e) = session.redo() { + self.message = e.to_string(); + } + self.model = session.model().clone(); + } + _ => { + if let Some(next) = self.redo.pop() { + self.undo.push(std::mem::replace(&mut self.model, next)); + } + } + } + self.dirty_since.get_or_insert_with(Instant::now); + } + + /// Writes the file (atomically). Live, in the kit's order: the save point (the model and the live document's + /// version it matches), the record to the agent, then the write. Without a save point (read-only, stopped, …) it + /// writes its own model with no record. + pub fn save(&mut self) { + let point = match &self.mode { + Mode::Live(session) => session.save_point().map_err(|e| self.message = e.to_string()).ok(), + _ => None, + }; + let bytes = model::to_file(point.as_ref().map_or(&self.model, |p| p.model())); + if let (Mode::Live(session), Some(point)) = (&mut self.mode, &point) { + session.record(point, &bytes); + } + if let Err(e) = write_atomic(&self.file, &bytes) { + self.message = format!("not saved: {e}"); + return; + } + if let Some(w) = &self.watch { + w.add_recent(&self.rel, true); + } + self.dirty_since = None; + self.behind_since = None; + self.message = "saved".into(); + } + + /// "Bring it in": the file's content (as read at open) replaces the live content, then the file is saved. When the + /// replace fails (stale, stalled, read-only), nothing is saved: the file keeps the outside edit, and the banner + /// stays to try again. + pub fn bring_in(&mut self) { + let file = (*self.read.0).clone(); + let Mode::Live(session) = &mut self.mode else { return }; + match session.replace_with_file(file) { + Ok(()) => self.model = session.model().clone(), + Err(e) => { + self.message = format!("not brought in: {e}"); + return; + } + } + self.save(); + self.outside_handled = true; + } + + /// The other machines' cursors in the story. + pub fn cursors(&self) -> Vec { + let Some(session) = self.session() else { return Vec::new() }; + let mut out: Vec = session + .peers() + .values() + .filter_map(|p| { + let text = std::str::from_utf8(&p.bytes).ok()?; + let (machine, at) = text.split_once('@')?; + Some(Cursor { machine: machine.to_owned(), at: at.parse().ok()? }) + }) + .collect(); + out.sort_by(|a, b| a.machine.cmp(&b.machine)); + out + } + + /// The layer to act on, or the first. + fn target(&self) -> Option { + self.selected.or_else(|| self.model.layers.first().map(|l| l.id)) + } + + pub fn add_layer(&mut self, name: &str) -> ElemId { + let id = ElemId::new(); + let layer = Arc::new(Layer { id, name: name.to_owned(), children: Vec::new() }); + let layers = match self.selected { + Some(parent) => model::insert(&self.model.layers, Some(parent), usize::MAX, layer), + None => model::insert(&self.model.layers, None, usize::MAX, layer), + }; + self.apply(DemoDoc { layers: Arc::new(layers), ..(*self.model).clone() }); + id + } + + pub fn select(&mut self, id: Option) { + self.selected = id; + self.rename = id.and_then(|id| model::find(&self.model.layers, id)).map(|l| l.name.clone()).unwrap_or_default(); + } + + fn layers_op(&mut self, op: impl FnOnce(&[Arc], ElemId) -> Vec>) { + let Some(id) = self.target() else { return }; + let layers = op(&self.model.layers, id); + self.apply(DemoDoc { layers: Arc::new(layers), ..(*self.model).clone() }); + } + + pub fn rename_selected(&mut self, name: &str) { + let name = name.to_owned(); + self.layers_op(|l, id| model::rename(l, id, &name)); + } + + /// The window's frame: poll, draw, autosave. + pub fn show(&mut self, ui: &mut Ui) { + self.poll(); + // Keys: undo and redo. + let (undo, redo) = ui.input(|i| { + let cmd = i.modifiers.command; + ( + cmd && !i.modifiers.shift && i.key_pressed(egui::Key::Z), + cmd && (i.key_pressed(egui::Key::Y) || (i.modifiers.shift && i.key_pressed(egui::Key::Z))), + ) + }); + // An IME composition holds remote changes back until it ends. + self.composing = ui.input(|i| { + i.events.iter().fold(self.composing, |on, e| match e { + egui::Event::Ime(egui::ImeEvent::Preedit { text, .. }) => !text.is_empty(), + egui::Event::Ime(egui::ImeEvent::Commit(_)) => false, + _ => on, + }) + }); + self.top(ui); + ui.separator(); + ui.columns(2, |cols| { + if let [story, layers] = cols { + self.story(story); + self.layers(layers); + } + }); + if undo { + self.undo(); + } + if redo { + self.redo(); + } + // Autosave when dirty or when the file is behind the live document, so the file catches up on its own. + if self.session().is_some_and(Session::file_behind) { + self.behind_since.get_or_insert_with(Instant::now); + } else { + self.behind_since = None; + } + let due = |since: Option| since.is_some_and(|at| at.elapsed() >= AUTOSAVE_AFTER); + if self.autosave && (due(self.dirty_since) || due(self.behind_since)) { + self.save(); + } + if self.dirty_since.is_some() || self.is_live() { + ui.ctx().request_repaint_after(Duration::from_millis(500)); + } + } + + fn top(&mut self, ui: &mut Ui) { + if let Some(w) = &mut self.watch { + craft_cloud_egui::account_line(ui, &w.state()); + } + ui.horizontal(|ui| { + ui.label(egui::RichText::new(&self.rel).strong()); + match &self.mode { + Mode::Opening(_) => { + ui.weak("opening live…"); + } + Mode::Live(session) => craft_cloud_egui::badge(ui, session.status()), + Mode::Local(why) => { + ui.weak(format!("plain file: {why}")); + } + } + ui.weak(format!("{} paragraphs", self.model.paragraphs)); + let live_undo = self.session().map(|s| (s.can_undo(), s.can_redo(), s.undo_kind())); + let (can_undo, can_redo) = match live_undo { + Some((u, r, UndoKind::Live)) => (u, r), + Some((_, _, UndoKind::None)) => (false, false), + None => (!self.undo.is_empty(), !self.redo.is_empty()), + }; + let blocked = self.session().and_then(Session::undo_blocked).unwrap_or_default(); + // A named step shows its name: "Undo Replace with the file's content". + let label = |what: &str, name: Option<&str>| name.map_or_else(|| what.to_owned(), |n| format!("{what} {n}")); + let (undo_label, redo_label) = + (label("Undo", self.session().and_then(|s| s.step_name(true))), label("Redo", self.session().and_then(|s| s.step_name(false)))); + if ui.add_enabled(can_undo, egui::Button::new(undo_label)).on_disabled_hover_text(blocked).clicked() { + self.undo(); + } + if ui.add_enabled(can_redo, egui::Button::new(redo_label)).clicked() { + self.redo(); + } + if ui.button("Save").clicked() { + self.save(); + } + ui.checkbox(&mut self.autosave, "Autosave"); + if !self.message.is_empty() { + ui.weak(&self.message); + } + }); + let outside = !self.outside_handled && self.session().is_some_and(|s| s.status().changed_outside); + if outside { + ui.horizontal(|ui| { + ui.colored_label( + Color32::from_rgb(0xb0, 0x80, 0x10), + "The file was changed outside Craft. Bringing it in replaces the live edits made since the file was \ + written, on every machine (Undo brings them back).", + ); + if ui.button("Bring it in").clicked() { + // The user's choice: the file's content replaces the live content as one ordinary change, named + // "Replace with the file's content", then saved, so the file and the document agree again. + self.bring_in(); + } + if ui.button("Keep the live document").clicked() { + // The next save writes the live content over the file. + self.outside_handled = true; + self.dirty_since.get_or_insert_with(Instant::now); + } + }); + } + if let Some(ro) = self.session().and_then(|s| s.status().read_only.clone()) { + ui.colored_label(Color32::from_rgb(0x40, 0x60, 0xb0), format!("Made with a newer kitdemo (layout {}): update it to edit.", ro.version)); + } + if let Some(w) = &mut self.watch { + let entries = w.entries().to_vec(); + ui.collapsing("Recent files", |ui| { + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_millis() as i64); + if let Some(path) = craft_cloud_egui::recents(ui, &entries, now) { + self.message = format!("open {path} with: craft-cloud-kit-demo \"{path}\""); + } + }); + } + } + + fn story(&mut self, ui: &mut Ui) { + let label = ui.label(egui::RichText::new("Story").strong()); + let editable = !matches!(self.mode, Mode::Opening(_)) && self.session().is_none_or(|s| s.status().read_only.is_none()); + ui.horizontal(|ui| { + let range = self.cursor.map(|(a, b)| a.min(b)..a.max(b)); + if ui.add_enabled(editable && range.as_ref().is_some_and(|r| !r.is_empty()), egui::Button::new("Bold")).clicked() + && let Some(r) = range + { + let on = model::has(&self.model.story, r.clone(), BOLD); + let story = model::format(&self.model.story, r, BOLD, if on { None } else { Some(Value::Bool(true)) }); + self.apply(DemoDoc { story: Arc::new(story), ..(*self.model).clone() }); + } + if ui.add_enabled(editable, egui::Button::new("Heading")).clicked() { + let para = model::paragraph_at(&self.model.story, self.cursor.map_or(0, |c| c.0)); + let on = self.model.story.paragraphs.get(para).is_some_and(|p| p.contains_key(HEADING)); + let story = model::heading(&self.model.story, para, if on { None } else { Some(1) }); + self.apply(DemoDoc { story: Arc::new(story), ..(*self.model).clone() }); + } + for c in self.cursors() { + ui.colored_label(Color32::from_rgb(0xc0, 0x30, 0x90), format!("{} at {}", c.machine, c.at)); + } + }); + let mut text = self.model.story.text.clone(); + let story = self.model.story.clone(); + let marks: Vec = self.cursors().iter().map(|c| c.at).collect(); + let mut layouter = move |ui: &Ui, buf: &dyn egui::TextBuffer, wrap: f32| { + let mut job = layout(&story, buf.as_str(), &marks, ui); + job.wrap.max_width = wrap; + ui.fonts_mut(|f| f.layout_job(job)) + }; + let out = + egui::TextEdit::multiline(&mut text).desired_width(f32::INFINITY).desired_rows(20).interactive(editable).layouter(&mut layouter).show(ui); + let response = out.response.response.clone().labelled_by(label.id); + if let Some(range) = out.cursor_range { + let (a, b) = (range.primary.index.0, range.secondary.index.0); + let to_byte = |chars: usize| text.char_indices().nth(chars).map_or(text.len(), |(i, _)| i); + let now = (to_byte(a), to_byte(b)); + if self.cursor != Some(now) { + self.cursor = Some(now); + if let Mode::Live(session) = &mut self.mode { + let _ = session.set_presence(Some(format!("{}@{}", self.machine, now.0).into_bytes())); + } + } + } + if response.changed() && text != self.model.story.text { + let story = model::edit_text(&self.model.story, &text); + self.apply(DemoDoc { story: Arc::new(story), ..(*self.model).clone() }); + } + } + + fn layers(&mut self, ui: &mut Ui) { + ui.label(egui::RichText::new("Layers").strong()); + let editable = !matches!(self.mode, Mode::Opening(_)) && self.session().is_none_or(|s| s.status().read_only.is_none()); + ui.add_enabled_ui(editable, |ui| { + ui.horizontal_wrapped(|ui| { + if ui.button("Add layer").clicked() { + let n = model::flatten(&self.model.layers).len() + 1; + let id = self.add_layer(&format!("Layer {n}")); + self.select(Some(id)); + } + if ui.button("Up").clicked() { + self.layers_op(|l, id| model::shift(l, id, -1)); + } + if ui.button("Down").clicked() { + self.layers_op(|l, id| model::shift(l, id, 1)); + } + if ui.button("Indent").clicked() { + self.layers_op(model::indent); + } + if ui.button("Outdent").clicked() { + self.layers_op(model::outdent); + } + if ui.button("Delete").clicked() { + self.layers_op(|l, id| model::remove(l, id).0); + self.select(None); + } + }); + if self.selected.is_some() { + ui.horizontal(|ui| { + let label = ui.label("Name"); + let r = ui.text_edit_singleline(&mut self.rename).labelled_by(label.id); + if (r.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter))) || ui.button("Rename").clicked() { + let name = self.rename.clone(); + self.rename_selected(&name); + } + }); + } + }); + egui::ScrollArea::vertical().id_salt("layers").show(ui, |ui| { + let mut clicked = None; + for (depth, layer) in model::flatten(&self.model.layers) { + ui.horizontal(|ui| { + ui.add_space(depth as f32 * 16.0); + if ui.selectable_label(self.selected == Some(layer.id), &layer.name).clicked() { + clicked = Some(layer.id); + } + }); + } + if let Some(id) = clicked { + self.select(if self.selected == Some(id) { None } else { Some(id) }); + } + }); + } +} + +/// The story's text drawn with its runs (bold as strong and underlined), headings larger, and other machines' +/// cursors marked: one section per stretch that looks the same. `text` is the text box's (a frame ahead of the story +/// while typing: then it is drawn plain). +fn layout(story: &craft_cloud_kit::RichText, text: &str, marks: &[usize], ui: &Ui) -> LayoutJob { + let mut job = LayoutJob::default(); + let normal = TextFormat { font_id: FontId::proportional(14.0), color: ui.visuals().text_color(), ..Default::default() }; + if story.text != text { + job.append(text, 0.0, normal); + return job; + } + // Where the look may change: runs, marks, and (when there are headings) paragraph breaks. + let mut cuts: Vec = vec![0, text.len()]; + for (r, a) in &story.runs { + if a.contains_key(BOLD) { + cuts.extend([r.start, r.end]); + } + } + for m in marks { + let end = text.get(*m..).and_then(|t| t.chars().next()).map_or(*m, |c| m + c.len_utf8()); + cuts.extend([*m, end]); + } + let headings = story.paragraphs.iter().any(|p| p.contains_key(HEADING)); + if headings { + cuts.extend(text.match_indices('\n').map(|(i, _)| i + 1)); + } + cuts.retain(|c| *c <= text.len() && text.is_char_boundary(*c)); + cuts.sort_unstable(); + cuts.dedup(); + for pair in cuts.windows(2) { + let (Some(&a), Some(&b)) = (pair.first(), pair.get(1)) else { continue }; + let mut format = normal.clone(); + if story.runs.iter().any(|(r, attrs)| r.start <= a && a < r.end && attrs.contains_key(BOLD)) { + format.color = ui.visuals().strong_text_color(); + format.underline = egui::Stroke::new(1.0, ui.visuals().strong_text_color()); + } + if headings && story.paragraphs.get(model::paragraph_at(story, a)).is_some_and(|p| p.contains_key(HEADING)) { + format.font_id = FontId::proportional(20.0); + } + if marks.contains(&a) { + format.background = Color32::from_rgb(0xf0, 0xa0, 0xd0); + } + job.append(text.get(a..b).unwrap_or_default(), 0.0, format); + } + job +} diff --git a/apps/craft-cloud-kit-demo/tests/speed.rs b/apps/craft-cloud-kit-demo/tests/speed.rs new file mode 100644 index 0000000..dce8930 --- /dev/null +++ b/apps/craft-cloud-kit-demo/tests/speed.rs @@ -0,0 +1,131 @@ +//! The kit on a big document, in release (`cargo test --release -p craft-cloud-kit-demo --test speed`): the demo with a +//! 1 MB story and 5,000 layers. A keystroke's `local_change` stays under 2 ms (median), a remote keystroke's `poll` +//! under 5 ms (median), and opening the document on the other machine under 300 ms. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use craft_cloud_app::fake::{Fake, Hub}; +use craft_cloud_kit::{Agent, ElemId, Open, RichText, Session, Value}; +use craft_cloud_kit_demo::model::{self, BOLD, DemoDoc, Layer}; + +fn big() -> DemoDoc { + let mut text = String::with_capacity(1_100_000); + let mut i = 0; + while text.len() < 1_000_000 { + text.push_str(&format!("Paragraph {i}: the quick brown fox jumps over the lazy dog, again and again, as stories do.\n")); + i += 1; + } + let mut story = RichText::plain(&text); + // A bold word every fifth paragraph, a heading every fiftieth. + let mut at = 0; + let mut runs = Vec::new(); + for (p, line) in text.split_inclusive('\n').enumerate() { + if p % 5 == 0 { + runs.push((at + 13..at + 18, [(BOLD.to_owned(), Value::Bool(true))].into())); + } + if p % 50 == 0 + && let Some(attrs) = story.paragraphs.get_mut(p) + { + attrs.insert(model::HEADING.to_owned(), Value::I64(1)); + } + at += line.len(); + } + story.runs = runs; + let layers: Vec> = (0..50) + .map(|g| { + let children = (0..99).map(|c| Arc::new(Layer { id: ElemId::new(), name: format!("Layer {g}.{c}"), children: Vec::new() })).collect(); + Arc::new(Layer { id: ElemId::new(), name: format!("Group {g}"), children }) + }) + .collect(); + let mut doc = DemoDoc { story: Arc::new(story), layers: Arc::new(layers), paragraphs: 0 }; + ::refresh_derived(&mut doc); + doc +} + +fn open(fake: &Fake, doc: DemoDoc) -> Session { + let client = fake.client(); + match Session::::open(&Agent::from_fn(move || Ok(client.clone())), "big.kitdemo", Arc::new(doc), None).unwrap() { + Open::Live(s) => *s, + Open::NotLive(why) => panic!("not live: {why:?}"), + } +} + +fn median(mut v: Vec) -> Duration { + v.sort(); + v[v.len() / 2] +} + +#[test] +#[cfg_attr(debug_assertions, ignore = "a speed test: run it in release")] +fn a_big_document_stays_quick() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let doc = big(); + assert!(doc.story.text.len() >= 1_000_000 && model::flatten(&doc.layers).len() == 5000); + let t = Instant::now(); + let mut a = open(&pc, doc); + let import = t.elapsed(); + let t = Instant::now(); + let mut b = open(&laptop, DemoDoc::default()); + let opened = t.elapsed(); + assert_eq!(b.model().story.text.len(), a.model().story.text.len()); + assert_eq!(model::flatten(&b.model().layers).len(), 5000); + eprintln!("SPEED import on the pc {import:?}; open on the laptop {opened:?}"); + assert!(opened < Duration::from_millis(300), "opening took {opened:?}"); + + let (mut local, mut remote) = (Vec::new(), Vec::new()); + let len = a.model().story.text.len(); + for k in 0..120usize { + // A keystroke somewhere in the story. + let at = (k * 7919) % len; + let at = (0..=at).rev().find(|i| a.model().story.text.is_char_boundary(*i)).unwrap_or(0); + let mut text = a.model().story.text.clone(); + text.insert(at, 'x'); + let old = a.model().clone(); + let new = Arc::new(DemoDoc { story: Arc::new(model::edit_text(&old.story, &text)), ..(*old).clone() }); + let t = Instant::now(); + a.local_change(&old, &new).unwrap(); + local.push(t.elapsed()); + // The laptop's poll that takes it. + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let t = Instant::now(); + let polled = b.poll(); + if polled.changed { + remote.push(t.elapsed()); + break; + } + assert!(Instant::now() < deadline, "the keystroke didn't come"); + std::thread::sleep(Duration::from_micros(200)); + } + } + assert_eq!(b.model().story.text, a.model().story.text); + let (l, r) = (median(local.clone()), median(remote.clone())); + eprintln!( + "SPEED keystroke local_change median {l:?} (worst {:?}); remote poll median {r:?} (worst {:?})", + local.iter().max(), + remote.iter().max() + ); + assert!(l < Duration::from_millis(2), "local_change median {l:?}"); + assert!(r < Duration::from_millis(5), "remote poll median {r:?}"); + + // A layer renamed: the other side rebuilds only that layer. + let first = model::flatten(&a.model().layers)[10].1.id; + let old = a.model().clone(); + let new = Arc::new(DemoDoc { layers: Arc::new(model::rename(&old.layers, first, "renamed")), ..(*old).clone() }); + let t = Instant::now(); + a.local_change(&old, &new).unwrap(); + let rename = t.elapsed(); + let deadline = Instant::now() + Duration::from_secs(10); + let poll = loop { + let t = Instant::now(); + if b.poll().changed { + break t.elapsed(); + } + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_micros(200)); + }; + eprintln!("SPEED a layer renamed: local_change {rename:?}, remote poll {poll:?}"); + assert_eq!(model::find(&b.model().layers, first).map(|l| l.name.as_str()), Some("renamed")); +} diff --git a/apps/craft-cloud-kit-demo/tests/two_demos.rs b/apps/craft-cloud-kit-demo/tests/two_demos.rs new file mode 100644 index 0000000..c3bb101 --- /dev/null +++ b/apps/craft-cloud-kit-demo/tests/two_demos.rs @@ -0,0 +1,154 @@ +//! Two demo windows, one per machine of an account on one fake hub, driven through their widgets as a person would: +//! a layer added on one shows on the other, typing shows there, a rename there comes back, undo takes back only +//! this window's edit, the other's cursor is marked, and saving writes the file with its save record. + +use std::path::Path; +use std::time::{Duration, Instant}; + +use craft_cloud_app::fake::{Fake, Hub}; +use craft_cloud_kit::Agent; +use craft_cloud_kit_demo::model; +use craft_cloud_kit_demo::ui::Demo; +use egui::accesskit::Role; +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; + +fn demo(fake: &Fake, folder: &Path, machine: &str) -> Demo { + let client = fake.client(); + Demo::open(Some(Agent::from_fn(move || Ok(client.clone()))), folder, "demo.kitdemo", machine, || {}) +} + +fn window(demo: Demo) -> Harness<'static, Demo> { + Harness::builder().with_size(egui::vec2(1000.0, 700.0)).build_ui_state(|ui, d: &mut Demo| d.show(ui), demo) +} + +/// Frames on both windows until `done` holds (20 s at most). +fn until(a: &mut Harness<'static, Demo>, b: &mut Harness<'static, Demo>, what: &str, done: impl Fn(&Demo, &Demo) -> bool) { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + a.run_steps(1); + b.run_steps(1); + if done(a.state(), b.state()) { + return; + } + assert!(Instant::now() < deadline, "timed out waiting for: {what}"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn names(d: &Demo) -> Vec { + model::flatten(&d.model().layers).into_iter().map(|(_, l)| l.name.clone()).collect() +} + +#[test] +fn two_windows_edit_one_live_document() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + let mut a = window(demo(&pc, dir_a.path(), "pc")); + let mut b = window(demo(&laptop, dir_b.path(), "laptop")); + until(&mut a, &mut b, "both live", |a, b| a.is_live() && b.is_live()); + + // A layer added on the pc shows on the laptop. + a.get_by_label("Add layer").click(); + until(&mut a, &mut b, "the layer on the laptop", |_, b| names(b) == ["Layer 1"]); + + // Typing on the pc shows on the laptop. + let story = a.get_by_role(Role::MultilineTextInput); + story.focus(); + a.run_steps(1); + a.get_by_role(Role::MultilineTextInput).type_text("Hello"); + until(&mut a, &mut b, "the text on the laptop", |_, b| b.model().story.text == "Hello"); + + // The laptop renames the layer; it comes back to the pc. + b.get_by_label("Layer 1").click(); + b.run_steps(2); + let name = b.get_by_role(Role::TextInput); + name.focus(); + b.run_steps(1); + b.get_by_role(Role::TextInput).type_text(" (background)"); + b.run_steps(1); + b.get_by_label("Rename").click(); + until(&mut a, &mut b, "the rename on the pc", |a, _| names(a) == ["Layer 1 (background)"]); + + // The laptop's cursor in the story is marked on the pc. + let story = b.get_by_role(Role::MultilineTextInput); + story.focus(); + until(&mut a, &mut b, "the laptop's cursor on the pc", |a, _| a.cursors().iter().any(|c| c.machine == "laptop")); + + // Undo on the pc takes back its typing (its last edit), never the laptop's rename. + a.get_by_label("Undo").click(); + until(&mut a, &mut b, "the pc's undo on both", |a, b| a.model().story.text.is_empty() && b.model().story.text.is_empty()); + assert_eq!(names(a.state()), ["Layer 1 (background)"]); + a.get_by_label("Redo").click(); + until(&mut a, &mut b, "the redo on both", |a, b| a.model().story.text == "Hello" && b.model().story.text == "Hello"); + + // Saving writes the file, and the agent gets the save record. + a.get_by_label("Save").click(); + a.run_steps(1); + let written = std::fs::read(dir_a.path().join("demo.kitdemo")).unwrap(); + let back = model::from_file(&written).unwrap(); + assert_eq!(back.story.text, "Hello"); + assert_eq!(model::flatten(&back.layers).len(), 1); + // (The save record names the content by BLAKE3, as memory blobs do.) + use craft_cloud_kit::BlobStore; + let hash: String = craft_cloud_kit::MemoryBlobs::default().put(&written).unwrap().hash.iter().map(|b| format!("{b:02x}")).collect(); + until(&mut a, &mut b, "the save record at the agent", |_, _| pc.saves().iter().any(|(h, _)| *h == hash)); +} + +/// The demo's model at a newer layout: the document opens read-only in the demo. +struct Newer; + +impl craft_cloud_kit::LiveDocument for Newer { + type Doc = model::DemoDoc; + const APP: &'static str = ::APP; + const SCHEMA: u32 = ::SCHEMA + 1; + fn mappings(doc: &model::DemoDoc) -> craft_cloud_kit::Mappings<'_> { + ::mappings(doc) + } + fn apply_remote( + old: &std::sync::Arc, + patch: &craft_cloud_kit::RemotePatch<'_>, + ) -> Result, craft_cloud_kit::KitError> { + ::apply_remote(old, patch) + } + fn normalise(doc: &mut model::DemoDoc) { + ::normalise(doc); + } + fn refresh_derived(doc: &mut model::DemoDoc) { + ::refresh_derived(doc); + } +} + +/// "Bring it in" when the replace can't be made (here the document is a newer demo's, read-only): nothing is saved, +/// the file keeps the outside edit, and the banner stays. +#[test] +fn bring_it_in_saves_nothing_when_the_replace_fails() { + let hub = Hub::new(); + let pc = hub.machine("pc").unwrap(); + let dir = tempfile::tempdir().unwrap(); + // A newer demo made the document live first, from other bytes. + let client = pc.client(); + let newer = craft_cloud_kit::Session::::open( + &Agent::from_fn(move || Ok(client.clone())), + "demo.kitdemo", + std::sync::Arc::new(model::DemoDoc::default()), + Some(b"the newer demo's file"), + ) + .unwrap(); + assert!(matches!(newer, craft_cloud_kit::Open::Live(_))); + // The file on disk was then edited outside Craft. + let edited = model::DemoDoc { story: std::sync::Arc::new(craft_cloud_kit::RichText::plain("written in an editor")), ..Default::default() }; + let outside = model::to_file(&edited); + std::fs::write(dir.path().join("demo.kitdemo"), &outside).unwrap(); + let mut a = window(demo(&pc, dir.path(), "pc")); + let mut b = window(demo(&pc, dir.path(), "pc again")); + until(&mut a, &mut b, "live, read-only, changed outside", |a, _| { + a.session().is_some_and(|s| s.status().read_only.is_some() && s.status().changed_outside) + }); + a.state_mut().bring_in(); + a.run_steps(1); + assert!(a.state().message.starts_with("not brought in"), "{}", a.state().message); + assert_eq!(std::fs::read(dir.path().join("demo.kitdemo")).unwrap(), outside, "nothing written over the outside edit"); + assert!(a.state().session().is_some_and(|s| s.status().changed_outside), "the banner stays"); +} diff --git a/apps/craft-cloud-server/src/api.rs b/apps/craft-cloud-server/src/api.rs index 00e6613..efc5392 100644 --- a/apps/craft-cloud-server/src/api.rs +++ b/apps/craft-cloud-server/src/api.rs @@ -55,6 +55,7 @@ pub fn router(state: Shared) -> Router { .route("/chunks/{id}", put(put_chunk).get(get_chunk).layer(DefaultBodyLimit::max(MAX_CHUNK_BYTES as usize))) .route("/manifests/{id}", put(put_manifest).get(get_manifest).layer(DefaultBodyLimit::max(MAX_MANIFEST_BYTES))) .route("/commit", post(commit).layer(DefaultBodyLimit::max(MAX_BATCH_JSON_BYTES))) + .route("/saves", post(attach_save).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) .route("/space", get(space)) .route("/tree", get(tree)) .route("/changes", get(changes)) @@ -362,6 +363,28 @@ async fn commit(State(state): State, Extension(auth): Extension, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: craft_cloud_proto::api::SaveAttach = read_json(&state, request, SMALL_JSON_BYTES).await?; + let task_state = state.clone(); + // As a commit: the publish is never skipped for a change that went through. + let outcome = tokio::spawn(async move { + let outcome = task_state.meta.attach_save(auth.space, req.node, req.version, &req.save).await?; + task_state.notifier.publish(auth.space, outcome.seq); + Ok::<_, ApiError>(outcome) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "save attach task failed"); + ApiError::internal() + })??; + Ok(Json(CommitResponse { seq: outcome.seq, epoch: outcome.epoch })) +} + fn cursor(after_seq: Option, after_node: Option) -> Result, ApiError> { match (after_seq, after_node) { (Some(seq), Some(node)) => Ok(Some(PageCursor { seq, node })), diff --git a/apps/craft-cloud-server/src/live.rs b/apps/craft-cloud-server/src/live.rs index 1cee86e..859d5c8 100644 --- a/apps/craft-cloud-server/src/live.rs +++ b/apps/craft-cloud-server/src/live.rs @@ -1130,7 +1130,7 @@ mod tests { self.meta.add_manifest(space, &manifest, 100, 0).await.unwrap(); let root = self.meta.space_info(space).await.unwrap().root; let node = NodeId::generate(); - let version = NewVersion { id: craft_cloud_proto::VersionId::generate(), manifest: manifest.id(), mtime_ms: 0, exec: false }; + let version = NewVersion { id: craft_cloud_proto::VersionId::generate(), manifest: manifest.id(), mtime_ms: 0, exec: false, save: None }; let op = Op::Create { node, parent: root, name: Name::new(name).unwrap(), version }; self.meta.commit(space, auth.device, &[op], 0).await.unwrap(); node diff --git a/apps/craft-cloud-server/tests/api.rs b/apps/craft-cloud-server/tests/api.rs index 163ad16..83b9514 100644 --- a/apps/craft-cloud-server/tests/api.rs +++ b/apps/craft-cloud-server/tests/api.rs @@ -265,7 +265,12 @@ async fn root(s: &Server, token: &str) -> NodeId { } fn create(node: NodeId, parent: NodeId, name: &str, manifest: ManifestId) -> Op { - Op::Create { node, parent, name: Name::new(name).unwrap(), version: NewVersion { id: VersionId::generate(), manifest, mtime_ms: 1, exec: false } } + Op::Create { + node, + parent, + name: Name::new(name).unwrap(), + version: NewVersion { id: VersionId::generate(), manifest, mtime_ms: 1, exec: false, save: None }, + } } // ---- scenarios ---- diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml new file mode 100644 index 0000000..e6be988 --- /dev/null +++ b/crates/api/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "craft-cloud-api" +description = "What Craft apps need of Craft Cloud's protocol types: IDs, names, limits and the agent's local API, with no binary wire encoding" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +caseless.workspace = true +serde.workspace = true +thiserror.workspace = true +unicode-normalization.workspace = true +uuid.workspace = true + +[dev-dependencies] +proptest.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/api/src/api.rs b/crates/api/src/api.rs new file mode 100644 index 0000000..699dfae --- /dev/null +++ b/crates/api/src/api.rs @@ -0,0 +1,89 @@ +//! The parts of the HTTP API's bodies (`/api/v1/…`) that apps see through the agent's local API: the error codes, the +//! error answer's code and message, the account's devices and add-a-computer codes. The rest of the HTTP API (commit +//! batches, manifests, the full error body) is the server's and the agent's, in `craft_cloud_proto::api`, which +//! re-exports these. + +use serde::{Deserialize, Serialize}; + +use crate::ids::DeviceId; + +/// Stable error codes, the `error` field of an error answer ([`ErrorAnswer`]). +pub mod codes { + /// A commit's operations clash with the current state; `conflicts` lists them. + pub const CONFLICT: &str = "conflict"; + /// Chunks must be uploaded first; `missing` lists them. + pub const MISSING_CHUNKS: &str = "missing_chunks"; + /// The client's change counter is stale or ahead of the server: list the whole tree again. + pub const RESYNC_REQUIRED: &str = "resync_required"; + pub const UNAUTHORIZED: &str = "unauthorized"; + pub const RATE_LIMITED: &str = "rate_limited"; + pub const INVALID: &str = "invalid"; + pub const NOT_FOUND: &str = "not_found"; + pub const TOO_LARGE: &str = "too_large"; + /// A request body took longer to arrive than the server allows for it (HTTP 408); send it again. + pub const TIMEOUT: &str = "timeout"; + pub const USERNAME_TAKEN: &str = "username_taken"; + /// A chunk or manifest would take the account over the bytes the server lets it store (HTTP 507); `quota` says + /// how much it stores and may store. + pub const QUOTA_EXCEEDED: &str = "quota_exceeded"; + /// `join`: the code is wrong, expired or used (never which). + pub const INVALID_CODE: &str = "invalid_code"; + /// `join`: an invite to an account without a password needs one; send it again with `password`. + pub const PASSWORD_REQUIRED: &str = "password_required"; + /// `join`: the new password breaks the server's rules (the message says which). + pub const WEAK_PASSWORD: &str = "weak_password"; + pub const INTERNAL: &str = "internal"; +} + +/// What a client reads of every error answer: `error`, a stable code (one of [`codes`], or the local API's), and +/// `message`, for people. Every other field is ignored (the full body is `craft_cloud_proto::api::ErrorBody`). +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct ErrorAnswer { + pub error: String, + pub message: String, +} + +/// Where an add-a-computer code stands, in [`PairingStatus`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PairingState { + /// Not used yet, and still good. + Waiting, + /// A computer joined with it; [`PairingStatus::device_name`] names it. + Used, + /// Its time ran out unused. + Expired, +} + +/// `GET /api/v1/pairings/{id}` (device auth, the account's own pairings only). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PairingStatus { + pub state: PairingState, + /// The device that joined, once `used`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_name: Option, +} + +/// One of the signed-in user's devices. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeviceInfo { + pub id: DeviceId, + pub name: String, + pub created_ms: i64, + pub last_used_ms: i64, + pub revoked: bool, + /// The device making this request. + pub current: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_error_answer_reads_any_error_body() { + let json = serde_json::json!({ "error": "conflict", "message": "m", "conflicts": [{ "op": 0, "kind": "root" }], "quota": { "used": 1, "limit": 2 } }); + let answer: ErrorAnswer = serde_json::from_value(json).unwrap(); + assert_eq!(answer, ErrorAnswer { error: "conflict".into(), message: "m".into() }); + } +} diff --git a/crates/proto/src/ids.rs b/crates/api/src/ids.rs similarity index 89% rename from crates/proto/src/ids.rs rename to crates/api/src/ids.rs index 15b55b9..42f44a8 100644 --- a/crates/proto/src/ids.rs +++ b/crates/api/src/ids.rs @@ -67,9 +67,9 @@ uuid_id!( Epoch ); uuid_id!( - /// A live document (see [`crate::live`]). Keyed by `(space, doc)` on the server, like nodes, so an ID never - /// reaches into another space. An app's recents document has an ID derived from the space and the app - /// ([`crate::live::recents_doc`]), so every machine finds the same one without asking. + /// A live document (see `craft_cloud_proto::live`). Keyed by `(space, doc)` on the server, like nodes, so an ID + /// never reaches into another space. An app's recents document has an ID derived from the space and the app + /// (`craft_cloud_proto::live::recents_doc`), so every machine finds the same one without asking. DocId ); diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs new file mode 100644 index 0000000..bb5682d --- /dev/null +++ b/crates/api/src/lib.rs @@ -0,0 +1,16 @@ +//! What Craft apps need of Craft Cloud's protocol types, with no binary wire encoding: IDs, names, limits, the agent's +//! local API ([`local`]) and the error codes and bodies it answers with ([`api`]). +//! +//! Pure: no I/O. `craft-cloud-app` and `craft-cloud-kit` (what apps link) depend on this crate alone; the server's and +//! the agent's wire types (the live protocol's frames, encoded with postcard, manifests, commit batches) are in +//! `craft-cloud-proto`, which re-exports everything here under the same paths. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod api; +pub mod ids; +pub mod limits; +pub mod local; +pub mod name; + +pub use ids::{DeviceId, DocId, Epoch, NodeId, SpaceId, VersionId}; +pub use name::{FoldKey, Name, NameError, UNICODE_VERSION}; diff --git a/crates/proto/src/limits.rs b/crates/api/src/limits.rs similarity index 67% rename from crates/proto/src/limits.rs rename to crates/api/src/limits.rs index 24854cb..bf46f86 100644 --- a/crates/proto/src/limits.rs +++ b/crates/api/src/limits.rs @@ -22,3 +22,14 @@ pub const MAX_BATCH_JSON_BYTES: usize = 1024 * 1024; pub const MAX_TREE_DEPTH: usize = 256; /// Most nodes one page of the tree or change feed may hold. pub const MAX_PAGE_NODES: u32 = 5000; + +/// Largest live update (a Loro update an app sends, or the server relays), in bytes. +pub const MAX_UPDATE_BYTES: usize = 1024 * 1024; +/// Largest snapshot of a live document (an import, or a compaction), in bytes (it travels through the chunk API, never +/// in a frame). +pub const MAX_SNAPSHOT_BYTES: usize = MAX_MANIFEST_BYTES; +/// Largest presence message (Loro's ephemeral state, or an app's own: a cursor, a selection, a machine name), in bytes. +pub const MAX_PRESENCE_BYTES: usize = 4096; +/// Largest blob an app may put through the local API's `/v1/blobs` (binary content kept out of a live document), in +/// bytes. +pub const MAX_BLOB_BYTES: usize = 64 * 1024 * 1024; diff --git a/crates/proto/src/local.rs b/crates/api/src/local.rs similarity index 75% rename from crates/proto/src/local.rs rename to crates/api/src/local.rs index 823a039..c2b9874 100644 --- a/crates/proto/src/local.rs +++ b/crates/api/src/local.rs @@ -15,8 +15,9 @@ //! `DELETE /v1/devices/{id}` → the list after; `POST /v1/pause`, `/v1/resume` and `/v1/quit` → [`Status`]; //! `POST /v1/autostart` ([`AutostartRequest`]) → [`Status`]; `GET /v1/recents`, upgraded to a WebSocket of JSON //! text messages: [`RecentsRequest`] in, [`RecentsEvent`] out; `GET /v1/docs`, upgraded to a WebSocket of live -//! documents ([`docs`]: [`DocsRequest`] and binary frames in, [`DocsEvent`] and binary frames out). Errors are the HTTP API's -//! [`ErrorBody`](crate::api::ErrorBody) with its codes and [`codes`]. +//! documents ([`docs`]: [`DocsRequest`] and binary frames in, [`DocsEvent`] and binary frames out); `PUT /v1/blobs` +//! and `GET /v1/blobs/{hash}`: binary content kept out of live documents ([`blobs`]). Errors are the HTTP API's error +//! body ([`ErrorAnswer`](crate::api::ErrorAnswer) reads its code and message) with its codes and [`codes`]. //! - **Versions**: [`Status::api`] is the local API version the agent speaks; a client refuses one outside //! [`LOCAL_API_MIN`]..=[`LOCAL_API_MAX`] when it connects. @@ -52,7 +53,7 @@ pub const MAX_MACHINE_BYTES: usize = 128; /// Largest JSON body or WebSocket message of the local API, either way. pub const MAX_LOCAL_JSON_BYTES: usize = 1024 * 1024; -/// Error codes of the local API (the `error` field of [`ErrorBody`](crate::api::ErrorBody)), beside the HTTP API's +/// Error codes of the local API (the `error` field of [`ErrorAnswer`](crate::api::ErrorAnswer)), beside the HTTP API's /// `invalid`, `unauthorized` (the token), `not_found`, `too_large` and `internal`. pub mod codes { /// A `Host` that isn't `127.0.0.1:`, or an `Origin` header: the request didn't come from a local program. @@ -83,6 +84,11 @@ pub mod codes { /// `join` or `login` without `adopt`: the folder already has files in it. Nothing was sent to the server, so the /// code still works; ask whether to sync those files, and send it again with `adopt` if so. pub const FOLDER_HAS_FILES: &str = "folder_has_files"; + /// `GET /v1/blobs/{hash}`: neither this machine nor the account's server has that blob. + pub const BLOB_NOT_FOUND: &str = "blob_not_found"; + /// Never sent by an agent: what a client reports when the agent answers a call it doesn't know (`not_found` for + /// the call itself), as one from before that call answers it. Update Craft Cloud. + pub const AGENT_TOO_OLD: &str = "agent_too_old"; } /// The paths of the local API. @@ -108,6 +114,69 @@ pub mod paths { pub const QUIT: &str = "/v1/quit"; /// `POST` ([`super::AutostartRequest`]): start, or don't start, the agent at login → [`super::Status`]. pub const AUTOSTART: &str = "/v1/autostart"; + /// `PUT`: a blob → [`super::blobs::Stored`]; `GET /v1/blobs/{hash}`: its bytes ([`super::blobs`]). + pub const BLOBS: &str = "/v1/blobs"; +} + +/// Binary content an app keeps out of a live document (images, audio, a font): `PUT /v1/blobs` with the bytes as the +/// body (`application/octet-stream`, 1 byte to [`MAX_BLOB_BYTES`](crate::limits::MAX_BLOB_BYTES)) is answered with +/// [`Stored`] once the blob is on the agent's disk; it goes up to the server as a file's content does (cut into +/// chunks, then a manifest), at once when online and later otherwise. `GET /v1/blobs/{hash}` answers the bytes: from +/// the agent's disk, or downloaded first when this machine doesn't have them yet (`offline` when it can't reach the +/// server; [`codes::BLOB_NOT_FOUND`](super::codes::BLOB_NOT_FOUND) when the account has no such blob). +/// +/// A blob is named by its content: `hash` is the BLAKE3 hash of the manifest the bytes cut into (with `fastcdc-v1`, +/// as the file sync cuts content), so the same bytes have the same name on every machine. An agent from before blobs +/// answers both calls with the local API's `not_found` for an unknown call; a client reads that as +/// [`codes::AGENT_TOO_OLD`](super::codes::AGENT_TOO_OLD). +pub mod blobs { + use serde::{Deserialize, Serialize}; + + /// The answer to a `PUT`: the blob's name and length. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] + pub struct Stored { + /// 64 lowercase hex digits. + #[serde(with = "hex32")] + pub hash: [u8; 32], + pub len: u64, + } + + /// `hash` written as 64 lowercase hex digits, for the `GET` path. + pub fn hex(hash: &[u8; 32]) -> String { + hash.iter().map(|b| format!("{b:02x}")).collect() + } + + /// 64 lowercase hex digits as a hash; `None` for anything else. + pub fn parse_hex(text: &str) -> Option<[u8; 32]> { + let bytes = text.as_bytes(); + if bytes.len() != 64 { + return None; + } + let mut out = [0u8; 32]; + for (slot, pair) in out.iter_mut().zip(bytes.chunks(2)) { + let digit = |c: u8| match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + _ => None, + }; + let (hi, lo) = (digit(*pair.first()?)?, digit(*pair.get(1)?)?); + *slot = (hi << 4) | lo; + } + Some(out) + } + + mod hex32 { + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(hash: &[u8; 32], s: S) -> Result { + s.serialize_str(&super::hex(hash)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<[u8; 32], D::Error> { + let text = String::deserialize(d)?; + super::parse_hex(&text).ok_or_else(|| serde::de::Error::custom("expected 64 lowercase hex digits")) + } + } } /// The agent's data folder: `$CRAFT_CLOUD_AGENT_DIR` if set, else [`default_data_dir`]. `env` reads an environment @@ -241,7 +310,7 @@ impl std::fmt::Debug for LoginRequest { #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct JoinRequest { /// As typed or pasted: `cloud.example.com/K7M4-Q9TX`, or the link `https://cloud.example.com/join/K7M4-Q9TX` - /// (see [`crate::join::JoinString`]). + /// (see `craft_cloud_proto::join::JoinString`). pub join: String, /// The folder to sync (absolute). Default: "Craft Cloud" in the home folder. #[serde(default)] @@ -396,7 +465,7 @@ pub mod docs { /// The header of a binary frame: kind and id. pub const HEADER_BYTES: usize = 9; /// Largest binary frame (a state or an import: a snapshot), either way. - pub const MAX_BINARY_BYTES: usize = crate::live::MAX_SNAPSHOT_BYTES + HEADER_BYTES; + pub const MAX_BINARY_BYTES: usize = crate::limits::MAX_SNAPSHOT_BYTES + HEADER_BYTES; /// A binary frame. pub fn encode_binary(kind: u8, id: u64, bytes: &[u8]) -> Vec { @@ -420,10 +489,71 @@ pub mod docs { #[serde(rename_all = "snake_case", tag = "type")] pub enum DocsRequest { /// Open the file at `path` (relative to the synced folder, `/`-separated, or absolute inside it) as a live document - /// for `app` (a lowercase app name), under the number `id` on this connection. - Open { id: u64, path: String, app: String }, + /// for `app` (a lowercase app name), under the number `id` on this connection. `file_hash`: the BLAKE3 hash (hex) + /// of the file's bytes as the app read them, for `opened` to say whether a Craft app saved them + /// ([`DocsEvent::Opened`]'s `changed_outside`). + Open { + id: u64, + path: String, + app: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + file_hash: Option, + }, /// The app is done with the document. Close { doc: crate::ids::DocId }, + /// The app is about to write the document's file: its content's BLAKE3 hash (hex) and the live document's + /// frontiers it was made from (Loro's heads, `[peer, counter]` operation IDs). Sent on the same stream after every + /// change it covers; the agent keeps it once all those operations are in its copy (one that comes first waits for + /// them, and is dropped if the app closes the document before), and records it with the file version it uploads + /// for that content ([`SaveRecord`]). + Saved { doc: crate::ids::DocId, hash: String, frontiers: Vec<(u64, u32)> }, + /// The app is about to write the document's file with content `hash` (hex), and its record (`saved`) follows once + /// the changes it covers have gone: sent at once, before the write. Until the record comes (or the app closes the + /// document), the agent's sync takes that content for a Craft save on its way: it doesn't upload it without its + /// record, nor make a conflict copy of it, but waits (at most the save-order wait, 10 minutes). + Saving { doc: crate::ids::DocId, hash: String }, +} + +/// Most heads a save record's frontiers may have: the live document's concurrent tips, one or two in practice +/// (a record with more is a hostile one and isn't kept). At most about 9 KiB of JSON. +pub const MAX_SAVE_HEADS: usize = 256; + +/// A file version a Craft app saved from a live document: what the version's content came from. Kept with the +/// version on the server (an older server or agent ignores it), so the file sync can tell two Craft saves of one live +/// document from an edit outside Craft. +/// +/// It names the live document's frontiers when saved (its heads: the last operations, as `(peer, counter)`), never a +/// version vector, so it stays small however many machines and sessions have edited the document. Whether one record +/// includes another is answered by an agent that holds the document, from its copy (see the sync's `SaveRecords`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SaveRecord { + /// The live document. + pub doc: crate::ids::DocId, + /// The live document's frontiers when saved: `(peer, counter)` operation IDs, sorted. + pub frontiers: Vec<(u64, u32)>, +} + +impl SaveRecord { + /// Whether the record is well formed: at most [`MAX_SAVE_HEADS`] heads, sorted and distinct. + pub fn is_valid(&self) -> bool { + self.frontiers.len() <= MAX_SAVE_HEADS && self.frontiers.windows(2).all(|w| matches!(w, [a, b] if a < b)) + } + + /// Reads an optional record field leniently (`#[serde(deserialize_with)]`): a record that doesn't read (an older + /// build's form, a damaged one, an unknown later one) or isn't well formed is no record, and only the record is + /// lost, never what holds it. + pub fn lenient<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum Any { + Record(SaveRecord), + Other(serde::de::IgnoredAny), + } + Ok(match Option::::deserialize(d)? { + Some(Any::Record(r)) if r.is_valid() => Some(r), + _ => None, + }) + } } /// How a document stands, as `opened` and `status` say it. @@ -436,6 +566,10 @@ pub struct DocState { /// The size of the binary state that follows `opened` (0 in `status`). #[serde(default)] pub bytes: u64, + /// How many `update` frames follow the state as part of it (updates still waiting for their history): an app + /// imports them with the state before it writes with its peer (0 in `status`, and from an older agent). + #[serde(default)] + pub updates: u64, } /// Why a file opens as a plain file rather than a live document. @@ -461,8 +595,17 @@ pub enum NotLiveReason { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] pub enum DocsEvent { - /// The document is live: its state follows as a binary frame of `state.bytes` bytes. - Opened { id: u64, doc: crate::ids::DocId, state: DocState }, + /// The document is live: its state follows as a binary frame of `state.bytes` bytes. `changed_outside`, for an + /// open that named its `file_hash`: whether no Craft app's save of this document (this machine's records, or the + /// file version's on the server) has that content: the file was edited outside Craft, and the app may offer to + /// bring it in. None: the agent can't tell. (Every app session writes with a fresh random Loro peer of its own.) + Opened { + id: u64, + doc: crate::ids::DocId, + state: DocState, + #[serde(default, skip_serializing_if = "Option::is_none")] + changed_outside: Option, + }, /// The file has never been live: send its content as a binary import frame. NeedsImport { id: u64 }, /// The file opens as a plain file. @@ -486,6 +629,15 @@ pub enum DocsEvent { #[serde(default)] stopped: bool, }, + /// The file of an open document, as it is on this machine's disk now: the live document's frontiers a Craft app + /// saved it from (the record of that save, this machine's or the server's version's), or none when no record is + /// known (edited outside Craft, not synced yet). Sent after `opened` and whenever it changes (the app saved, the + /// sync brought another machine's save). An app whose live document is strictly ahead of it saves again. + File { + doc: crate::ids::DocId, + #[serde(default)] + frontiers: Option>, + }, /// A request or a binary frame was refused (`id` and `seq` name it when they can). `code` is one of /// [`crate::api::codes`] or [`codes`]. Error { @@ -517,6 +669,17 @@ pub fn recent_path(path: &str) -> Option { mod tests { use super::*; + /// A save record names frontiers, sorted and distinct, at most [`MAX_SAVE_HEADS`] of them. + #[test] + fn a_save_record_is_its_frontiers() { + let doc = crate::ids::DocId::generate(); + assert!(SaveRecord { doc, frontiers: vec![(1, 5), (2, 0)] }.is_valid()); + assert!(SaveRecord { doc, frontiers: Vec::new() }.is_valid(), "an empty document"); + assert!(!SaveRecord { doc, frontiers: vec![(2, 0), (1, 5)] }.is_valid(), "unsorted"); + assert!(!SaveRecord { doc, frontiers: vec![(1, 5), (1, 5)] }.is_valid(), "twice"); + assert!(!SaveRecord { doc, frontiers: (0..=MAX_SAVE_HEADS as u64).map(|p| (p, 1)).collect() }.is_valid()); + } + /// A list's event says when its push stopped; without the field (an older agent) it hasn't. #[test] fn a_list_says_when_its_push_stopped() { @@ -633,7 +796,11 @@ mod tests { assert_eq!(docs::decode_binary(&frame[..8]), None, "too short for a header"); assert_eq!(docs::decode_binary(&[]), None); let open: DocsRequest = serde_json::from_value(serde_json::json!({ "type": "open", "id": 1, "path": "a.txt", "app": "wordcraft" })).unwrap(); - assert_eq!(open, DocsRequest::Open { id: 1, path: "a.txt".into(), app: "wordcraft".into() }); + assert_eq!(open, DocsRequest::Open { id: 1, path: "a.txt".into(), app: "wordcraft".into(), file_hash: None }); + let plain = + serde_json::to_value(DocsEvent::Opened { id: 1, doc: crate::ids::DocId::generate(), state: DocState::default(), changed_outside: None }) + .unwrap(); + assert!(plain.get("changed_outside").is_none(), "none: no field (as an older agent sends it)"); let not_live = serde_json::to_value(DocsEvent::NotLive { id: 1, reason: NotLiveReason::NotSynced, message: None }).unwrap(); assert_eq!(not_live, serde_json::json!({ "type": "not_live", "id": 1, "reason": "not_synced", "message": null })); let status: DocsEvent = diff --git a/crates/proto/src/name.rs b/crates/api/src/name.rs similarity index 98% rename from crates/proto/src/name.rs rename to crates/api/src/name.rs index 6f96884..76dbf09 100644 --- a/crates/proto/src/name.rs +++ b/crates/api/src/name.rs @@ -10,7 +10,7 @@ use unicode_normalization::UnicodeNormalization; use crate::limits::MAX_NAME_BYTES; /// Characters Windows forbids in names (besides control characters, which are rejected separately). -pub(crate) const FORBIDDEN: [char; 9] = ['/', '\\', '<', '>', ':', '"', '|', '?', '*']; +pub const FORBIDDEN: [char; 9] = ['/', '\\', '<', '>', ':', '"', '|', '?', '*']; /// The Unicode version behind [`Name`]'s NFC form and [`FoldKey`]. Both are only stable within one Unicode /// version: anything that stores fold keys must store this version with them and recompute the keys when it diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 7004111..eee563e 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -1,14 +1,18 @@ [package] name = "craft-cloud-app" -description = "What Craft apps and the launcher link to reach the Craft Cloud agent on this machine: account status, sign-in, joining, recents" +description = "What Craft apps and the launcher link to reach the Craft Cloud agent on this machine: account status, sign-in, joining, recents, live documents and blobs" version.workspace = true edition.workspace = true license.workspace = true rust-version.workspace = true publish = false +[features] +# `craft_cloud_app::fake`: an agent in the test's own process (documents and blobs in memory, on a shared hub). +fake = [] + [dependencies] -craft-cloud-proto.workspace = true +craft-cloud-api.workspace = true httparse.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/app/src/blobs.rs b/crates/app/src/blobs.rs new file mode 100644 index 0000000..2769870 --- /dev/null +++ b/crates/app/src/blobs.rs @@ -0,0 +1,94 @@ +//! Binary content kept out of live documents, through the agent's `/v1/blobs` ([`craft_cloud_api::local::blobs`]): an +//! app puts the bytes and keeps the [`Stored`] name in its document; any machine of the account gets the bytes back by +//! that name. Blocking; a get may wait for a download (up to [`GET_TIMEOUT`]), so call it off the UI thread. +//! +//! An agent from before blobs answers [`Error::Refused`] with [`codes::AGENT_TOO_OLD`] here (and only here: the rest +//! of the local API works with it as before). + +use std::time::Duration; + +use craft_cloud_api::limits::MAX_BLOB_BYTES; +use craft_cloud_api::local::blobs::hex; +use craft_cloud_api::local::{codes, paths}; + +pub use craft_cloud_api::local::blobs::Stored; + +use crate::{Client, Error}; + +/// How long a put may take (the agent writes the bytes to its disk and answers; the upload follows by itself). +pub const PUT_TIMEOUT: Duration = Duration::from_secs(60); +/// How long a get may take (it may download the blob first). +pub const GET_TIMEOUT: Duration = Duration::from_secs(120); + +/// The agent's blob store. +#[derive(Debug, Clone)] +pub struct Blobs { + client: Client, +} + +/// What an agent from before blobs answers. +fn not_found() -> Error { + Error::Refused { code: craft_cloud_api::api::codes::NOT_FOUND.into(), message: "no such call".into() } +} + +/// An older agent's "no such call" as [`codes::AGENT_TOO_OLD`]. +fn too_old(e: Error) -> Error { + match e { + Error::Refused { code, .. } if code == craft_cloud_api::api::codes::NOT_FOUND => Error::Refused { + code: codes::AGENT_TOO_OLD.into(), + message: "the Craft Cloud agent on this computer is too old for binary content in live documents: update Craft Cloud".into(), + }, + other => other, + } +} + +impl Blobs { + pub(crate) fn new(client: Client) -> Self { + Self { client } + } + + /// Whether the agent knows `/v1/blobs`, asked once per client with a cheap `GET` of a name no blob has (an agent + /// from before blobs answers its `not_found` for an unknown call before reading a body, so a big put to it would + /// only end in a broken connection). A question that gets no answer is asked again next time. + fn check_agent(&self) -> Result<(), Error> { + if let Some(known) = self.client.blobs_known.get() { + return if *known { Ok(()) } else { Err(too_old(not_found())) }; + } + let path = format!("{}/{}", paths::BLOBS, hex(&[0; 32])); + let known = match self.client.binary("GET", &path, None, MAX_BLOB_BYTES, crate::http::QUICK_TIMEOUT) { + Err(Error::Refused { code, .. }) if code == craft_cloud_api::api::codes::NOT_FOUND => false, + // `blob_not_found`, `not_signed_in`, `offline`: an agent that knows blobs. + Ok(_) | Err(Error::Refused { .. }) => true, + Err(e) => return Err(e), + }; + let _ = self.client.blobs_known.set(known); + if known { Ok(()) } else { Err(too_old(not_found())) } + } + + /// Keeps `bytes` (1 byte to [`MAX_BLOB_BYTES`]) and names them. Answered once they are on the agent's disk. An + /// agent too old for blobs is found out before a byte of them is sent ([`codes::AGENT_TOO_OLD`]). + pub fn put(&self, bytes: &[u8]) -> Result { + if bytes.is_empty() || bytes.len() > MAX_BLOB_BYTES { + return Err(Error::Refused { code: "too_large".into(), message: format!("a blob is 1 byte to {MAX_BLOB_BYTES} bytes") }); + } + self.check_agent()?; + let answer = + self.client.binary("PUT", paths::BLOBS, Some(bytes), craft_cloud_api::local::MAX_LOCAL_JSON_BYTES, PUT_TIMEOUT).map_err(too_old)?; + let stored: Stored = serde_json::from_slice(&answer.body).map_err(|e| Error::Protocol(e.to_string()))?; + if stored.len != bytes.len() as u64 { + return Err(Error::Protocol("the agent named a blob of another length".into())); + } + Ok(stored) + } + + /// The bytes named `hash`, of `len` bytes (checked). + pub fn get(&self, hash: &[u8; 32], len: u64) -> Result, Error> { + self.check_agent()?; + let path = format!("{}/{}", paths::BLOBS, hex(hash)); + let answer = self.client.binary("GET", &path, None, MAX_BLOB_BYTES, GET_TIMEOUT).map_err(too_old)?; + if answer.body.len() as u64 != len { + return Err(Error::Protocol(format!("the agent's blob is {} bytes, not {len}", answer.body.len()))); + } + Ok(answer.body) + } +} diff --git a/crates/app/src/fake.rs b/crates/app/src/fake.rs new file mode 100644 index 0000000..ee5b8de --- /dev/null +++ b/crates/app/src/fake.rs @@ -0,0 +1,899 @@ +//! A fake agent for tests (feature `fake`): it serves the local API's `/v1/status`, `/v1/docs` and `/v1/blobs` on a +//! loopback port of this process, from memory, so an app (or the app kit) is tested through the real client code with +//! no agent and no server. +//! +//! Every [`Fake`] is one machine. Fakes made from one [`Hub`] are machines of one account: the hub stands in for the +//! server, holding each document (its import and every update after it, in order) and the blobs. A machine keeps its +//! own copy of each document it opened (as the agent's journal does): an app's update goes into that copy and is acked +//! at once, and goes to the hub while the machine is online ([`Fake::set_online`]); offline it waits there and goes up +//! when the machine is back, and the other machines' updates come down then. What the bytes mean is never read: the +//! state an app opens is the import followed by the updates as `update` frames (the local API allows that). +//! +//! [`Fake::restart`] drops every app connection and refuses new ones until [`Fake::start`] (an agent that stopped): +//! what the machine held stays, as the agent's files do. [`Fake::too_old`] makes a machine answer the blob calls as an +//! agent from before blobs (`not_found` for the call). [`Hub::set_live`] holds live updates back between machines while +//! saved files' records still go (a proxy that blocks WebSockets). + +use std::collections::{HashMap, VecDeque}; +use std::io::{Read, Write}; +use std::net::{Ipv4Addr, Shutdown, SocketAddr, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use craft_cloud_api::DocId; +use craft_cloud_api::limits::{MAX_BLOB_BYTES, MAX_PRESENCE_BYTES, MAX_UPDATE_BYTES}; +use craft_cloud_api::local::blobs::{Stored, hex, parse_hex}; +use craft_cloud_api::local::docs::{IMPORT, MAX_BINARY_BYTES, PRESENCE, STATE, UPDATE, decode_binary, encode_binary}; +use craft_cloud_api::local::{DocState, DocsEvent, DocsRequest, LOCAL_API_MAX, MAX_LOCAL_JSON_BYTES, NotLiveReason, RunState, Status, paths}; +use tungstenite::protocol::{Role, WebSocketConfig}; +use tungstenite::{Message, WebSocket}; + +use crate::Client; + +/// The account the fakes of one hub share: the server's part. +#[derive(Clone, Default)] +pub struct Hub { + inner: Arc>, +} + +impl std::fmt::Debug for Hub { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Hub").finish_non_exhaustive() + } +} + +#[derive(Default)] +struct Inner { + docs: HashMap, + blobs: HashMap<[u8; 32], Vec>, + machines: HashMap, + next_machine: usize, + next_conn: u64, + /// The record of each path's file on the server (as if every save were written and synced at once: the newest + /// upload wins); `None` for one edited outside Craft ([`Hub::edit_outside`]). + files: HashMap>, + /// The live relay is down while the file sync goes on ([`Hub::set_live`]). + live_down: bool, +} + +struct HubDoc { + id: DocId, + import: Vec, + /// Every update after the import, with the machine it came from. + log: Vec<(usize, Vec)>, +} + +/// Where an update in a machine's copy came from. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Origin { + /// An app connection of this machine. + Conn(u64), + /// Another machine. + Remote, +} + +/// A machine's copy of a document. +struct MachineDoc { + path: String, + import: Vec, + log: Vec<(Origin, Vec)>, + /// How much of the hub's log this copy has taken. + hub_seen: usize, + /// Positions in `log` of this machine's updates the hub doesn't have yet. + pending: VecDeque, +} + +struct Machine { + name: String, + token: String, + online: bool, + running: bool, + too_old: bool, + docs: HashMap, + blobs: HashMap<[u8; 32], Vec>, + pending_blobs: Vec<[u8; 32]>, + conns: HashMap, + /// The save records apps sent: document, content hash (hex), frontiers. + saves: Vec<(DocId, String, Version)>, + /// Codes the next updates are refused with ([`Fake::refuse_next`]). + refuse: VecDeque, + /// Saves made here that haven't gone up yet (offline): each path's file record on this machine until they do. + file_pending: HashMap, +} + +/// A live document's frontiers: `(peer, counter)` heads. +type Version = Vec<(u64, u32)>; + +/// One app connection to the docs socket. +struct Conn { + writer: Arc>>, + stream: TcpStream, + /// Each document number opened: the document, how much of the machine's copy went out, and the app's updates. + open: HashMap, + /// A `needs_import` waiting for its import, by number: the path, and the document it will be. + importing: HashMap, + /// The hash (hex) of the file each open read, by number. + apps: HashMap>, + told: HashMap, + /// The file record each document's apps were last told. + told_file: HashMap>, +} + +struct Opened { + doc: DocId, + sent: usize, + seq: u64, +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl Hub { + pub fn new() -> Self { + Self::default() + } + + /// A new machine of this account, signed in and online, its local API listening on a loopback port. + pub fn machine(&self, name: &str) -> std::io::Result { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?; + let addr = listener.local_addr()?; + let token = format!("fake-{}-{}", name, addr.port()); + let id = { + let mut inner = lock(&self.inner); + let id = inner.next_machine; + inner.next_machine += 1; + inner.machines.insert( + id, + Machine { + name: name.to_owned(), + token: token.clone(), + online: true, + running: true, + too_old: false, + docs: HashMap::new(), + blobs: HashMap::new(), + pending_blobs: Vec::new(), + conns: HashMap::new(), + saves: Vec::new(), + refuse: VecDeque::new(), + file_pending: HashMap::new(), + }, + ); + id + }; + let stop = Arc::new(AtomicBool::new(false)); + let (hub, stopping) = (self.clone(), stop.clone()); + std::thread::Builder::new().name(format!("fake-agent-{name}")).spawn(move || { + for stream in listener.incoming() { + if stopping.load(Ordering::Relaxed) { + return; + } + let Ok(stream) = stream else { continue }; + let hub = hub.clone(); + let _ = std::thread::Builder::new().name("fake-agent-conn".into()).spawn(move || hub.serve(id, stream)); + } + })?; + Ok(Fake { hub: self.clone(), id, addr, token, stop }) + } + + /// Another file takes `path`'s place (the old one renamed away while agents were down): with `Some(import)`, a + /// live document of its own (a new ID); with `None`, one that was never live. Every machine forgets its copy of + /// the old one there. + pub fn replace(&self, path: &str, import: Option>) { + let mut inner = lock(&self.inner); + match import { + Some(import) => { + inner.docs.insert(path.to_owned(), HubDoc { id: DocId::generate(), import, log: Vec::new() }); + } + None => { + inner.docs.remove(path); + } + } + for m in inner.machines.values_mut() { + m.docs.retain(|_, d| d.path != path); + } + } + + /// Whether live updates go between machines (`false`: a proxy that blocks WebSockets, the relay out) while the + /// file sync goes on (saved files' records still reach every machine). + pub fn set_live(&self, on: bool) { + let mut inner = lock(&self.inner); + inner.live_down = !on; + sync(&mut inner); + } + + /// Another program writes `path`'s file (no save record): every machine's apps hear it has none. + pub fn edit_outside(&self, path: &str) { + let mut inner = lock(&self.inner); + inner.files.insert(path.to_owned(), None); + sync(&mut inner); + } + + /// The record of `path`'s file on the server: the live document's version a Craft app saved it from. + pub fn file(&self, path: &str) -> Option> { + lock(&self.inner).files.get(path).cloned().flatten() + } + + /// How many updates the hub holds of the document at `path` (0 when it has none). + pub fn updates(&self, path: &str) -> usize { + lock(&self.inner).docs.get(path).map_or(0, |d| d.log.len()) + } + + fn serve(&self, machine: usize, mut stream: TcpStream) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let (token, running, too_old, name) = { + let inner = lock(&self.inner); + let Some(m) = inner.machines.get(&machine) else { return }; + (m.token.clone(), m.running, m.too_old, m.name.clone()) + }; + // An agent from before blobs answers an unknown call from its fallback without reading the body. + let Ok(Some(request)) = read_request(&mut stream, !too_old) else { return }; + if too_old && request.path.starts_with(paths::BLOBS) { + respond_error(&mut stream, 404, "not_found", "no such call"); + let _ = stream.shutdown(Shutdown::Both); + return; + } + if !running { + let _ = stream.shutdown(Shutdown::Both); + return; + } + if request.auth.as_deref() != Some(&format!("Bearer {token}")) { + return respond_error(&mut stream, 401, "unauthorized", "the token is wrong"); + } + let blob_get = request.path.strip_prefix(paths::BLOBS).and_then(|p| p.strip_prefix('/')).map(str::to_owned); + match (request.method.as_str(), request.path.as_str()) { + ("GET", paths::STATUS) => { + let online = lock(&self.inner).machines.get(&machine).is_some_and(|m| m.online); + let status = Status { + api: LOCAL_API_MAX, + signed_in: true, + account: Some("fake".into()), + server: Some("https://fake.invalid".into()), + folder: None, + machine: Some(name), + online, + run_state: RunState::Idle, + autostart: None, + last_sync_ms: None, + removed: false, + }; + respond_json(&mut stream, 200, &status); + } + ("GET", paths::DOCS) => self.docs_socket(machine, stream, &request), + ("PUT", paths::BLOBS) | ("GET", _) if too_old && (request.path == paths::BLOBS || blob_get.is_some()) => { + respond_error(&mut stream, 404, "not_found", "no such call"); + } + ("PUT", paths::BLOBS) => match self.put_blob(machine, &request.body) { + Ok(stored) => respond_json(&mut stream, 200, &stored), + Err((status, code, message)) => respond_error(&mut stream, status, code, &message), + }, + ("GET", _) if blob_get.is_some() => { + let hash = blob_get.as_deref().and_then(parse_hex); + match hash.map(|h| self.get_blob(machine, &h)) { + Some(Ok(bytes)) => respond(&mut stream, 200, "application/octet-stream", &bytes), + Some(Err((status, code, message))) => respond_error(&mut stream, status, code, &message), + None => respond_error(&mut stream, 400, "invalid", "not a blob's name"), + } + } + _ => respond_error(&mut stream, 404, "not_found", "no such call"), + } + } + + fn put_blob(&self, machine: usize, bytes: &[u8]) -> Result { + if bytes.is_empty() || bytes.len() > MAX_BLOB_BYTES { + return Err((413, "too_large", format!("a blob is 1 byte to {MAX_BLOB_BYTES} bytes"))); + } + let hash = fake_hash(bytes); + let mut inner = lock(&self.inner); + let Some(m) = inner.machines.get_mut(&machine) else { return Err((500, "internal", "no machine".into())) }; + m.blobs.insert(hash, bytes.to_vec()); + m.pending_blobs.push(hash); + sync(&mut inner); + Ok(Stored { hash, len: bytes.len() as u64 }) + } + + fn get_blob(&self, machine: usize, hash: &[u8; 32]) -> Result, (u16, &'static str, String)> { + let mut inner = lock(&self.inner); + let hub_copy = inner.blobs.get(hash).cloned(); + let Some(m) = inner.machines.get_mut(&machine) else { return Err((500, "internal", "no machine".into())) }; + if let Some(bytes) = m.blobs.get(hash) { + return Ok(bytes.clone()); + } + if !m.online { + return Err((503, "offline", "the server can't be reached to fetch the blob".into())); + } + match hub_copy { + Some(bytes) => { + m.blobs.insert(*hash, bytes.clone()); + Ok(bytes) + } + None => Err((404, "blob_not_found", "no such blob".into())), + } + } + + /// The docs WebSocket: the handshake by hand, then a reader here and writes from wherever state changes. + fn docs_socket(&self, machine: usize, mut stream: TcpStream, request: &Request) { + let Some(key) = request.ws_key.as_deref() else { + return respond_error(&mut stream, 400, "invalid", "not a WebSocket request"); + }; + let accept = tungstenite::handshake::derive_accept_key(key.as_bytes()); + let head = format!("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\n\r\n"); + if stream.write_all(head.as_bytes()).is_err() { + return; + } + let _ = stream.set_nodelay(true); + let _ = stream.set_read_timeout(None); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + let limit = MAX_BINARY_BYTES.max(MAX_LOCAL_JSON_BYTES); + let config = WebSocketConfig::default().max_message_size(Some(limit)).max_frame_size(Some(limit)); + let (Ok(write_side), Ok(shut)) = (stream.try_clone(), stream.try_clone()) else { return }; + let mut reader = WebSocket::from_raw_socket(stream, Role::Server, Some(config)); + let writer = Arc::new(Mutex::new(WebSocket::from_raw_socket(write_side, Role::Server, Some(config)))); + let conn = { + let mut inner = lock(&self.inner); + let conn = inner.next_conn; + inner.next_conn += 1; + let Some(m) = inner.machines.get_mut(&machine) else { return }; + m.conns.insert( + conn, + Conn { + writer, + stream: shut, + open: HashMap::new(), + importing: HashMap::new(), + apps: HashMap::new(), + told: HashMap::new(), + told_file: HashMap::new(), + }, + ); + conn + }; + loop { + match reader.read() { + Ok(Message::Text(text)) => { + if let Ok(request) = serde_json::from_str::(text.as_str()) { + self.on_request(machine, conn, request); + } + } + Ok(Message::Binary(bytes)) => { + if let Some((kind, id, body)) = decode_binary(&bytes) { + self.on_binary(machine, conn, kind, id, body); + } + } + Ok(Message::Close(_)) | Err(_) => break, + Ok(_) => {} + } + } + let mut inner = lock(&self.inner); + if let Some(m) = inner.machines.get_mut(&machine) { + m.conns.remove(&conn); + } + } + + fn on_request(&self, machine: usize, conn: u64, request: DocsRequest) { + let mut inner = lock(&self.inner); + match request { + DocsRequest::Open { id, path, app: _, file_hash } => { + if let Some(c) = inner.machines.get_mut(&machine).and_then(|m| m.conns.get_mut(&conn)) { + c.apps.insert(id, file_hash); + } + open(&mut inner, machine, conn, id, &path); + } + DocsRequest::Close { doc } => { + if let Some(c) = inner.machines.get_mut(&machine).and_then(|m| m.conns.get_mut(&conn)) { + c.open.retain(|_, o| o.doc != doc); + } + } + DocsRequest::Saved { doc, hash, frontiers } => { + let Some(m) = inner.machines.get_mut(&machine) else { return }; + m.saves.push((doc, hash, frontiers.clone())); + // The file is written and goes up at once (offline: once the machine is back). (The fake reads no + // Loro: the record comes after the changes it covers on the app's stream, which the copy took first.) + let Some(path) = m.docs.get(&doc).map(|d| d.path.clone()) else { return }; + m.file_pending.insert(path, frontiers); + sync(&mut inner); + } + // (The fake's file sync is the records themselves: nothing waits for a save on its way.) + DocsRequest::Saving { .. } => {} + } + } + + fn on_binary(&self, machine: usize, conn: u64, kind: u8, id: u64, body: &[u8]) { + let mut inner = lock(&self.inner); + match kind { + IMPORT => { + let Some((path, new_id)) = + inner.machines.get_mut(&machine).and_then(|m| m.conns.get_mut(&conn)).and_then(|c| c.importing.remove(&id)) + else { + return; + }; + let online = inner.machines.get(&machine).is_some_and(|m| m.online); + if !online { + return tell(&inner, machine, conn, &DocsEvent::NotLive { id, reason: NotLiveReason::Offline, message: None }); + } + // The first import is kept; a later one joins it. + inner.docs.entry(path.clone()).or_insert_with(|| HubDoc { id: new_id, import: body.to_vec(), log: Vec::new() }); + open(&mut inner, machine, conn, id, &path); + } + UPDATE => { + let Some(m) = inner.machines.get_mut(&machine) else { return }; + let Some(c) = m.conns.get_mut(&conn) else { return }; + let Some(opened) = c.open.get_mut(&id) else { return }; + opened.seq += 1; + let (seq, doc) = (opened.seq, opened.doc); + if body.is_empty() || body.len() > MAX_UPDATE_BYTES { + let message = format!("an update is 1 byte to {MAX_UPDATE_BYTES} bytes"); + return tell(&inner, machine, conn, &DocsEvent::Error { id: Some(id), seq: Some(seq), code: "too_large".into(), message }); + } + // A refusal asked for ([`Fake::refuse_next`]): the update isn't kept. + if let Some(code) = m.refuse.pop_front() { + let message = format!("refused as asked ({code})"); + return tell(&inner, machine, conn, &DocsEvent::Error { id: Some(id), seq: Some(seq), code, message }); + } + if let Some(d) = m.docs.get_mut(&doc) { + d.log.push((Origin::Conn(conn), body.to_vec())); + d.pending.push_back(d.log.len() - 1); + } + tell(&inner, machine, conn, &DocsEvent::Ack { id, seq }); + sync(&mut inner); + } + PRESENCE => { + if body.len() > MAX_PRESENCE_BYTES { + return; + } + let Some(doc) = inner.machines.get(&machine).and_then(|m| m.conns.get(&conn)).and_then(|c| c.open.get(&id)).map(|o| o.doc) else { + return; + }; + let from_online = inner.machines.get(&machine).is_some_and(|m| m.online); + for (mid, m) in &inner.machines { + if *mid != machine && !(m.online && from_online) { + continue; + } + for (cid, c) in &m.conns { + if (*mid, *cid) == (machine, conn) { + continue; + } + for (number, o) in &c.open { + if o.doc == doc { + let _ = lock(&c.writer).send(Message::Binary(encode_binary(PRESENCE, *number, body).into())); + } + } + } + } + } + _ => {} + } + } +} + +/// Opens the document at `path` on `machine` for `conn` under `id` (the machine takes the hub's copy if it has none). +fn open(inner: &mut Inner, machine: usize, conn: u64, id: u64, path: &str) { + let hub_doc = inner.docs.get(path).map(|d| (d.id, d.import.clone())); + let Some(m) = inner.machines.get_mut(&machine) else { return }; + let known = m.docs.iter().find(|(_, d)| d.path == path).map(|(doc, _)| *doc); + let doc = match (known, hub_doc) { + (Some(doc), _) => doc, + (None, Some((doc, import))) if m.online => { + m.docs.insert(doc, MachineDoc { path: path.to_owned(), import, log: Vec::new(), hub_seen: 0, pending: VecDeque::new() }); + doc + } + (None, None) if m.online => { + let new_id = DocId::generate(); + if let Some(c) = m.conns.get_mut(&conn) { + c.importing.insert(id, (path.to_owned(), new_id)); + } + return tell(inner, machine, conn, &DocsEvent::NeedsImport { id }); + } + (None, _) => return tell(inner, machine, conn, &DocsEvent::NotLive { id, reason: NotLiveReason::Offline, message: None }), + }; + // Take what the hub has first (when online), so the state is as fresh as it can be. + sync(inner); + // Edited outside Craft unless some machine's app saved that content (the server's versions keep every record). + let file_hash = inner.machines.get(&machine).and_then(|m| m.conns.get(&conn)).and_then(|c| c.apps.get(&id)).and_then(Clone::clone); + let changed_outside = file_hash.map(|h| !inner.machines.values().any(|m| m.saves.iter().any(|(d, saved, _)| *d == doc && *saved == h))); + let Some(m) = inner.machines.get_mut(&machine) else { return }; + let online = m.online; + let Some(d) = m.docs.get(&doc) else { return }; + let state = DocState { online, waiting: d.pending.len() as u64, bytes: d.import.len() as u64, updates: d.log.len() as u64 }; + let (import, log): (Vec, Vec>) = (d.import.clone(), d.log.iter().map(|(_, b)| b.clone()).collect()); + let Some(c) = m.conns.get_mut(&conn) else { return }; + c.open.insert(id, Opened { doc, sent: log.len(), seq: 0 }); + c.told.insert(doc, (online, state.waiting)); + let mut w = lock(&c.writer); + let opened = DocsEvent::Opened { id, doc, state, changed_outside }; + if let Ok(text) = serde_json::to_string(&opened) { + let _ = w.send(Message::Text(text.into())); + } + let _ = w.send(Message::Binary(encode_binary(STATE, id, &import).into())); + for bytes in log { + let _ = w.send(Message::Binary(encode_binary(UPDATE, id, &bytes).into())); + } + drop(w); + // The file's record, after the state. + sync(inner); +} + +/// Sends one control message to one connection. +fn tell(inner: &Inner, machine: usize, conn: u64, event: &DocsEvent) { + let Some(c) = inner.machines.get(&machine).and_then(|m| m.conns.get(&conn)) else { return }; + if let Ok(text) = serde_json::to_string(event) { + let _ = lock(&c.writer).send(Message::Text(text.into())); + } +} + +/// Moves updates and blobs between the online machines and the hub, then sends every connection what it lacks. +fn sync(inner: &mut Inner) { + let live_down = inner.live_down; + let Inner { docs, blobs, machines, files, .. } = inner; + // Up: every online machine's waiting updates, blobs and saved files. + let mut ids: Vec = machines.keys().copied().collect(); + ids.sort_unstable(); + for mid in &ids { + let Some(m) = machines.get_mut(mid) else { continue }; + if !m.online { + continue; + } + for (path, version) in m.file_pending.drain() { + files.insert(path, Some(version)); + } + for hash in m.pending_blobs.drain(..) { + if let Some(bytes) = m.blobs.get(&hash) { + blobs.insert(hash, bytes.clone()); + } + } + for d in m.docs.values_mut() { + let Some(hub) = docs.get_mut(&d.path) else { continue }; + if live_down { + continue; + } + while let Some(at) = d.pending.pop_front() { + if let Some((_, bytes)) = d.log.get(at) { + hub.log.push((*mid, bytes.clone())); + } + } + } + } + // Down: what the others sent. + for mid in &ids { + let Some(m) = machines.get_mut(mid) else { continue }; + if !m.online { + continue; + } + for d in m.docs.values_mut() { + let Some(hub) = docs.get(&d.path) else { continue }; + if live_down { + continue; + } + for (from, bytes) in hub.log.iter().skip(d.hub_seen) { + if from != mid { + d.log.push((Origin::Remote, bytes.clone())); + } + } + d.hub_seen = hub.log.len(); + } + } + // Out to the apps: the copy's new updates (an app's own not back to it), and status when it changed. + for m in machines.values_mut() { + let online = m.online; + for (cid, c) in m.conns.iter_mut() { + let mut w = lock(&c.writer); + for (number, o) in c.open.iter_mut() { + let Some(d) = m.docs.get(&o.doc) else { continue }; + for (origin, bytes) in d.log.iter().skip(o.sent) { + if *origin != Origin::Conn(*cid) { + let _ = w.send(Message::Binary(encode_binary(UPDATE, *number, bytes).into())); + } + } + o.sent = d.log.len(); + let now = (online, d.pending.len() as u64); + if c.told.get(&o.doc) != Some(&now) { + c.told.insert(o.doc, now); + let status = DocsEvent::Status { doc: o.doc, online, waiting: now.1, deleted: false, stopped: false }; + if let Ok(text) = serde_json::to_string(&status) { + let _ = w.send(Message::Text(text.into())); + } + } + // The file here: this machine's save not up yet, else the server's (offline: as it was). + let file = match m.file_pending.get(&d.path) { + Some(v) => Some(v.clone()), + None if online => files.get(&d.path).cloned().flatten(), + None => c.told_file.get(&o.doc).cloned().flatten(), + }; + if c.told_file.get(&o.doc) != Some(&file) { + c.told_file.insert(o.doc, file.clone()); + if let Ok(text) = serde_json::to_string(&DocsEvent::File { doc: o.doc, frontiers: file }) { + let _ = w.send(Message::Text(text.into())); + } + } + } + } + } +} + +/// One machine of a [`Hub`]. Dropping it stops its local API (its app connections end). +pub struct Fake { + hub: Hub, + id: usize, + addr: SocketAddr, + token: String, + stop: Arc, +} + +impl std::fmt::Debug for Fake { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Fake").field("addr", &self.addr).finish_non_exhaustive() + } +} + +impl Fake { + /// A client of this machine's agent, as [`Client::connect`] gives one. + pub fn client(&self) -> Client { + Client::at(self.addr, self.token.clone(), std::env::temp_dir()) + } + + /// The machine reaches the hub (`true`) or not: offline, its updates and blobs wait, and others' don't come. + pub fn set_online(&self, online: bool) { + let mut inner = lock(&self.hub.inner); + if let Some(m) = inner.machines.get_mut(&self.id) { + m.online = online; + } + sync(&mut inner); + } + + /// Whether this machine answers the local API ([`Fake::restart`] stops it until [`Fake::start`]). + pub fn running(&self) -> bool { + lock(&self.hub.inner).machines.get(&self.id).is_some_and(|m| m.running) + } + + /// The agent stops: every app connection ends, and new ones are refused until [`Fake::start`]. What the machine + /// holds (its copies, its waiting updates) stays, as the agent's files do. + pub fn restart(&self) { + let mut inner = lock(&self.hub.inner); + if let Some(m) = inner.machines.get_mut(&self.id) { + m.running = false; + for (_, c) in m.conns.drain() { + let _ = c.stream.shutdown(Shutdown::Both); + } + } + } + + /// The agent runs again. + pub fn start(&self) { + if let Some(m) = lock(&self.hub.inner).machines.get_mut(&self.id) { + m.running = true; + } + } + + /// Every app connection ends (the agent dropped them), but the agent keeps running. + pub fn drop_connections(&self) { + if let Some(m) = lock(&self.hub.inner).machines.get_mut(&self.id) { + for (_, c) in m.conns.drain() { + let _ = c.stream.shutdown(Shutdown::Both); + } + } + } + + /// Refuses the next `count` updates any app of this machine sends with `code`, as the agent does: `internal` when + /// it can't write a change to disk (worth sending again), `invalid` when Loro doesn't take it (never). + pub fn refuse_next(&self, code: &str, count: usize) { + if let Some(m) = lock(&self.hub.inner).machines.get_mut(&self.id) { + m.refuse.extend(std::iter::repeat_n(code.to_owned(), count)); + } + } + + /// Answer the blob calls as an agent from before blobs. + pub fn too_old(&self) { + if let Some(m) = lock(&self.hub.inner).machines.get_mut(&self.id) { + m.too_old = true; + } + } + + /// The save records this machine's apps sent: content hash (hex) and frontiers, in order. + pub fn saves(&self) -> Vec<(String, Vec<(u64, u32)>)> { + lock(&self.hub.inner).machines.get(&self.id).map(|m| m.saves.iter().map(|(_, h, v)| (h.clone(), v.clone())).collect()).unwrap_or_default() + } + + /// How many of this machine's updates of the document at `path` the hub doesn't have yet. + pub fn waiting(&self, path: &str) -> usize { + let inner = lock(&self.hub.inner); + inner.machines.get(&self.id).map_or(0, |m| m.docs.values().filter(|d| d.path == path).map(|d| d.pending.len()).sum()) + } +} + +impl Drop for Fake { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(m) = lock(&self.hub.inner).machines.get_mut(&self.id) { + m.running = false; + for (_, c) in m.conns.drain() { + let _ = c.stream.shutdown(Shutdown::Both); + } + } + // Wake the accept loop so it sees the stop. + let _ = TcpStream::connect_timeout(&self.addr, Duration::from_millis(200)); + } +} + +/// A content name for the fake's blobs: 32 bytes, the same for the same bytes (not a cryptographic hash; the real +/// agent names blobs by their manifest's BLAKE3 hash). +fn fake_hash(bytes: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + for (lane, slot) in out.chunks_mut(8).enumerate() { + let mut h: u64 = 0xcbf2_9ce4_8422_2325 ^ (lane as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15); + for &b in bytes { + h ^= u64::from(b); + h = h.wrapping_mul(0x0100_0000_01b3 ^ ((lane as u64) << 40)); + } + h ^= bytes.len() as u64; + slot.copy_from_slice(&h.to_le_bytes()); + } + out +} + +struct Request { + method: String, + path: String, + auth: Option, + ws_key: Option, + body: Vec, +} + +/// One HTTP request: its head (at most 16 KiB) and, with `body`, its body (by `Content-Length`, at most the blob +/// limit). +fn read_request(stream: &mut TcpStream, body: bool) -> std::io::Result> { + let mut buf = Vec::new(); + let mut chunk = vec![0u8; 64 * 1024]; + loop { + let n = stream.read(&mut chunk)?; + if n == 0 { + return Ok(None); + } + buf.extend_from_slice(chunk.get(..n).unwrap_or_default()); + let mut headers = [httparse::EMPTY_HEADER; 32]; + let mut parsed = httparse::Request::new(&mut headers); + match parsed.parse(&buf) { + Ok(httparse::Status::Complete(head)) => { + let header = |name: &str| { + parsed + .headers + .iter() + .find(|h| h.name.eq_ignore_ascii_case(name)) + .and_then(|h| std::str::from_utf8(h.value).ok()) + .map(str::to_owned) + }; + let length = header("content-length").and_then(|v| v.trim().parse::().ok()).unwrap_or(0); + if length > MAX_BLOB_BYTES { + return Ok(None); + } + let mut request = Request { + method: parsed.method.unwrap_or_default().to_owned(), + path: parsed.path.unwrap_or_default().to_owned(), + auth: header("authorization"), + ws_key: header("sec-websocket-key"), + body: buf.split_off(head.min(buf.len())), + }; + if !body { + return Ok(Some(request)); + } + while request.body.len() < length { + let n = stream.read(&mut chunk)?; + if n == 0 { + return Ok(None); + } + request.body.extend_from_slice(chunk.get(..n).unwrap_or_default()); + } + request.body.truncate(length); + return Ok(Some(request)); + } + Ok(httparse::Status::Partial) if buf.len() <= 16 * 1024 => {} + _ => return Ok(None), + } + } +} + +fn respond(stream: &mut TcpStream, status: u16, content_type: &str, body: &[u8]) { + let head = format!("HTTP/1.1 {status} X\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(body); + let _ = stream.flush(); +} + +fn respond_json(stream: &mut TcpStream, status: u16, value: &impl serde::Serialize) { + let body = serde_json::to_vec(value).unwrap_or_default(); + respond(stream, status, "application/json", &body); +} + +fn respond_error(stream: &mut TcpStream, status: u16, code: &str, message: &str) { + respond_json(stream, status, &serde_json::json!({ "error": code, "message": message })); +} + +/// The blob name a fake gives `bytes`, for tests that compare. +pub fn blob_name(bytes: &[u8]) -> String { + hex(&fake_hash(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::live::{Event, Opened}; + + fn next_update(doc: &crate::live::Document) -> Vec { + for _ in 0..200 { + if let Some(Event::Update(bytes)) = doc.recv_timeout(Duration::from_millis(25)) { + return bytes; + } + } + panic!("no update came"); + } + + fn acked(doc: &crate::live::Document, seq: u64) { + for _ in 0..200 { + if let Some(Event::Acked(s)) = doc.recv_timeout(Duration::from_millis(25)) + && s == seq + { + return; + } + } + panic!("no ack came"); + } + + /// Two machines of one hub: an import on one is live on the other; updates go both ways, and wait while a machine + /// is offline; a restarted machine keeps what it held; blobs go up and come down; an old agent's blob calls are + /// `agent_too_old`. + #[test] + fn two_fakes_share_documents_and_blobs() { + let hub = Hub::new(); + let (a, b) = (hub.machine("a").unwrap(), hub.machine("b").unwrap()); + let docs_a = a.client().documents().unwrap(); + let Opened::NeedsImport(importer) = docs_a.open("x.txt", "test").unwrap() else { panic!("not new") }; + let doc_a = importer.import(b"snapshot").unwrap(); + assert_eq!(doc_a.state(), b"snapshot"); + let docs_b = b.client().documents().unwrap(); + let Opened::Live(doc_b) = docs_b.open("x.txt", "test").unwrap() else { panic!("not live on b") }; + assert_eq!(doc_b.state(), b"snapshot"); + doc_a.send(b"u1").unwrap(); + assert_eq!(next_update(&doc_b), b"u1"); + b.set_online(false); + acked(&doc_b, doc_b.send(b"u2").unwrap()); + acked(&doc_a, doc_a.send(b"u3").unwrap()); + assert_eq!(b.waiting("x.txt"), 1); + b.set_online(true); + assert_eq!(next_update(&doc_a), b"u2"); + assert_eq!(next_update(&doc_b), b"u3"); + assert_eq!(hub.updates("x.txt"), 3); + // A restart: b's connections end; it keeps its copy. + b.set_online(false); + acked(&doc_b, doc_b.send(b"u4").unwrap()); + b.restart(); + assert!(b.client().documents().is_err() || !b.running()); + b.start(); + let docs_b = b.client().documents().unwrap(); + let Opened::Live(again) = docs_b.open("x.txt", "test").unwrap() else { panic!("not live after the restart") }; + assert_eq!(again.opened_state().waiting, 1); + // Blobs. + let stored = a.client().blobs().put(b"blob bytes").unwrap(); + assert_eq!( + b.client().blobs().get(&stored.hash, stored.len).unwrap_err(), + crate::Error::Refused { code: "offline".into(), message: "the server can't be reached to fetch the blob".into() } + ); + b.set_online(true); + assert_eq!(b.client().blobs().get(&stored.hash, stored.len).unwrap(), b"blob bytes"); + assert!(matches!(b.client().blobs().get(&[1; 32], 3), Err(crate::Error::Refused { code, .. }) if code == "blob_not_found")); + b.too_old(); + assert!(matches!(b.client().blobs().put(b"x"), Err(crate::Error::Refused { code, .. }) if code == "agent_too_old")); + // A big put too: an old agent answers before reading the body, and the client asks before sending it. + let put = b.client().blobs().put(&vec![9u8; 8 * 1024 * 1024]); + assert!(matches!(&put, Err(crate::Error::Refused { code, .. }) if code == "agent_too_old"), "{put:?}"); + assert!(matches!(b.client().blobs().get(&stored.hash, stored.len), Err(crate::Error::Refused { code, .. }) if code == "agent_too_old")); + assert!(b.client().status().is_ok(), "the rest still works"); + let big = vec![0u8; MAX_UPDATE_BYTES + 1]; + let seq = doc_a.send(&big).unwrap(); + let refused = (0..200).find_map(|_| match doc_a.recv_timeout(Duration::from_millis(25)) { + Some(Event::Refused { seq: s, .. }) => Some(s), + _ => None, + }); + assert_eq!(refused, Some(seq), "an update over 1 MiB is refused"); + } +} diff --git a/crates/app/src/http.rs b/crates/app/src/http.rs index 8b67027..48aad8d 100644 --- a/crates/app/src/http.rs +++ b/crates/app/src/http.rs @@ -5,7 +5,7 @@ use std::io::{Read, Write}; use std::net::{SocketAddr, TcpStream}; use std::time::Duration; -use craft_cloud_proto::local::MAX_LOCAL_JSON_BYTES; +use craft_cloud_api::local::MAX_LOCAL_JSON_BYTES; use crate::Error; @@ -30,19 +30,34 @@ pub(crate) struct Answer { /// Sends `method path` with the token and an optional JSON body to the agent at `addr`; the answer must come within /// `timeout`. pub(crate) fn request(addr: SocketAddr, token: &str, method: &str, path: &str, body: Option<&[u8]>, timeout: Duration) -> Result { + request_with(addr, token, method, path, body, "application/json", MAX_LOCAL_JSON_BYTES, timeout) +} + +/// [`request`] with a body of `content_type` and an answer of at most `max` bytes. +#[allow(clippy::too_many_arguments)] +pub(crate) fn request_with( + addr: SocketAddr, + token: &str, + method: &str, + path: &str, + body: Option<&[u8]>, + content_type: &str, + max: usize, + timeout: Duration, +) -> Result { let mut stream = TcpStream::connect_timeout(&addr, CONNECT_TIMEOUT).map_err(|e| Error::NotRunning(format!("can't reach it: {e}")))?; stream.set_read_timeout(Some(timeout)).map_err(io)?; stream.set_write_timeout(Some(timeout)).map_err(io)?; let body = body.unwrap_or_default(); let head = format!( - "{method} {path} HTTP/1.1\r\nHost: {addr}\r\nAuthorization: Bearer {token}\r\nContent-Type: application/json\r\n\ + "{method} {path} HTTP/1.1\r\nHost: {addr}\r\nAuthorization: Bearer {token}\r\nContent-Type: {content_type}\r\n\ Content-Length: {}\r\nConnection: close\r\n\r\n", body.len() ); stream.write_all(head.as_bytes()).map_err(io)?; stream.write_all(body).map_err(io)?; stream.flush().map_err(io)?; - read_answer(&mut stream) + read_answer_max(&mut stream, max) } fn io(e: std::io::Error) -> Error { @@ -50,7 +65,13 @@ fn io(e: std::io::Error) -> Error { } /// Reads one answer (the connection closes after it). +#[cfg(test)] pub(crate) fn read_answer(stream: &mut impl Read) -> Result { + read_answer_max(stream, MAX_LOCAL_JSON_BYTES) +} + +/// Reads one answer of at most `max` bytes of body (the connection closes after it). +pub(crate) fn read_answer_max(stream: &mut impl Read, max: usize) -> Result { let mut buf = Vec::with_capacity(4096); let mut chunk = [0u8; 4096]; let (status, length, head_len) = loop { @@ -80,7 +101,7 @@ pub(crate) fn read_answer(stream: &mut impl Read) -> Result { Err(e) => return Err(Error::Protocol(format!("the answer doesn't parse: {e}"))), } }; - if length.is_some_and(|n| n > MAX_LOCAL_JSON_BYTES) { + if length.is_some_and(|n| n > max) { return Err(Error::Protocol("the answer is too long".into())); } let mut body = buf.split_off(head_len.min(buf.len())); @@ -93,7 +114,7 @@ pub(crate) fn read_answer(stream: &mut impl Read) -> Result { break; } body.extend_from_slice(chunk.get(..n).unwrap_or_default()); - if body.len() > MAX_LOCAL_JSON_BYTES { + if body.len() > max { return Err(Error::Protocol("the answer is too long".into())); } } diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 70c24e5..3f5d9af 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -39,11 +39,19 @@ //! when the agent stops or signs out it says so ([`RecentsEvent::Unavailable`]) and connects again by itself, with a //! growing pause, to whichever agent runs next ([`RecentsEvent::Available`]). //! -//! Dependencies: `craft-cloud-proto` (the local API types; serde, uuid, blake3, postcard and Unicode tables), `serde_json`, +//! Binary content kept out of live documents goes through [`blobs`] (the agent's `/v1/blobs`). +//! +//! With the `fake` feature (tests only), [`fake`] is an agent in this process: its documents and blobs live in memory, +//! shared by every fake on one [`fake::Hub`], so two app instances can be tested together with no server. +//! +//! Dependencies: `craft-cloud-api` (the local API types; serde, uuid and Unicode tables, no postcard), `serde_json`, //! `thiserror`, `httparse`, and `tungstenite` without TLS (with `http`, `sha1`, `rand`, `data-encoding`, `utf-8`). No //! TLS: the agent is on this machine. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod blobs; +#[cfg(feature = "fake")] +pub mod fake; mod http; pub mod live; mod recents; @@ -52,12 +60,12 @@ use std::net::{Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::time::Duration; -use craft_cloud_proto::api::ErrorBody; -use craft_cloud_proto::local::{AutostartRequest, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_MIN, LocalApiFile, LogoutRequest, paths}; +use craft_cloud_api::api::ErrorAnswer; +use craft_cloud_api::local::{AutostartRequest, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_MIN, LocalApiFile, LogoutRequest, paths}; -pub use craft_cloud_proto::DeviceId; -pub use craft_cloud_proto::api::{DeviceInfo, PairingState, PairingStatus}; -pub use craft_cloud_proto::local::{JoinRequest, LoginRequest, NewRecent, PairingInfo, RecentEntry, RunState, Status, codes}; +pub use craft_cloud_api::DeviceId; +pub use craft_cloud_api::api::{DeviceInfo, PairingState, PairingStatus}; +pub use craft_cloud_api::local::{JoinRequest, LoginRequest, NewRecent, PairingInfo, RecentEntry, RunState, Status, codes}; pub use recents::{RecentsEvent, RecentsWatch}; pub use uuid::Uuid; @@ -88,12 +96,14 @@ pub struct Client { token: String, /// Where `local-api.json` is: a watch looks there again to reconnect. dir: PathBuf, + /// Whether this agent knows `/v1/blobs`, once asked ([`blobs::Blobs`]): shared by the client's clones. + blobs_known: std::sync::Arc>, } impl Client { /// Finds the running agent: its data folder from `$CRAFT_CLOUD_AGENT_DIR` or the per-user data folder. pub fn connect() -> Result { - let dir = craft_cloud_proto::local::data_dir(|name| std::env::var_os(name)) + let dir = craft_cloud_api::local::data_dir(|name| std::env::var_os(name)) .ok_or_else(|| Error::NotRunning("no user data folder to look in".into()))?; Self::connect_in(&dir) } @@ -105,7 +115,12 @@ impl Client { let record: LocalApiFile = serde_json::from_slice(&bytes).map_err(|e| Error::NotRunning(format!("{} doesn't read: {e}", file.display())))?; // A file a crashed agent left behind names a port nobody, or someone else, answers on: the connection tells. // (Its pid isn't used for that: an app in a sandbox with a PID namespace of its own can't see the agent's.) - let client = Self { addr: SocketAddr::from((Ipv4Addr::LOCALHOST, record.port)), token: record.token, dir: dir.to_owned() }; + let client = Self { + addr: SocketAddr::from((Ipv4Addr::LOCALHOST, record.port)), + token: record.token, + dir: dir.to_owned(), + blobs_known: Default::default(), + }; match client.status() { Ok(status) if (LOCAL_API_MIN..=LOCAL_API_MAX).contains(&status.api) => Ok(client), Ok(status) => Err(Error::Version(status.api)), @@ -140,14 +155,19 @@ impl Client { /// The answer if it succeeded, else the agent's refusal. fn answer(&self, method: &str, path: &str, body: Option>, timeout: Duration) -> Result { let answer = http::request(self.addr, &self.token, method, path, body.as_deref(), timeout)?; - if (200..300).contains(&answer.status) { - Ok(answer) - } else { - match serde_json::from_slice::(&answer.body) { - Ok(body) => Err(Error::Refused { code: body.error, message: body.message }), - Err(_) => Err(Error::Protocol(format!("HTTP {}", answer.status))), - } - } + refusal(answer) + } + + /// A call with a binary body and a binary answer of at most `max` bytes (blobs). + pub(crate) fn binary(&self, method: &str, path: &str, body: Option<&[u8]>, max: usize, timeout: Duration) -> Result { + let answer = http::request_with(self.addr, &self.token, method, path, body, "application/octet-stream", max, timeout)?; + refusal(answer) + } + + /// A client of the agent at `addr` with `token` (a fake agent's, in tests), whose `local-api.json` is in `dir`. + #[cfg(feature = "fake")] + pub(crate) fn at(addr: SocketAddr, token: String, dir: PathBuf) -> Self { + Self { addr, token, dir, blobs_known: Default::default() } } /// Whether the agent is signed in, as whom, to which server, whether it is online and what it is doing. Quick @@ -256,6 +276,23 @@ impl Client { pub fn documents(&self) -> Result { live::Documents::connect(self) } + + /// The agent's blob store ([`blobs::Blobs`]): binary content kept out of live documents. + pub fn blobs(&self) -> blobs::Blobs { + blobs::Blobs::new(self.clone()) + } +} + +/// `answer` if it succeeded, else the agent's refusal (its code and message). +fn refusal(answer: http::Answer) -> Result { + if (200..300).contains(&answer.status) { + Ok(answer) + } else { + match serde_json::from_slice::(&answer.body) { + Ok(body) => Err(Error::Refused { code: body.error, message: body.message }), + Err(_) => Err(Error::Protocol(format!("HTTP {}", answer.status))), + } + } } #[cfg(test)] @@ -339,11 +376,11 @@ mod tests { #[test] fn each_call_sends_its_request_and_reads_its_answer() { let id = uuid::Uuid::now_v7(); - let device = craft_cloud_proto::DeviceId::generate(); + let device = craft_cloud_api::DeviceId::generate(); let info = PairingInfo { id, join: "c.example/K7M4-Q9TX".into(), link: "https://c.example/join/K7M4-Q9TX".into(), expires_ms: 9 }; let used = PairingStatus { state: PairingState::Used, device_name: Some("lptp1".into()) }; let list = vec![DeviceInfo { id: device, name: "lptp1".into(), created_ms: 1, last_used_ms: 2, revoked: false, current: false }]; - let refused = json(&ErrorBody { error: "password_required".into(), message: "choose a password".into(), ..ErrorBody::default() }); + let refused = json(&ErrorAnswer { error: "password_required".into(), message: "choose a password".into() }); let script = vec![ (400, refused), (200, json(&status(true))), diff --git a/crates/app/src/live.rs b/crates/app/src/live.rs index 5745b97..1e3eba6 100644 --- a/crates/app/src/live.rs +++ b/crates/app/src/live.rs @@ -42,20 +42,23 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::thread::JoinHandle; use std::time::Duration; -use craft_cloud_proto::DocId; -use craft_cloud_proto::api::ErrorBody; -use craft_cloud_proto::local::docs::{IMPORT, MAX_BINARY_BYTES, PRESENCE, STATE, UPDATE, decode_binary, encode_binary}; -use craft_cloud_proto::local::{DocState, DocsEvent, DocsRequest, MAX_LOCAL_JSON_BYTES, paths}; +use craft_cloud_api::DocId; +use craft_cloud_api::api::ErrorAnswer as ErrorBody; +use craft_cloud_api::local::docs::{IMPORT, MAX_BINARY_BYTES, PRESENCE, STATE, UPDATE, decode_binary, encode_binary}; +use craft_cloud_api::local::{DocState, DocsEvent, DocsRequest, MAX_LOCAL_JSON_BYTES, paths}; use tungstenite::client::IntoClientRequest; use tungstenite::handshake::HandshakeError; use tungstenite::protocol::WebSocketConfig; use tungstenite::{Message, WebSocket}; -pub use craft_cloud_proto::local::NotLiveReason; +pub use craft_cloud_api::local::NotLiveReason; use crate::http::CONNECT_TIMEOUT; use crate::{Client, Error}; +/// What a document calls when an event came for it (from the reader's thread): an egui app's `request_repaint`. +pub type Waker = Arc; + /// How long an open may take: finding the file's node (the agent may sync it first, up to 5 s), joining the document on /// the server, and its state. pub const OPEN_TIMEOUT: Duration = Duration::from_secs(30); @@ -73,13 +76,17 @@ pub enum Event { Presence(Vec), /// This app's change number `seq` ([`Document::send`]'s answer) is on the agent's disk. Acked(u64), - /// This app's change number `seq` was refused (it doesn't decode, or is over 1 MiB): the agent's copy doesn't have - /// it. - Refused { seq: u64, message: String }, + /// This app's change number `seq` was refused: the agent's copy doesn't have it. `code` says whether sending it + /// again can help: `invalid` (it doesn't decode) and `too_large` (over 1 MiB) never will; anything else (`internal`: + /// the agent couldn't write it to disk) is worth sending again later. + Refused { seq: u64, code: String, message: String }, /// How the document stands: `online` (edits go out as they are made), changes waiting for the server, and whether /// its file was deleted (the document still works; a later open of the path is a new file), and whether the agent /// stopped following it until it restarts (Loro failed on it twice; changes made meanwhile are kept). Status { online: bool, waiting: u64, deleted: bool, stopped: bool }, + /// The document's file on this machine's disk now: the live document's frontiers (`(peer, counter)` heads) a Craft + /// app saved it from, or none when no save record is known for it (see `DocsEvent::File`). + File(Option>), /// The connection to the agent ended. Closed(String), } @@ -95,9 +102,28 @@ pub enum Opened { NotLive(NotLiveReason), } +/// What an app may tell the agent when it opens a file ([`Documents::open_with`]). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OpenOptions { + /// The BLAKE3 hash of the file's bytes as the app read them: [`Document::changed_outside`] says whether a Craft + /// app saved them. + pub file_hash: Option<[u8; 32]>, +} + +/// What `opened` says, before its state comes. +#[derive(Debug, Clone)] +struct Head { + doc: DocId, + state: DocState, + changed_outside: Option, +} + +/// An open's state so far: what `opened` said, the snapshot, and the updates that are part of it. +type Filling = (Head, Vec, Vec>); + /// What the reader hands an open in progress. enum Answer { - Live { doc: DocId, state: DocState, snapshot: Vec }, + Live { head: Head, snapshot: Vec, updates: Vec> }, NeedsImport, NotLive(NotLiveReason), Refused(Error), @@ -111,9 +137,13 @@ struct Shared { routes: Mutex>>, answers: Mutex>>, /// An `opened` whose state hasn't come yet, by number. - opening: Mutex>, + opening: Mutex>, + /// An open whose state came and whose state's updates are still coming, by number. + filling: Mutex>, /// Which numbers a document is open as (its status names the document). numbers: Mutex>>, + /// What to call after an event went to a document's channel, by number ([`Document::set_waker`]). + wakers: Mutex>, closed: AtomicBool, next_id: AtomicU64, /// A clone of the stream: dropping the connection shuts it down (which wakes the reader). @@ -145,6 +175,14 @@ impl Shared { if let Some(events) = lock(&self.routes).get(&id) { let _ = events.send(event); } + self.wake(id); + } + + fn wake(&self, id: u64) { + let waker = lock(&self.wakers).get(&id).cloned(); + if let Some(waker) = waker { + waker(); + } } fn answer(&self, id: u64, answer: Answer) { @@ -157,8 +195,8 @@ impl Shared { fn on_message(&self, message: Message) { match message { Message::Text(text) => match serde_json::from_str::(text.as_str()) { - Ok(DocsEvent::Opened { id, doc, state }) => { - lock(&self.opening).insert(id, (doc, state)); + Ok(DocsEvent::Opened { id, doc, state, changed_outside }) => { + lock(&self.opening).insert(id, Head { doc, state, changed_outside }); } Ok(DocsEvent::NeedsImport { id }) => self.answer(id, Answer::NeedsImport), Ok(DocsEvent::NotLive { id, reason, .. }) => self.answer(id, Answer::NotLive(reason)), @@ -169,7 +207,13 @@ impl Shared { self.route(id, Event::Status { online, waiting, deleted, stopped }); } } - Ok(DocsEvent::Error { id: Some(id), seq: Some(seq), message, .. }) => self.route(id, Event::Refused { seq, message }), + Ok(DocsEvent::File { doc, frontiers }) => { + let ids = lock(&self.numbers).get(&doc).cloned().unwrap_or_default(); + for id in ids { + self.route(id, Event::File(frontiers.clone())); + } + } + Ok(DocsEvent::Error { id: Some(id), seq: Some(seq), code, message }) => self.route(id, Event::Refused { seq, code, message }), Ok(DocsEvent::Error { id: Some(id), seq: None, code, message }) => self.answer(id, Answer::Refused(Error::Refused { code, message })), Ok(DocsEvent::Error { id: None, .. }) | Err(_) => {} }, @@ -177,12 +221,31 @@ impl Shared { let Some((kind, id, body)) = decode_binary(&bytes) else { return }; match kind { STATE => { - if let Some((doc, state)) = lock(&self.opening).remove(&id) { - lock(&self.numbers).entry(doc).or_default().push(id); - self.answer(id, Answer::Live { doc, state, snapshot: body.to_vec() }); + if let Some(head) = lock(&self.opening).remove(&id) { + lock(&self.numbers).entry(head.doc).or_default().push(id); + if head.state.updates == 0 { + self.answer(id, Answer::Live { head, snapshot: body.to_vec(), updates: Vec::new() }); + } else { + lock(&self.filling).insert(id, (head, body.to_vec(), Vec::new())); + } } } - UPDATE => self.route(id, Event::Update(body.to_vec())), + UPDATE => { + // An update that is part of an open's state: kept with it until the last has come. + let mut filling = lock(&self.filling); + if let Some((head, _, updates)) = filling.get_mut(&id) { + updates.push(body.to_vec()); + if updates.len() as u64 >= head.state.updates + && let Some((head, snapshot, updates)) = filling.remove(&id) + { + drop(filling); + self.answer(id, Answer::Live { head, snapshot, updates }); + } + return; + } + drop(filling); + self.route(id, Event::Update(body.to_vec())); + } PRESENCE => self.route(id, Event::Presence(body.to_vec())), _ => {} } @@ -197,6 +260,11 @@ impl Shared { for (_, events) in lock(&self.routes).drain() { let _ = events.send(Event::Closed(why.to_owned())); } + let wakers: Vec = lock(&self.wakers).drain().map(|(_, w)| w).collect(); + for waker in wakers { + waker(); + } + lock(&self.filling).clear(); for (_, waiting) in lock(&self.answers).drain() { let _ = waiting.send(Answer::Refused(Error::Io(why.to_owned()))); } @@ -270,7 +338,9 @@ impl Documents { routes: Mutex::new(HashMap::new()), answers: Mutex::new(HashMap::new()), opening: Mutex::new(HashMap::new()), + filling: Mutex::new(HashMap::new()), numbers: Mutex::new(HashMap::new()), + wakers: Mutex::new(HashMap::new()), closed: AtomicBool::new(false), next_id: AtomicU64::new(1), stream, @@ -286,10 +356,17 @@ impl Documents { /// Opens the file at `path` (relative to the synced folder, `/`-separated, or absolute inside it) for `app` (its /// lowercase name, as in recents). Waits for the answer, [`OPEN_TIMEOUT`] at most. pub fn open(&self, path: &str, app: &str) -> Result { + self.open_with(path, app, &OpenOptions::default()) + } + + /// [`Documents::open`], with what the app knows already ([`OpenOptions`]). + pub fn open_with(&self, path: &str, app: &str, options: &OpenOptions) -> Result { let id = self.shared.next_id.fetch_add(1, Ordering::Relaxed); let (events_tx, events) = mpsc::channel(); lock(&self.shared.routes).insert(id, events_tx); - let answer = self.ask(id, || self.shared.send_json(&DocsRequest::Open { id, path: path.to_owned(), app: app.to_owned() })); + let file_hash = options.file_hash.map(|h| h.iter().map(|b| format!("{b:02x}")).collect()); + let request = DocsRequest::Open { id, path: path.to_owned(), app: app.to_owned(), file_hash }; + let answer = self.ask(id, || self.shared.send_json(&request)); self.opened(id, answer, events) } @@ -316,8 +393,8 @@ impl Documents { lock(&self.shared.routes).remove(&id); }; match answer { - Ok(Answer::Live { doc, state, snapshot }) => { - Ok(Opened::Live(Document { id, doc, state, snapshot, events, shared: self.shared.clone(), seq: AtomicU64::new(0) })) + Ok(Answer::Live { head, snapshot, updates }) => { + Ok(Opened::Live(Document { id, head, snapshot, updates, events, shared: self.shared.clone(), seq: AtomicU64::new(0) })) } Ok(Answer::NeedsImport) => Ok(Opened::NeedsImport(Importer { id, events, shared: self.shared.clone() })), Ok(Answer::NotLive(reason)) => { @@ -370,8 +447,8 @@ impl Importer { } let answer = answer.recv_timeout(OPEN_TIMEOUT).map_err(|_| Error::Io("the agent didn't answer the import in time".into()))?; match answer { - Answer::Live { doc, state, snapshot } => { - Ok(Document { id: self.id, doc, state, snapshot, events: self.events, shared: self.shared, seq: AtomicU64::new(0) }) + Answer::Live { head, snapshot, updates } => { + Ok(Document { id: self.id, head, snapshot, updates, events: self.events, shared: self.shared, seq: AtomicU64::new(0) }) } Answer::NotLive(reason) => { Err(Error::Refused { code: "not_live".into(), message: format!("the file opens as a plain file ({reason:?})") }) @@ -385,9 +462,9 @@ impl Importer { /// An open live document. pub struct Document { id: u64, - doc: DocId, - state: DocState, + head: Head, snapshot: Vec, + updates: Vec>, events: Receiver, shared: Arc, /// This app's changes sent so far (the agent numbers them the same way). @@ -396,25 +473,36 @@ pub struct Document { impl std::fmt::Debug for Document { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Document").field("doc", &self.doc).field("state", &self.state).finish() + f.debug_struct("Document").field("doc", &self.head.doc).field("state", &self.head.state).finish() } } impl Document { - /// The document as it was when opened: a Loro snapshot (updates still waiting for their history come as the first - /// [`Event::Update`]s). + /// The document as it was when opened: a Loro snapshot, then [`Document::state_updates`] (from an older agent, + /// updates still waiting for their history come as the first [`Event::Update`]s instead). pub fn state(&self) -> &[u8] { &self.snapshot } + /// The updates that are part of the state (still waiting for their history), to import after it, in order. + pub fn state_updates(&self) -> &[Vec] { + &self.updates + } + /// Whether it was online, and how many changes waited, when opened. pub fn opened_state(&self) -> DocState { - self.state + self.head.state } /// The live document's ID. pub fn id(&self) -> DocId { - self.doc + self.head.doc + } + + /// For an open that named the file's hash ([`OpenOptions::file_hash`]): whether no Craft app's save of this + /// document has that content (the file was edited outside Craft). None: the agent can't tell (or is older). + pub fn changed_outside(&self) -> Option { + self.head.changed_outside } /// Sends this app's change (a Loro update, at most 1 MiB); returns its number, which [`Event::Acked`] names once @@ -425,11 +513,37 @@ impl Document { Ok(seq) } + /// Tells the agent the app is about to write the document's file: the content's BLAKE3 `hash`, and the live + /// document's frontiers it was made from (`(peer, counter)` heads). Send it after every change it covers + /// ([`Document::send`]): the agent keeps it once those are in its copy, and records it with the file version it + /// uploads for that content. Doesn't wait for an answer. + pub fn saved(&self, hash: [u8; 32], frontiers: &[(u64, u32)]) -> Result<(), Error> { + let hex = |b: &[u8]| b.iter().map(|x| format!("{x:02x}")).collect::(); + let mut frontiers = frontiers.to_vec(); + frontiers.sort_unstable(); + frontiers.dedup(); + self.shared.send_json(&DocsRequest::Saved { doc: self.head.doc, hash: hex(&hash), frontiers }) + } + + /// Tells the agent, at once and before the file is written, that the app is writing the document's file with + /// content `hash`, and that its record ([`Document::saved`]) follows: until it comes, the agent's sync waits for it + /// rather than take the content for an edit outside Craft. An older agent answers `error`; nothing else changes. + pub fn saving(&self, hash: [u8; 32]) -> Result<(), Error> { + let hex = hash.iter().map(|x| format!("{x:02x}")).collect::(); + self.shared.send_json(&DocsRequest::Saving { doc: self.head.doc, hash: hex }) + } + /// Sends this app's presence (at most 4 KiB, at most 20 a second; more are dropped). pub fn presence(&self, bytes: &[u8]) -> Result<(), Error> { self.shared.send_binary(PRESENCE, self.id, bytes) } + /// Calls `waker` (from the connection's reader thread) after each event that comes for this document from now on, + /// and once when the connection ends: an app's UI repaints, then takes the events with [`Document::try_recv`]. + pub fn set_waker(&self, waker: Waker) { + lock(&self.shared.wakers).insert(self.id, waker); + } + /// The next event, if one came. pub fn try_recv(&self) -> Option { self.events.try_recv().ok() @@ -449,15 +563,17 @@ impl Document { impl Drop for Document { fn drop(&mut self) { lock(&self.shared.routes).remove(&self.id); + lock(&self.shared.wakers).remove(&self.id); let mut numbers = lock(&self.shared.numbers); - let remaining = numbers.get_mut(&self.doc).map(|ids| { + let doc = self.head.doc; + let remaining = numbers.get_mut(&doc).map(|ids| { ids.retain(|id| *id != self.id); ids.len() }); if remaining == Some(0) { - numbers.remove(&self.doc); + numbers.remove(&doc); drop(numbers); - let _ = self.shared.send_json(&DocsRequest::Close { doc: self.doc }); + let _ = self.shared.send_json(&DocsRequest::Close { doc }); } } } @@ -476,7 +592,9 @@ mod tests { routes: Mutex::new(HashMap::new()), answers: Mutex::new(HashMap::new()), opening: Mutex::new(HashMap::new()), + filling: Mutex::new(HashMap::new()), numbers: Mutex::new(HashMap::new()), + wakers: Mutex::new(HashMap::new()), closed: AtomicBool::new(false), next_id: AtomicU64::new(1), stream, @@ -484,6 +602,33 @@ mod tests { (shared, listener) } + /// A state with updates that are part of it: answered once the last of them came, with them; the updates after + /// it are the document's events. + #[test] + fn an_open_waits_for_the_updates_of_its_state() { + let (shared, _listener) = shared(); + let (answer_tx, answer) = mpsc::channel(); + lock(&shared.answers).insert(5, answer_tx); + let (events_tx, events) = mpsc::channel(); + lock(&shared.routes).insert(5, events_tx); + let opened = DocsEvent::Opened { + id: 5, + doc: DocId::generate(), + state: DocState { online: true, waiting: 0, bytes: 1, updates: 2 }, + changed_outside: None, + }; + shared.on_message(Message::Text(serde_json::to_string(&opened).unwrap().into())); + shared.on_message(Message::Binary(encode_binary(STATE, 5, b"s").into())); + shared.on_message(Message::Binary(encode_binary(UPDATE, 5, b"1").into())); + assert!(answer.try_recv().is_err(), "one of its updates is still to come"); + shared.on_message(Message::Binary(encode_binary(UPDATE, 5, b"2").into())); + shared.on_message(Message::Binary(encode_binary(UPDATE, 5, b"3").into())); + let Ok(Answer::Live { updates, .. }) = answer.try_recv() else { panic!("no answer") }; + assert_eq!(updates, vec![b"1".to_vec(), b"2".to_vec()]); + assert_eq!(events.try_recv().unwrap(), Event::Update(b"3".to_vec())); + assert!(events.try_recv().is_err()); + } + /// `opened` then the state: one answer; the events after it go to the document, its status by document ID. #[test] fn an_open_is_answered_once_its_state_came() { @@ -493,12 +638,14 @@ mod tests { let (events_tx, events) = mpsc::channel(); lock(&shared.routes).insert(4, events_tx); let doc = DocId::generate(); - let opened = DocsEvent::Opened { id: 4, doc, state: DocState { online: true, waiting: 0, bytes: 3 } }; + let opened = + DocsEvent::Opened { id: 4, doc, state: DocState { online: true, waiting: 0, bytes: 3, updates: 0 }, changed_outside: Some(true) }; shared.on_message(Message::Text(serde_json::to_string(&opened).unwrap().into())); assert!(answer.try_recv().is_err(), "not before the state"); shared.on_message(Message::Binary(encode_binary(STATE, 4, b"abc").into())); - let Ok(Answer::Live { snapshot, .. }) = answer.try_recv() else { panic!("no answer") }; + let Ok(Answer::Live { snapshot, head, .. }) = answer.try_recv() else { panic!("no answer") }; assert_eq!(snapshot, b"abc"); + assert_eq!(head.changed_outside, Some(true)); shared.on_message(Message::Binary(encode_binary(UPDATE, 4, b"u").into())); let status = DocsEvent::Status { doc, online: false, waiting: 2, deleted: true, stopped: true }; shared.on_message(Message::Text(serde_json::to_string(&status).unwrap().into())); diff --git a/crates/app/src/recents.rs b/crates/app/src/recents.rs index 2e62f6b..30552fe 100644 --- a/crates/app/src/recents.rs +++ b/crates/app/src/recents.rs @@ -10,8 +10,8 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; -use craft_cloud_proto::api::ErrorBody; -use craft_cloud_proto::local::{MAX_LOCAL_JSON_BYTES, NewRecent, RecentEntry, RecentsEvent as Wire, RecentsRequest, codes, paths}; +use craft_cloud_api::api::ErrorAnswer as ErrorBody; +use craft_cloud_api::local::{MAX_LOCAL_JSON_BYTES, NewRecent, RecentEntry, RecentsEvent as Wire, RecentsRequest, codes, paths}; use tungstenite::client::IntoClientRequest; use tungstenite::handshake::HandshakeError; use tungstenite::protocol::WebSocketConfig; diff --git a/crates/egui/Cargo.toml b/crates/egui/Cargo.toml new file mode 100644 index 0000000..64c3316 --- /dev/null +++ b/crates/egui/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "craft-cloud-egui" +description = "Craft Cloud's piece of a Craft app's window (egui): the account and status line, the shared recent files, and a live document's badge" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +craft-cloud-app.workspace = true +craft-cloud-kit.workspace = true +egui.workspace = true + +[dev-dependencies] +egui_kittest.workspace = true + +[lints] +workspace = true diff --git a/crates/egui/src/lib.rs b/crates/egui/src/lib.rs new file mode 100644 index 0000000..3871173 --- /dev/null +++ b/crates/egui/src/lib.rs @@ -0,0 +1,326 @@ +//! Craft Cloud's piece of a Craft app's window, on egui 0.36 as every Craft app is: +//! +//! - [`account_line`]: who is signed in where, and whether Craft Cloud is online (or not running); +//! - [`recents`]: the app's recent files, shared by every machine of the account, newest first (a click opens one); +//! - [`badge`]: a live document's state (live, waiting N, read-only, offline), next to its title. +//! +//! The widgets only draw what they are given. [`CloudWatch`] keeps it current off the UI thread (the agent's status +//! every few seconds, the recents as they change) and wakes the UI when it changed (`request_repaint`). + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use craft_cloud_app::{Client, NewRecent, RecentEntry, RecentsEvent, RecentsWatch, RunState, Status}; +use craft_cloud_kit::Badge; +use egui::{Color32, RichText, Ui}; + +/// How often [`CloudWatch`] asks the agent how it stands. +pub const STATUS_EVERY: Duration = Duration::from_secs(3); + +/// What the account line shows. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AgentState { + /// Not known yet (the first look is under way). + Looking, + /// Craft Cloud isn't running on this computer (or this app can't reach it): live features are off. + NotRunning, + SignedOut, + SignedIn { + account: String, + server: String, + machine: String, + online: bool, + paused: bool, + }, +} + +impl AgentState { + pub fn from_status(status: &Status) -> Self { + if !status.signed_in { + return Self::SignedOut; + } + Self::SignedIn { + account: status.account.clone().unwrap_or_default(), + server: host(status.server.as_deref().unwrap_or_default()), + machine: status.machine.clone().unwrap_or_default(), + online: status.online, + paused: matches!(status.run_state, RunState::PausedByUser | RunState::Paused { .. }), + } + } +} + +/// `https://cloud.example.com/` as `cloud.example.com`. +fn host(server: &str) -> String { + let rest = server.split_once("://").map_or(server, |(_, r)| r); + rest.trim_end_matches('/').to_owned() +} + +/// The words a badge shows. +pub fn badge_text(badge: Badge) -> String { + match badge { + Badge::Live => "Live".to_owned(), + Badge::Waiting(n) => format!("Waiting {n}"), + Badge::ReadOnly => "Read-only".to_owned(), + Badge::Offline => "Offline".to_owned(), + Badge::Stalled => "Stalled".to_owned(), + Badge::Stopped => "Stopped".to_owned(), + } +} + +/// The account and status line: "Craft Cloud · ana on cloud.example.com · online". +pub fn account_line(ui: &mut Ui, state: &AgentState) { + ui.horizontal(|ui| { + ui.label(RichText::new("Craft Cloud").strong()); + match state { + AgentState::Looking => { + ui.weak("…"); + } + AgentState::NotRunning => { + ui.weak("isn't running on this computer"); + } + AgentState::SignedOut => { + ui.weak("not signed in"); + } + AgentState::SignedIn { account, server, machine, online, paused } => { + ui.label(format!("{account} on {server}")); + if !machine.is_empty() { + ui.weak(format!("({machine})")); + } + let (word, color) = match (paused, online) { + (true, _) => ("paused", Color32::from_rgb(0xb0, 0x80, 0x10)), + (false, true) => ("online", Color32::from_rgb(0x2a, 0x8a, 0x3e)), + (false, false) => ("offline", Color32::from_rgb(0xb0, 0x40, 0x30)), + }; + ui.colored_label(color, word); + } + } + }); +} + +/// The recent files, newest first, as buttons: the path of the one clicked. +pub fn recents(ui: &mut Ui, entries: &[RecentEntry], now_ms: i64) -> Option { + let mut clicked = None; + if entries.is_empty() { + ui.weak("No recent files"); + return None; + } + for entry in entries { + ui.horizontal(|ui| { + if ui.button(&entry.title).on_hover_text(&entry.path).clicked() { + clicked = Some(entry.path.clone()); + } + ui.weak(format!("{} · {}", entry.machine, ago(now_ms, entry.saved_ms.unwrap_or(entry.opened_ms)))); + }); + } + clicked +} + +/// How long ago, as people say it. +pub fn ago(now_ms: i64, then_ms: i64) -> String { + let secs = now_ms.saturating_sub(then_ms).max(0) / 1000; + match secs { + 0..=59 => "just now".to_owned(), + 60..=3599 => format!("{} min ago", secs / 60), + 3600..=86_399 => format!("{} h ago", secs / 3600), + _ => format!("{} d ago", secs / 86_400), + } +} + +/// A live document's badge (with what it means on hover). +pub fn badge(ui: &mut Ui, status: &craft_cloud_kit::Status) { + let b = status.badge(); + let (color, hover) = match b { + Badge::Live => (Color32::from_rgb(0x2a, 0x8a, 0x3e), "Edits go to your other computers as you make them".to_owned()), + Badge::Waiting(n) => (Color32::from_rgb(0xb0, 0x80, 0x10), format!("{n} changes are waiting to reach Craft Cloud")), + Badge::ReadOnly => ( + Color32::from_rgb(0x40, 0x60, 0xb0), + match &status.read_only { + Some(ro) if !ro.app.is_empty() => format!("Made with a newer version (layout {}): update the app to edit it", ro.version), + _ => "This document can't be edited by this app".to_owned(), + }, + ), + Badge::Offline => (Color32::from_rgb(0xb0, 0x40, 0x30), "Edits are kept and go out when Craft Cloud is back".to_owned()), + Badge::Stalled => ( + Color32::from_rgb(0xc0, 0x20, 0x20), + "Stalled: too much is waiting for Craft Cloud, so new changes are not kept until it is back".to_owned(), + ), + Badge::Stopped => ( + Color32::from_rgb(0xc0, 0x20, 0x20), + format!("Stopped: changes no longer go out ({}); save the document to keep them", status.error.as_deref().unwrap_or("see the agent")), + ), + }; + ui.colored_label(color, badge_text(b)).on_hover_text(hover); +} + +/// Keeps an [`AgentState`] and an app's recents current, off the UI thread. +pub struct CloudWatch { + state: Arc>, + recents: Option, + entries: Vec, + stop: Arc, +} + +impl std::fmt::Debug for CloudWatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CloudWatch").finish_non_exhaustive() + } +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl CloudWatch { + /// Watches the agent `connect` finds (again and again: it may start later) for `app` (its lowercase name); + /// `wake` is called when something changed (egui's `request_repaint`). + pub fn start(app: &str, connect: impl Fn() -> Result + Send + 'static, wake: impl Fn() + Send + 'static) -> Self { + let state = Arc::new(Mutex::new(AgentState::Looking)); + let stop = Arc::new(AtomicBool::new(false)); + let recents = connect().ok().and_then(|c| c.watch_recents(app).ok()); + let (s, halt) = (state.clone(), stop.clone()); + let _ = std::thread::Builder::new().name("craft-cloud-status".into()).spawn(move || { + while !halt.load(Ordering::Relaxed) { + let now = match connect().and_then(|c| c.status()) { + Ok(status) => AgentState::from_status(&status), + Err(_) => AgentState::NotRunning, + }; + let changed = { + let mut held = lock(&s); + let changed = *held != now; + *held = now; + changed + }; + if changed { + wake(); + } + std::thread::sleep(STATUS_EVERY); + } + }); + let entries = recents.as_ref().map(RecentsWatch::entries).unwrap_or_default(); + Self { state, recents, entries, stop } + } + + /// How the agent stands now. + pub fn state(&self) -> AgentState { + lock(&self.state).clone() + } + + /// The recents now (takes what changed since the last frame). + pub fn entries(&mut self) -> &[RecentEntry] { + if let Some(watch) = &self.recents { + while let Ok(event) = watch.events().try_recv() { + if let RecentsEvent::Changed(list) = event { + self.entries = list; + } + } + } + &self.entries + } + + /// Notes a file this app opened or saved in its recents (on every machine). + pub fn add_recent(&self, path: &str, saved: bool) { + if let Some(watch) = &self.recents { + let _ = watch.add(NewRecent { path: path.to_owned(), title: None, saved }); + } + } +} + +impl Drop for CloudWatch { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use egui_kittest::Harness; + use egui_kittest::kittest::Queryable; + + fn signed_in(online: bool) -> AgentState { + AgentState::SignedIn { account: "ana".into(), server: "cloud.example.com".into(), machine: "lptp1".into(), online, paused: false } + } + + #[test] + fn the_account_line_says_how_craft_cloud_stands() { + for (state, words) in [ + (signed_in(true), vec!["ana on cloud.example.com", "online", "(lptp1)"]), + (signed_in(false), vec!["offline"]), + (AgentState::NotRunning, vec!["isn't running on this computer"]), + (AgentState::SignedOut, vec!["not signed in"]), + ] { + let mut h = Harness::new_ui_state(|ui, s: &mut AgentState| account_line(ui, s), state); + h.run(); + for w in words { + assert!(h.query_by_label(w).is_some(), "{w}"); + } + } + let status = Status { + api: 1, + signed_in: true, + account: Some("ana".into()), + server: Some("https://cloud.example.com/".into()), + folder: None, + machine: Some("BEAST".into()), + online: true, + run_state: RunState::PausedByUser, + autostart: None, + last_sync_ms: None, + removed: false, + }; + assert!(matches!(AgentState::from_status(&status), AgentState::SignedIn { paused: true, ref server, .. } if server == "cloud.example.com")); + } + + #[test] + fn a_recent_file_opens_on_a_click() { + let entries = vec![ + RecentEntry { + node: None, + path: "Notes/a.kitdemo".into(), + title: "a.kitdemo".into(), + opened_ms: 0, + saved_ms: Some(50_000), + machine: "BEAST".into(), + }, + RecentEntry { node: None, path: "b.kitdemo".into(), title: "b.kitdemo".into(), opened_ms: 0, saved_ms: None, machine: "lptp1".into() }, + ]; + let mut h = Harness::new_ui_state( + |ui, (entries, clicked): &mut (Vec, Option)| { + if let Some(p) = recents(ui, entries, 120_000) { + *clicked = Some(p); + } + }, + (entries, None), + ); + h.run(); + assert!(h.query_by_label("BEAST · 1 min ago").is_some()); + h.get_by_label("b.kitdemo").click(); + h.run(); + assert_eq!(h.state().1.as_deref(), Some("b.kitdemo")); + let mut empty = Harness::new_ui(|ui| { + recents(ui, &[], 0); + }); + empty.run(); + assert!(empty.query_by_label("No recent files").is_some()); + } + + #[test] + fn the_badge_says_live_waiting_read_only_or_offline() { + let live = craft_cloud_kit::Status { connected: true, online: true, ..Default::default() }; + let cases = [ + (live.clone(), "Live"), + (craft_cloud_kit::Status { waiting: 3, ..live.clone() }, "Waiting 3"), + (craft_cloud_kit::Status { read_only: Some(craft_cloud_kit::ReadOnly { app: "demo".into(), version: 2 }), ..live.clone() }, "Read-only"), + (craft_cloud_kit::Status { connected: false, ..live.clone() }, "Offline"), + ]; + for (status, word) in cases { + let mut h = Harness::new_ui_state(|ui, s: &mut craft_cloud_kit::Status| badge(ui, s), status); + h.run(); + assert!(h.query_by_label(word).is_some(), "{word}"); + } + assert_eq!(ago(10_000_000, 10_000_000 - 7_200_000), "2 h ago"); + } +} diff --git a/crates/kit/Cargo.toml b/crates/kit/Cargo.toml new file mode 100644 index 0000000..ad63500 --- /dev/null +++ b/crates/kit/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "craft-cloud-kit" +description = "The Craft Cloud app kit: a Craft app's model live through the agent, mapped onto Loro (tree, rich text, table, timeline, opaque values), with undo while live and binary content by reference" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[features] +# `craft_cloud_kit::testing`: the fake agent of `craft-cloud-app`, re-exported for an app's adapter tests. +fake = ["craft-cloud-app/fake"] + +[dependencies] +blake3.workspace = true +craft-cloud-api.workspace = true +craft-cloud-app.workspace = true +getrandom.workspace = true +loro.workspace = true +thiserror.workspace = true + +[dev-dependencies] +craft-cloud-app = { workspace = true, features = ["fake"] } +proptest.workspace = true + +[lints] +workspace = true diff --git a/crates/kit/src/error.rs b/crates/kit/src/error.rs new file mode 100644 index 0000000..d69cccf --- /dev/null +++ b/crates/kit/src/error.rs @@ -0,0 +1,64 @@ +//! What can go wrong in the kit. Nothing here panics: a Loro error, a hostile or broken document, a lost agent all +//! become a [`KitError`], and the app keeps its model. + +/// Why a kit call failed. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum KitError { + /// The document was made by a newer version of the app (or by another app): it is shown, but nothing is sent. + #[error("this document needs a newer {app} (made with layout {version}); it opens read-only")] + ReadOnly { app: String, version: u32 }, + /// More than the queue holds is waiting for the agent: no more changes until it is back. + #[error("{0} bytes of changes are waiting for Craft Cloud; reconnect before changing more")] + Stalled(usize), + /// `old` isn't the model the session holds (a remote change came in between, or the app passed another one). + #[error("the change was made from an older model than the session's; take the session's model and make it again")] + Stale, + /// A value too big for a live document: binary content goes in as a chunk reference instead. + #[error("{what} is {len} bytes; at most {max} go in a live document (use a chunk reference)")] + TooLarge { what: &'static str, len: usize, max: usize }, + /// A mapping's key, or a property's, that the kit doesn't take. + #[error("{0}")] + Invalid(String), + /// Loro refused something (the document stays as it was before the change, or as the session rebuilt it). + #[error("the live document: {0}")] + Loro(String), + /// The document's content isn't what the mapping expects (a hostile or broken peer); the part is skipped. + #[error("the live document doesn't read: {0}")] + Corrupt(String), + /// The agent: not running, refused, or the connection broke. + #[error("Craft Cloud: {0}")] + Agent(String), + /// The app's own codec failed. + #[error("{0}")] + App(String), + /// The model couldn't take a change from the live document (`apply_remote` or a repair failed): it is behind, so + /// no local change is taken from it until the session has rebuilt it whole (it tries at each poll). + #[error("model_stale: the model is behind the live document; wait for the session to rebuild it")] + ModelStale, + /// A save now carries no record ([`crate::Session::save_point`] says why): the app saves its own model as it + /// would without Craft Cloud, and the file syncs as any file does (a conflict copy if it meets another edit). + #[error("this save can't carry a record: {0}")] + NoRecord(String), + /// The step changed more text than can be undone while others edit (Loro's undo of it would freeze the app). It + /// stays on the undo stack as a barrier. + #[error("too large to undo while others edit ({bytes} bytes of text)")] + UndoTooLarge { bytes: usize }, +} + +impl From for KitError { + fn from(e: loro::LoroError) -> Self { + Self::Loro(e.to_string()) + } +} + +impl From for KitError { + fn from(e: loro::LoroEncodeError) -> Self { + Self::Loro(e.to_string()) + } +} + +impl From for KitError { + fn from(e: craft_cloud_app::Error) -> Self { + Self::Agent(e.to_string()) + } +} diff --git a/crates/kit/src/lib.rs b/crates/kit/src/lib.rs new file mode 100644 index 0000000..5693589 --- /dev/null +++ b/crates/kit/src/lib.rs @@ -0,0 +1,111 @@ +//! The Craft Cloud app kit: what a Craft app adds to make its documents live. +//! +//! An app's whole change is one adapter: +//! 1. implement [`LiveDocument`] for its model: which parts are shared ([`Mappings`]: a tree, rich text, a table, a +//! timeline, opaque values), how to rebuild the parts a remote change touched ([`LiveDocument::apply_remote`]), and +//! how to repair what a merge can break ([`LiveDocument::normalise`]); +//! 2. open the file through a [`Session`], and call [`Session::local_change`] at the end of each top-level command +//! (old and new model); +//! 3. call [`Session::poll`] each frame, and take [`Session::model`] when it says the model changed; +//! 4. route undo and redo through the session while the document is live ([`Session::undo`]). +//! +//! Binary content never goes into the live document: it goes through a [`BlobStore`] (the agent's, [`AgentBlobs`]) +//! and is referenced by [`ChunkRef`]. Nothing here starts an async runtime, and nothing panics: errors are +//! [`KitError`]s, and the app keeps its model. +//! +//! See `docs/kit.md` for the walk-through with the demo app. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod error; +pub(crate) mod list; +mod mapping; +pub(crate) mod opaque; +mod props; +mod session; +pub(crate) mod table; +pub(crate) mod text; +pub(crate) mod timeline; +pub(crate) mod tree; +mod value; +pub(crate) mod writer; + +pub use error::KitError; +pub use mapping::{Attrs, Mappings, OpaqueSource, RemotePatch, Span, TableSource, TextSource, TimelineSource, Touched, TreeSource}; +pub use opaque::Opaque; +pub use props::{Props, PropsRead}; +pub use session::{ + Agent, Badge, CATCH_UP_EVERY, LiveDocument, MAX_BACKOFF, MAX_QUEUE_BYTES, MAX_UNDO_TEXT_BYTES, NotLive, Open, PRESENCE_EVERY, PRESENCE_TTL, Peer, + Polled, REPLACE_WITH_FILE, ReadOnly, SavePoint, SaveRecord, Session, Status, UndoKind, +}; +pub use table::{TableTouch, TableView}; +pub use text::RichText; +pub use timeline::{ItemRead, TimelineTouch, TimelineView}; +pub use tree::{TreeTouch, TreeView}; +pub use value::{AgentBlobs, BlobStore, ChunkRef, ElemId, MAX_VALUE_BYTES, MemoryBlobs, OpaqueCodec, Value}; +pub use writer::{OP_OVERHEAD, SPLIT_BYTES}; + +/// For an app's adapter tests: the fake agent of `craft-cloud-app` (feature `fake`). +#[cfg(feature = "fake")] +pub mod testing { + pub use craft_cloud_app::fake::{Fake, Hub}; +} + +/// For the kit's own tests and simulations: the mappings without a session. +#[doc(hidden)] +pub mod raw { + use loro::LoroDoc; + + use crate::session::{Ctx, LiveDocument, write_diff}; + use crate::writer::{LOCAL_ORIGIN, Writer}; + use crate::{KitError, RemotePatch, Touched}; + + /// A Loro document with the mappings' state, as a session keeps it. + pub struct Replica { + pub doc: LoroDoc, + ctx: Ctx, + } + + impl Default for Replica { + fn default() -> Self { + let doc = LoroDoc::new(); + doc.config_default_text_style(Some(loro::StyleConfig { expand: loro::ExpandType::After })); + Self { doc, ctx: Ctx::default() } + } + } + + impl Replica { + /// Writes `old` → `new`; the updates (each commit's export) and how many commits. + pub fn write(&mut self, old: Option<&D::Doc>, new: &D::Doc) -> Result<(Vec>, usize), KitError> { + let mut w = Writer::new(&self.doc, LOCAL_ORIGIN); + write_diff::(&mut w, old, new, &mut self.ctx)?; + w.finish() + } + + /// Imports updates and indexes the trees again. + pub fn import(&mut self, updates: &[Vec], trees: &[&str]) -> Result<(), KitError> { + for u in updates { + self.doc.import(u)?; + } + self.reindex(trees); + Ok(()) + } + + pub fn reindex(&mut self, trees: &[&str]) { + for key in trees { + self.ctx.trees.insert((*key).to_owned(), crate::tree::TreeIndex::build(&self.doc.get_tree(*key))); + } + } + + /// The whole document read back through `D::apply_remote`, from `base`. + pub fn read(&self, base: &std::sync::Arc) -> Result, KitError> { + let touched = Touched::everything(); + let patch = RemotePatch { doc: &self.doc, touched: &touched, trees: &self.ctx.trees, old: None }; + D::apply_remote(base, &patch) + } + + /// How many operations this document's oplog holds. + pub fn ops(&self) -> i64 { + self.doc.oplog_vv().values().map(|c| i64::from(*c)).sum() + } + } +} diff --git a/crates/kit/src/list.rs b/crates/kit/src/list.rs new file mode 100644 index 0000000..fad869b --- /dev/null +++ b/crates/kit/src/list.rs @@ -0,0 +1,151 @@ +//! Keeping a `LoroMovableList` of element IDs in a given order with few operations: what isn't wanted (or is there +//! twice, or isn't an ID) is deleted, the longest run already in order stays, and only the rest moves or is inserted, +//! so concurrent reorders on two machines merge rather than fight. + +use std::collections::{HashMap, HashSet}; + +use loro::{LoroMovableList, ValueOrContainer}; + +use crate::writer::Writer; +use crate::{ElemId, KitError}; + +const ID_WEIGHT: usize = 16; + +/// The list's IDs as stored (anything else is `None`). +pub(crate) fn read(list: &LoroMovableList) -> Vec> { + (0..list.len()) + .map(|i| match list.get(i) { + Some(ValueOrContainer::Value(v)) => ElemId::from_loro(&v), + _ => None, + }) + .collect() +} + +/// The list's IDs, each once (first place wins), skipping anything that isn't one. +pub(crate) fn ids(list: &LoroMovableList) -> Vec { + let mut seen = HashSet::new(); + read(list).into_iter().flatten().filter(|id| seen.insert(*id)).collect() +} + +/// Makes `list` hold exactly `desired` (IDs, each once), in order. Returns the IDs it removed. +pub(crate) fn sync(w: &mut Writer<'_>, list: &LoroMovableList, desired: &[ElemId]) -> Result, KitError> { + let want: HashSet = desired.iter().copied().collect(); + let stored = read(list); + // Which places to keep: a wanted ID's first place. + let mut seen = HashSet::new(); + let keep: Vec = stored.iter().map(|id| id.is_some_and(|id| want.contains(&id) && seen.insert(id))).collect(); + let mut removed = Vec::new(); + for (i, kept) in keep.iter().enumerate().rev() { + if !kept { + if let Some(Some(id)) = stored.get(i) + && !want.contains(id) + { + removed.push(*id); + } + w.spend(ID_WEIGHT)?; + list.delete(i, 1)?; + } + } + let mut cur: Vec = stored.iter().zip(&keep).filter(|(_, k)| **k).filter_map(|(id, _)| *id).collect(); + if cur.as_slice() == desired { + return Ok(removed); + } + // The longest run of what is there already in the wanted order stays put. + let rank: HashMap = desired.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + let ranks: Vec = cur.iter().filter_map(|id| rank.get(id).copied()).collect(); + let steady: HashSet = longest_increasing(&ranks).into_iter().filter_map(|i| desired.get(i).copied()).collect(); + let mut prev: Option = None; + for id in desired { + if !steady.contains(id) { + let to = match prev { + Some(p) => cur.iter().position(|c| *c == p).map_or(0, |i| i + 1), + None => 0, + }; + w.spend(ID_WEIGHT)?; + match cur.iter().position(|c| c == id) { + Some(from) => { + cur.remove(from); + let to = if from < to { to - 1 } else { to }; + list.mov(from, to)?; + cur.insert(to, *id); + } + None => { + list.insert(to, id.to_loro())?; + cur.insert(to, *id); + } + } + } + prev = Some(*id); + } + Ok(removed) +} + +/// The values of the longest strictly increasing subsequence of `seq`. +pub(crate) fn longest_increasing(seq: &[usize]) -> Vec { + // Patience sorting: `tails[k]` is the index (into `seq`) ending the best run of length k + 1. + let mut tails: Vec = Vec::new(); + let mut back: Vec> = vec![None; seq.len()]; + for (i, v) in seq.iter().enumerate() { + let k = tails.partition_point(|&t| seq.get(t).is_some_and(|tv| tv < v)); + if k > 0 + && let Some(slot) = back.get_mut(i) + { + *slot = tails.get(k - 1).copied(); + } + if k == tails.len() { + tails.push(i); + } else if let Some(slot) = tails.get_mut(k) { + *slot = i; + } + } + let mut out = Vec::new(); + let mut at = tails.last().copied(); + while let Some(i) = at { + if let Some(v) = seq.get(i) { + out.push(*v); + } + at = back.get(i).copied().flatten(); + } + out.reverse(); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use loro::LoroDoc; + + fn ids(n: u128) -> Vec { + (0..n).map(ElemId).collect() + } + + #[test] + fn a_list_is_brought_to_order_with_few_moves() { + assert_eq!(longest_increasing(&[3, 0, 1, 2]), vec![0, 1, 2]); + assert_eq!(longest_increasing(&[]), Vec::::new()); + let doc = LoroDoc::new(); + let list = doc.get_movable_list("l"); + let all = ids(6); + for (from, to) in [ + (vec![], all.clone()), + (all.clone(), vec![all[1], all[2], all[3], all[4], all[5], all[0]]), + (all.clone(), vec![all[5], all[0], all[1]]), + (all.clone(), vec![all[2], all[0], all[4], all[1]]), + (all.clone(), all.iter().rev().copied().collect()), + ] { + let mut w = Writer::new(&doc, crate::writer::LOCAL_ORIGIN); + sync(&mut w, &list, &from).unwrap(); + w.finish().unwrap(); + assert_eq!(super::ids(&list), from); + let before = doc.oplog_vv(); + let mut w = Writer::new(&doc, crate::writer::LOCAL_ORIGIN); + sync(&mut w, &list, &to).unwrap(); + w.finish().unwrap(); + assert_eq!(super::ids(&list), to, "from {from:?}"); + let ops: i32 = doc.oplog_vv().iter().map(|(p, c)| c - before.get(p).copied().unwrap_or(0)).sum(); + if from.len() == 6 && to.len() == 6 && to.first() == Some(&all[1]) { + assert_eq!(ops, 1, "one move: the first to the end"); + } + } + } +} diff --git a/crates/kit/src/mapping.rs b/crates/kit/src/mapping.rs new file mode 100644 index 0000000..e9783d7 --- /dev/null +++ b/crates/kit/src/mapping.rs @@ -0,0 +1,401 @@ +//! How an app's model maps onto the live document: the parts it shares ([`Mappings`]), each wired to a shape (tree, +//! rich text, table, timeline, opaque value) through a source trait the app implements over its own types, and what a +//! remote change touched ([`RemotePatch`]), for the app to rebuild only that. +//! +//! Each part has a key (`layers`, `story`); its Loro containers are named after it (`layers`, `story`, `cells.rows`, +//! …), so two parts never share one. + +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::ops::Range; + +use loro::LoroDoc; + +use crate::props::{Props, PropsRead}; +use crate::table::{TableTouch, TableView}; +use crate::text::RichText; +use crate::timeline::{TimelineTouch, TimelineView}; +use crate::tree::{TreeIndex, TreeTouch, TreeView}; +use crate::{ElemId, KitError, Value}; + +/// A part key: 1 to 64 of `a-z`, `0-9`, `_` and `-`, not `root`. +pub(crate) fn check_part_key(key: &str) -> Result<(), KitError> { + let ok = + (1..=64).contains(&key.len()) && key.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_' || b == b'-') && key != "root"; + if ok { Ok(()) } else { Err(KitError::Invalid(format!("{key:?} isn't a part key (1-64 of a-z, 0-9, '_', '-'; not \"root\")"))) } +} + +/// A tree of elements (layers, objects, bins): `LoroTree`, a property map per node. Subtrees whose +/// [`subtree_key`](TreeSource::subtree_key) is the same in the old and new model are skipped. +pub trait TreeSource { + /// The same value in two models when the whole tree is the same (`Arc::as_ptr` of its root list); `None`: compare. + fn key(&self) -> Option { + None + } + /// The top-level elements, in order. + fn roots(&self) -> Vec; + /// An element's children, in order (none for a leaf or an unknown ID). + fn children(&self, id: ElemId) -> Vec; + /// The same value in two models when the element and everything under it are the same (`Arc::as_ptr` as + /// `usize`; both models are alive while the kit compares, so an address can't be reused). + fn subtree_key(&self, _id: ElemId) -> Option { + None + } + /// Writes the element's properties (only the ones that differ from what is stored are written). + fn write(&self, id: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError>; +} + +/// Character attributes of a run, or paragraph attributes: key to value (`bold` → `true`, `font` → `"Inter"`). +pub type Attrs = BTreeMap; + +/// Rich text (a story): one `LoroText`, character attributes as marks, paragraph attributes as marks on each +/// paragraph's break (`\n`; the last paragraph's, which has none, beside it). +pub trait TextSource { + /// The same value in two models when the text and its attributes are the same; `None`: compare. + fn key(&self) -> Option { + None + } + fn text(&self) -> &str; + /// Runs with character attributes: byte ranges of [`TextSource::text`] on character boundaries, in order, not + /// overlapping. What no run covers has none. (Borrowed when the app keeps them so: a keystroke copies nothing.) + fn runs(&self) -> Cow<'_, [(Range, Attrs)]>; + /// Each paragraph's attributes, in order: one more than the text has `\n`s (missing ones have none). + fn paragraphs(&self) -> Cow<'_, [Attrs]>; +} + +/// A table (or keyed records, with one column): movable lists of row and column IDs, and a map of cells by row and +/// column, so inserting a row doesn't renumber cells and two inserts at once keep both. +pub trait TableSource { + /// The same value in two models when the table is the same; `None`: compare. + fn key(&self) -> Option { + None + } + fn rows(&self) -> Vec; + fn columns(&self) -> Vec; + fn cell(&self, row: ElemId, column: ElemId) -> Option; + /// Every cell that holds something. + fn cells(&self) -> Vec<(ElemId, ElemId, Value)>; + /// The cells changed since the old model of this change, when the app knows (GridCraft's commit does): then only + /// those are compared. `None`: every cell is. + fn changed_cells(&self) -> Option> { + None + } +} + +/// An item's place on its track, in the app's time unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Span { + pub start: i64, + pub len: i64, +} + +/// A timeline: a movable list of tracks (each with properties), items by ID (each on a track, with a start, a length +/// and properties), and keyframes by ID (each on an item, at a time, with a value), so two keyframes added at once on +/// one item are both kept. +pub trait TimelineSource { + /// The same value in two models when the timeline is the same; `None`: compare. + fn key(&self) -> Option { + None + } + fn tracks(&self) -> Vec; + /// The same value in two models when a track and its items are the same; `None`: compare. + fn track_key(&self, _track: ElemId) -> Option { + None + } + fn write_track(&self, track: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError>; + /// The items on a track (any order: they read back by start). + fn items(&self, track: ElemId) -> Vec; + fn span(&self, item: ElemId) -> Option; + /// The same value in two models when an item and its keyframes are the same; `None`: compare. + fn item_key(&self, _item: ElemId) -> Option { + None + } + fn write_item(&self, item: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError>; + /// An item's keyframes: ID, time and value. + fn keyframes(&self, item: ElemId) -> Vec<(ElemId, i64, Value)>; +} + +/// A value kept whole, last writer wins: settings, styles, plugin state ([`crate::OpaqueCodec`] encodes it). +pub trait OpaqueSource { + /// The same value in two models when the value is the same; `None`: compare the bytes. + fn key(&self) -> Option { + None + } + /// The value's bytes (`None`: there is none), at most [`crate::MAX_VALUE_BYTES`]. + fn encode(&self) -> Result>, KitError>; +} + +// A borrowed source is a source: `Mappings::text("story", &doc.story)`. +impl TreeSource for &T { + fn key(&self) -> Option { + (**self).key() + } + fn roots(&self) -> Vec { + (**self).roots() + } + fn children(&self, id: ElemId) -> Vec { + (**self).children(id) + } + fn subtree_key(&self, id: ElemId) -> Option { + (**self).subtree_key(id) + } + fn write(&self, id: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError> { + (**self).write(id, props) + } +} + +impl TextSource for &T { + fn key(&self) -> Option { + (**self).key() + } + fn text(&self) -> &str { + (**self).text() + } + fn runs(&self) -> Cow<'_, [(Range, Attrs)]> { + (**self).runs() + } + fn paragraphs(&self) -> Cow<'_, [Attrs]> { + (**self).paragraphs() + } +} + +impl TableSource for &T { + fn key(&self) -> Option { + (**self).key() + } + fn rows(&self) -> Vec { + (**self).rows() + } + fn columns(&self) -> Vec { + (**self).columns() + } + fn cell(&self, row: ElemId, column: ElemId) -> Option { + (**self).cell(row, column) + } + fn cells(&self) -> Vec<(ElemId, ElemId, Value)> { + (**self).cells() + } + fn changed_cells(&self) -> Option> { + (**self).changed_cells() + } +} + +impl TimelineSource for &T { + fn key(&self) -> Option { + (**self).key() + } + fn tracks(&self) -> Vec { + (**self).tracks() + } + fn track_key(&self, track: ElemId) -> Option { + (**self).track_key(track) + } + fn write_track(&self, track: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError> { + (**self).write_track(track, props) + } + fn items(&self, track: ElemId) -> Vec { + (**self).items(track) + } + fn span(&self, item: ElemId) -> Option { + (**self).span(item) + } + fn item_key(&self, item: ElemId) -> Option { + (**self).item_key(item) + } + fn write_item(&self, item: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError> { + (**self).write_item(item, props) + } + fn keyframes(&self, item: ElemId) -> Vec<(ElemId, i64, Value)> { + (**self).keyframes(item) + } +} + +impl OpaqueSource for &T { + fn key(&self) -> Option { + (**self).key() + } + fn encode(&self) -> Result>, KitError> { + (**self).encode() + } +} + +/// One shared part of the model. +pub(crate) enum Part<'a> { + Tree(Box), + Text(Box), + Table(Box), + Timeline(Box), + Opaque(Box), +} + +impl Part<'_> { + pub(crate) fn kind(&self) -> Kind { + match self { + Part::Tree(_) => Kind::Tree, + Part::Text(_) => Kind::Text, + Part::Table(_) => Kind::Table, + Part::Timeline(_) => Kind::Timeline, + Part::Opaque(_) => Kind::Opaque, + } + } +} + +/// A part's shape. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + Tree, + Text, + Table, + Timeline, + Opaque, +} + +/// The parts of an app's model the live document holds, each by key. Everything else in the model (view state, +/// selection, caches, derived fields) stays the app's own. +#[derive(Default)] +pub struct Mappings<'a> { + pub(crate) parts: Vec<(&'static str, Part<'a>)>, +} + +impl<'a> Mappings<'a> { + pub fn new() -> Self { + Self { parts: Vec::new() } + } + + pub fn tree(mut self, key: &'static str, source: impl TreeSource + 'a) -> Self { + self.parts.push((key, Part::Tree(Box::new(source)))); + self + } + + pub fn text(mut self, key: &'static str, source: impl TextSource + 'a) -> Self { + self.parts.push((key, Part::Text(Box::new(source)))); + self + } + + pub fn table(mut self, key: &'static str, source: impl TableSource + 'a) -> Self { + self.parts.push((key, Part::Table(Box::new(source)))); + self + } + + pub fn timeline(mut self, key: &'static str, source: impl TimelineSource + 'a) -> Self { + self.parts.push((key, Part::Timeline(Box::new(source)))); + self + } + + pub fn opaque(mut self, key: &'static str, source: impl OpaqueSource + 'a) -> Self { + self.parts.push((key, Part::Opaque(Box::new(source)))); + self + } + + /// The parts' keys and shapes, checked (valid, each once). + pub(crate) fn layout(&self) -> Result, KitError> { + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for (key, part) in &self.parts { + check_part_key(key)?; + if !seen.insert(*key) { + return Err(KitError::Invalid(format!("the part {key:?} is mapped twice"))); + } + out.push((*key, part.kind())); + } + Ok(out) + } +} + +/// What a remote change (or an undo) touched, by part. +#[derive(Debug, Default, Clone)] +pub struct Touched { + /// Everything (the document was just opened): every part reads whole. + pub(crate) all: bool, + pub(crate) trees: BTreeMap, + pub(crate) texts: BTreeSet, + /// Loro's text deltas, by text, in the order they came (to patch the model's text rather than read it whole). + pub(crate) text_deltas: BTreeMap>>, + pub(crate) tables: BTreeMap, + pub(crate) timelines: BTreeMap, + pub(crate) opaques: BTreeSet, + /// The schema (or the save marks) changed. + pub(crate) kit: bool, +} + +impl Touched { + pub(crate) fn everything() -> Self { + Self { all: true, ..Self::default() } + } + + /// Whether any part was touched. + pub fn any(&self) -> bool { + self.all + || !self.trees.is_empty() + || !self.texts.is_empty() + || !self.tables.is_empty() + || !self.timelines.is_empty() + || !self.opaques.is_empty() + } +} + +/// A remote change, as the app's [`crate::LiveDocument::apply_remote`] gets it: per part, whether it was touched and +/// what of it, with the part as it is now. Parts not touched return `None`: keep them as they are. +pub struct RemotePatch<'a> { + pub(crate) doc: &'a LoroDoc, + pub(crate) touched: &'a Touched, + pub(crate) trees: &'a HashMap, + /// The model's parts before the change (texts are patched from them). + pub(crate) old: Option<&'a Mappings<'a>>, +} + +impl RemotePatch<'_> { + /// Whether this is the whole document (the first build, at open): every part is `Some`. + pub fn is_whole(&self) -> bool { + self.touched.all + } + + /// The tree `key`, if the change touched it. + pub fn tree(&self, key: &str) -> Option> { + let touch = if self.touched.all { TreeTouch::whole() } else { self.touched.trees.get(key)?.clone() }; + let index = self.trees.get(key)?; + Some(TreeView::new(self.doc, key, index, touch)) + } + + /// The rich text `key` as it is now, if the change touched it. + pub fn text(&self, key: &str) -> Option { + if !self.touched.all && !self.touched.texts.contains(key) { + return None; + } + // From the old text and Loro's deltas when it can (only what changed), else read whole. + if !self.touched.all + && let Some(Part::Text(old)) = self.old.and_then(|m| m.parts.iter().find(|(k, _)| *k == key)).map(|(_, p)| p) + { + let deltas = self.touched.text_deltas.get(key).map_or(&[][..], Vec::as_slice); + if let Some(patched) = crate::text::patch(self.doc, key, old.as_ref(), deltas) { + return Some(patched); + } + } + Some(crate::text::read(self.doc, key)) + } + + /// The table `key`, if the change touched it. + pub fn table(&self, key: &str) -> Option> { + let touch = if self.touched.all { TableTouch::whole() } else { self.touched.tables.get(key)?.clone() }; + Some(TableView::new(self.doc, key, touch)) + } + + /// The timeline `key`, if the change touched it. + pub fn timeline(&self, key: &str) -> Option> { + let touch = if self.touched.all { TimelineTouch::whole() } else { self.touched.timelines.get(key)?.clone() }; + Some(TimelineView::new(self.doc, key, touch)) + } + + /// The opaque value `key` as it is now (`Some(None)`: it was removed), if the change touched it. + pub fn opaque(&self, key: &str) -> Option>> { + if !self.touched.all && !self.touched.opaques.contains(key) { + return None; + } + Some(crate::opaque::read(self.doc, key)) + } +} + +/// Reads back an element's properties: for tree nodes, tracks and items ([`TreeView::props`] and the others). +pub(crate) fn props_of(map: loro::LoroMap) -> PropsRead { + PropsRead { map } +} diff --git a/crates/kit/src/opaque.rs b/crates/kit/src/opaque.rs new file mode 100644 index 0000000..7e5f07a --- /dev/null +++ b/crates/kit/src/opaque.rs @@ -0,0 +1,58 @@ +//! Opaque values: one key of the root map `$opaque` per part, the value's bytes, last writer wins. + +use loro::{LoroDoc, LoroValue, ValueOrContainer}; + +use crate::mapping::OpaqueSource; +use crate::writer::Writer; +use crate::{KitError, MAX_VALUE_BYTES}; + +/// The root map of every opaque part. +pub(crate) const ROOT: &str = "$opaque"; + +/// Writes `new`'s value unless `old` has the same key, or the stored bytes are the same. +pub(crate) fn write(w: &mut Writer<'_>, key: &str, old: Option<&dyn OpaqueSource>, new: &dyn OpaqueSource) -> Result<(), KitError> { + if let (Some(old), Some(k)) = (old, new.key()) + && old.key() == Some(k) + { + return Ok(()); + } + let map = w.doc.get_map(ROOT); + match new.encode()? { + Some(bytes) => { + if bytes.len() > MAX_VALUE_BYTES { + return Err(KitError::TooLarge { what: "an opaque value", len: bytes.len(), max: MAX_VALUE_BYTES }); + } + let weight = bytes.len(); + w.set_raw(&map, key, LoroValue::from(bytes), weight)?; + } + None => { + w.delete(&map, key)?; + } + } + Ok(()) +} + +/// The value as stored (`None`: there is none, or it isn't bytes). +pub(crate) fn read(doc: &LoroDoc, key: &str) -> Option> { + match doc.get_map(ROOT).get(key)? { + ValueOrContainer::Value(LoroValue::Binary(bytes)) => Some(bytes.to_vec()), + _ => None, + } +} + +/// An [`OpaqueSource`] over a value with an [`crate::OpaqueCodec`]; `key` is the value's address when the app shares +/// it by `Arc` (unchanged values are then skipped without encoding them). +pub struct Opaque<'a, T: crate::OpaqueCodec> { + pub value: Option<&'a T>, + pub key: Option, +} + +impl OpaqueSource for Opaque<'_, T> { + fn key(&self) -> Option { + self.key + } + + fn encode(&self) -> Result>, KitError> { + Ok(self.value.map(crate::OpaqueCodec::encode)) + } +} diff --git a/crates/kit/src/props.rs b/crates/kit/src/props.rs new file mode 100644 index 0000000..0463bb5 --- /dev/null +++ b/crates/kit/src/props.rs @@ -0,0 +1,109 @@ +//! A model element's properties as a Loro map: what the app's codec writes ([`Props`], only what changed) and reads +//! back ([`PropsRead`]). Keys starting with `$` are the kit's own. + +use loro::{LoroMap, ValueOrContainer}; + +use crate::writer::Writer; +use crate::{KitError, Value}; + +/// Checks an app's property key: 1 to 128 bytes, not starting with `$` (the kit's) or `¶` (paragraph marks). +pub(crate) fn check_key(key: &str) -> Result<(), KitError> { + if key.is_empty() || key.len() > 128 || key.starts_with('$') || key.starts_with('¶') { + return Err(KitError::Invalid(format!("{key:?} isn't a property key (1-128 bytes, not starting with '$' or '¶')"))); + } + Ok(()) +} + +/// The write side of one element's properties: set what the element has; what it no longer has goes when the writer +/// is done. A value equal to the stored one writes nothing. +pub struct Props<'a, 'd> { + pub(crate) map: LoroMap, + pub(crate) w: &'a mut Writer<'d>, + pub(crate) written: Vec, +} + +impl<'a, 'd> Props<'a, 'd> { + pub(crate) fn new(map: LoroMap, w: &'a mut Writer<'d>) -> Self { + Self { map, w, written: Vec::new() } + } + + /// Sets the property `key`. + pub fn set(&mut self, key: &str, value: impl Into) -> Result<(), KitError> { + check_key(key)?; + let value = value.into(); + self.w.set(&self.map, key, &value)?; + self.written.push(key.to_owned()); + Ok(()) + } + + /// Deletes the properties not set (the kit's own keys stay). + pub(crate) fn finish(self) -> Result<(), KitError> { + let mut stale = Vec::new(); + self.map.for_each(|key, _| { + if !key.starts_with('$') && !self.written.iter().any(|w| w == key) { + stale.push(key.to_owned()); + } + }); + for key in stale { + self.w.delete(&self.map, &key)?; + } + Ok(()) + } +} + +/// One element's properties as stored. +#[derive(Debug, Clone)] +pub struct PropsRead { + pub(crate) map: LoroMap, +} + +impl PropsRead { + /// The property `key`, if it holds a value an app writes. + pub fn get(&self, key: &str) -> Option { + match self.map.get(key)? { + ValueOrContainer::Value(v) => Value::from_loro(&v), + ValueOrContainer::Container(_) => None, + } + } + + pub fn str(&self, key: &str) -> Option { + self.get(key).and_then(|v| v.as_str().map(str::to_owned)) + } + + pub fn i64(&self, key: &str) -> Option { + self.get(key).and_then(|v| v.as_i64()) + } + + pub fn f64(&self, key: &str) -> Option { + self.get(key).and_then(|v| v.as_f64()) + } + + pub fn bool(&self, key: &str) -> Option { + self.get(key).and_then(|v| v.as_bool()) + } + + pub fn bytes(&self, key: &str) -> Option> { + self.get(key).and_then(|v| v.as_bytes().map(<[u8]>::to_vec)) + } + + pub fn chunk(&self, key: &str) -> Option { + self.get(key).and_then(|v| v.as_chunk()) + } + + /// Every app property, sorted by key. + pub fn all(&self) -> Vec<(String, Value)> { + let mut out = Vec::new(); + self.map.for_each(|key, value| { + if key.starts_with('$') { + return; + } + if let ValueOrContainer::Value(v) = value + && let Some(v) = Value::from_loro(&v) + { + out.push((key.to_owned(), v)); + } + }); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } +} diff --git a/crates/kit/src/session.rs b/crates/kit/src/session.rs new file mode 100644 index 0000000..3a55e98 --- /dev/null +++ b/crates/kit/src/session.rs @@ -0,0 +1,1472 @@ +//! The session: one open live document, tying the app's model to a Loro document and the agent. +//! +//! - [`Session::open`] opens a file of the synced folder by path. A file that has never been live is imported from the +//! app's model (the model it read from the file); otherwise the model is built from the live document. +//! - [`Session::local_change`] (the hook, at the end of each top-level command) writes the difference between the +//! model before and after as Loro commits with this machine's origin, one undo step, and sends them. +//! - [`Session::poll`] (once per frame) takes what came from the agent and applies remote changes: the app rebuilds +//! the touched parts ([`LiveDocument::apply_remote`]), then the kit runs [`LiveDocument::normalise`] (writing back +//! what it repaired) and [`LiveDocument::refresh_derived`], and hands the new model over. It never blocks, and +//! never applies anything while the app holds remote changes back ([`Session::hold_remote`]: a drag, an IME +//! composition). +//! - Undo and redo go through Loro's undo manager ([`Session::undo`]): only this machine's changes are taken back, and +//! the result comes back like a remote change. [`Session::group`] makes several changes one step (typing). +//! - When the agent goes, the model stays editable: changes queue in memory (at most [`MAX_QUEUE_BYTES`]; past that, +//! [`KitError::Stalled`]) and go out when the session is back, which it tries by itself with a growing pause, off +//! the UI thread. + +use std::collections::{HashMap, VecDeque}; +use std::sync::mpsc::{Receiver, TryRecvError}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use craft_cloud_app::Client; +use craft_cloud_app::live::{Document, Documents, Event, NotLiveReason, OpenOptions, Opened, Waker}; +use loro::event::{Diff, DiffEvent}; +use loro::{ContainerID, ExpandType, Index, LoroDoc, LoroValue, StyleConfig, StyleConfigMap, TreeID, UndoManager, ValueOrContainer}; + +use crate::mapping::{Kind, Mappings, Part, RemotePatch, Touched}; +use crate::table::{TableTouch, parse_cell_key}; +use crate::timeline::TimelineTouch; +use crate::tree::{TreeIndex, TreeTouch}; +use crate::writer::{KIT_ORIGIN, LOCAL_ORIGIN, Updates, Writer, export_since}; +use crate::{ElemId, KitError}; + +/// The most bytes of changes a session keeps in memory for the agent; at or past it, local changes are refused +/// ([`KitError::Stalled`]) until the agent takes them. +pub const MAX_QUEUE_BYTES: usize = 64 * 1024 * 1024; +/// How long the pause between reconnects grows to. +pub const MAX_BACKOFF: Duration = Duration::from_secs(8); +const FIRST_BACKOFF: Duration = Duration::from_millis(200); +/// Presence is sent again this often while set, and a peer's is dropped after this long without one. +pub const PRESENCE_EVERY: Duration = Duration::from_secs(3); +pub const PRESENCE_TTL: Duration = Duration::from_secs(10); +/// Undo steps kept. +const UNDO_STEPS: usize = 500; +/// The most text (inserted and deleted, in bytes) one step may change and still be undone while live. Loro's undo of a +/// run of text takes time growing with its square: about 35 ms at 64 KiB, 2.4 s at 512 KiB, 10 s at 1 MB. +pub const MAX_UNDO_TEXT_BYTES: usize = 64 * 1024; + +/// The root map of the kit's own keys: `schema` (`{app, version}`) and `import` (the hash, hex, of the file's bytes +/// the document was imported from). +const ROOT: &str = "root"; +const SCHEMA: &str = "schema"; +const IMPORT: &str = "import"; + +/// An app's model, mapped onto a live document. The app implements it once, over its own document type. +pub trait LiveDocument { + /// The app's document model (shared by `Arc`, cloned copy-on-write). + type Doc: Clone; + /// The app's lowercase name, as in recents (`deckcraft`). + const APP: &'static str; + /// The layout version the app's mappings write: a document with a newer one opens read-only. + const SCHEMA: u32; + /// The parts of the model the live document holds. + fn mappings(doc: &Self::Doc) -> Mappings<'_>; + /// The model after a remote change: `old` with the touched parts rebuilt from `patch` (no undo step). + fn apply_remote(old: &Arc, patch: &RemotePatch<'_>) -> Result, KitError>; + /// Repairs what a merge can break (duplicate names, dangling references, missing cells): the result must be a + /// model the app could have made itself. What it changes in shared parts is written back. + fn normalise(doc: &mut Self::Doc); + /// Recomputes what is derived from shared state and never synced (layout, caches). + fn refresh_derived(doc: &mut Self::Doc); +} + +/// How to reach the agent (again, after it went): the running agent's client, or a fake's in tests. +#[derive(Clone)] +pub struct Agent { + connect: Arc Result + Send + Sync>, +} + +impl std::fmt::Debug for Agent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Agent") + } +} + +impl Agent { + /// The agent running on this machine for this user ([`Client::connect`]). + pub fn local() -> Self { + Self::from_fn(Client::connect) + } + + /// An agent found by `connect` each time (a fake's client in tests). + pub fn from_fn(connect: impl Fn() -> Result + Send + Sync + 'static) -> Self { + Self { connect: Arc::new(connect) } + } + + pub fn client(&self) -> Result { + (self.connect)() + } +} + +/// Why a file opened as a plain file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NotLive { + /// Craft Cloud isn't running here (or refused): the message says why. + NoAgent(String), + /// The agent's reason (offline before the first live open, not synced yet, …). + Reason(NotLiveReason), +} + +/// What opening a file gives. +pub enum Open { + Live(Box>), + /// Open and save it as a plain file, with the app's own undo. + NotLive(NotLive), +} + +/// A document made by a newer app (or another one): it is shown, but nothing is sent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReadOnly { + pub app: String, + pub version: u32, +} + +/// How the session stands. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Status { + /// Connected to the agent, with the document open. + pub connected: bool, + /// The agent reaches the server (edits go out to the other machines as they are made). + pub online: bool, + /// Changes the agent has that the server hasn't acked. + pub waiting: u64, + /// Changes kept here that the agent hasn't acked (while disconnected: everything since). + pub queued: usize, + pub queued_bytes: usize, + /// The document needs a newer app: show "update ". + pub read_only: Option, + /// Too much waits for the agent: local changes are refused until it is back. + pub stalled: bool, + /// The file was deleted (the document still works). + pub deleted: bool, + /// Nothing more goes out: the agent stopped following the document until it restarts, or refused a change for + /// good, or the document was replaced (the error says which). The model stays editable: save it to keep the work. + pub stopped: bool, + /// `document_replaced`: when the session came back, its path was another document (renamed away while the agent + /// was down, another file in its place). It is never merged into it; save the model as a new file. + pub replaced: bool, + /// The file on disk wasn't saved by a Craft app from this document (edited outside): the app may bring it in as a + /// local change, or keep the live content. + pub changed_outside: bool, + /// The last error, for the app to show. + pub error: Option, +} + +/// The one-word state a document badge shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Badge { + Live, + Waiting(u64), + ReadOnly, + Offline, + /// Too much waits for the agent: new changes are refused until it is back. + Stalled, + /// Nothing more goes out (`Status::error` says why): save the document to keep the work. + Stopped, +} + +impl Status { + pub fn badge(&self) -> Badge { + if self.read_only.is_some() { + Badge::ReadOnly + } else if self.stopped { + Badge::Stopped + } else if self.stalled { + Badge::Stalled + } else if !self.connected || !self.online { + Badge::Offline + } else if self.waiting > 0 || self.queued > 0 { + Badge::Waiting(self.waiting.max(self.queued as u64)) + } else { + Badge::Live + } + } +} + +/// What a poll did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Polled { + /// The model changed (a remote change, a reconnect): take [`Session::model`] and redraw. + pub changed: bool, + pub status: Status, +} + +/// What undo means for the document right now. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UndoKind { + /// Through the session ([`Session::undo`]): only this machine's changes. + Live, + /// The document is read-only: nothing to undo here. + None, +} + +/// What a save gives the agent: the content's hash and the live document's frontiers (its heads, `(peer, counter)` +/// operation IDs, sorted) it was made from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SaveRecord { + pub hash: [u8; 32], + pub frontiers: Vec<(u64, u32)>, +} + +/// What a save is made from ([`Session::save_point`]): the model to write, and the live document's frontiers it +/// matches. +pub struct SavePoint { + model: Arc, + frontiers: Vec<(u64, u32)>, + /// This session's counter then: the record goes to the agent after every change up to it. + end: i32, +} + +impl SavePoint { + /// The model to write to the file. + pub fn model(&self) -> &Arc { + &self.model + } + + /// The live document's frontiers (`(peer, counter)` heads, sorted) the model matches. + pub fn frontiers(&self) -> &[(u64, u32)] { + &self.frontiers + } +} + +impl Clone for SavePoint { + fn clone(&self) -> Self { + Self { model: self.model.clone(), frontiers: self.frontiers.clone(), end: self.end } + } +} + +impl std::fmt::Debug for SavePoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SavePoint").field("frontiers", &self.frontiers).finish_non_exhaustive() + } +} + +/// How many save records a session keeps to tell the agent (again) when it is back. +const KEPT_RECORDS: usize = 16; +/// The name of the undo step [`Session::replace_with_file`] makes. +pub const REPLACE_WITH_FILE: &str = "Replace with the file's content"; +/// At most one save that `file_behind` asks for per this long (a backstop against a loop between machines). +pub const CATCH_UP_EVERY: Duration = Duration::from_secs(5); + +/// A save record kept for the agent: sent after every change it covers. +#[derive(Debug, Clone)] +struct Kept { + record: SaveRecord, + /// This session's counter at the save: every change up to it goes first. + end: i32, + /// Sent on the current connection. + sent: bool, + /// Announced on the current connection (`saving`: at once, before the write and before the changes go). + announced: bool, +} + +/// Another machine's (or app's) presence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Peer { + pub bytes: Vec, + pub seen: Instant, +} + +/// One change waiting for the agent. +struct Out { + bytes: Vec, + /// This peer's counter after it (what the agent has once it holds this). + end: i32, + /// Its number on the current connection, once sent. + seq: Option, +} + +/// A raw touch from Loro's events, sorted out after an import. +#[derive(Debug, Clone, Default)] +struct Raw { + root: String, + nodes: Vec, + keys: Vec, + tree: bool, + /// A text's delta (the text root's own). + text: Option>, +} + +#[derive(Default)] +struct Collector { + on: bool, + raw: Vec, +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn root_name(id: &ContainerID) -> Option { + match id { + ContainerID::Root { name, .. } => Some(name.to_string()), + ContainerID::Normal { .. } => None, + } +} + +/// What the mappings need between changes: the trees' indexes and the text styles seen. +#[derive(Default)] +pub(crate) struct Ctx { + pub(crate) trees: HashMap, + pub(crate) styles: StyleConfigMap, +} + +/// The connection to the agent: the document and its documents connection (kept alive). The document goes first, so +/// the agent hears it closed before the connection ends. +struct Link { + doc: Document, + _docs: Documents, +} + +/// Why a reopen didn't attach. +enum Reopen { + /// Not now (the agent isn't there, the file isn't live now): tried again after a pause. + Retry(String), + /// The path is another document now (renamed away while the agent was down, another file in its place): never + /// merged into or imported over. + Replaced(String), +} + +type Reopened = Result<(Documents, Document), Reopen>; + +/// One open live document. +pub struct Session { + doc: LoroDoc, + undo: UndoManager, + model: Arc, + ctx: Ctx, + layout: Vec<(&'static str, Kind)>, + collector: Arc>, + _events: loro::Subscription, + agent: Agent, + path: String, + link: Option, + reopening: Option>, + next_try: Instant, + backoff: Duration, + outbox: VecDeque, + outbox_bytes: usize, + incoming: Vec>, + held: bool, + grouping: bool, + status: Status, + peers: HashMap, + presence: Option>, + presence_sent: Option, + waker: Option, + /// The agent refused a change for good (it doesn't decode, it is over the cap): nothing more is sent, and the + /// session says it is stopped; the model stays editable, and saving to the file (without a record) keeps the work. + refused: Option, + /// The live document this session is of (a reopen must find the same one at its path). + doc_id: Option, + /// Undos (`true`) and redos asked for while remote changes were held: run once the gesture ends. + deferred: Vec, + /// The text bytes of each undo step, by the counter it starts at (the undo manager's items carry that start). + step_sizes: HashMap, + /// The start of the group of changes under way. + group_key: Option, + /// A rebuild failed: the live document is ahead of the model, which takes no local change until a whole rebuild + /// succeeds. + model_stale: bool, + /// The agent refused a change for now (its disk): the queue goes again, in order, from this moment. + resend_at: Instant, + resend_backoff: Duration, + /// The last saves' records, told the agent again on every reconnect (it may have missed them; it keeps one once), + /// each after the changes it covers. + records: VecDeque, + /// The record of the document's file as it is on disk (its frontiers): the agent's word, or this session's last + /// save since. + file_version: Option>, + /// When the last save that `file_behind` asked for was recorded ([`CATCH_UP_EVERY`]). + caught_up_at: Option, + /// A local change (or undo) was made since the last save. + changed_since_save: bool, + /// The names of undo steps that have one ([`Session::replace_with_file`]), by the counter they start at. + step_names: HashMap, +} + +impl std::fmt::Debug for Session { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Session").field("path", &self.path).field("status", &self.status).finish_non_exhaustive() + } +} + +/// A fresh Loro document with the kit's text styles. +fn new_doc() -> LoroDoc { + let doc = LoroDoc::new(); + doc.config_default_text_style(Some(StyleConfig { expand: ExpandType::After })); + doc +} + +/// The undo manager of a session's document: this machine's changes only, each step carrying where it starts, which +/// names its size (`step_sizes`). +fn new_undo(doc: &LoroDoc) -> UndoManager { + let mut undo = UndoManager::new(doc); + undo.add_exclude_origin_prefix(KIT_ORIGIN); + undo.set_max_undo_steps(UNDO_STEPS); + undo.set_on_push(Some(Box::new(|_, span, _| { + let mut meta = loro::UndoItemMeta::new(); + meta.set_value(LoroValue::I64(i64::from(span.start))); + meta + }))); + undo +} + +/// Writes the difference of two models (everything, when `old` is `None`) through `w`. +pub(crate) fn write_diff(w: &mut Writer<'_>, old: Option<&D::Doc>, new: &D::Doc, ctx: &mut Ctx) -> Result<(), KitError> { + let new_maps = D::mappings(new); + new_maps.layout()?; + let old_maps = old.map(D::mappings); + for (key, part) in &new_maps.parts { + let old_part = old_maps.as_ref().and_then(|m| m.parts.iter().find(|(k, _)| k == key)).map(|(_, p)| p); + match part { + Part::Tree(n) => { + let o = match old_part { + Some(Part::Tree(o)) => Some(o.as_ref()), + _ => None, + }; + let index = ctx.trees.entry((*key).to_owned()).or_default(); + crate::tree::write(w, key, o, n.as_ref(), index)?; + } + Part::Text(n) => { + let o = match old_part { + Some(Part::Text(o)) => Some(o.as_ref()), + _ => None, + }; + crate::text::write(w, key, o, n.as_ref(), &mut ctx.styles)?; + } + Part::Table(n) => { + let o = match old_part { + Some(Part::Table(o)) => Some(o.as_ref()), + _ => None, + }; + crate::table::write(w, key, o, n.as_ref())?; + } + Part::Timeline(n) => { + let o = match old_part { + Some(Part::Timeline(o)) => Some(o.as_ref()), + _ => None, + }; + crate::timeline::write(w, key, o, n.as_ref())?; + } + Part::Opaque(n) => { + let o = match old_part { + Some(Part::Opaque(o)) => Some(o.as_ref()), + _ => None, + }; + crate::opaque::write(w, key, o, n.as_ref())?; + } + } + } + Ok(()) +} + +/// The document's schema: `(app, version)`. +fn schema(doc: &LoroDoc) -> Option<(String, u32)> { + let ValueOrContainer::Value(LoroValue::Map(map)) = doc.get_map(ROOT).get(SCHEMA)? else { return None }; + let app = match map.get("app")? { + LoroValue::String(s) => s.to_string(), + _ => return None, + }; + let version = match map.get("version")? { + LoroValue::I64(v) => u32::try_from(*v).ok()?, + _ => return None, + }; + Some((app, version)) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// The hash (hex) of the file bytes the document was imported from, if the import said. +fn import_hash(doc: &LoroDoc) -> Option { + match doc.get_map(ROOT).get(IMPORT)? { + ValueOrContainer::Value(LoroValue::String(s)) => Some(s.to_string()), + _ => None, + } +} + +impl Session { + /// Opens the file at `path` (relative to the synced folder) live. `file` is the app's model of the file as it read + /// it, and `file_bytes` the bytes it read (to tell whether it was changed outside Craft). Blocking (the agent may + /// wait for the file's first sync, up to 30 s): call it where the app opens files. + pub fn open(agent: &Agent, path: &str, file: Arc, file_bytes: Option<&[u8]>) -> Result, KitError> { + let not_running = |e: craft_cloud_app::Error| Ok(Open::NotLive(NotLive::NoAgent(e.to_string()))); + let client = match agent.client() { + Ok(c) => c, + Err(e) => return not_running(e), + }; + let docs = match client.documents() { + Ok(d) => d, + Err(e) => return not_running(e), + }; + let file_hash = file_bytes.map(|b| *blake3::hash(b).as_bytes()); + let opened = match docs.open_with(path, D::APP, &OpenOptions { file_hash }) { + Ok(o) => o, + Err(craft_cloud_app::Error::Refused { code, message }) => return Err(KitError::Agent(format!("{code}: {message}"))), + Err(e) => return not_running(e), + }; + let (document, snapshot) = match opened { + Opened::NotLive(reason) => return Ok(Open::NotLive(NotLive::Reason(reason))), + Opened::Live(document) => (document, None), + Opened::NeedsImport(importer) => { + let snapshot = Self::import_snapshot(&file, file_hash.as_ref())?; + match importer.import(&snapshot) { + Ok(document) => (document, Some(snapshot)), + Err(craft_cloud_app::Error::Refused { code, .. }) if code == "not_live" => { + return Ok(Open::NotLive(NotLive::Reason(NotLiveReason::Offline))); + } + Err(e) => return Err(e.into()), + } + } + }; + let ours = snapshot.as_deref().is_some_and(|s| s == document.state()) && document.state_updates().is_empty(); + let mut session = Self::start(agent.clone(), path, file, (document.state(), document.state_updates()), ours)?; + // Decided once, here: the agent knows every Craft save of the file (its records, the server's versions'); the + // bytes the document was imported from are no edit either. + session.status.changed_outside = + snapshot.is_none() && document.changed_outside() == Some(true) && file_hash.is_some_and(|h| import_hash(&session.doc) != Some(hex(&h))); + session.attach(Link { _docs: docs, doc: document }); + Ok(Open::Live(Box::new(session))) + } + + /// The snapshot that makes `file` a live document: its schema, the hash of the bytes it was read from, its parts. + fn import_snapshot(file: &D::Doc, hash: Option<&[u8; 32]>) -> Result, KitError> { + let doc = new_doc(); + let mut schema = HashMap::new(); + schema.insert("app".to_owned(), LoroValue::from(D::APP)); + schema.insert("version".to_owned(), LoroValue::from(i64::from(D::SCHEMA))); + doc.get_map(ROOT).insert(SCHEMA, LoroValue::from(schema))?; + if let Some(hash) = hash { + doc.get_map(ROOT).insert(IMPORT, LoroValue::from(hex(hash)))?; + } + let mut ctx = Ctx::default(); + let mut w = Writer::new(&doc, KIT_ORIGIN); + write_diff::(&mut w, None, file, &mut ctx)?; + w.finish()?; + Ok(doc.export(loro::ExportMode::Snapshot)?) + } + + /// A session over the document `state` (a snapshot), for `file` (kept as the model when `ours`: the state is the + /// import just made from it). + fn start(agent: Agent, path: &str, file: Arc, state: (&[u8], &[Vec]), ours: bool) -> Result { + let layout = D::mappings(&file).layout()?; + // A fresh random Loro peer of its own: never one another session may have written with. + let doc = new_doc(); + doc.import(state.0)?; + // The updates that are part of the state. + let mut unread = None; + for update in state.1 { + if let Err(e) = doc.import(update) { + unread = Some(format!("a change from another machine doesn't read: {e}")); + } + } + let collector = Arc::new(Mutex::new(Collector::default())); + let c = collector.clone(); + let events = doc.subscribe_root(Arc::new(move |event: DiffEvent<'_>| collect(&c, event))); + let undo = new_undo(&doc); + let mut session = Self { + doc, + undo, + model: file, + ctx: Ctx::default(), + layout, + collector, + _events: events, + agent, + path: path.to_owned(), + link: None, + reopening: None, + next_try: Instant::now(), + backoff: FIRST_BACKOFF, + outbox: VecDeque::new(), + outbox_bytes: 0, + incoming: Vec::new(), + held: false, + grouping: false, + status: Status::default(), + peers: HashMap::new(), + presence: None, + presence_sent: None, + waker: None, + refused: None, + doc_id: None, + deferred: Vec::new(), + step_sizes: HashMap::new(), + group_key: None, + model_stale: false, + resend_at: Instant::now(), + resend_backoff: FIRST_BACKOFF, + records: VecDeque::new(), + file_version: None, + caught_up_at: None, + changed_since_save: false, + step_names: HashMap::new(), + }; + session.reindex_trees(); + session.check_schema(); + session.status.error = unread; + if !ours { + session.rebuild(Touched::everything())?; + } + Ok(session) + } + + fn reindex_trees(&mut self) { + for (key, kind) in &self.layout { + if *kind == Kind::Tree { + self.ctx.trees.insert((*key).to_owned(), TreeIndex::build(&self.doc.get_tree(*key))); + } + } + } + + fn check_schema(&mut self) { + self.status.read_only = match schema(&self.doc) { + Some((app, version)) if app == D::APP && version <= D::SCHEMA => None, + Some((app, version)) => Some(ReadOnly { app, version }), + None => Some(ReadOnly { app: String::new(), version: 0 }), + }; + } + + fn attach(&mut self, link: Link) { + let state = link.doc.opened_state(); + self.doc_id.get_or_insert(link.doc.id()); + self.status.connected = true; + self.status.online = state.online; + self.status.waiting = state.waiting; + if let Some(waker) = &self.waker { + link.doc.set_waker(waker.clone()); + } + // What the agent's copy has of this peer's changes: the rest goes again. (The peer is this session's alone, + // random and never handed out, so whatever the agent has under it is this session's.) + let ours = self.doc.peer_id(); + let held = std::iter::once(link.doc.state()) + .chain(link.doc.state_updates().iter().map(Vec::as_slice)) + .filter_map(|bytes| loro::LoroDoc::decode_import_blob_meta(bytes, false).ok()?.partial_end_vv.get(&ours).copied()) + .max() + .unwrap_or(0); + self.outbox.retain(|o| o.end > held); + for out in &mut self.outbox { + out.seq = None; + } + self.outbox_bytes = self.outbox.iter().map(|o| o.bytes.len()).sum(); + // Saves made while the agent was away, and the last ones again (the agent keeps a record once): each after the + // changes it covers (announced at once). + for kept in &mut self.records { + kept.sent = false; + kept.announced = false; + } + self.link = Some(link); + self.backoff = FIRST_BACKOFF; + self.send_outbox(); + } + + /// The model as it is now (after the last local change or applied remote one). + pub fn model(&self) -> &Arc { + &self.model + } + + pub fn status(&self) -> &Status { + &self.status + } + + /// The live document's path, as opened. + pub fn path(&self) -> &str { + &self.path + } + + /// What undo means now. A document that isn't live has no session: the app keeps its own undo. + pub fn undo_kind(&self) -> UndoKind { + if self.status.read_only.is_some() { UndoKind::None } else { UndoKind::Live } + } + + /// Calls `waker` (from another thread) whenever something comes from the agent: an egui app passes its + /// `request_repaint`, and polls on the next frame. + pub fn set_waker(&mut self, waker: Waker) { + if let Some(link) = &self.link { + link.doc.set_waker(waker.clone()); + } + self.waker = Some(waker); + } + + /// While on, remote changes are kept and applied only once it is off again (at the next poll): for a gesture + /// (a drag, an IME composition) whose `old` model must stay the session's until its command ends. + pub fn hold_remote(&mut self, on: bool) { + self.held = on; + } + + /// The hook: `old` is the model before the command (the session's), `new` after. Writes the difference as one undo + /// step and sends it. Nothing is written for a read-only or stalled document, or from an `old` that isn't the + /// session's model ([`KitError::Stale`]: take [`Session::model`] and make the change again). + pub fn local_change(&mut self, old: &Arc, new: &Arc) -> Result<(), KitError> { + if let Some(ro) = &self.status.read_only { + return Err(KitError::ReadOnly { app: ro.app.clone(), version: ro.version }); + } + if self.model_stale { + return Err(KitError::ModelStale); + } + if !Arc::ptr_eq(old, &self.model) { + return Err(KitError::Stale); + } + if Arc::ptr_eq(old, new) { + return Ok(()); + } + if self.outbox_bytes >= MAX_QUEUE_BYTES { + self.status.stalled = true; + return Err(KitError::Stalled(self.outbox_bytes)); + } + let before = self.doc.oplog_vv(); + self.changed_since_save = true; + let outer = !self.grouping; + if outer { + let _ = self.undo.group_start(); + } + let mut w = Writer::new(&self.doc, LOCAL_ORIGIN); + let wrote = write_diff::(&mut w, Some(old.as_ref()), new.as_ref(), &mut self.ctx); + let text_bytes = w.text_bytes; + let finished = w.finish(); + if outer { + self.undo.group_end(); + } + // The step's size, by where it starts (the undo manager's item names that start; a group's first change). + let start = before.get(&self.doc.peer_id()).copied().unwrap_or(0); + let key = if self.grouping { *self.group_key.get_or_insert(start) } else { start }; + *self.step_sizes.entry(key).or_default() += text_bytes; + let updates = match finished { + Ok((updates, _)) => updates, + Err(_) => { + let mut rest = Vec::new(); + let _ = export_since(&self.doc, &before, &mut rest); + rest + } + }; + self.queue(updates); + match wrote { + Ok(()) => { + self.model = new.clone(); + Ok(()) + } + Err(e) => { + // What was written stays (and goes out); the model is built again from the document, so they agree. + if self.doc.oplog_vv() != before { + self.reindex_trees(); + let _ = self.rebuild(Touched::everything()); + } + self.status.error = Some(e.to_string()); + Err(e) + } + } + } + + /// Makes every change `f` makes one undo step (typing a word, a drag's previews). + pub fn group(&mut self, f: impl FnOnce(&mut Self) -> R) -> R { + let outer = self.begin_group(); + let r = f(self); + if outer { + self.end_group(); + } + r + } + + /// Starts a group of changes that undo as one step, until [`Session::end_group`]. False if one is open already. + pub fn begin_group(&mut self) -> bool { + if self.grouping { + return false; + } + let _ = self.undo.group_start(); + self.grouping = true; + self.group_key = None; + true + } + + pub fn end_group(&mut self) { + if self.grouping { + self.undo.group_end(); + self.grouping = false; + self.group_key = None; + } + } + + /// Whether there is a step to undo now (a step too big to undo while live isn't one: see + /// [`Session::undo_blocked`]). + pub fn can_undo(&self) -> bool { + self.status.read_only.is_none() && self.undo.can_undo() && self.too_big(true).is_none() + } + + pub fn can_redo(&self) -> bool { + self.status.read_only.is_none() && self.undo.can_redo() && self.too_big(false).is_none() + } + + /// When the next undo is of a step whose text changes are over [`MAX_UNDO_TEXT_BYTES`]: why it can't be undone + /// (for the undo menu). Such a step is a barrier: nothing before it can be undone either. + pub fn undo_blocked(&self) -> Option { + self.too_big(true).map(|bytes| KitError::UndoTooLarge { bytes }.to_string()) + } + + /// The text bytes of the top undo (or redo) step, when over the limit. + fn too_big(&self, undo: bool) -> Option { + let start = self.top_start(undo)?; + self.step_sizes.get(&start).copied().filter(|bytes| *bytes > MAX_UNDO_TEXT_BYTES) + } + + /// Where the step at the top of the undo (or redo) stack starts. + fn top_start(&self, undo: bool) -> Option { + match if undo { self.undo.top_undo_value() } else { self.undo.top_redo_value() }? { + LoroValue::I64(start) => i32::try_from(start).ok(), + _ => None, + } + } + + /// The name of the step undo (`true`) or redo would take back, if it has one (as "Replace with the file's + /// content", [`Session::replace_with_file`]): for the app's Undo and Redo labels. + pub fn step_name(&self, undo: bool) -> Option<&'static str> { + self.top_start(undo).and_then(|s| self.step_names.get(&s)).copied() + } + + /// The user chose to bring the file's content in (the app showed [`Status::changed_outside`] and offered it; the + /// kit never does by itself): `file` (the app's model of the file as it read it) replaces the live content, as one + /// ordinary local change, one undo step named [`REPLACE_WITH_FILE`], that goes to every machine. It replaces the + /// live edits made since the file was written; undo brings them back. + pub fn replace_with_file(&mut self, file: D::Doc) -> Result<(), KitError> { + let mut file = file; + D::refresh_derived(&mut file); + let old = self.model.clone(); + let start = self.doc.oplog_vv().get(&self.doc.peer_id()).copied().unwrap_or(0); + let key = if self.grouping { self.group_key.unwrap_or(start) } else { start }; + self.local_change(&old, &Arc::new(file))?; + // Named only if it made a step (the file may be the live content already: then the next edit starts there). + if self.doc.oplog_vv().get(&self.doc.peer_id()).copied().unwrap_or(0) > start { + self.step_names.insert(key, REPLACE_WITH_FILE); + } + self.status.changed_outside = false; + Ok(()) + } + + /// Takes back this machine's last change (remote changes since stay). The model changes as after a remote change: + /// take [`Session::model`]. False if there was nothing to undo. While the app holds remote changes back + /// ([`Session::hold_remote`]: a gesture), the undo waits: it answers false now and runs at the first poll after the + /// gesture ends (which then says the model changed). + pub fn undo(&mut self) -> Result { + self.undo_redo(true) + } + + pub fn redo(&mut self) -> Result { + self.undo_redo(false) + } + + fn undo_redo(&mut self, undo: bool) -> Result { + if let Some(ro) = &self.status.read_only { + return Err(KitError::ReadOnly { app: ro.app.clone(), version: ro.version }); + } + if self.outbox_bytes >= MAX_QUEUE_BYTES { + self.status.stalled = true; + return Err(KitError::Stalled(self.outbox_bytes)); + } + // Mid-gesture nothing changes the model under the app (its `old` must stay the session's): the undo waits for + // the gesture's end, and runs at the first poll after it. + if self.held { + self.deferred.push(undo); + return Ok(false); + } + self.end_group(); + // A step whose text is too big for Loro's undo while live (it takes time growing with the square of the text): + // refused, and it stays on the stack as a barrier. + if let Some(bytes) = self.too_big(undo) { + return Err(KitError::UndoTooLarge { bytes }); + } + let top = self.top_start(undo); + let size = top.and_then(|s| self.step_sizes.get(&s)).copied().unwrap_or(0); + let name = top.and_then(|s| self.step_names.get(&s)).copied(); + // What came meanwhile first, so the step applies to the newest document. + self.apply_incoming()?; + let before = self.doc.oplog_vv(); + lock(&self.collector).on = true; + let done = if undo { self.undo.undo() } else { self.undo.redo() }; + let raw = std::mem::take(&mut lock(&self.collector).raw); + lock(&self.collector).on = false; + let done = done?; + self.changed_since_save |= done; + // The step that takes this one back (on the other stack) starts where this undo's commit does: as big, and + // with its name. + let start = before.get(&self.doc.peer_id()).copied().unwrap_or(0); + self.step_sizes.insert(start, size); + if let Some(name) = name { + self.step_names.insert(start, name); + } + let mut updates = Vec::new(); + export_since(&self.doc, &before, &mut updates)?; + self.queue(updates); + if done { + let touched = self.classify(raw); + self.rebuild(touched)?; + } + Ok(done) + } + + /// Takes what came from the agent and applies remote changes (unless held). Never blocks; call it once per frame. + pub fn poll(&mut self) -> Polled { + self.take_events(); + self.reconnect(); + let mut changed = if self.held { + false + } else { + match self.apply_incoming() { + Ok(changed) => changed, + Err(e) => { + self.status.error = Some(e.to_string()); + false + } + } + }; + // A model left behind by a failed rebuild: a whole rebuild, tried again each poll. + if self.model_stale && !self.held { + self.reindex_trees(); + if self.rebuild(Touched::everything()).is_ok() { + self.status.error = None; + changed = true; + } + } + // Undos asked for during a gesture, now that it is over, in order. + if !self.held && !self.model_stale { + for undo in std::mem::take(&mut self.deferred) { + match self.undo_redo(undo) { + Ok(done) => changed |= done, + Err(e) => self.status.error = Some(e.to_string()), + } + } + } + self.keep_presence(); + // What a refusal for now held back goes again once its pause is over. + if self.outbox.iter().any(|o| o.seq.is_none()) { + self.send_outbox(); + } + self.status.queued = self.outbox.len(); + self.status.queued_bytes = self.outbox_bytes; + self.status.stalled = self.outbox_bytes >= MAX_QUEUE_BYTES; + self.status.stopped |= self.refused.is_some() || self.status.replaced; + Polled { changed, status: self.status.clone() } + } + + fn take_events(&mut self) { + let mut closed = None; + let mut events = Vec::new(); + if let Some(link) = &self.link { + while let Some(event) = link.doc.try_recv() { + events.push(event); + } + } + for event in events { + match event { + Event::Update(bytes) => self.incoming.push(bytes), + Event::Presence(bytes) => { + if let Some((peer, payload)) = bytes.split_first_chunk::<8>() { + let peer = u64::from_be_bytes(*peer); + if peer != self.doc.peer_id() { + self.peers.insert(peer, Peer { bytes: payload.to_vec(), seen: Instant::now() }); + } + } + } + Event::Acked(seq) => { + if let Some(at) = self.outbox.iter().position(|o| o.seq == Some(seq)) + && let Some(out) = self.outbox.remove(at) + { + self.outbox_bytes = self.outbox_bytes.saturating_sub(out.bytes.len()); + self.resend_backoff = FIRST_BACKOFF; + } + } + Event::Refused { seq, code, message } => { + // Never dropped: every later change depends on it. + let Some(at) = self.outbox.iter().position(|o| o.seq == Some(seq)) else { continue }; + if code == "invalid" || code == "too_large" { + // It can never be taken: nothing more goes out (what follows needs it), and the app hears + // it is stopped. The model stays editable; saving the file (without a record) keeps the work. + let why = format!("Craft Cloud can't take this document's changes ({code}: {message}); save it to the file"); + self.refused = Some(why.clone()); + self.status.error = Some(why); + } else { + // For now (the agent's disk): it and everything after it go again, in order, after a pause. + for out in self.outbox.iter_mut().skip(at) { + out.seq = None; + } + self.resend_at = Instant::now() + self.resend_backoff; + self.resend_backoff = (self.resend_backoff * 2).min(MAX_BACKOFF); + self.status.error = Some(format!("Craft Cloud couldn't keep a change yet ({code}: {message}); trying again")); + } + } + Event::Status { online, waiting, deleted, stopped } => { + self.status.online = online; + self.status.waiting = waiting; + self.status.deleted = deleted; + self.status.stopped = stopped || self.refused.is_some() || self.status.replaced; + } + Event::File(frontiers) => self.file_version = frontiers, + Event::Closed(why) => closed = Some(why), + } + } + if let Some(why) = closed { + self.drop_link(&why); + } + } + + fn drop_link(&mut self, why: &str) { + self.link = None; + self.status.connected = false; + self.status.online = false; + self.status.error = Some(format!("Craft Cloud: {why}")); + self.next_try = Instant::now() + self.backoff; + for out in &mut self.outbox { + out.seq = None; + } + for kept in &mut self.records { + kept.sent = false; + kept.announced = false; + } + } + + /// Tries to reach the agent again at the next poll, without waiting out the pause (the app learnt the agent is + /// back: Craft Cloud's status says so, say). + pub fn reconnect_now(&mut self) { + self.next_try = Instant::now(); + self.backoff = FIRST_BACKOFF; + } + + /// Opens the document again off the UI thread, with a growing pause between tries. + fn reconnect(&mut self) { + if self.link.is_some() || self.status.replaced { + return; + } + if let Some(rx) = &self.reopening { + match rx.try_recv() { + Ok(Ok((docs, doc))) => { + self.reopening = None; + self.incoming.push(doc.state().to_vec()); + self.incoming.extend(doc.state_updates().iter().cloned()); + self.status.error = None; + self.attach(Link { _docs: docs, doc }); + if let Some(waker) = &self.waker { + waker(); + } + } + Ok(Err(Reopen::Retry(why))) => { + self.reopening = None; + self.status.error = Some(why); + self.next_try = Instant::now() + self.backoff; + self.backoff = (self.backoff * 2).min(MAX_BACKOFF); + } + Ok(Err(Reopen::Replaced(why))) => { + // Stopped for good: the model stays, editable, for the app to save elsewhere. + self.reopening = None; + self.status.replaced = true; + self.status.stopped = true; + self.status.error = Some(format!("document_replaced: {why}; save this document as a new file")); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => self.reopening = None, + } + return; + } + if Instant::now() < self.next_try { + return; + } + let (tx, rx) = std::sync::mpsc::channel(); + let (agent, path) = (self.agent.clone(), self.path.clone()); + // The document as this machine has it, in case the agent lost it (it asks for an import then). + let snapshot = self.doc.export(loro::ExportMode::Snapshot).unwrap_or_default(); + let waker = self.waker.clone(); + let expected = self.doc_id; + let options = OpenOptions::default(); + let spawned = std::thread::Builder::new().name("craft-cloud-kit-reopen".into()).spawn(move || { + let result = (|| -> Reopened { + let retry = |e: craft_cloud_app::Error| Reopen::Retry(e.to_string()); + let client = agent.client().map_err(retry)?; + let docs = client.documents().map_err(retry)?; + match docs.open_with(&path, D::APP, &options).map_err(retry)? { + // The same document, or none known yet (it was never attached). + Opened::Live(doc) if expected.is_none_or(|id| id == doc.id()) => Ok((docs, doc)), + Opened::Live(_) => Err(Reopen::Replaced(format!("{path} is another document now"))), + // The path's file was never live: another file took its place. Nothing is imported over it. + Opened::NeedsImport(_) if expected.is_some() => { + Err(Reopen::Replaced(format!("{path} is another file now, or Craft Cloud lost its copy of the document"))) + } + Opened::NeedsImport(importer) => importer.import(&snapshot).map(|doc| (docs, doc)).map_err(retry), + Opened::NotLive(reason) => Err(Reopen::Retry(format!("the document can't be live now ({reason:?})"))), + } + })(); + let _ = tx.send(result); + if let Some(waker) = waker { + waker(); + } + }); + match spawned { + Ok(_) => self.reopening = Some(rx), + Err(e) => { + self.status.error = Some(e.to_string()); + self.next_try = Instant::now() + self.backoff; + } + } + } + + /// Imports what came and hands the app the touched parts. True if the model changed. + fn apply_incoming(&mut self) -> Result { + if self.incoming.is_empty() { + return Ok(false); + } + let batch = std::mem::take(&mut self.incoming); + // One by one: Loro's batch import of much text takes time growing with its square, and one at a time also + // means only an update Loro refuses is lost (the agent checked them all already). + lock(&self.collector).on = true; + for bytes in &batch { + if let Err(e) = self.doc.import(bytes) { + self.status.error = Some(format!("a change from another machine doesn't read: {e}")); + } + } + let raw = std::mem::take(&mut lock(&self.collector).raw); + lock(&self.collector).on = false; + let touched = self.classify(raw); + let was_read_only = self.status.read_only.clone(); + if touched.kit { + self.check_schema(); + } + if !touched.any() { + return Ok(was_read_only != self.status.read_only); + } + self.rebuild(touched)?; + Ok(true) + } + + /// Sorts Loro's raw touches out by part. + fn classify(&mut self, raw: Vec) -> Touched { + let mut touched = Touched::default(); + let mut reindex = Vec::new(); + for r in raw { + if r.root == ROOT { + touched.kit = true; + continue; + } + if r.root == crate::opaque::ROOT { + for k in r.keys { + if self.layout.iter().any(|(key, kind)| *key == k && *kind == Kind::Opaque) { + touched.opaques.insert(k); + } + } + continue; + } + let (part, suffix) = r.root.split_once('.').map_or((r.root.as_str(), ""), |(p, s)| (p, s)); + let Some((key, kind)) = self.layout.iter().find(|(k, _)| *k == part).copied() else { continue }; + let ids = || r.keys.iter().filter_map(|k| ElemId::from_hex(k)).collect::>(); + match kind { + Kind::Tree => { + let t = touched.trees.entry(key.to_owned()).or_insert_with(TreeTouch::default); + if r.tree { + t.structure = true; + reindex.push(key); + } + t.nodes.extend(r.nodes.iter().filter_map(|n| self.ctx.trees.get(key).and_then(|i| i.elem_of.get(n)).copied())); + } + Kind::Text => { + touched.texts.insert(key.to_owned()); + if let Some(delta) = r.text.clone() + && suffix.is_empty() + { + touched.text_deltas.entry(key.to_owned()).or_default().push(delta); + } + } + Kind::Table => { + let t = touched.tables.entry(key.to_owned()).or_insert_with(TableTouch::default); + match suffix { + "rows" => t.rows = true, + "cols" => t.columns = true, + _ => t.cells.extend(r.keys.iter().filter_map(|k| parse_cell_key(k))), + } + } + Kind::Timeline => { + let t = touched.timelines.entry(key.to_owned()).or_insert_with(TimelineTouch::default); + match suffix { + "tracks" => t.tracks = true, + "track" => t.track_props.extend(ids()), + "items" => t.items.extend(ids()), + _ => t.keyframes.extend(ids()), + } + } + Kind::Opaque => {} + } + } + for key in reindex { + let tree = self.doc.get_tree(key); + let index = TreeIndex::build(&tree); + if let Some(t) = touched.trees.get_mut(key) { + // Created nodes: their IDs are known only now. + for (id, _) in index.tid_of.iter().filter(|(id, _)| !self.ctx.trees.get(key).is_some_and(|old| old.tid_of.contains_key(id))) { + t.nodes.insert(*id); + } + } + self.ctx.trees.insert(key.to_owned(), index); + } + touched + } + + /// Rebuilds the model from the touched parts (see [`Session::rebuild_parts`]). If that fails, the document has + /// moved on and the model hasn't: the model is marked stale, local changes are refused ([`KitError::ModelStale`]) + /// so no diff is ever written from it, and each poll tries a whole rebuild until one succeeds. + fn rebuild(&mut self, touched: Touched) -> Result<(), KitError> { + let whole = touched.all; + match self.rebuild_parts(touched) { + Ok(()) => { + // Only a whole rebuild catches the model up: a part rebuilt since leaves the failed one behind. + if whole { + self.model_stale = false; + } + Ok(()) + } + Err(e) => { + self.model_stale = true; + self.status.error = Some(format!("the model couldn't take a change from the live document: {e}")); + Err(e) + } + } + } + + /// Rebuilds the model from the touched parts, normalises it (writing back what that repaired), refreshes what is + /// derived, and makes it the session's. + fn rebuild_parts(&mut self, touched: Touched) -> Result<(), KitError> { + let touched_all = touched.all; + let raw = { + // The old parts, for the texts to be patched rather than read whole. + let old_parts = D::mappings(&self.model); + let patch = RemotePatch { doc: &self.doc, touched: &touched, trees: &self.ctx.trees, old: (!touched.all).then_some(&old_parts) }; + D::apply_remote(&self.model, &patch)? + }; + let mut fixed = raw.clone(); + D::normalise(Arc::make_mut(&mut fixed)); + // A repair is written back only from a model that has the whole document: one still stale in a part not + // rebuilt would write that part's old state over the document (deleting what the other machine added). + if self.status.read_only.is_none() && (touched_all || !self.model_stale) { + let mut w = Writer::new(&self.doc, KIT_ORIGIN); + let wrote = write_diff::(&mut w, Some(raw.as_ref()), fixed.as_ref(), &mut self.ctx); + let (updates, _) = w.finish()?; + self.queue(updates); + wrote?; + } + D::refresh_derived(Arc::make_mut(&mut fixed)); + self.model = fixed; + Ok(()) + } + + /// Keeps changes for the agent and sends them if connected. + fn queue(&mut self, updates: Updates) { + if updates.is_empty() || self.status.read_only.is_some() { + return; + } + let end = self.doc.oplog_vv().get(&self.doc.peer_id()).copied().unwrap_or(0); + for bytes in updates { + self.outbox_bytes = self.outbox_bytes.saturating_add(bytes.len()); + self.outbox.push_back(Out { bytes, end, seq: None }); + } + self.send_outbox(); + } + + fn send_outbox(&mut self) { + let Some(link) = &self.link else { return }; + // Saves on their way, announced at once (before the changes they cover, which may wait): the agent's sync then + // waits for their records rather than take the files for edits outside Craft. + // (Never while refused for good: those records can't come, and the agent's grace ends their wait.) + for kept in self.records.iter_mut().filter(|k| !k.announced && !k.sent && self.refused.is_none()) { + if link.doc.saving(kept.record.hash).is_ok() { + kept.announced = true; + } + } + // Refused for good: nothing after it can be taken. Refused for now: not before the pause is over. + if self.refused.is_some() || Instant::now() < self.resend_at { + return; + } + let mut failed = None; + for out in self.outbox.iter_mut().filter(|o| o.seq.is_none()) { + match link.doc.send(&out.bytes) { + Ok(seq) => out.seq = Some(seq), + Err(e) => { + failed = Some(e.to_string()); + break; + } + } + } + // Save records, on the same stream after every change they cover (the agent keeps one once those are in). + if failed.is_none() { + let unsent_from = self.outbox.iter().filter(|o| o.seq.is_none()).map(|o| o.end).min(); + for kept in self.records.iter_mut().filter(|k| !k.sent && unsent_from.is_none_or(|from| k.end < from)) { + match link.doc.saved(kept.record.hash, &kept.record.frontiers) { + Ok(()) => kept.sent = true, + Err(e) => { + failed = Some(e.to_string()); + break; + } + } + } + } + if let Some(why) = failed { + self.drop_link(&why); + } + } + + /// Shows this app's cursor or selection to the document's other machines and apps (the bytes are the app's, at + /// most 4 KiB less 8); `None` stops showing it. Sent now and again every [`PRESENCE_EVERY`]. + pub fn set_presence(&mut self, bytes: Option>) -> Result<(), KitError> { + if let Some(b) = &bytes + && b.len() + 8 > craft_cloud_api::limits::MAX_PRESENCE_BYTES + { + return Err(KitError::TooLarge { what: "a presence", len: b.len(), max: craft_cloud_api::limits::MAX_PRESENCE_BYTES - 8 }); + } + self.presence = bytes; + self.presence_sent = None; + self.keep_presence(); + Ok(()) + } + + fn keep_presence(&mut self) { + let now = Instant::now(); + self.peers.retain(|_, p| now.duration_since(p.seen) < PRESENCE_TTL); + let (Some(link), Some(bytes)) = (&self.link, &self.presence) else { return }; + if self.presence_sent.is_some_and(|at| now.duration_since(at) < PRESENCE_EVERY) { + return; + } + let mut frame = self.doc.peer_id().to_be_bytes().to_vec(); + frame.extend_from_slice(bytes); + if link.doc.presence(&frame).is_ok() { + self.presence_sent = Some(now); + } + } + + /// The other machines' and apps' presence, by their peer, as they sent it (dropped after [`PRESENCE_TTL`] quiet). + pub fn peers(&self) -> &HashMap { + &self.peers + } + + /// Where a save starts: the model to write to the file and the live document's frontiers it matches. Refused + /// ([`KitError::NoRecord`]) when a save now can't say what it holds: the document is read-only (a newer app's), the + /// session is stalled or stopped, or the model is behind the live document (a remote change it couldn't take + /// yet). The app then saves its own model without a record, as it would without Craft Cloud. + /// + /// The order of a save: `save_point`, the bytes from [`SavePoint::model`], [`Session::record`] with them, and only + /// then the write to disk (the agent has to know the content before its sync pass sees it). + pub fn save_point(&self) -> Result, KitError> { + if let Some(why) = self.no_record() { + return Err(KitError::NoRecord(why.to_owned())); + } + let end = self.doc.oplog_vv().get(&self.doc.peer_id()).copied().unwrap_or(0); + Ok(SavePoint { model: self.model.clone(), frontiers: self.frontiers(), end }) + } + + /// The live document's frontiers (its heads), sorted. + fn frontiers(&self) -> Vec<(u64, u32)> { + let mut heads: Vec<(u64, u32)> = self.doc.oplog_frontiers().iter().map(|id| (id.peer, u32::try_from(id.counter).unwrap_or(0))).collect(); + heads.sort_unstable(); + heads + } + + /// Why a save now can't carry a record, if it can't. + fn no_record(&self) -> Option<&'static str> { + if self.status.read_only.is_some() { + Some("the document is a newer app's (read-only here)") + } else if self.outbox_bytes >= MAX_QUEUE_BYTES { + Some("too much waits for Craft Cloud (stalled)") + } else if self.status.stopped || self.refused.is_some() || self.status.replaced { + Some("the live document stopped here") + } else if self.model_stale { + Some("the model is behind the live document") + } else { + None + } + } + + /// Whether the live document is strictly ahead of the file's record: everything the file was saved from is in it, + /// and it has more. + fn ahead_of_file(&self) -> bool { + let Some(file) = &self.file_version else { return false }; + let vv = self.doc.oplog_vv(); + let held = file.iter().all(|(peer, counter)| i32::try_from(*counter).is_ok_and(|c| vv.includes_id(loro::ID::new(*peer, c)))); + held && *file != self.frontiers() + } + + /// Whether the file on disk lacks edits the live document has: the record of the file as it is now (the agent's + /// word, or this session's last save) is included in the live document's version and isn't it. An app autosaves + /// when it is dirty or this says so, so the file catches up on its own with edits from other machines, and with + /// its own edits after another machine's save won over this one's. + /// + /// False when the file's record has anything this document lacks (saved concurrently elsewhere, ahead of it, or + /// unknown): this session is the one behind, and waits for the live updates; only the side that is ahead saves, + /// and a file ahead of the live document is never written over by itself. False too when no record of the file is + /// known (an edit outside Craft; not synced yet; an older agent), when a save couldn't carry a record, and for + /// [`CATCH_UP_EVERY`] after a save it asked for (a backstop: at most one such save in that time). + pub fn file_behind(&self) -> bool { + if self.no_record().is_some() || self.caught_up_at.is_some_and(|at| at.elapsed() < CATCH_UP_EVERY) { + return false; + } + self.ahead_of_file() + } + + /// The app is about to write `bytes` (made from `point`'s model) to the document's file: tells the agent, on the + /// same stream after every change the save covers (while it is away, the record is kept and told once it is back; + /// it is added to the file's version then), so the version the agent uploads for that content carries the record. + /// Call it before the write. + pub fn record(&mut self, point: &SavePoint, bytes: &[u8]) -> SaveRecord { + let hash = *blake3::hash(bytes).as_bytes(); + // The file is a Craft save from now on (nothing is written to the document: the record is the agent's). + self.status.changed_outside = false; + if !self.changed_since_save && self.ahead_of_file() { + // A save only the file being behind called for (no edit here since the last): the next waits a while. + self.caught_up_at = Some(Instant::now()); + } + self.changed_since_save = false; + let record = SaveRecord { hash, frontiers: point.frontiers.clone() }; + // The file is this save from now on (until the agent says otherwise: the sync brought another one). + self.file_version = Some(record.frontiers.clone()); + self.records.push_back(Kept { record: record.clone(), end: point.end, sent: false, announced: false }); + while self.records.len() > KEPT_RECORDS { + self.records.pop_front(); + } + self.send_outbox(); + record + } + + /// The live document's state as a Loro snapshot (for tests and diagnostics). + pub fn snapshot(&self) -> Result, KitError> { + Ok(self.doc.export(loro::ExportMode::Snapshot)?) + } + + /// This session's Loro peer. + pub fn peer(&self) -> u64 { + self.doc.peer_id() + } +} + +/// Loro's events, noted while the collector is on (an import, an undo). +fn collect(collector: &Mutex, event: DiffEvent<'_>) { + let mut c = lock(collector); + if !c.on { + return; + } + for diff in &event.events { + let first = diff.path.first().map(|(id, _)| id).unwrap_or(diff.target); + let Some(root) = root_name(first) else { continue }; + let mut raw = Raw { root, ..Raw::default() }; + for (_, index) in diff.path { + match index { + Index::Node(t) => raw.nodes.push(*t), + Index::Key(k) => raw.keys.push(k.to_string()), + Index::Seq(_) => {} + } + } + match &diff.diff { + Diff::Map(delta) => raw.keys.extend(delta.updated.keys().map(|k| k.to_string())), + Diff::Tree(tree) => { + raw.tree = true; + raw.nodes.extend(tree.diff.iter().map(|item| item.target)); + } + Diff::Text(delta) => raw.text = Some(delta.clone()), + _ => {} + } + c.raw.push(raw); + } +} diff --git a/crates/kit/src/table.rs b/crates/kit/src/table.rs new file mode 100644 index 0000000..fd60fa5 --- /dev/null +++ b/crates/kit/src/table.rs @@ -0,0 +1,176 @@ +//! Tables: `.rows` and `.cols` (movable lists of IDs) and `.cells` (a map from `row:column` to the +//! cell's value). A cell whose row or column is gone (deleted on one machine while edited on another) is never read +//! back, and goes with the next change to its row or column. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use loro::{LoroDoc, LoroMap, LoroMovableList, ValueOrContainer}; + +use crate::mapping::TableSource; +use crate::writer::Writer; +use crate::{ElemId, KitError, Value}; + +pub(crate) fn rows_name(key: &str) -> String { + format!("{key}.rows") +} + +pub(crate) fn cols_name(key: &str) -> String { + format!("{key}.cols") +} + +pub(crate) fn cells_name(key: &str) -> String { + format!("{key}.cells") +} + +pub(crate) fn cell_key(row: ElemId, col: ElemId) -> String { + format!("{}:{}", row.hex(), col.hex()) +} + +pub(crate) fn parse_cell_key(key: &str) -> Option<(ElemId, ElemId)> { + let (r, c) = key.split_once(':')?; + Some((ElemId::from_hex(r)?, ElemId::from_hex(c)?)) +} + +struct Containers { + rows: LoroMovableList, + cols: LoroMovableList, + cells: LoroMap, +} + +fn containers(doc: &LoroDoc, key: &str) -> Containers { + Containers { rows: doc.get_movable_list(rows_name(key)), cols: doc.get_movable_list(cols_name(key)), cells: doc.get_map(cells_name(key)) } +} + +fn set_cell(w: &mut Writer<'_>, cells: &LoroMap, row: ElemId, col: ElemId, value: Option) -> Result<(), KitError> { + let key = cell_key(row, col); + match value { + Some(v) => w.set(cells, &key, &v).map(|_| ()), + None => w.delete(cells, &key).map(|_| ()), + } +} + +/// Writes what changed from `old` to `new` (everything, when `old` is `None`). +pub(crate) fn write(w: &mut Writer<'_>, key: &str, old: Option<&dyn TableSource>, new: &dyn TableSource) -> Result<(), KitError> { + if let (Some(old), Some(k)) = (old, new.key()) + && old.key() == Some(k) + { + return Ok(()); + } + let c = containers(w.doc, key); + let stored_cols = crate::list::ids(&c.cols); + let stored_rows = crate::list::ids(&c.rows); + let (new_rows, new_cols) = (new.rows(), new.columns()); + let removed_rows = crate::list::sync(w, &c.rows, &new_rows)?; + let removed_cols = crate::list::sync(w, &c.cols, &new_cols)?; + // The cells of removed rows and columns. + for row in &removed_rows { + for col in &stored_cols { + set_cell(w, &c.cells, *row, *col, None)?; + } + } + for col in &removed_cols { + for row in &stored_rows { + set_cell(w, &c.cells, *row, *col, None)?; + } + } + match (old, new.changed_cells()) { + (Some(_), Some(hint)) => { + for (row, col) in hint { + set_cell(w, &c.cells, row, col, new.cell(row, col))?; + } + } + (Some(old), None) => { + let before: HashMap<(ElemId, ElemId), Value> = old.cells().into_iter().map(|(r, c, v)| ((r, c), v)).collect(); + let mut now: HashSet<(ElemId, ElemId)> = HashSet::new(); + for (row, col, value) in new.cells() { + now.insert((row, col)); + if before.get(&(row, col)) != Some(&value) { + set_cell(w, &c.cells, row, col, Some(value))?; + } + } + for (row, col) in before.keys() { + if !now.contains(&(*row, *col)) { + set_cell(w, &c.cells, *row, *col, None)?; + } + } + } + (None, _) => { + for (row, col, value) in new.cells() { + set_cell(w, &c.cells, row, col, Some(value))?; + } + } + } + Ok(()) +} + +/// What a remote change touched of a table. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TableTouch { + /// Everything (read it whole). + pub all: bool, + pub rows: bool, + pub columns: bool, + /// The cells written or removed. + pub cells: BTreeSet<(ElemId, ElemId)>, +} + +impl TableTouch { + pub(crate) fn whole() -> Self { + Self { all: true, ..Self::default() } + } +} + +/// A table as stored, after a remote change. +pub struct TableView<'a> { + c: Containers, + touch: TableTouch, + _doc: std::marker::PhantomData<&'a LoroDoc>, +} + +impl<'a> TableView<'a> { + pub(crate) fn new(doc: &'a LoroDoc, key: &str, touch: TableTouch) -> Self { + Self { c: containers(doc, key), touch, _doc: std::marker::PhantomData } + } + + pub fn touch(&self) -> &TableTouch { + &self.touch + } + + /// The rows, in order, each once. + pub fn rows(&self) -> Vec { + crate::list::ids(&self.c.rows) + } + + /// The columns, in order, each once. + pub fn columns(&self) -> Vec { + crate::list::ids(&self.c.cols) + } + + /// A cell's value (`None` when empty or unreadable). + pub fn cell(&self, row: ElemId, col: ElemId) -> Option { + match self.c.cells.get(&cell_key(row, col))? { + ValueOrContainer::Value(v) => Value::from_loro(&v), + ValueOrContainer::Container(_) => None, + } + } + + /// Every readable cell of a row and column that are there. + pub fn cells(&self) -> Vec<(ElemId, ElemId, Value)> { + let rows: HashSet = self.rows().into_iter().collect(); + let cols: HashSet = self.columns().into_iter().collect(); + let mut out = Vec::new(); + self.c.cells.for_each(|key, value| { + let Some((r, c)) = parse_cell_key(key) else { return }; + if !rows.contains(&r) || !cols.contains(&c) { + return; + } + if let ValueOrContainer::Value(v) = value + && let Some(v) = Value::from_loro(&v) + { + out.push((r, c, v)); + } + }); + out.sort_by_key(|a| (a.0, a.1)); + out + } +} diff --git a/crates/kit/src/text.rs b/crates/kit/src/text.rs new file mode 100644 index 0000000..69e00a7 --- /dev/null +++ b/crates/kit/src/text.rs @@ -0,0 +1,697 @@ +//! Rich text: one `LoroText` per part. Character attributes are marks (expanding after, as bold does when one types +//! on); paragraph attributes are marks named `¶` on each paragraph's `\n` (never expanding), and the last +//! paragraph's, which has no `\n`, sit in the root map `.last`. +//! +//! The diff takes the common prefix and suffix of the old and new text (so a keystroke in a 1 MB story touches one +//! spot), replaces what is between (in pieces, so a 5 MB paste splits into commits under the cap), then compares the +//! attributes run by run: the old ones moved past the edit against the new ones, and in the inserted text what Loro +//! actually gave it (what it inherited by expanding) against what the app wants. Only differences are marked. + +use std::borrow::Cow; +use std::ops::Range; + +use loro::cursor::PosType; +use loro::{ExpandType, LoroDoc, LoroText, StyleConfig, StyleConfigMap, TextDelta, ValueOrContainer}; + +use crate::mapping::{Attrs, TextSource}; +use crate::writer::Writer; +use crate::{KitError, Value}; + +/// A character run: a byte range with its attributes. +type Run = (Range, Attrs); + +/// The prefix of paragraph marks. +const PARA: char = '¶'; + +pub(crate) fn last_name(key: &str) -> String { + format!("{key}.last") +} + +/// A story as plain data: text, character runs and paragraph attributes. It is a [`TextSource`] too. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct RichText { + pub text: String, + /// Byte ranges with their character attributes (in order, not overlapping; none where no run is). + pub runs: Vec<(Range, Attrs)>, + /// One per paragraph (one more than the text has `\n`s). + pub paragraphs: Vec, +} + +impl RichText { + /// Plain text: no attributes. + pub fn plain(text: &str) -> Self { + let paragraphs = vec![Attrs::new(); text.matches('\n').count() + 1]; + Self { text: text.to_owned(), runs: Vec::new(), paragraphs } + } + + /// The character attributes at byte `at`. + pub fn attrs_at(&self, at: usize) -> Attrs { + self.runs.iter().find(|(r, _)| r.contains(&at)).map(|(_, a)| a.clone()).unwrap_or_default() + } +} + +impl TextSource for RichText { + fn text(&self) -> &str { + &self.text + } + + fn runs(&self) -> Cow<'_, [(Range, Attrs)]> { + Cow::Borrowed(&self.runs) + } + + fn paragraphs(&self) -> Cow<'_, [Attrs]> { + Cow::Borrowed(&self.paragraphs) + } +} + +/// The style config the kit gives every key: character marks expand after, paragraph marks never. +pub(crate) fn configure(doc: &LoroDoc, styles: &mut StyleConfigMap, key: &str) { + if key.starts_with(PARA) && styles.get(&key.into()).is_none() { + styles.insert(key.into(), StyleConfig { expand: ExpandType::None }); + doc.config_text_style(styles.clone()); + } +} + +/// The longest common prefix of `a` and `b`, in bytes, on a character boundary. +fn common_prefix(a: &str, b: &str) -> usize { + let (x, y) = (a.as_bytes(), b.as_bytes()); + let max = x.len().min(y.len()); + let mut i = 0; + // Whole blocks first (compared as slices: fast), then byte by byte. + const BLOCK: usize = 4096; + while i + BLOCK <= max && x.get(i..i + BLOCK) == y.get(i..i + BLOCK) { + i += BLOCK; + } + while i < max && x.get(i) == y.get(i) { + i += 1; + } + while i > 0 && !a.is_char_boundary(i) { + i -= 1; + } + i +} + +/// The longest common suffix of `a` and `b` of at most `max` bytes, on a character boundary. +fn common_suffix(a: &str, b: &str, max: usize) -> usize { + let (x, y) = (a.as_bytes(), b.as_bytes()); + let mut n = 0; + const BLOCK: usize = 4096; + while n + BLOCK <= max && x.get(x.len() - n - BLOCK..x.len() - n) == y.get(y.len() - n - BLOCK..y.len() - n) { + n += BLOCK; + } + while n < max && x.get(x.len() - n - 1) == y.get(y.len() - n - 1) { + n += 1; + } + while n > 0 && !a.is_char_boundary(a.len() - n) { + n -= 1; + } + n +} + +/// Checks runs: in order, not overlapping, inside the text, on character boundaries, not empty. +fn check_runs(text: &str, runs: &[(Range, Attrs)]) -> Result<(), KitError> { + let mut at = 0; + for (r, attrs) in runs { + if r.start < at || r.end <= r.start || r.end > text.len() || !text.is_char_boundary(r.start) || !text.is_char_boundary(r.end) { + return Err(KitError::Invalid(format!("the run {r:?} isn't in order, inside the text and on character boundaries"))); + } + for (k, v) in attrs { + crate::props::check_key(k)?; + v.check()?; + } + at = r.end; + } + Ok(()) +} + +/// Loro's attributes as the kit's, character ones and paragraph ones apart. +fn split_attrs<'a>(attributes: impl IntoIterator) -> (Attrs, Attrs) { + let (mut chars, mut paras) = (Attrs::new(), Attrs::new()); + for (k, v) in attributes { + let Some(v) = Value::from_loro(v) else { continue }; + if matches!(v, Value::Null) { + continue; + } + match k.strip_prefix(PARA) { + Some(k) => paras.insert(k.to_owned(), v), + None => chars.insert(k.clone(), v), + }; + } + (chars, paras) +} + +/// The text's character runs between bytes `lo` and `hi`, as Loro holds them. +fn stored_runs(text: &LoroText, lo: usize, hi: usize) -> Result, Attrs)>, KitError> { + if hi <= lo { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + let mut at = lo; + for delta in text.slice_delta(lo, hi, PosType::Bytes)? { + if let TextDelta::Insert { insert, attributes } = delta { + let end = at + insert.len(); + let (chars, _) = split_attrs(attributes.iter().flatten()); + if !chars.is_empty() { + out.push((at..end, chars)); + } + at = end; + } + } + Ok(out) +} + +/// The old runs as they lie in the new text: the prefix's as they were, the suffix's moved by the edit, and in the +/// replaced middle what Loro holds there now. +fn moved_runs( + old: &[(Range, Attrs)], + p: usize, + old_end: usize, + new_end: usize, + middle: Vec<(Range, Attrs)>, +) -> Vec<(Range, Attrs)> { + let mut out = Vec::new(); + for (r, a) in old { + if r.start < p { + out.push((r.start..r.end.min(p), a.clone())); + } + } + out.extend(middle); + for (r, a) in old { + if r.end > old_end { + let start = r.start.max(old_end) - old_end + new_end; + let end = r.end - old_end + new_end; + out.push((start..end, a.clone())); + } + } + out +} + +/// Where `desired` and `current` (sorted runs) differ: each stretch with the keys to set (`Some`) or remove. +type Changes = Vec<(Range, Vec<(String, Option)>)>; + +fn differences(desired: &[(Range, Attrs)], current: &[(Range, Attrs)]) -> Changes { + let mut cuts: Vec = Vec::with_capacity((desired.len() + current.len()) * 2); + for (r, _) in desired.iter().chain(current) { + cuts.push(r.start); + cuts.push(r.end); + } + cuts.sort_unstable(); + cuts.dedup(); + let empty = Attrs::new(); + let at = |runs: &[(Range, Attrs)], i: &mut usize, x: usize| -> Attrs { + while runs.get(*i).is_some_and(|(r, _)| r.end <= x) { + *i += 1; + } + match runs.get(*i) { + Some((r, a)) if r.start <= x => a.clone(), + _ => empty.clone(), + } + }; + let (mut di, mut ci) = (0, 0); + let mut out: Changes = Vec::new(); + for pair in cuts.windows(2) { + let (Some(&x), Some(&y)) = (pair.first(), pair.get(1)) else { continue }; + let (want, have) = (at(desired, &mut di, x), at(current, &mut ci, x)); + if want == have { + continue; + } + let mut ops = Vec::new(); + for (k, v) in &want { + if have.get(k) != Some(v) { + ops.push((k.clone(), Some(v.clone()))); + } + } + for k in have.keys() { + if !want.contains_key(k) { + ops.push((k.clone(), None)); + } + } + // A stretch right after one with the same changes grows it. + match out.last_mut() { + Some((r, last)) if r.end == x && *last == ops => r.end = y, + _ => out.push((x..y, ops)), + } + } + out +} + +fn apply(w: &mut Writer<'_>, text: &LoroText, styles: &mut StyleConfigMap, changes: Changes, prefix: &str) -> Result<(), KitError> { + for (range, ops) in changes { + for (k, v) in ops { + let key = format!("{prefix}{k}"); + configure(w.doc, styles, &key); + match v { + Some(v) => { + w.spend(16 + key.len() + v.weight())?; + text.mark_utf8(range.clone(), &key, v.to_loro())?; + } + None => { + w.spend(16 + key.len())?; + let start = text.convert_pos(range.start, PosType::Bytes, PosType::Unicode); + let end = text.convert_pos(range.end, PosType::Bytes, PosType::Unicode); + if let (Some(start), Some(end)) = (start, end) { + text.unmark(start..end, &key)?; + } + } + } + } + } + Ok(()) +} + +/// The byte positions of the `\n`s of `text`. +fn breaks(text: &str) -> Vec { + text.bytes().enumerate().filter(|(_, b)| *b == b'\n').map(|(i, _)| i).collect() +} + +/// Writes what changed from `old` to `new` (everything, when `old` is `None`) into the text `key`. +pub(crate) fn write( + w: &mut Writer<'_>, + key: &str, + old: Option<&dyn TextSource>, + new: &dyn TextSource, + styles: &mut StyleConfigMap, +) -> Result<(), KitError> { + if let (Some(old), Some(k)) = (old, new.key()) + && old.key() == Some(k) + { + return Ok(()); + } + let text = w.doc.get_text(key); + let new_text = new.text(); + let new_runs = new.runs(); + check_runs(new_text, &new_runs)?; + let new_paras = new.paragraphs(); + for attrs in new_paras.iter() { + for (k, v) in attrs { + crate::props::check_key(k)?; + v.check()?; + } + } + // What Loro holds: the old model, unless it is out of step (then Loro's own). + let stored; + let (old_text, old_runs, old_paras): (Cow<'_, str>, Cow<'_, [Run]>, Cow<'_, [Attrs]>) = match old { + Some(o) if text.len_utf8() == o.text().len() => (Cow::Borrowed(o.text()), o.runs(), o.paragraphs()), + _ => { + stored = read(w.doc, key); + (Cow::Owned(stored.text), Cow::Owned(stored.runs), Cow::Owned(stored.paragraphs)) + } + }; + let p = common_prefix(&old_text, new_text); + let s = common_suffix(&old_text, new_text, old_text.len().min(new_text.len()) - p); + let (old_end, new_end) = (old_text.len() - s, new_text.len() - s); + // The text. + if old_end > p { + w.spend(old_end - p)?; + text.delete_utf8(p, old_end - p)?; + w.text_bytes = w.text_bytes.saturating_add(old_end - p); + } + let mut at = p; + while at < new_end { + let mut room = w.room(); + if room == 0 { + w.cut()?; + room = w.room(); + } + let mut end = (at + room.max(1)).min(new_end); + while end > at && !new_text.is_char_boundary(end) { + end -= 1; + } + if end == at { + // One character bigger than the room: it goes alone. + end = at + new_text.get(at..).and_then(|t| t.chars().next()).map_or(1, char::len_utf8); + } + let piece = new_text.get(at..end).ok_or_else(|| KitError::Invalid("the text's edit isn't on character boundaries".into()))?; + w.spend(piece.len())?; + text.insert_utf8(at, piece)?; + w.text_bytes = w.text_bytes.saturating_add(piece.len()); + at = end; + } + // Character attributes. + let middle = stored_runs(&text, p, new_end)?; + let current = moved_runs(&old_runs, p, old_end, new_end, middle); + apply(w, &text, styles, differences(&new_runs, ¤t), "")?; + // Paragraph attributes, on the `\n`s. + // (Paragraphs the app didn't give attributes for have none.) + let new_count = new_text.matches('\n').count(); + let empty = Attrs::new(); + let want_at = |j: usize| new_paras.get(j).unwrap_or(&empty); + let inserted = new_text.get(p..new_end).map_or(0, |t| t.matches('\n').count()); + let removed = old_text.get(p..old_end).map_or(0, |t| t.matches('\n').count()); + let mut differ: Vec = Vec::new(); + if inserted == 0 && removed == 0 { + for j in 0..new_count { + if old_paras.get(j).unwrap_or(&empty) != want_at(j) { + differ.push(j); + } + } + } else { + let before = new_text.get(..p).map_or(0, |t| t.matches('\n').count()); + for j in 0..new_count { + let want = want_at(j); + let have = if j < before { + old_paras.get(j) + } else if j < before + inserted { + None + } else { + old_paras.get(j + removed - inserted) + }; + if have.unwrap_or(&empty) != want { + differ.push(j); + } + } + } + if !differ.is_empty() { + let positions = breaks(new_text); + let mut desired = Vec::new(); + let mut current = Vec::new(); + for j in differ { + let Some(&x) = positions.get(j) else { continue }; + let range = x..x + 1; + desired.push((range.clone(), want_at(j).clone())); + // What the `\n` holds now. + let mut held = Attrs::new(); + for delta in text.slice_delta(x, x + 1, PosType::Bytes)? { + if let TextDelta::Insert { attributes, .. } = delta { + held = split_attrs(attributes.iter().flatten()).1; + } + } + current.push((range, held)); + } + apply(w, &text, styles, differences(&desired, ¤t), &PARA.to_string())?; + } + // The last paragraph's. + let last = w.doc.get_map(last_name(key)); + let want = new_paras.get(new_count).cloned().unwrap_or_default(); + for (k, v) in &want { + w.set(&last, k, v)?; + } + let mut stale = Vec::new(); + last.for_each(|k, _| { + if !want.contains_key(k) { + stale.push(k.to_owned()); + } + }); + for k in stale { + w.delete(&last, &k)?; + } + Ok(()) +} + +/// The text `key` as stored. +pub(crate) fn read(doc: &LoroDoc, key: &str) -> RichText { + let text = doc.get_text(key); + let mut out = RichText::default(); + for delta in text.to_delta() { + let TextDelta::Insert { insert, attributes } = delta else { continue }; + let start = out.text.len(); + let (chars, paras) = split_attrs(attributes.iter().flatten()); + for _ in insert.matches('\n') { + out.paragraphs.push(paras.clone()); + } + out.text.push_str(&insert); + let end = out.text.len(); + if chars.is_empty() { + continue; + } + match out.runs.last_mut() { + Some((r, a)) if r.end == start && *a == chars => r.end = end, + _ => out.runs.push((start..end, chars)), + } + } + out.paragraphs.push(last_paragraph(doc, key)); + out +} + +/// The last paragraph's attributes, as stored beside the text. +fn last_paragraph(doc: &LoroDoc, key: &str) -> Attrs { + let mut last = Attrs::new(); + doc.get_map(last_name(key)).for_each(|k, v| { + if let ValueOrContainer::Value(v) = v + && let Some(v) = Value::from_loro(&v) + { + last.insert(k.to_owned(), v); + } + }); + last +} + +/// The byte `chars` characters after byte `from` of `text`; `None` past its end. +fn byte_after(text: &str, from: usize, chars: usize) -> Option { + if chars == 0 { + return Some(from); + } + let bytes = text.as_bytes().get(from..)?; + let mut seen = 0usize; + for (i, b) in bytes.iter().enumerate() { + // A character starts at every byte that isn't a continuation byte. + if b & 0xC0 != 0x80 { + if seen == chars { + return Some(from + i); + } + seen += 1; + } + } + (seen == chars).then_some(text.len()) +} + +/// Sets or removes attribute changes (Loro's: a null value removes) on `range` of `runs` (sorted), splitting runs. +fn change_runs(runs: Vec<(Range, Attrs)>, range: Range, changes: &Attrs, removed: &[String]) -> Vec<(Range, Attrs)> { + let mut out = Vec::with_capacity(runs.len() + 2); + let mut at = range.start; + let apply = |attrs: &mut Attrs| { + for (k, v) in changes { + attrs.insert(k.clone(), v.clone()); + } + for k in removed { + attrs.remove(k); + } + }; + for (r, attrs) in runs { + if r.end <= range.start || r.start >= range.end { + out.push((r, attrs)); + continue; + } + // A gap before this run inside the range. + if r.start > at { + let mut a = Attrs::new(); + apply(&mut a); + out.push((at..r.start, a)); + } + if r.start < range.start { + out.push((r.start..range.start, attrs.clone())); + } + let mut inside = attrs.clone(); + apply(&mut inside); + out.push((r.start.max(range.start)..r.end.min(range.end), inside)); + if r.end > range.end { + out.push((range.end..r.end, attrs)); + } + at = r.end.min(range.end); + } + if at < range.end { + let mut a = Attrs::new(); + apply(&mut a); + out.push((at..range.end, a)); + } + out.sort_by_key(|(r, _)| r.start); + tidy(out) +} + +/// Adjacent runs with the same attributes merged, empty ones dropped. +fn tidy(runs: Vec<(Range, Attrs)>) -> Vec<(Range, Attrs)> { + let mut out: Vec<(Range, Attrs)> = Vec::with_capacity(runs.len()); + for (r, a) in runs { + if r.start >= r.end || a.is_empty() { + continue; + } + match out.last_mut() { + Some((last, la)) if last.end == r.start && *la == a => last.end = r.end, + _ => out.push((r, a)), + } + } + out +} + +/// Loro's attribute changes on a retain, char and paragraph ones apart: what to set, and what a null removes. +fn changes<'a>(attributes: impl IntoIterator) -> ((Attrs, Vec), (Attrs, Vec)) { + let (mut chars, mut paras) = ((Attrs::new(), Vec::new()), (Attrs::new(), Vec::new())); + for (k, v) in attributes { + let (target, name) = match k.strip_prefix(PARA) { + Some(name) => (&mut paras, name.to_owned()), + None => (&mut chars, k.clone()), + }; + match Value::from_loro(v) { + Some(Value::Null) | None => target.1.push(name), + Some(v) => { + target.0.insert(name, v); + } + } + } + (chars, paras) +} + +/// `old` (the text as the model had it before the change) with Loro's event `deltas` applied, in order: only what +/// changed is touched (a remote keystroke in a 1 MB story copies the text once and moves its runs, without reading the +/// whole text back from Loro). `None` when the deltas don't fit `old` (the model was out of step): read it whole. +pub(crate) fn patch(doc: &LoroDoc, key: &str, old: &dyn TextSource, deltas: &[Vec]) -> Option { + let mut text = old.text().to_owned(); + let mut runs: Vec<(Range, Attrs)> = old.runs().into_owned(); + let mut paras: Vec = old.paragraphs().into_owned(); + paras.resize(text.matches('\n').count() + 1, Attrs::new()); + paras.pop(); + for delta in deltas { + let mut out = String::with_capacity(text.len() + 64); + // (old byte range, new byte start) of every piece kept, to move the runs. + let mut kept: Vec<(Range, usize)> = Vec::new(); + let mut added: Vec<(Range, Attrs)> = Vec::new(); + let mut marked: Vec<(Range, Attrs, Vec)> = Vec::new(); + let mut new_paras: Vec = Vec::with_capacity(paras.len() + 1); + let (mut ob, mut pi) = (0usize, 0usize); + for op in delta { + match op { + TextDelta::Retain { retain, attributes } => { + let end = byte_after(&text, ob, *retain)?; + let piece = text.get(ob..end)?; + let start = out.len(); + out.push_str(piece); + kept.push((ob..end, start)); + let breaks = piece.matches('\n').count(); + let ((set, unset), (pset, punset)) = changes(attributes.iter().flatten()); + for _ in 0..breaks { + let mut p = paras.get(pi)?.clone(); + for (k, v) in &pset { + p.insert(k.clone(), v.clone()); + } + for k in &punset { + p.remove(k); + } + new_paras.push(p); + pi += 1; + } + if !set.is_empty() || !unset.is_empty() { + marked.push((start..out.len(), set, unset)); + } + ob = end; + } + TextDelta::Insert { insert, attributes } => { + let start = out.len(); + out.push_str(insert); + let (chars, ps) = split_attrs(attributes.iter().flatten()); + for _ in insert.matches('\n') { + new_paras.push(ps.clone()); + } + if !chars.is_empty() { + added.push((start..out.len(), chars)); + } + } + TextDelta::Delete { delete } => { + let end = byte_after(&text, ob, *delete)?; + pi += text.get(ob..end)?.matches('\n').count(); + ob = end; + } + } + } + // What the delta leaves out is kept as it is. + let rest = text.get(ob..)?; + let start = out.len(); + out.push_str(rest); + kept.push((ob..text.len(), start)); + new_paras.extend(paras.get(pi..)?.iter().cloned()); + // The runs, moved with the pieces they were in. + let mut moved: Vec<(Range, Attrs)> = Vec::with_capacity(runs.len() + added.len()); + let mut k = 0; + for (r, a) in runs { + while kept.get(k).is_some_and(|(o, _)| o.end <= r.start) { + k += 1; + } + let mut j = k; + while let Some((o, n)) = kept.get(j) { + if o.start >= r.end { + break; + } + let (s, e) = (r.start.max(o.start), r.end.min(o.end)); + if s < e { + moved.push((s - o.start + n..e - o.start + n, a.clone())); + } + j += 1; + } + } + moved.extend(added); + moved.sort_by_key(|(r, _)| r.start); + runs = tidy(moved); + for (range, set, unset) in marked { + runs = change_runs(runs, range, &set, &unset); + } + text = out; + paras = new_paras; + } + // Out of step with what Loro holds: no answer. + if text.len() != doc.get_text(key).len_utf8() || paras.len() != text.matches('\n').count() { + return None; + } + paras.push(last_paragraph(doc, key)); + Some(RichText { text, runs, paragraphs: paras }) +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use loro::{ExportMode, LoroDoc}; + + use super::*; + + /// A text patched from Loro's deltas is the text read back whole: inserts, deletes and marks among characters of + /// one to four bytes, paragraph marks, a deleted break, several updates in one go. + #[test] + fn a_patched_text_is_the_text_read_whole() { + let a = LoroDoc::new(); + a.config_default_text_style(Some(StyleConfig { expand: ExpandType::After })); + let mut styles = StyleConfigMap::new(); + configure(&a, &mut styles, "¶h"); + let t = a.get_text("t"); + t.insert(0, "héllo 日本\nwörld 🙂 end\nlast").unwrap(); + a.commit(); + let b = LoroDoc::new(); + b.import(&a.export(ExportMode::Snapshot).unwrap()).unwrap(); + let deltas: Arc>>> = Arc::default(); + let d = deltas.clone(); + let _sub = b.subscribe_root(Arc::new(move |e| { + for diff in e.events { + if let loro::event::Diff::Text(delta) = diff.diff { + d.lock().unwrap().push(delta.clone()); + } + } + })); + type Edit = Box; + let edits: Vec = vec![ + Box::new(|t| t.insert(3, "XY🙂").unwrap()), + Box::new(|t| t.mark(2..9, "b", true).unwrap()), + Box::new(|t| t.mark(10..11, "¶h", 1i64).unwrap()), + Box::new(|t| t.delete(9, 3).unwrap()), + Box::new(|t| t.unmark(4..6, "b").unwrap()), + Box::new(|t| t.insert(0, "日\n").unwrap()), + ]; + let mut sent = Vec::new(); + for edit in &edits { + let before = a.oplog_vv(); + edit(&a.get_text("t")); + a.commit(); + sent.push(a.export(ExportMode::updates(&before)).unwrap()); + } + // Two updates at a time, then the rest. + for chunk in sent.chunks(2) { + let old = read(&b, "t"); + deltas.lock().unwrap().clear(); + for u in chunk { + b.import(u).unwrap(); + } + let got = patch(&b, "t", &old, &deltas.lock().unwrap()).expect("patched"); + assert_eq!(got, read(&b, "t")); + } + // Deltas that don't fit the old text: no answer (read it whole). + let wrong = RichText::plain("short"); + assert!(patch(&b, "t", &wrong, &[vec![TextDelta::Retain { retain: 999, attributes: None }]]).is_none()); + } +} diff --git a/crates/kit/src/timeline.rs b/crates/kit/src/timeline.rs new file mode 100644 index 0000000..946ba1e --- /dev/null +++ b/crates/kit/src/timeline.rs @@ -0,0 +1,301 @@ +//! Timelines: `.tracks` (a movable list of track IDs), `.track` (a map from track ID to the track's +//! properties), `.items` (a map from item ID to its track, start, length and properties) and `.keys` (a map +//! from keyframe ID to its item, time and value). Items and keyframes are keyed, never positional, so two machines +//! adding keyframes to one item keep both, and an item moved to another track on one machine while trimmed on another +//! keeps both changes. Items read back by start (then ID); keyframes by time (then ID). An item whose track is gone, +//! or a keyframe whose item is gone, is never read back. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use loro::{LoroDoc, LoroMap, LoroMovableList, ValueOrContainer}; + +use crate::mapping::{Span, TimelineSource, props_of}; +use crate::props::{Props, PropsRead}; +use crate::writer::Writer; +use crate::{ElemId, KitError, Value}; + +const TRACK: &str = "$track"; +const START: &str = "$start"; +const LEN: &str = "$len"; +const OWNER: &str = "$item"; +const AT: &str = "$at"; +const VALUE: &str = "$value"; + +pub(crate) fn tracks_name(key: &str) -> String { + format!("{key}.tracks") +} +pub(crate) fn track_props_name(key: &str) -> String { + format!("{key}.track") +} +pub(crate) fn items_name(key: &str) -> String { + format!("{key}.items") +} +pub(crate) fn keys_name(key: &str) -> String { + format!("{key}.keys") +} + +struct Containers { + tracks: LoroMovableList, + track_props: LoroMap, + items: LoroMap, + keys: LoroMap, +} + +fn containers(doc: &LoroDoc, key: &str) -> Containers { + Containers { + tracks: doc.get_movable_list(tracks_name(key)), + track_props: doc.get_map(track_props_name(key)), + items: doc.get_map(items_name(key)), + keys: doc.get_map(keys_name(key)), + } +} + +fn child_map(w: &mut Writer<'_>, parent: &LoroMap, key: &str) -> Result { + if let Some(ValueOrContainer::Container(loro::Container::Map(map))) = parent.get(key) { + return Ok(map); + } + w.spend(key.len())?; + // Mergeable: two machines making the same key's map at once share one. + Ok(parent.ensure_mergeable_map(key)?) +} + +fn existing_map(parent: &LoroMap, key: &str) -> Option { + match parent.get(key)? { + ValueOrContainer::Container(loro::Container::Map(map)) => Some(map), + _ => None, + } +} + +/// Every item of a timeline: its track and whether it was skipped. +fn all_items(src: &dyn TimelineSource) -> HashMap { + let mut out = HashMap::new(); + for track in src.tracks() { + for item in src.items(track) { + out.insert(item, track); + } + } + out +} + +/// Writes what changed from `old` to `new` (everything, when `old` is `None`) into the timeline `key`. +pub(crate) fn write(w: &mut Writer<'_>, key: &str, old: Option<&dyn TimelineSource>, new: &dyn TimelineSource) -> Result<(), KitError> { + if let (Some(old), Some(k)) = (old, new.key()) + && old.key() == Some(k) + { + return Ok(()); + } + let c = containers(w.doc, key); + let tracks = new.tracks(); + let removed_tracks = crate::list::sync(w, &c.tracks, &tracks)?; + for track in removed_tracks { + w.delete(&c.track_props, &track.hex())?; + } + let new_items = all_items(new); + let old_items = old.map(all_items).unwrap_or_default(); + for track in &tracks { + let same_track = old.is_some_and(|o| new.track_key(*track).is_some() && o.track_key(*track) == new.track_key(*track)); + if same_track { + continue; + } + let map = child_map(w, &c.track_props, &track.hex())?; + let mut props = Props::new(map, w); + new.write_track(*track, &mut props)?; + props.finish()?; + for item in new.items(*track) { + let same_item = + old.is_some_and(|o| new.item_key(item).is_some() && o.item_key(item) == new.item_key(item)) && old_items.get(&item) == Some(track); + if same_item { + continue; + } + write_item(w, &c, item, *track, old, new)?; + } + } + // Items gone, with their keyframes. + for (item, _) in old_items.iter().filter(|(i, _)| !new_items.contains_key(i)) { + if let Some(o) = old { + for (k, _, _) in o.keyframes(*item) { + w.delete(&c.keys, &k.hex())?; + } + } + w.delete(&c.items, &item.hex())?; + } + Ok(()) +} + +fn write_item( + w: &mut Writer<'_>, + c: &Containers, + item: ElemId, + track: ElemId, + old: Option<&dyn TimelineSource>, + new: &dyn TimelineSource, +) -> Result<(), KitError> { + let Some(Span { start, len }) = new.span(item) else { + return Err(KitError::Invalid(format!("the item {item} has no span"))); + }; + let map = child_map(w, &c.items, &item.hex())?; + w.set(&map, TRACK, &Value::Bytes(track.0.to_le_bytes().to_vec()))?; + w.set(&map, START, &Value::I64(start))?; + w.set(&map, LEN, &Value::I64(len))?; + let mut props = Props::new(map, w); + new.write_item(item, &mut props)?; + props.finish()?; + // Keyframes: by ID; only the ones that changed. + let before: HashMap = + old.map(|o| o.keyframes(item)).unwrap_or_default().into_iter().map(|(k, at, v)| (k, (at, v))).collect(); + let mut now = HashSet::new(); + for (k, at, value) in new.keyframes(item) { + now.insert(k); + if old.is_some() && before.get(&k) == Some(&(at, value.clone())) { + continue; + } + let map = child_map(w, &c.keys, &k.hex())?; + w.set(&map, OWNER, &Value::Bytes(item.0.to_le_bytes().to_vec()))?; + w.set(&map, AT, &Value::I64(at))?; + w.set(&map, VALUE, &value)?; + } + for k in before.keys().filter(|k| !now.contains(k)) { + w.delete(&c.keys, &k.hex())?; + } + Ok(()) +} + +fn id_value(map: &LoroMap, key: &str) -> Option { + match map.get(key)? { + ValueOrContainer::Value(loro::LoroValue::Binary(b)) => <[u8; 16]>::try_from(b.as_slice()).ok().map(|a| ElemId(u128::from_le_bytes(a))), + _ => None, + } +} + +fn i64_value(map: &LoroMap, key: &str) -> Option { + match map.get(key)? { + ValueOrContainer::Value(loro::LoroValue::I64(i)) => Some(i), + _ => None, + } +} + +/// What a remote change touched of a timeline. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TimelineTouch { + /// Everything (read it whole). + pub all: bool, + /// The track list changed. + pub tracks: bool, + /// Tracks whose properties changed. + pub track_props: BTreeSet, + /// Items written or removed (their keyframes too). + pub items: BTreeSet, + /// Keyframes written or removed. + pub keyframes: BTreeSet, +} + +impl TimelineTouch { + pub(crate) fn whole() -> Self { + Self { all: true, ..Self::default() } + } +} + +/// One item as stored. +#[derive(Debug, Clone)] +pub struct ItemRead { + pub id: ElemId, + pub track: ElemId, + pub span: Span, + pub props: PropsRead, +} + +/// A timeline as stored, after a remote change. +pub struct TimelineView<'a> { + c: Containers, + touch: TimelineTouch, + _doc: std::marker::PhantomData<&'a LoroDoc>, +} + +impl<'a> TimelineView<'a> { + pub(crate) fn new(doc: &'a LoroDoc, key: &str, touch: TimelineTouch) -> Self { + Self { c: containers(doc, key), touch, _doc: std::marker::PhantomData } + } + + pub fn touch(&self) -> &TimelineTouch { + &self.touch + } + + /// The tracks, in order. + pub fn tracks(&self) -> Vec { + crate::list::ids(&self.c.tracks) + } + + /// A track's properties. + pub fn track_props(&self, track: ElemId) -> Option { + existing_map(&self.c.track_props, &track.hex()).map(props_of) + } + + /// An item, if it is on a track that is there. + pub fn item(&self, id: ElemId) -> Option { + let map = existing_map(&self.c.items, &id.hex())?; + let track = id_value(&map, TRACK)?; + if !self.tracks().contains(&track) { + return None; + } + let span = Span { start: i64_value(&map, START)?, len: i64_value(&map, LEN)? }; + Some(ItemRead { id, track, span, props: props_of(map) }) + } + + /// Every item on a track that is there, by track, each by start then ID. + pub fn items(&self) -> HashMap> { + let tracks: HashSet = self.tracks().into_iter().collect(); + let mut out: HashMap> = HashMap::new(); + let mut ids = Vec::new(); + self.c.items.for_each(|k, _| ids.extend(ElemId::from_hex(k))); + for id in ids { + if let Some(item) = self.item(id) + && tracks.contains(&item.track) + { + out.entry(item.track).or_default().push(item); + } + } + for list in out.values_mut() { + list.sort_by_key(|i| (i.span.start, i.id)); + } + out + } + + /// An item's keyframes: ID, time and value, by time then ID. + pub fn keyframes(&self, item: ElemId) -> Vec<(ElemId, i64, Value)> { + let mut out = Vec::new(); + let mut ids = Vec::new(); + self.c.keys.for_each(|k, _| ids.extend(ElemId::from_hex(k))); + for id in ids { + let Some(map) = existing_map(&self.c.keys, &id.hex()) else { continue }; + if id_value(&map, OWNER) != Some(item) { + continue; + } + let (Some(at), Some(ValueOrContainer::Value(v))) = (i64_value(&map, AT), map.get(VALUE)) else { continue }; + if let Some(v) = Value::from_loro(&v) { + out.push((id, at, v)); + } + } + out.sort_by_key(|(id, at, _)| (*at, *id)); + out + } + + /// Every keyframe, by item. + pub fn all_keyframes(&self) -> HashMap> { + let mut out: HashMap> = HashMap::new(); + let mut ids = Vec::new(); + self.c.keys.for_each(|k, _| ids.extend(ElemId::from_hex(k))); + for id in ids { + let Some(map) = existing_map(&self.c.keys, &id.hex()) else { continue }; + let (Some(owner), Some(at), Some(ValueOrContainer::Value(v))) = (id_value(&map, OWNER), i64_value(&map, AT), map.get(VALUE)) else { + continue; + }; + if let Some(v) = Value::from_loro(&v) { + out.entry(owner).or_default().push((id, at, v)); + } + } + for list in out.values_mut() { + list.sort_by_key(|(id, at, _)| (*at, *id)); + } + out + } +} diff --git a/crates/kit/src/tree.rs b/crates/kit/src/tree.rs new file mode 100644 index 0000000..24980fe --- /dev/null +++ b/crates/kit/src/tree.rs @@ -0,0 +1,267 @@ +//! Trees: one `LoroTree` per part (fractional index on, so children keep their order), each node's meta map holding +//! the app's properties and the element's ID under `$id`. Moves and reparenting merge without duplicates, and a move +//! that would make a cycle with a concurrent one is dropped by Loro. +//! +//! The diff walks the new model from its roots, skipping every subtree whose [`TreeSource::subtree_key`] is the same in +//! the old one; under each parent it visits, it creates what is missing, moves only what is out of order (the longest +//! run in order stays), and deletes, at the end, what left and went nowhere else. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use loro::{LoroDoc, LoroTree, TreeID, TreeParentId, ValueOrContainer}; + +use crate::mapping::{TreeSource, props_of}; +use crate::props::{Props, PropsRead}; +use crate::writer::Writer; +use crate::{ElemId, KitError}; + +/// The meta key that holds a node's element ID. +const ID_KEY: &str = "$id"; +const NODE_WEIGHT: usize = 24; + +/// Which Loro node is which element, for one tree. +#[derive(Debug, Clone, Default)] +pub struct TreeIndex { + pub(crate) tid_of: HashMap, + pub(crate) elem_of: HashMap, +} + +impl TreeIndex { + /// Every live node of `tree` with an ID (the first node with an ID wins; a second is never read). + pub(crate) fn build(tree: &LoroTree) -> Self { + let mut index = Self::default(); + let mut nodes = tree.nodes(); + nodes.sort(); + for tid in nodes { + if tree.is_node_deleted(&tid).unwrap_or(true) { + continue; + } + if let Some(id) = node_id(tree, tid) + && !index.tid_of.contains_key(&id) + { + index.tid_of.insert(id, tid); + index.elem_of.insert(tid, id); + } + } + index + } + + fn insert(&mut self, id: ElemId, tid: TreeID) { + self.tid_of.insert(id, tid); + self.elem_of.insert(tid, id); + } + + pub(crate) fn len(&self) -> usize { + self.tid_of.len() + } +} + +fn node_id(tree: &LoroTree, tid: TreeID) -> Option { + let meta = tree.get_meta(tid).ok()?; + match meta.get(ID_KEY)? { + ValueOrContainer::Value(v) => ElemId::from_loro(&v), + ValueOrContainer::Container(_) => None, + } +} + +/// Writes what changed from `old` to `new` (everything, when `old` is `None`) into the tree `key`. +pub(crate) fn write( + w: &mut Writer<'_>, + key: &str, + old: Option<&dyn TreeSource>, + new: &dyn TreeSource, + index: &mut TreeIndex, +) -> Result<(), KitError> { + if let (Some(old), Some(k)) = (old, new.key()) + && old.key() == Some(k) + { + return Ok(()); + } + let tree = w.doc.get_tree(key); + tree.enable_fractional_index(0); + let mut visited: HashSet = HashSet::new(); + let mut leaving: Vec = Vec::new(); + let mut deleted_any = false; + let mut skipped: Vec = Vec::new(); + // (parent, its children in the new model), depth first. + let mut stack: Vec<(TreeParentId, Vec)> = vec![(TreeParentId::Root, new.roots())]; + while let Some((parent, desired)) = stack.pop() { + let mut created: HashSet = HashSet::new(); + for id in &desired { + if !visited.insert(*id) { + return Err(KitError::Invalid(format!("the element {id} is in the tree twice"))); + } + if !index.tid_of.contains_key(id) { + w.spend(NODE_WEIGHT)?; + let tid = tree.create(parent)?; + tree.get_meta(tid)?.insert(ID_KEY, id.to_loro())?; + index.insert(*id, tid); + created.insert(*id); + } + } + order(w, &tree, parent, &desired, index, &mut leaving)?; + for id in desired.iter().rev() { + let Some(tid) = index.tid_of.get(id).copied() else { continue }; + let fresh = created.contains(id); + let unchanged = !fresh && old.is_some_and(|old| old.subtree_key(*id).is_some() && old.subtree_key(*id) == new.subtree_key(*id)); + if unchanged { + skipped.push(tid); + continue; + } + let mut props = Props::new(tree.get_meta(tid)?, w); + new.write(*id, &mut props)?; + props.finish()?; + stack.push((TreeParentId::Node(tid), new.children(*id))); + } + } + if !leaving.is_empty() { + // Nothing under an unchanged subtree leaves (it would have changed): kept even if the app says otherwise. + for tid in skipped { + mark_subtree(&tree, tid, index, &mut visited); + } + } + for tid in leaving { + let Some(id) = index.elem_of.get(&tid).copied() else { continue }; + if visited.contains(&id) || tree.is_node_deleted(&tid).unwrap_or(true) { + continue; + } + w.spend(NODE_WEIGHT)?; + tree.delete(tid)?; + deleted_any = true; + } + if deleted_any { + index.tid_of.retain(|_, tid| !tree.is_node_deleted(tid).unwrap_or(true)); + index.elem_of.retain(|tid, _| !tree.is_node_deleted(tid).unwrap_or(true)); + } + Ok(()) +} + +/// Marks everything under `tid` as placed (an unchanged subtree). +fn mark_subtree(tree: &LoroTree, tid: TreeID, index: &TreeIndex, visited: &mut HashSet) { + let mut stack = vec![tid]; + while let Some(t) = stack.pop() { + for child in tree.children(TreeParentId::Node(t)).unwrap_or_default() { + if let Some(id) = index.elem_of.get(&child) { + visited.insert(*id); + } + stack.push(child); + } + } +} + +/// Puts `desired` (all existing nodes) under `parent` in that order, moving as few as it can; the children there +/// that aren't wanted go to `leaving`. +fn order( + w: &mut Writer<'_>, + tree: &LoroTree, + parent: TreeParentId, + desired: &[ElemId], + index: &TreeIndex, + leaving: &mut Vec, +) -> Result<(), KitError> { + let current = tree.children(parent).unwrap_or_default(); + let rank: HashMap = desired.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + let mut here: Vec = Vec::new(); + for tid in ¤t { + match index.elem_of.get(tid).and_then(|id| rank.get(id)) { + Some(r) => here.push(*r), + None => leaving.push(*tid), + } + } + let in_order = here.windows(2).all(|p| p.first() < p.get(1)) && here.len() == desired.len(); + if in_order { + return Ok(()); + } + let steady: HashSet = crate::list::longest_increasing(&here).into_iter().collect(); + let mut prev: Option = None; + for (r, id) in desired.iter().enumerate() { + let Some(tid) = index.tid_of.get(id).copied() else { continue }; + if !steady.contains(&r) { + w.spend(NODE_WEIGHT)?; + match prev { + Some(p) => tree.mov_after(tid, p)?, + None => tree.mov_to(tid, parent, 0)?, + } + } + prev = Some(tid); + } + Ok(()) +} + +/// What a remote change touched of a tree. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TreeTouch { + /// Everything (read it whole). + pub all: bool, + /// Nodes were created, moved or deleted: read the structure again. + pub structure: bool, + /// The nodes whose properties changed (and the ones created). + pub nodes: BTreeSet, +} + +impl TreeTouch { + pub(crate) fn whole() -> Self { + Self { all: true, structure: true, nodes: BTreeSet::new() } + } +} + +/// A tree as stored, after a remote change. +pub struct TreeView<'a> { + tree: LoroTree, + index: &'a TreeIndex, + touch: TreeTouch, +} + +impl<'a> TreeView<'a> { + pub(crate) fn new(doc: &'a LoroDoc, key: &str, index: &'a TreeIndex, touch: TreeTouch) -> Self { + Self { tree: doc.get_tree(key), index, touch } + } + + pub fn touch(&self) -> &TreeTouch { + &self.touch + } + + fn known(&self, tids: Vec) -> Vec { + tids.into_iter().filter_map(|t| self.index.elem_of.get(&t).copied()).collect() + } + + /// The top-level elements, in order. + pub fn roots(&self) -> Vec { + self.known(self.tree.children(TreeParentId::Root).unwrap_or_default()) + } + + /// An element's children, in order. + pub fn children(&self, id: ElemId) -> Vec { + match self.index.tid_of.get(&id) { + Some(tid) => self.known(self.tree.children(TreeParentId::Node(*tid)).unwrap_or_default()), + None => Vec::new(), + } + } + + /// An element's parent (`None` for a top-level one or an unknown ID). + pub fn parent(&self, id: ElemId) -> Option { + match self.tree.parent(*self.index.tid_of.get(&id)?)? { + TreeParentId::Node(p) => self.index.elem_of.get(&p).copied(), + _ => None, + } + } + + pub fn contains(&self, id: ElemId) -> bool { + self.index.tid_of.contains_key(&id) + } + + /// An element's properties. + pub fn props(&self, id: ElemId) -> Option { + let tid = self.index.tid_of.get(&id)?; + self.tree.get_meta(*tid).ok().map(props_of) + } + + /// How many elements the tree has. + pub fn len(&self) -> usize { + self.index.len() + } + + pub fn is_empty(&self) -> bool { + self.index.len() == 0 + } +} diff --git a/crates/kit/src/value.rs b/crates/kit/src/value.rs new file mode 100644 index 0000000..faa4753 --- /dev/null +++ b/crates/kit/src/value.rs @@ -0,0 +1,336 @@ +//! The values the mappings carry: stable element IDs, scalar values, chunk references for binary content (with the +//! blob store behind them) and the opaque-value codec. + +use std::sync::Arc; + +use loro::LoroValue; + +use crate::KitError; + +/// A model element's stable identity across machines: the app keeps it in its own model (a layer's, a paragraph's, a +/// row's ID). The kit never makes IDs from positions; [`ElemId::new`] makes a random one for apps that have none. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ElemId(pub u128); + +impl ElemId { + /// A new random ID (from the OS's random source; a time-and-counter mix if that fails). + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + let mut bytes = [0u8; 16]; + if getrandom::fill(&mut bytes).is_err() { + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + return Self(now ^ (u128::from(n) << 64) ^ u128::from(std::process::id()) << 96); + } + Self(u128::from_le_bytes(bytes)) + } + + /// The ID as 32 lowercase hex digits (the key it has in Loro maps). + pub fn hex(&self) -> String { + format!("{:032x}", self.0) + } + + /// 32 lowercase hex digits as an ID. + pub fn from_hex(text: &str) -> Option { + if text.len() != 32 || !text.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return None; + } + u128::from_str_radix(text, 16).ok().map(Self) + } + + pub(crate) fn to_loro(self) -> LoroValue { + LoroValue::from(self.0.to_le_bytes().to_vec()) + } + + pub(crate) fn from_loro(value: &LoroValue) -> Option { + match value { + LoroValue::Binary(bytes) => { + let array: [u8; 16] = bytes.as_slice().try_into().ok()?; + Some(Self(u128::from_le_bytes(array))) + } + _ => None, + } + } +} + +impl std::fmt::Display for ElemId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:032x}", self.0) + } +} + +/// Binary content kept out of the live document: its name (the agent's content hash) and length. The bytes go +/// through a [`BlobStore`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ChunkRef { + pub hash: [u8; 32], + pub len: u64, +} + +/// Where binary content lives: the agent's `/v1/blobs` ([`AgentBlobs`]), or memory in tests ([`MemoryBlobs`]). Blocking: +/// a get may download, so call it off the UI thread. +pub trait BlobStore: Send + Sync { + fn put(&self, bytes: &[u8]) -> Result; + fn get(&self, chunk: &ChunkRef) -> Result, KitError>; +} + +/// The agent's blob store, through `craft-cloud-app`. An agent too old for blobs answers [`KitError::Agent`] with +/// `agent_too_old` in the message. +#[derive(Debug, Clone)] +pub struct AgentBlobs(pub craft_cloud_app::blobs::Blobs); + +impl BlobStore for AgentBlobs { + fn put(&self, bytes: &[u8]) -> Result { + let stored = self.0.put(bytes).map_err(agent_error)?; + Ok(ChunkRef { hash: stored.hash, len: stored.len }) + } + + fn get(&self, chunk: &ChunkRef) -> Result, KitError> { + Ok(self.0.get(&chunk.hash, chunk.len).map_err(agent_error)?.into()) + } +} + +/// An agent's refusal, its code kept in the message (`agent_too_old: …`). +fn agent_error(e: craft_cloud_app::Error) -> KitError { + match e { + craft_cloud_app::Error::Refused { code, message } => KitError::Agent(format!("{code}: {message}")), + other => KitError::from(other), + } +} + +/// Blobs in memory (tests, and apps that run without Craft Cloud), named by BLAKE3. +#[derive(Debug, Default)] +pub struct MemoryBlobs(std::sync::Mutex>>); + +impl BlobStore for MemoryBlobs { + fn put(&self, bytes: &[u8]) -> Result { + let hash = *blake3::hash(bytes).as_bytes(); + self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).insert(hash, bytes.into()); + Ok(ChunkRef { hash, len: bytes.len() as u64 }) + } + + fn get(&self, chunk: &ChunkRef) -> Result, KitError> { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&chunk.hash) + .cloned() + .ok_or_else(|| KitError::Agent("blob_not_found: no such blob".into())) + } +} + +/// A value that merges as a whole (last writer wins): settings, styles, plugin state. +pub trait OpaqueCodec: Sized { + fn encode(&self) -> Vec; + fn decode(bytes: &[u8]) -> Result; +} + +impl OpaqueCodec for Vec { + fn encode(&self) -> Vec { + self.clone() + } + + fn decode(bytes: &[u8]) -> Result { + Ok(bytes.to_vec()) + } +} + +impl OpaqueCodec for String { + fn encode(&self) -> Vec { + self.as_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Result { + String::from_utf8(bytes.to_vec()).map_err(|e| KitError::App(e.to_string())) + } +} + +/// Largest opaque value or property value in a live document, in bytes (bigger content goes in as a [`ChunkRef`]). +pub const MAX_VALUE_BYTES: usize = 256 * 1024; + +/// A property or cell value. +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + Null, + Bool(bool), + I64(i64), + F64(f64), + Str(String), + Bytes(Vec), + Chunk(ChunkRef), +} + +const CHUNK_HASH: &str = "$chunk"; +const CHUNK_LEN: &str = "$len"; + +impl Value { + /// Its size as the change-size estimate counts it. + pub(crate) fn weight(&self) -> usize { + match self { + Value::Null | Value::Bool(_) | Value::I64(_) | Value::F64(_) => 9, + Value::Str(s) => s.len(), + Value::Bytes(b) => b.len(), + Value::Chunk(_) => 48, + } + } + + /// Refuses a value too big for a live document. + pub(crate) fn check(&self) -> Result<(), KitError> { + let len = self.weight(); + if len > MAX_VALUE_BYTES { + return Err(KitError::TooLarge { what: "a value", len, max: MAX_VALUE_BYTES }); + } + Ok(()) + } + + pub(crate) fn to_loro(&self) -> LoroValue { + match self { + Value::Null => LoroValue::Null, + Value::Bool(b) => LoroValue::from(*b), + Value::I64(i) => LoroValue::from(*i), + Value::F64(f) => LoroValue::from(*f), + Value::Str(s) => LoroValue::from(s.as_str()), + Value::Bytes(b) => LoroValue::from(b.clone()), + Value::Chunk(c) => { + let mut map = std::collections::HashMap::new(); + map.insert(CHUNK_HASH.to_owned(), LoroValue::from(c.hash.to_vec())); + map.insert(CHUNK_LEN.to_owned(), LoroValue::from(i64::try_from(c.len).unwrap_or(i64::MAX))); + LoroValue::from(map) + } + } + } + + /// A Loro value as a property value; `None` for one no app wrote (a list, a container, a broken chunk). + pub(crate) fn from_loro(value: &LoroValue) -> Option { + Some(match value { + LoroValue::Null => Value::Null, + LoroValue::Bool(b) => Value::Bool(*b), + LoroValue::I64(i) => Value::I64(*i), + LoroValue::Double(f) => Value::F64(*f), + LoroValue::String(s) => Value::Str(s.to_string()), + LoroValue::Binary(b) => Value::Bytes(b.to_vec()), + LoroValue::Map(map) => { + let hash = match map.get(CHUNK_HASH)? { + LoroValue::Binary(b) => <[u8; 32]>::try_from(b.as_slice()).ok()?, + _ => return None, + }; + let len = match map.get(CHUNK_LEN)? { + LoroValue::I64(n) => u64::try_from(*n).ok()?, + _ => return None, + }; + Value::Chunk(ChunkRef { hash, len }) + } + LoroValue::List(_) | LoroValue::Container(_) => return None, + }) + } + + pub fn as_str(&self) -> Option<&str> { + match self { + Value::Str(s) => Some(s), + _ => None, + } + } + + pub fn as_i64(&self) -> Option { + match self { + Value::I64(i) => Some(*i), + _ => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + Value::F64(f) => Some(*f), + Value::I64(i) => Some(*i as f64), + _ => None, + } + } + + pub fn as_bool(&self) -> Option { + match self { + Value::Bool(b) => Some(*b), + _ => None, + } + } + + pub fn as_bytes(&self) -> Option<&[u8]> { + match self { + Value::Bytes(b) => Some(b), + _ => None, + } + } + + pub fn as_chunk(&self) -> Option { + match self { + Value::Chunk(c) => Some(*c), + _ => None, + } + } +} + +impl From for Value { + fn from(v: bool) -> Self { + Value::Bool(v) + } +} + +impl From for Value { + fn from(v: i64) -> Self { + Value::I64(v) + } +} + +impl From for Value { + fn from(v: f64) -> Self { + Value::F64(v) + } +} + +impl From<&str> for Value { + fn from(v: &str) -> Self { + Value::Str(v.to_owned()) + } +} + +impl From for Value { + fn from(v: String) -> Self { + Value::Str(v) + } +} + +impl From> for Value { + fn from(v: Vec) -> Self { + Value::Bytes(v) + } +} + +impl From for Value { + fn from(v: ChunkRef) -> Self { + Value::Chunk(v) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ids_and_values_round_trip() { + let id = ElemId::new(); + assert_ne!(id, ElemId::new()); + assert_eq!(ElemId::from_hex(&id.hex()), Some(id)); + assert_eq!(ElemId::from_loro(&id.to_loro()), Some(id)); + assert_eq!(ElemId::from_hex("XYZ"), None); + let chunk = ChunkRef { hash: [3; 32], len: 9 }; + for v in [Value::Null, Value::Bool(true), Value::I64(-3), Value::F64(1.5), Value::from("é"), Value::Bytes(vec![1, 2]), Value::Chunk(chunk)] { + assert_eq!(Value::from_loro(&v.to_loro()), Some(v)); + } + assert!(Value::Bytes(vec![0; MAX_VALUE_BYTES]).check().is_ok()); + assert!(Value::Bytes(vec![0; MAX_VALUE_BYTES + 1]).check().is_err()); + let blobs = MemoryBlobs::default(); + let r = blobs.put(b"abc").unwrap(); + assert_eq!(&*blobs.get(&r).unwrap(), b"abc"); + } +} diff --git a/crates/kit/src/writer.rs b/crates/kit/src/writer.rs new file mode 100644 index 0000000..fc4ea0d --- /dev/null +++ b/crates/kit/src/writer.rs @@ -0,0 +1,149 @@ +//! Writing one local change into Loro: the mappings write through a [`Writer`], which counts what they write and +//! commits before a commit's export would pass [`SPLIT_BYTES`] (a 5 MB paste becomes several commits, each well under +//! the agent's 1 MiB cap, one undo step together). Each commit's export is taken at once ([`export_since`]); an export +//! that is still over the limit (an undo of a big delete, which Loro makes as one commit) is cut by counter range, so +//! no update the kit sends is over the cap. + +use loro::{IdSpan, LoroDoc, LoroMap, LoroValue, ValueOrContainer, VersionVector}; + +use crate::{KitError, Value}; + +/// A commit's estimated export is kept at or under this (the agent refuses an update over 1 MiB). +pub const SPLIT_BYTES: usize = 512 * 1024; +/// What one operation adds to the estimate beside its payload (IDs, positions, keys). +pub const OP_OVERHEAD: usize = 32; +/// The origin of the kit's commits of local changes (the undo manager records them). +pub const LOCAL_ORIGIN: &str = "craft-cloud-kit:local"; +/// The origin of the kit's own repairs (a normalised merge, a save mark): never an undo step. +pub const KIT_ORIGIN: &str = "craft-cloud-kit:kit"; + +/// The updates one change made: the exports of its commits, in order, each at most [`SPLIT_BYTES`] (one operation +/// over it alone, at most the agent's 1 MiB cap: values and text pieces are capped below it). +pub type Updates = Vec>; + +/// The write side of one change. +pub struct Writer<'d> { + pub(crate) doc: &'d LoroDoc, + origin: &'static str, + /// The estimate of what the open commit holds. + spent: usize, + /// Where the open commit started. + from: VersionVector, + out: Updates, + commits: usize, + /// The text this change inserted and deleted, in bytes (what an undo of it would have Loro go through). + pub(crate) text_bytes: usize, +} + +impl<'d> Writer<'d> { + pub(crate) fn new(doc: &'d LoroDoc, origin: &'static str) -> Self { + Self { doc, origin, spent: 0, from: doc.oplog_vv(), out: Vec::new(), commits: 0, text_bytes: 0 } + } + + /// Room left in the open commit before it is cut. + pub(crate) fn room(&self) -> usize { + SPLIT_BYTES.saturating_sub(self.spent).saturating_sub(OP_OVERHEAD) + } + + /// Counts an operation of `payload` bytes, committing first if it wouldn't fit in the open commit. + pub(crate) fn spend(&mut self, payload: usize) -> Result<(), KitError> { + let cost = payload.saturating_add(OP_OVERHEAD); + if self.spent > 0 && self.spent.saturating_add(cost) > SPLIT_BYTES { + self.cut()?; + } + self.spent = self.spent.saturating_add(cost); + Ok(()) + } + + /// Commits what is open and takes its export. + pub(crate) fn cut(&mut self) -> Result<(), KitError> { + if self.spent == 0 { + return Ok(()); + } + self.doc.set_next_commit_origin(self.origin); + self.doc.commit(); + self.commits += 1; + export_since(self.doc, &self.from, &mut self.out)?; + self.from = self.doc.oplog_vv(); + self.spent = 0; + Ok(()) + } + + /// The change's updates (the last commit made now). + pub(crate) fn finish(mut self) -> Result<(Updates, usize), KitError> { + self.cut()?; + // Anything written without being counted (there should be none) still goes out. + self.doc.set_next_commit_origin(self.origin); + self.doc.commit(); + export_since(self.doc, &self.from, &mut self.out)?; + Ok((self.out, self.commits)) + } + + /// Sets `key` of `map` to `value` unless it holds that already. + pub(crate) fn set(&mut self, map: &LoroMap, key: &str, value: &Value) -> Result { + value.check()?; + let new = value.to_loro(); + if let Some(ValueOrContainer::Value(old)) = map.get(key) + && old == new + { + return Ok(false); + } + self.spend(value.weight().saturating_add(key.len()))?; + map.insert(key, new)?; + Ok(true) + } + + /// Sets `key` of `map` to a raw Loro value unless it holds it already. + pub(crate) fn set_raw(&mut self, map: &LoroMap, key: &str, value: LoroValue, weight: usize) -> Result { + if let Some(ValueOrContainer::Value(old)) = map.get(key) + && old == value + { + return Ok(false); + } + self.spend(weight.saturating_add(key.len()))?; + map.insert(key, value)?; + Ok(true) + } + + /// Deletes `key` of `map` if it is there. + pub(crate) fn delete(&mut self, map: &LoroMap, key: &str) -> Result { + if map.get(key).is_none() { + return Ok(false); + } + self.spend(key.len())?; + map.delete(key)?; + Ok(true) + } +} + +/// The export of every change since `from` (all made by this peer: a local change, an undo), cut by counter range +/// where it is over [`SPLIT_BYTES`], appended to `out`. +pub(crate) fn export_since(doc: &LoroDoc, from: &VersionVector, out: &mut Updates) -> Result<(), KitError> { + let now = doc.oplog_vv(); + let mut spans = Vec::new(); + for (peer, end) in now.iter() { + let start = from.get(peer).copied().unwrap_or(0); + if *end > start { + spans.push(IdSpan::new(*peer, start, *end)); + } + } + for span in spans { + slice(doc, span, out)?; + } + Ok(()) +} + +fn slice(doc: &LoroDoc, span: IdSpan, out: &mut Updates) -> Result<(), KitError> { + let bytes = doc.export(loro::ExportMode::updates_in_range(vec![span]))?; + let (start, end) = (span.counter.start, span.counter.end); + if bytes.len() <= SPLIT_BYTES || end - start <= 1 { + // (One operation over the cap can't happen: values and text pieces are capped before they are written. Were + // it to, it still goes: the commit is made, and the agent's refusal stops the session where the app sees it, + // rather than leaving a hole every later change depends on.) + out.push(bytes); + return Ok(()); + } + let mid = start + (end - start) / 2; + slice(doc, IdSpan::new(span.peer, start, mid), out)?; + slice(doc, IdSpan::new(span.peer, mid, end), out) +} diff --git a/crates/kit/tests/common/mod.rs b/crates/kit/tests/common/mod.rs new file mode 100644 index 0000000..a81f15a --- /dev/null +++ b/crates/kit/tests/common/mod.rs @@ -0,0 +1,161 @@ +//! A small model for the kit's tests: a title (an opaque value) and a list of notes (a table with one column), plus +//! helpers to drive sessions on the fake agent. +#![allow(dead_code)] + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use craft_cloud_app::fake::Fake; +use craft_cloud_kit::{Agent, ElemId, KitError, LiveDocument, Mappings, Opaque, Open, RemotePatch, Session, TableSource, Value}; + +/// The one column of the list. +pub const TEXT: ElemId = ElemId(1); + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Notes { + pub title: Arc, + pub items: Arc>, + /// Cells known changed since the previous model (the hint), when the change says. + pub hint: Option>, +} + +impl Notes { + pub fn new(title: &str, items: &[&str]) -> Self { + Self { title: Arc::new(title.into()), items: Arc::new(items.iter().map(|t| (ElemId::new(), (*t).to_owned())).collect()), hint: None } + } + + pub fn texts(&self) -> Vec { + self.items.iter().map(|(_, t)| t.clone()).collect() + } + + pub fn with_title(&self, title: &str) -> Self { + Self { title: Arc::new(title.into()), hint: None, ..self.clone() } + } + + pub fn with_item(&self, at: usize, text: &str) -> Self { + let mut items = (*self.items).clone(); + let id = ElemId::new(); + items.insert(at.min(items.len()), (id, text.to_owned())); + Self { items: Arc::new(items), hint: Some(vec![id]), ..self.clone() } + } + + pub fn with_edit(&self, at: usize, text: &str) -> Self { + let mut items = (*self.items).clone(); + let mut hint = Vec::new(); + if let Some(item) = items.get_mut(at) { + item.1 = text.to_owned(); + hint.push(item.0); + } + Self { items: Arc::new(items), hint: Some(hint), ..self.clone() } + } + + pub fn without(&self, at: usize) -> Self { + let mut items = (*self.items).clone(); + if at < items.len() { + items.remove(at); + } + Self { items: Arc::new(items), hint: Some(Vec::new()), ..self.clone() } + } +} + +pub struct Items<'a>(&'a Notes); + +impl TableSource for Items<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(&self.0.items) as usize) + } + + fn rows(&self) -> Vec { + self.0.items.iter().map(|(id, _)| *id).collect() + } + + fn columns(&self) -> Vec { + vec![TEXT] + } + + fn cell(&self, row: ElemId, column: ElemId) -> Option { + if column != TEXT { + return None; + } + self.0.items.iter().find(|(id, _)| *id == row).map(|(_, t)| Value::from(t.as_str())) + } + + fn cells(&self) -> Vec<(ElemId, ElemId, Value)> { + self.0.items.iter().map(|(id, t)| (*id, TEXT, Value::from(t.as_str()))).collect() + } + + fn changed_cells(&self) -> Option> { + self.0.hint.as_ref().map(|ids| ids.iter().map(|id| (*id, TEXT)).collect()) + } +} + +/// The notes app, at layout version `V`. +pub struct NotesApp; + +impl LiveDocument for NotesApp { + type Doc = Notes; + const APP: &'static str = "notes"; + const SCHEMA: u32 = V; + + fn mappings(doc: &Notes) -> Mappings<'_> { + Mappings::new().opaque("title", Opaque { value: Some(&*doc.title), key: Some(Arc::as_ptr(&doc.title) as usize) }).table("items", Items(doc)) + } + + fn apply_remote(old: &Arc, patch: &RemotePatch<'_>) -> Result, KitError> { + let mut new = (**old).clone(); + new.hint = None; + if let Some(title) = patch.opaque("title") { + new.title = Arc::new(String::from_utf8_lossy(&title.unwrap_or_default()).into_owned()); + } + if let Some(table) = patch.table("items") { + let items = table + .rows() + .into_iter() + .map(|row| (row, table.cell(row, TEXT).and_then(|v| v.as_str().map(str::to_owned)).unwrap_or_default())) + .collect(); + new.items = Arc::new(items); + } + Ok(Arc::new(new)) + } + + fn normalise(_doc: &mut Notes) {} + + fn refresh_derived(_doc: &mut Notes) {} +} + +pub type App = NotesApp<1>; + +pub fn agent(fake: &Fake) -> Agent { + let client = fake.client(); + Agent::from_fn(move || Ok(client.clone())) +} + +/// Opens `path` live on `fake`'s machine (importing `file` if it was never live). +pub fn open(fake: &Fake, path: &str, file: D::Doc) -> Session { + match Session::::open(&agent(fake), path, Arc::new(file), None).unwrap() { + Open::Live(s) => *s, + Open::NotLive(why) => panic!("not live: {why:?}"), + } +} + +/// Polls every session until `done` holds (10 s at most). +pub fn until(sessions: &mut [&mut Session], what: &str, mut done: impl FnMut(&[&mut Session]) -> bool) { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + for s in sessions.iter_mut() { + s.poll(); + } + if done(sessions) { + return; + } + assert!(Instant::now() < deadline, "timed out waiting for: {what}"); + std::thread::sleep(Duration::from_millis(5)); + } +} + +/// Makes a change from the session's model. +pub fn change(s: &mut Session, f: impl FnOnce(&D::Doc) -> D::Doc) -> Result<(), KitError> { + let old = s.model().clone(); + let new = Arc::new(f(&old)); + s.local_change(&old, &new) +} diff --git a/crates/kit/tests/kit_sim.rs b/crates/kit/tests/kit_sim.rs new file mode 100644 index 0000000..0c2c369 --- /dev/null +++ b/crates/kit/tests/kit_sim.rs @@ -0,0 +1,193 @@ +//! The kit simulation: app sessions on three machines of one fake hub (two apps on the third), each making random +//! edits of every shape (tree, rich text, table, timeline, opaque), undoing and redoing, saving, while machines go +//! offline, drop their apps' connections and restart, and the live relay goes down while files still sync. Healed, +//! every session ends with the same model, every model is one the app could make (normalised), and the file catches up +//! with the autosave rule within a save per window. +//! +//! `SIM_SEEDS=5000 SIM_FIRST=0 cargo test --release -p craft-cloud-kit --test kit_sim` runs seeds 0 to 4999 on +//! `SIM_JOBS` threads (default 4); `SIM_SEED=17` runs one. A seed's history is random but not replayed exactly: the +//! fake agent's threads and the sessions' reconnects run on the clock. + +mod common; +mod model; + +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use common::{change, open}; +use craft_cloud_app::fake::{Fake, Hub}; +use craft_cloud_kit::{KitError, Session}; +use model::{Doc, Rng, SHAPES, TestApp, edit, valid}; + +const PATH: &str = "sim/story.kitdemo"; +const STEPS: usize = 80; +/// Which machine each session runs on: the third machine has two apps open on the document. +const ON: [usize; 4] = [0, 1, 2, 2]; + +fn run_seed(seed: u64) { + let hub = Hub::new(); + let fakes: Vec = (0..3).map(|i| hub.machine(&format!("m{i}")).unwrap()).collect(); + let mut rng = Rng(seed); + let start = model::random_doc(&mut rng, 12); + let mut sessions: Vec> = Vec::new(); + for (k, m) in ON.iter().enumerate() { + sessions.push(open::(&fakes[*m], PATH, if k == 0 { start.clone() } else { Doc::empty() })); + } + let mut stopped = [false; 3]; + for step in 0..STEPS { + let i = rng.below(sessions.len()); + let m = ON[i]; + match rng.below(24) { + 0..=11 => { + let shape = SHAPES[rng.below(SHAPES.len())]; + let mut r = rng.clone(); + match change(&mut sessions[i], |d| edit(d, shape, &mut r)) { + Ok(()) | Err(KitError::Stalled(_)) => {} + Err(e) => panic!("seed {seed} step {step}: session {i}'s change failed: {e}"), + } + rng = r; + } + 12 | 13 => { + sessions[i].undo().unwrap_or_else(|e| panic!("seed {seed} step {step}: undo: {e}")); + } + 14 => { + sessions[i].redo().unwrap_or_else(|e| panic!("seed {seed} step {step}: redo: {e}")); + } + 15 => { + if let Ok(point) = sessions[i].save_point() { + sessions[i].record(&point, format!("seed {seed} step {step}").as_bytes()); + } + } + 16 => fakes[m].set_online(false), + 17 | 18 => fakes[m].set_online(true), + 19 => fakes[m].drop_connections(), + 20 => { + fakes[m].restart(); + stopped[m] = true; + } + 21 => { + fakes[m].start(); + stopped[m] = false; + } + // The live relay goes down (files still sync) or comes back. + 22 => hub.set_live(rng.below(2) == 0), + _ => std::thread::sleep(Duration::from_millis(rng.below(3) as u64)), + } + for s in &mut sessions { + s.poll(); + } + } + // Healed: every machine runs and is online, the live relay is up, and every session tries at once. + hub.set_live(true); + for f in &fakes { + f.start(); + f.set_online(true); + } + for s in &mut sessions { + s.reconnect_now(); + } + let deadline = Instant::now() + Duration::from_secs(60); + loop { + for s in &mut sessions { + s.poll(); + } + let quiet = sessions.iter().all(|s| s.status().connected && s.status().queued == 0) && fakes.iter().all(|f| f.waiting(PATH) == 0); + if quiet { + let first = sessions[0].model().clone(); + if sessions.iter().all(|s| **s.model() == *first) { + break; + } + } + if Instant::now() > deadline { + let views: Vec = sessions.iter().map(|s| format!("{:?}\n {:?}", s.status(), s.model())).collect(); + panic!("seed {seed}: the sessions never agreed:\n{}", views.join("\n")); + } + std::thread::sleep(Duration::from_millis(2)); + } + // The file catches up on its own: one window saves, then each saves while its file is behind (as an app's + // autosave does), and soon none is; the file then has the version every session has. + if let Ok(point) = sessions[0].save_point() { + sessions[0].record(&point, b"healed"); + } + let heads = |s: &Session| { + let doc = loro::LoroDoc::new(); + doc.import(&s.snapshot().unwrap()).unwrap(); + let mut v: Vec<(u64, u32)> = doc.oplog_frontiers().iter().map(|id| (id.peer, u32::try_from(id.counter).unwrap())).collect(); + v.sort_unstable(); + v + }; + let (mut saves, mut quiet) = (0usize, 0); + // (A save the file being behind asks for may wait out `CATCH_UP_EVERY` after the window's last one.) + while quiet < 30 || sessions.iter().any(|s| hub.file(PATH) != Some(heads(s))) { + let mut saved = false; + for s in &mut sessions { + s.poll(); + if s.file_behind() + && let Ok(point) = s.save_point() + { + s.record(&point, format!("caught up {saves}").as_bytes()); + saves += 1; + saved = true; + } + } + quiet = if saved { 0 } else { quiet + 1 }; + assert!(Instant::now() < deadline + Duration::from_secs(30), "seed {seed}: the file never caught up ({saves} saves)"); + std::thread::sleep(Duration::from_millis(2)); + } + assert!(saves <= sessions.len(), "seed {seed}: {saves} saves to catch up"); + for (i, s) in sessions.iter().enumerate() { + assert_eq!(hub.file(PATH), Some(heads(s)), "seed {seed}: session {i}'s edits aren't all in the file"); + } + for (i, s) in sessions.iter().enumerate() { + if let Err(why) = valid(s.model()) { + panic!("seed {seed}: session {i}'s model isn't one the app could make: {why}"); + } + assert!(s.status().read_only.is_none()); + // The model the session built change by change (texts patched from Loro's deltas) is the one its document + // reads back whole. + let mut replica = craft_cloud_kit::raw::Replica::default(); + replica.import(&[s.snapshot().unwrap()], &["layers"]).unwrap(); + let mut whole = (*replica.read::(&std::sync::Arc::new(Doc::empty())).unwrap()).clone(); + ::normalise(&mut whole); + ::refresh_derived(&mut whole); + assert!(whole == **s.model(), "seed {seed}: session {i}'s model isn't what its document reads back:\n{whole:?}\n{:?}", s.model()); + } +} + +/// The seeds the environment asks for, on threads; the lowest failing seed is reported. +#[test] +fn kit_sessions_converge() { + if let Ok(seed) = std::env::var("SIM_SEED") { + run_seed(seed.parse().unwrap()); + return; + } + let seeds: u64 = std::env::var("SIM_SEEDS").ok().and_then(|s| s.parse().ok()).unwrap_or(16); + let first: u64 = std::env::var("SIM_FIRST").ok().and_then(|s| s.parse().ok()).unwrap_or(0); + let jobs: usize = std::env::var("SIM_JOBS").ok().and_then(|s| s.parse().ok()).unwrap_or(4); + let next = AtomicU64::new(first); + let failed: Mutex> = Mutex::new(None); + std::thread::scope(|scope| { + for _ in 0..jobs { + scope.spawn(|| { + loop { + let seed = next.fetch_add(1, Ordering::Relaxed); + if seed >= first + seeds || failed.lock().unwrap().as_ref().is_some_and(|(s, _)| *s < seed) { + return; + } + if let Err(e) = std::panic::catch_unwind(|| run_seed(seed)) { + let message = + e.downcast_ref::().cloned().or_else(|| e.downcast_ref::<&str>().map(|s| (*s).to_owned())).unwrap_or_default(); + let mut f = failed.lock().unwrap(); + if f.as_ref().is_none_or(|(s, _)| seed < *s) { + *f = Some((seed, message)); + } + } + } + }); + } + }); + if let Some((seed, message)) = failed.into_inner().unwrap() { + panic!("seed {seed} failed (SIM_SEED={seed} replays it, as near as the clock allows): {message}"); + } +} diff --git a/crates/kit/tests/mappings.rs b/crates/kit/tests/mappings.rs new file mode 100644 index 0000000..f55eb93 --- /dev/null +++ b/crates/kit/tests/mappings.rs @@ -0,0 +1,321 @@ +//! The mappings, without a session: every shape round-trips (model, Loro, model), merges (two replicas with random +//! edits end equal, and normalised to a model the app could make), and writes nothing for what didn't change. Then the +//! cases each shape has to get right: marks across a split and a join, typing at one spot at once, rows inserted at +//! once, moves that would make a cycle, keyframes added at once. + +mod model; + +use std::sync::Arc; + +use craft_cloud_kit::raw::Replica; +use craft_cloud_kit::{RichText, Value}; +use model::{Doc, Grid, Item, Key, Node, Rng, SHAPES, Shape, TestApp, Timeline, edit, valid}; +use proptest::prelude::*; + +const TREES: &[&str] = &["layers"]; + +/// Reads a replica's whole model and normalises it as a session does (writing back what that repaired). +fn settle(r: &mut Replica) -> Doc { + let base = Arc::new(Doc::empty()); + let raw = r.read::(&base).unwrap(); + let mut fixed = (*raw).clone(); + ::normalise(&mut fixed); + r.write::(Some(&raw), &fixed).unwrap(); + ::refresh_derived(&mut fixed); + fixed +} + +fn shape_doc(rng: &mut Rng, shape: Shape, n: usize) -> Doc { + let mut d = Doc::empty(); + for _ in 0..n { + d = edit(&d, shape, rng); + } + d +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 96, .. ProptestConfig::default() })] + + /// Model to Loro and back: an import, then every change after it, read back exactly. + #[test] + fn every_shape_round_trips(seed in any::(), shape in 0usize..6) { + let mut rng = Rng(seed); + let mut doc = match SHAPES.get(shape) { + Some(s) => shape_doc(&mut rng, *s, 12), + None => model::random_doc(&mut rng, 30), + }; + let mut r = Replica::default(); + r.write::(None, &doc).unwrap(); + r.reindex(TREES); + prop_assert_eq!(&settle(&mut r), &doc); + for _ in 0..12 { + let s = SHAPES.get(shape).copied().unwrap_or(SHAPES[rng.below(5)]); + let next = edit(&doc, s, &mut rng); + r.write::(Some(&doc), &next).unwrap(); + doc = settle(&mut r); + prop_assert_eq!(&doc, &next); + } + } + + /// Two replicas from one import, random edits on both, then each takes the other's: both end equal, and valid. + #[test] + fn every_shape_merges(seed in any::(), shape in 0usize..6) { + let mut rng = Rng(seed); + let start = model::random_doc(&mut rng, 15); + let mut a = Replica::default(); + let (import, _) = a.write::(None, &start).unwrap(); + a.reindex(TREES); + let mut b = Replica::default(); + b.import(&import, TREES).unwrap(); + let (mut da, mut db) = (settle(&mut a), settle(&mut b)); + for _round in 0..3 { + let (mut out_a, mut out_b) = (Vec::new(), Vec::new()); + for _ in 0..rng.below(6) { + let s = SHAPES.get(shape).copied().unwrap_or(SHAPES[rng.below(5)]); + let next = edit(&da, s, &mut rng); + out_a.extend(a.write::(Some(&da), &next).unwrap().0); + da = next; + } + for _ in 0..rng.below(6) { + let s = SHAPES.get(shape).copied().unwrap_or(SHAPES[rng.below(5)]); + let next = edit(&db, s, &mut rng); + out_b.extend(b.write::(Some(&db), &next).unwrap().0); + db = next; + } + a.import(&out_b, TREES).unwrap(); + b.import(&out_a, TREES).unwrap(); + settle(&mut a); + settle(&mut b); + // What normalising wrote on each goes to the other too. + let (fa, fb) = (a.doc.export(loro_updates(&b)).unwrap(), b.doc.export(loro_updates(&a)).unwrap()); + b.import(&[fa], TREES).unwrap(); + a.import(&[fb], TREES).unwrap(); + da = settle(&mut a); + db = settle(&mut b); + prop_assert_eq!(&da, &db); + prop_assert!(valid(&da).is_ok(), "{:?}", valid(&da)); + } + } +} + +fn loro_updates(other: &Replica) -> loro::ExportMode<'static> { + loro::ExportMode::updates_owned(other.doc.oplog_vv()) +} + +/// Ops one change made. +fn ops(r: &mut Replica, old: &Doc, new: &Doc) -> i64 { + let before = r.ops(); + r.write::(Some(old), new).unwrap(); + r.ops() - before +} + +/// Unchanged parts write nothing: the same content in new allocations writes nothing, and one small change in one +/// part writes one operation (unchanged subtrees, cells and items aren't even compared). +#[test] +fn unchanged_parts_write_nothing() { + let mut rng = Rng(7); + let mut r = Replica::default(); + let doc = model::random_doc(&mut rng, 60); + let doc = { + let mut d = doc; + if model::flatten(&d.layers).is_empty() { + d.layers = Arc::new(vec![Arc::new(Node { id: rng.id(), name: "l".into(), children: Vec::new() })]); + } + d + }; + r.write::(None, &doc).unwrap(); + r.reindex(TREES); + let doc = settle(&mut r); + // Equal content, every part in a new allocation: compared, nothing written. + let copy = Doc { + layers: Arc::new(doc.layers.iter().map(|n| Arc::new((**n).clone())).collect()), + story: Arc::new((*doc.story).clone()), + grid: Arc::new((*doc.grid).clone()), + timeline: Arc::new((*doc.timeline).clone()), + settings: Arc::new((*doc.settings).clone()), + words: doc.words, + }; + assert_eq!(ops(&mut r, &doc, ©), 0, "the same content writes nothing"); + let doc = copy; + // One keystroke. + let typed = Doc { story: Arc::new(model::type_at(&doc.story, 0, "Z")), ..doc.clone() }; + assert_eq!(ops(&mut r, &doc, &typed), 1, "a keystroke is one insert"); + let doc = typed; + // One layer renamed. + let first = model::flatten(&doc.layers)[0].0; + let renamed = Doc { layers: Arc::new(model::rename(&doc.layers, first, "renamed")), ..doc.clone() }; + assert_eq!(ops(&mut r, &doc, &renamed), 1, "one property"); + let doc = renamed; + // One cell, one keyframe value, the settings. + let mut g = (*doc.grid).clone(); + if g.rows.is_empty() || g.cols.is_empty() { + g.rows.push(rng.id()); + g.cols.push(rng.id()); + let next = Doc { grid: Arc::new(g.clone()), ..doc.clone() }; + r.write::(Some(&doc), &next).unwrap(); + } + let doc = Doc { grid: Arc::new(g.clone()), ..doc }; + g.cells.insert((g.rows[0], g.cols[0]), "cell!".into()); + let celled = Doc { grid: Arc::new(g), ..doc.clone() }; + assert_eq!(ops(&mut r, &doc, &celled), 1, "one cell"); + let doc = celled; + let settings = Doc { settings: Arc::new(vec![1, 2, 3]), ..doc.clone() }; + assert_eq!(ops(&mut r, &doc, &settings), 1, "one value"); +} + +/// Two replicas of `doc`. +fn pair(doc: &Doc) -> (Replica, Replica, Doc) { + let mut a = Replica::default(); + let (import, _) = a.write::(None, doc).unwrap(); + a.reindex(TREES); + let mut b = Replica::default(); + b.import(&import, TREES).unwrap(); + let d = settle(&mut a); + (a, b, d) +} + +/// Each takes the other's changes; both models, equal. +fn exchange(a: &mut Replica, b: &mut Replica) -> Doc { + let (fa, fb) = (a.doc.export(loro_updates(b)).unwrap(), b.doc.export(loro_updates(a)).unwrap()); + b.import(&[fa], TREES).unwrap(); + a.import(&[fb], TREES).unwrap(); + let (da, db) = (settle(a), settle(b)); + assert_eq!(da, db); + assert!(valid(&da).is_ok(), "{:?}", valid(&da)); + da +} + +fn with_story(text: RichText) -> Doc { + Doc { story: Arc::new(text), ..Doc::empty() } +} + +/// Character marks across a paragraph split and a join, made on one machine while the other formats the same words: +/// both end with the bold across both paragraphs, then across the joined one; the split paragraph keeps its heading +/// on both halves. +#[test] +fn marks_survive_a_split_and_a_join() { + let base = model::format_para(&RichText::plain("Hello world"), 0, "h", Some(Value::I64(1))); + let (mut a, mut b, doc) = pair(&with_story(base)); + // A splits the paragraph after "Hello"; B makes "lo wo" bold. + let split = Doc { story: Arc::new(model::type_at(&doc.story, 5, "\n")), ..doc.clone() }; + a.write::(Some(&doc), &split).unwrap(); + let bold = Doc { story: Arc::new(model::format(&doc.story, 3..8, "b", Some(Value::Bool(true)))), ..doc.clone() }; + b.write::(Some(&doc), &bold).unwrap(); + let merged = exchange(&mut a, &mut b); + assert_eq!(merged.story.text, "Hello\n world"); + let bolded: String = merged.story.text.char_indices().filter(|(i, _)| merged.story.attrs_at(*i).contains_key("b")).map(|(_, c)| c).collect(); + assert_eq!(bolded.replace('\n', ""), "lo wo", "the bold is on both sides of the split"); + assert_eq!(merged.story.paragraphs.len(), 2); + assert!(merged.story.paragraphs.iter().all(|p| p.get("h") == Some(&Value::I64(1))), "{:?}", merged.story.paragraphs); + // A joins them again; B italicises "world". + let joined = Doc { story: Arc::new(model::delete(&merged.story, 5..6)), ..merged.clone() }; + a.write::(Some(&merged), &joined).unwrap(); + let italic = Doc { story: Arc::new(model::format(&merged.story, 7..12, "i", Some(Value::Bool(true)))), ..merged.clone() }; + b.write::(Some(&merged), &italic).unwrap(); + let merged = exchange(&mut a, &mut b); + assert_eq!(merged.story.text, "Hello world"); + assert_eq!(merged.story.paragraphs.len(), 1); + let bolded: String = merged.story.text.char_indices().filter(|(i, _)| merged.story.attrs_at(*i).contains_key("b")).map(|(_, c)| c).collect(); + assert_eq!(bolded, "lo wo"); + let italic: String = merged.story.text.char_indices().filter(|(i, _)| merged.story.attrs_at(*i).contains_key("i")).map(|(_, c)| c).collect(); + assert_eq!(italic, "world"); +} + +/// Two machines typing at the same spot at once: both words, each whole (not interleaved). +#[test] +fn typing_at_one_spot_at_once_keeps_both_words() { + let (mut a, mut b, doc) = pair(&with_story(RichText::plain("ab"))); + let ta = Doc { story: Arc::new(model::type_at(&doc.story, 1, "XYZ")), ..doc.clone() }; + a.write::(Some(&doc), &ta).unwrap(); + let tb = Doc { story: Arc::new(model::type_at(&doc.story, 1, "123")), ..doc.clone() }; + b.write::(Some(&doc), &tb).unwrap(); + let merged = exchange(&mut a, &mut b); + assert!(merged.story.text == "aXYZ123b" || merged.story.text == "a123XYZb", "{}", merged.story.text); +} + +/// Two machines inserting a row at the same place at once: both rows, each with its cells. +#[test] +fn rows_inserted_at_once_are_both_kept() { + let mut rng = Rng(3); + let (r0, c0) = (rng.id(), rng.id()); + let grid = Grid { rows: vec![r0], cols: vec![c0], cells: [((r0, c0), "first".to_owned())].into() }; + let (mut a, mut b, doc) = pair(&Doc { grid: Arc::new(grid), ..Doc::empty() }); + let (ra, rb) = (rng.id(), rng.id()); + let mut ga = (*doc.grid).clone(); + ga.rows.insert(0, ra); + ga.cells.insert((ra, c0), "from a".into()); + let mut gb = (*doc.grid).clone(); + gb.rows.insert(0, rb); + gb.cells.insert((rb, c0), "from b".into()); + a.write::(Some(&doc), &Doc { grid: Arc::new(ga), ..doc.clone() }).unwrap(); + b.write::(Some(&doc), &Doc { grid: Arc::new(gb), ..doc.clone() }).unwrap(); + let merged = exchange(&mut a, &mut b); + assert_eq!(merged.grid.rows.len(), 3); + assert_eq!(merged.grid.rows.last(), Some(&r0)); + assert_eq!(merged.grid.cells.get(&(ra, c0)).map(String::as_str), Some("from a")); + assert_eq!(merged.grid.cells.get(&(rb, c0)).map(String::as_str), Some("from b")); + // A row deleted on one machine while its cell is edited on the other: the cell isn't read back. + let mut gd = (*merged.grid).clone(); + gd.rows.retain(|r| *r != ra); + gd.cells.retain(|(r, _), _| *r != ra); + let mut ge = (*merged.grid).clone(); + ge.cells.insert((ra, c0), "edited".into()); + a.write::(Some(&merged), &Doc { grid: Arc::new(gd), ..merged.clone() }).unwrap(); + b.write::(Some(&merged), &Doc { grid: Arc::new(ge), ..merged.clone() }).unwrap(); + let merged = exchange(&mut a, &mut b); + assert!(!merged.grid.rows.contains(&ra) && !merged.grid.cells.contains_key(&(ra, c0))); +} + +/// Two machines moving two layers under each other at once: Loro drops one move, both end with the same tree and no +/// cycle, and both layers are kept. +#[test] +fn moves_that_would_make_a_cycle_end_in_one_tree() { + let mut rng = Rng(11); + let (x, y) = (rng.id(), rng.id()); + let layers = + vec![Arc::new(Node { id: x, name: "x".into(), children: Vec::new() }), Arc::new(Node { id: y, name: "y".into(), children: Vec::new() })]; + let (mut a, mut b, doc) = pair(&Doc { layers: Arc::new(layers), ..Doc::empty() }); + let (rest, taken) = model::remove(&doc.layers, x); + let a_moves = Doc { layers: Arc::new(model::insert(&rest, Some(y), 0, taken.unwrap())), ..doc.clone() }; + let (rest, taken) = model::remove(&doc.layers, y); + let b_moves = Doc { layers: Arc::new(model::insert(&rest, Some(x), 0, taken.unwrap())), ..doc.clone() }; + a.write::(Some(&doc), &a_moves).unwrap(); + b.write::(Some(&doc), &b_moves).unwrap(); + let merged = exchange(&mut a, &mut b); + let all = model::flatten(&merged.layers); + assert_eq!(all.len(), 2, "both layers kept: {all:?}"); + assert_eq!(merged.layers.len(), 1, "one under the other, no cycle"); +} + +/// Two machines adding keyframes to one item at once: both kept; and an item trimmed on one machine while moved to +/// another track on the other keeps both changes. +#[test] +fn keyframes_added_at_once_on_one_item_are_both_kept() { + let mut rng = Rng(5); + let (t1, t2, item) = (rng.id(), rng.id(), rng.id()); + let tl = Timeline { + tracks: vec![(t1, "video".into()), (t2, "audio".into())], + items: [(item, Item { track: t1, start: 10, len: 50, name: "clip".into() })].into(), + keys: Default::default(), + }; + let (mut a, mut b, doc) = pair(&Doc { timeline: Arc::new(tl), ..Doc::empty() }); + let (ka, kb) = (rng.id(), rng.id()); + let mut ta = (*doc.timeline).clone(); + ta.keys.insert(ka, Key { item, at: 5, value: 1 }); + let mut tb = (*doc.timeline).clone(); + tb.keys.insert(kb, Key { item, at: 5, value: 2 }); + a.write::(Some(&doc), &Doc { timeline: Arc::new(ta), ..doc.clone() }).unwrap(); + b.write::(Some(&doc), &Doc { timeline: Arc::new(tb), ..doc.clone() }).unwrap(); + let merged = exchange(&mut a, &mut b); + assert_eq!(merged.timeline.keys.len(), 2); + let mut ta = (*merged.timeline).clone(); + ta.items.get_mut(&item).unwrap().len = 20; + let mut tb = (*merged.timeline).clone(); + tb.items.get_mut(&item).unwrap().track = t2; + a.write::(Some(&merged), &Doc { timeline: Arc::new(ta), ..merged.clone() }).unwrap(); + b.write::(Some(&merged), &Doc { timeline: Arc::new(tb), ..merged.clone() }).unwrap(); + let merged = exchange(&mut a, &mut b); + let i = &merged.timeline.items[&item]; + assert_eq!((i.track, i.len), (t2, 20), "both changes kept"); + assert_eq!(merged.timeline.keys.len(), 2, "its keyframes go with it"); +} diff --git a/crates/kit/tests/model/mod.rs b/crates/kit/tests/model/mod.rs new file mode 100644 index 0000000..5ad551f --- /dev/null +++ b/crates/kit/tests/model/mod.rs @@ -0,0 +1,710 @@ +//! A model with every shape the kit maps, for the mapping tests and the kit simulation: a layer tree, a story (rich +//! text), a grid (table), a timeline and settings (opaque), and random edits of each the way an app makes them +//! (copy-on-write, unchanged parts shared by `Arc`). +#![allow(dead_code)] + +use std::collections::{BTreeMap, HashMap}; +use std::ops::Range; +use std::sync::Arc; + +use craft_cloud_kit::{ + Attrs, ElemId, KitError, LiveDocument, Mappings, Opaque, Props, RemotePatch, RichText, Span, TableSource, TextSource, TimelineSource, TreeSource, + Value, +}; + +/// A small deterministic random source (SplitMix64). +#[derive(Debug, Clone)] +pub struct Rng(pub u64); + +impl Rng { + pub fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + pub fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { (self.next() % n as u64) as usize } + } + + pub fn chance(&mut self, percent: u64) -> bool { + self.next() % 100 < percent + } + + pub fn id(&mut self) -> ElemId { + ElemId(u128::from(self.next()) << 64 | u128::from(self.next())) + } + + pub fn word(&mut self) -> String { + const WORDS: &[&str] = &["a", "bc", "déf", "ghij", " ", "x", "日本", "🙂", "yz", "kl m"]; + WORDS[self.below(WORDS.len())].to_owned() + } +} + +// The tree. + +#[derive(Debug, Clone, PartialEq)] +pub struct Node { + pub id: ElemId, + pub name: String, + pub children: Vec>, +} + +pub type Roots = Arc>>; + +pub struct LayersSrc<'a> { + roots: &'a Roots, + index: HashMap>, +} + +impl<'a> LayersSrc<'a> { + pub fn new(roots: &'a Roots) -> Self { + let mut index = HashMap::new(); + let mut stack: Vec<&'a Arc> = roots.iter().collect(); + while let Some(n) = stack.pop() { + index.insert(n.id, n); + stack.extend(n.children.iter()); + } + Self { roots, index } + } +} + +impl TreeSource for LayersSrc<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(self.roots) as usize) + } + fn roots(&self) -> Vec { + self.roots.iter().map(|n| n.id).collect() + } + fn children(&self, id: ElemId) -> Vec { + self.index.get(&id).map(|n| n.children.iter().map(|c| c.id).collect()).unwrap_or_default() + } + fn subtree_key(&self, id: ElemId) -> Option { + self.index.get(&id).map(|n| Arc::as_ptr(n) as usize) + } + fn write(&self, id: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError> { + if let Some(n) = self.index.get(&id) { + props.set("name", n.name.as_str())?; + } + Ok(()) + } +} + +/// Every node: id, parent, depth-first. +pub fn flatten(roots: &[Arc]) -> Vec<(ElemId, Option)> { + let mut out = Vec::new(); + let mut stack: Vec<(&Arc, Option)> = roots.iter().rev().map(|n| (n, None)).collect(); + while let Some((n, parent)) = stack.pop() { + out.push((n.id, parent)); + stack.extend(n.children.iter().rev().map(|c| (c, Some(n.id)))); + } + out +} + +fn contains(node: &Node, id: ElemId) -> bool { + node.id == id || node.children.iter().any(|c| contains(c, id)) +} + +/// `roots` without `id`'s subtree, which is returned. +pub fn remove(roots: &[Arc], id: ElemId) -> (Vec>, Option>) { + let mut taken = None; + let mut out = Vec::new(); + for n in roots { + if n.id == id { + taken = Some(n.clone()); + } else if taken.is_none() && contains(n, id) { + let (children, t) = remove(&n.children, id); + taken = t; + out.push(Arc::new(Node { id: n.id, name: n.name.clone(), children })); + } else { + out.push(n.clone()); + } + } + (out, taken) +} + +/// `roots` with `node` under `parent` (top level when `None`) at `at`. +pub fn insert(roots: &[Arc], parent: Option, at: usize, node: Arc) -> Vec> { + match parent { + None => { + let mut out = roots.to_vec(); + out.insert(at.min(out.len()), node); + out + } + Some(p) => roots + .iter() + .map(|n| { + if n.id == p { + let mut children = n.children.clone(); + children.insert(at.min(children.len()), node.clone()); + Arc::new(Node { id: n.id, name: n.name.clone(), children }) + } else if contains(n, p) { + Arc::new(Node { id: n.id, name: n.name.clone(), children: insert(&n.children, parent, at, node.clone()) }) + } else { + n.clone() + } + }) + .collect(), + } +} + +pub fn rename(roots: &[Arc], id: ElemId, name: &str) -> Vec> { + roots + .iter() + .map(|n| { + if n.id == id { + Arc::new(Node { id, name: name.to_owned(), children: n.children.clone() }) + } else if contains(n, id) { + Arc::new(Node { id: n.id, name: n.name.clone(), children: rename(&n.children, id, name) }) + } else { + n.clone() + } + }) + .collect() +} + +pub fn find(roots: &[Arc], id: ElemId) -> Option<&Arc> { + for n in roots { + if n.id == id { + return Some(n); + } + if let Some(found) = find(&n.children, id) { + return Some(found); + } + } + None +} + +// The story, as characters with attributes (small texts only: tests). + +pub struct Exploded { + pub chars: Vec, + pub attrs: Vec, + pub paras: Vec, +} + +pub fn explode(t: &RichText) -> Exploded { + let mut chars = Vec::new(); + let mut attrs = Vec::new(); + for (i, c) in t.text.char_indices() { + chars.push(c); + attrs.push(t.attrs_at(i)); + } + let mut paras = t.paragraphs.clone(); + paras.resize(t.text.matches('\n').count() + 1, Attrs::new()); + Exploded { chars, attrs, paras } +} + +pub fn implode(e: &Exploded) -> RichText { + let mut text = String::new(); + let mut runs: Vec<(Range, Attrs)> = Vec::new(); + for (c, a) in e.chars.iter().zip(&e.attrs) { + let start = text.len(); + text.push(*c); + let end = text.len(); + if a.is_empty() { + continue; + } + match runs.last_mut() { + Some((r, last)) if r.end == start && last == a => r.end = end, + _ => runs.push((start..end, a.clone())), + } + } + RichText { text, runs, paragraphs: e.paras.clone() } +} + +/// Whether a story's runs are canonical (in order, none empty, adjacent ones different) and its paragraphs match its +/// breaks: cheap, by runs. +pub fn runs_canonical(t: &RichText) -> bool { + let ordered = t.runs.windows(2).all(|w| w[0].0.end <= w[1].0.start && !(w[0].0.end == w[1].0.start && w[0].1 == w[1].1)); + ordered && t.runs.iter().all(|(r, a)| !a.is_empty() && r.start < r.end) && t.paragraphs.len() == t.text.matches('\n').count() + 1 +} + +/// The canonical form of a story (runs merged, none empty), as the app keeps it. +pub fn canonical(t: &RichText) -> RichText { + implode(&explode(t)) +} + +pub struct StorySrc<'a>(pub &'a Arc); + +impl TextSource for StorySrc<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(self.0) as usize) + } + fn text(&self) -> &str { + &self.0.text + } + fn runs(&self) -> std::borrow::Cow<'_, [(Range, Attrs)]> { + std::borrow::Cow::Borrowed(&self.0.runs) + } + fn paragraphs(&self) -> std::borrow::Cow<'_, [Attrs]> { + std::borrow::Cow::Borrowed(&self.0.paragraphs) + } +} + +/// Inserts `s` at character `at`, as typing does: it takes the attributes of the character before, and new +/// paragraphs take the attributes of the one they split. +pub fn type_at(t: &RichText, at: usize, s: &str) -> RichText { + let mut e = explode(t); + let at = at.min(e.chars.len()); + let inherited = if at > 0 { e.attrs.get(at - 1).cloned().unwrap_or_default() } else { Attrs::new() }; + let para = e.chars.iter().take(at).filter(|c| **c == '\n').count(); + let split = s.matches('\n').count(); + let attrs = e.paras.get(para).cloned().unwrap_or_default(); + for _ in 0..split { + e.paras.insert(para, attrs.clone()); + } + for (k, c) in s.chars().enumerate() { + e.chars.insert(at + k, c); + e.attrs.insert(at + k, inherited.clone()); + } + implode(&e) +} + +/// Deletes characters `range`: the paragraphs it joins keep the first one's attributes. +pub fn delete(t: &RichText, range: Range) -> RichText { + let mut e = explode(t); + let end = range.end.min(e.chars.len()); + let start = range.start.min(end); + let para = e.chars.iter().take(start).filter(|c| **c == '\n').count(); + let joined = e.chars.get(start..end).map_or(0, |cs| cs.iter().filter(|c| **c == '\n').count()); + for _ in 0..joined { + if para + 1 < e.paras.len() { + e.paras.remove(para + 1); + } + } + e.chars.drain(start..end); + e.attrs.drain(start..end); + implode(&e) +} + +/// Sets (or, with `None`, removes) a character attribute on characters `range`. +pub fn format(t: &RichText, range: Range, key: &str, value: Option) -> RichText { + let mut e = explode(t); + let end = range.end.min(e.chars.len()); + for a in e.attrs.iter_mut().take(end).skip(range.start) { + match &value { + Some(v) => a.insert(key.to_owned(), v.clone()), + None => a.remove(key), + }; + } + implode(&e) +} + +/// Sets (or removes) a paragraph attribute. +pub fn format_para(t: &RichText, para: usize, key: &str, value: Option) -> RichText { + let mut e = explode(t); + if let Some(p) = e.paras.get_mut(para) { + match value { + Some(v) => p.insert(key.to_owned(), v), + None => p.remove(key), + }; + } + implode(&e) +} + +// The grid. + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Grid { + pub rows: Vec, + pub cols: Vec, + pub cells: BTreeMap<(ElemId, ElemId), String>, +} + +pub struct GridSrc<'a>(pub &'a Arc); + +impl TableSource for GridSrc<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(self.0) as usize) + } + fn rows(&self) -> Vec { + self.0.rows.clone() + } + fn columns(&self) -> Vec { + self.0.cols.clone() + } + fn cell(&self, row: ElemId, column: ElemId) -> Option { + self.0.cells.get(&(row, column)).map(|s| Value::from(s.as_str())) + } + fn cells(&self) -> Vec<(ElemId, ElemId, Value)> { + self.0.cells.iter().map(|((r, c), s)| (*r, *c, Value::from(s.as_str()))).collect() + } +} + +// The timeline. + +#[derive(Debug, Clone, PartialEq)] +pub struct Item { + pub track: ElemId, + pub start: i64, + pub len: i64, + pub name: String, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Key { + pub item: ElemId, + pub at: i64, + pub value: i64, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Timeline { + pub tracks: Vec<(ElemId, String)>, + pub items: BTreeMap, + pub keys: BTreeMap, +} + +pub struct TimelineSrc<'a>(pub &'a Arc); + +impl TimelineSource for TimelineSrc<'_> { + fn key(&self) -> Option { + Some(Arc::as_ptr(self.0) as usize) + } + fn tracks(&self) -> Vec { + self.0.tracks.iter().map(|(id, _)| *id).collect() + } + fn write_track(&self, track: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError> { + if let Some((_, name)) = self.0.tracks.iter().find(|(id, _)| *id == track) { + props.set("name", name.as_str())?; + } + Ok(()) + } + fn items(&self, track: ElemId) -> Vec { + self.0.items.iter().filter(|(_, i)| i.track == track).map(|(id, _)| *id).collect() + } + fn span(&self, item: ElemId) -> Option { + self.0.items.get(&item).map(|i| Span { start: i.start, len: i.len }) + } + fn write_item(&self, item: ElemId, props: &mut Props<'_, '_>) -> Result<(), KitError> { + if let Some(i) = self.0.items.get(&item) { + props.set("name", i.name.as_str())?; + } + Ok(()) + } + fn keyframes(&self, item: ElemId) -> Vec<(ElemId, i64, Value)> { + self.0.keys.iter().filter(|(_, k)| k.item == item).map(|(id, k)| (*id, k.at, Value::I64(k.value))).collect() + } +} + +// The whole document. + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Doc { + pub layers: Roots, + pub story: Arc, + pub grid: Arc, + pub timeline: Arc, + pub settings: Arc>, + /// Derived from the story, never synced. + pub words: usize, +} + +impl Doc { + pub fn empty() -> Self { + let mut d = Self { story: Arc::new(RichText::plain("")), ..Self::default() }; + d.words = count_words(&d.story.text); + d + } +} + +fn count_words(text: &str) -> usize { + text.split_whitespace().count() +} + +pub struct TestApp; + +impl LiveDocument for TestApp { + type Doc = Doc; + const APP: &'static str = "kittest"; + const SCHEMA: u32 = 1; + + fn mappings(doc: &Doc) -> Mappings<'_> { + Mappings::new() + .tree("layers", LayersSrc::new(&doc.layers)) + .text("story", StorySrc(&doc.story)) + .table("grid", GridSrc(&doc.grid)) + .timeline("timeline", TimelineSrc(&doc.timeline)) + .opaque("settings", Opaque { value: Some(&*doc.settings), key: Some(Arc::as_ptr(&doc.settings) as usize) }) + } + + fn apply_remote(old: &Arc, patch: &RemotePatch<'_>) -> Result, KitError> { + let mut new = (**old).clone(); + if let Some(tree) = patch.tree("layers") { + fn build(tree: &craft_cloud_kit::TreeView<'_>, ids: Vec) -> Vec> { + ids.into_iter() + .map(|id| { + let name = tree.props(id).and_then(|p| p.str("name")).unwrap_or_default(); + Arc::new(Node { id, name, children: build(tree, tree.children(id)) }) + }) + .collect() + } + new.layers = Arc::new(build(&tree, tree.roots())); + } + if let Some(story) = patch.text("story") { + new.story = Arc::new(story); + } + if let Some(table) = patch.table("grid") { + let cells = table.cells().into_iter().filter_map(|(r, c, v)| v.as_str().map(|s| ((r, c), s.to_owned()))).collect(); + new.grid = Arc::new(Grid { rows: table.rows(), cols: table.columns(), cells }); + } + if let Some(view) = patch.timeline("timeline") { + let tracks: Vec<(ElemId, String)> = + view.tracks().into_iter().map(|t| (t, view.track_props(t).and_then(|p| p.str("name")).unwrap_or_default())).collect(); + let mut items = BTreeMap::new(); + for (_, list) in view.items() { + for i in list { + items.insert(i.id, Item { track: i.track, start: i.span.start, len: i.span.len, name: i.props.str("name").unwrap_or_default() }); + } + } + let mut keys = BTreeMap::new(); + for (item, list) in view.all_keyframes() { + for (id, at, v) in list { + keys.insert(id, Key { item, at, value: v.as_i64().unwrap_or_default() }); + } + } + new.timeline = Arc::new(Timeline { tracks, items, keys }); + } + if let Some(settings) = patch.opaque("settings") { + new.settings = Arc::new(settings.unwrap_or_default()); + } + Ok(Arc::new(new)) + } + + fn normalise(doc: &mut Doc) { + // Keyframes of items that are gone, and items of tracks that are gone, aren't the app's. + let tracks: Vec = doc.timeline.tracks.iter().map(|(t, _)| *t).collect(); + let tl = &doc.timeline; + if tl.items.values().any(|i| !tracks.contains(&i.track)) || tl.keys.values().any(|k| !tl.items.contains_key(&k.item)) { + let mut t = (**tl).clone(); + t.items.retain(|_, i| tracks.contains(&i.track)); + let items = t.items.clone(); + t.keys.retain(|_, k| items.contains_key(&k.item)); + doc.timeline = Arc::new(t); + } + if !runs_canonical(&doc.story) { + doc.story = Arc::new(canonical(&doc.story)); + } + } + + fn refresh_derived(doc: &mut Doc) { + doc.words = count_words(&doc.story.text); + } +} + +/// Whether a model is one the app could make: unique layer IDs, canonical story, paragraphs matching, no orphans. +pub fn valid(doc: &Doc) -> Result<(), String> { + let ids = flatten(&doc.layers); + let mut seen = std::collections::HashSet::new(); + for (id, _) in &ids { + if !seen.insert(*id) { + return Err(format!("layer {id} twice")); + } + } + if canonical(&doc.story) != *doc.story { + return Err("the story isn't canonical".into()); + } + if doc.story.paragraphs.len() != doc.story.text.matches('\n').count() + 1 { + return Err("paragraphs don't match the breaks".into()); + } + let tracks: Vec = doc.timeline.tracks.iter().map(|(t, _)| *t).collect(); + if doc.timeline.items.values().any(|i| !tracks.contains(&i.track)) { + return Err("an item on a missing track".into()); + } + if doc.timeline.keys.values().any(|k| !doc.timeline.items.contains_key(&k.item)) { + return Err("a keyframe of a missing item".into()); + } + for (r, c) in doc.grid.cells.keys() { + if !doc.grid.rows.contains(r) || !doc.grid.cols.contains(c) { + return Err("a cell of a missing row or column".into()); + } + } + if doc.words != count_words(&doc.story.text) { + return Err("derived words are stale".into()); + } + Ok(()) +} + +/// Which part an edit goes to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Shape { + Tree, + Text, + Table, + Timeline, + Opaque, +} + +pub const SHAPES: [Shape; 5] = [Shape::Tree, Shape::Text, Shape::Table, Shape::Timeline, Shape::Opaque]; + +/// A random edit of `shape`, as a command of the app makes it (the result is refreshed). +pub fn edit(doc: &Doc, shape: Shape, rng: &mut Rng) -> Doc { + let mut d = doc.clone(); + match shape { + Shape::Tree => d.layers = Arc::new(edit_tree(&doc.layers, rng)), + Shape::Text => d.story = Arc::new(edit_text(&doc.story, rng)), + Shape::Table => d.grid = Arc::new(edit_grid(&doc.grid, rng)), + Shape::Timeline => d.timeline = Arc::new(edit_timeline(&doc.timeline, rng)), + Shape::Opaque => d.settings = Arc::new((0..rng.below(20)).map(|_| rng.next() as u8).collect()), + } + TestApp::refresh_derived(&mut d); + d +} + +pub fn edit_tree(roots: &[Arc], rng: &mut Rng) -> Vec> { + let all = flatten(roots); + let pick = |rng: &mut Rng| all.get(rng.below(all.len())).map(|(id, _)| *id); + match if all.is_empty() { 0 } else { rng.below(4) } { + 0 => { + let parent = if rng.chance(40) { None } else { pick(rng) }; + let n = Arc::new(Node { id: rng.id(), name: rng.word(), children: Vec::new() }); + let at = rng.below(8); + insert(roots, parent, at, n) + } + 1 => match pick(rng) { + Some(id) => rename(roots, id, &rng.word()), + None => roots.to_vec(), + }, + 2 => match pick(rng) { + Some(id) => remove(roots, id).0, + None => roots.to_vec(), + }, + _ => { + let (Some(id), parent) = (pick(rng), pick(rng)) else { return roots.to_vec() }; + let Some(node) = find(roots, id) else { return roots.to_vec() }; + let parent = parent.filter(|p| !contains(node, *p)); + let (rest, taken) = remove(roots, id); + match taken { + Some(n) => insert(&rest, parent, rng.below(8), n), + None => roots.to_vec(), + } + } + } +} + +pub fn edit_text(t: &RichText, rng: &mut Rng) -> RichText { + let len = t.text.chars().count(); + let range = |rng: &mut Rng| { + let a = rng.below(len + 1); + let b = (a + rng.below(6)).min(len); + a..b + }; + match rng.below(7) { + 0 | 1 => { + let s = if rng.chance(25) { format!("{}\n{}", rng.word(), rng.word()) } else { rng.word() }; + type_at(t, rng.below(len + 1), &s) + } + 2 => delete(t, range(rng)), + 3 => format(t, range(rng), "b", if rng.chance(70) { Some(Value::Bool(true)) } else { None }), + 4 => format(t, range(rng), "size", Some(Value::I64(8 + rng.below(4) as i64))), + 5 => { + let paras = t.text.matches('\n').count() + 1; + format_para(t, rng.below(paras), "h", if rng.chance(70) { Some(Value::I64(1 + rng.below(3) as i64)) } else { None }) + } + _ => format(t, range(rng), "b", None), + } +} + +pub fn edit_grid(g: &Grid, rng: &mut Rng) -> Grid { + let mut g = g.clone(); + match rng.below(7) { + 0 => { + let at = rng.below(g.rows.len() + 1); + g.rows.insert(at, rng.id()); + } + 1 => { + let at = rng.below(g.cols.len() + 1); + g.cols.insert(at, rng.id()); + } + 2 if !g.rows.is_empty() => { + let r = g.rows.remove(rng.below(g.rows.len())); + g.cells.retain(|(row, _), _| *row != r); + } + 3 if !g.cols.is_empty() => { + let c = g.cols.remove(rng.below(g.cols.len())); + g.cells.retain(|(_, col), _| *col != c); + } + 4 if !g.rows.is_empty() => { + let r = g.rows.remove(rng.below(g.rows.len())); + let at = rng.below(g.rows.len() + 1); + g.rows.insert(at, r); + } + 5 | 6 if !g.rows.is_empty() && !g.cols.is_empty() => { + let (r, c) = (g.rows[rng.below(g.rows.len())], g.cols[rng.below(g.cols.len())]); + if rng.chance(80) { + g.cells.insert((r, c), rng.word()); + } else { + g.cells.remove(&(r, c)); + } + } + _ => g.rows.push(rng.id()), + } + g +} + +pub fn edit_timeline(t: &Timeline, rng: &mut Rng) -> Timeline { + let mut t = t.clone(); + let item_ids: Vec = t.items.keys().copied().collect(); + let key_ids: Vec = t.keys.keys().copied().collect(); + match rng.below(9) { + 0 => { + let at = rng.below(t.tracks.len() + 1); + t.tracks.insert(at, (rng.id(), rng.word())); + } + 1 if !t.tracks.is_empty() => { + let (track, _) = t.tracks.remove(rng.below(t.tracks.len())); + t.items.retain(|_, i| i.track != track); + let items = t.items.clone(); + t.keys.retain(|_, k| items.contains_key(&k.item)); + } + 2 | 3 if !t.tracks.is_empty() => { + let track = t.tracks[rng.below(t.tracks.len())].0; + t.items.insert(rng.id(), Item { track, start: rng.below(1000) as i64, len: 1 + rng.below(100) as i64, name: rng.word() }); + } + 4 if !item_ids.is_empty() && !t.tracks.is_empty() => { + let track = t.tracks[rng.below(t.tracks.len())].0; + if let Some(i) = t.items.get_mut(&item_ids[rng.below(item_ids.len())]) { + i.track = track; + } + } + 5 if !item_ids.is_empty() => { + let start = rng.below(1000) as i64; + if let Some(i) = t.items.get_mut(&item_ids[rng.below(item_ids.len())]) { + i.start = start; + i.name = format!("{}*", i.name); + } + } + 6 if !item_ids.is_empty() => { + let id = item_ids[rng.below(item_ids.len())]; + t.items.remove(&id); + t.keys.retain(|_, k| k.item != id); + } + 7 if !item_ids.is_empty() => { + let item = item_ids[rng.below(item_ids.len())]; + t.keys.insert(rng.id(), Key { item, at: rng.below(100) as i64, value: rng.below(10) as i64 }); + } + 8 if !key_ids.is_empty() => { + let id = key_ids[rng.below(key_ids.len())]; + if rng.chance(50) { + t.keys.remove(&id); + } else if let Some(k) = t.keys.get_mut(&id) { + k.value += 1; + } + } + _ => t.tracks.push((rng.id(), rng.word())), + } + t +} + +/// A random document: `n` edits of every shape from empty. +pub fn random_doc(rng: &mut Rng, n: usize) -> Doc { + let mut d = Doc::empty(); + for _ in 0..n { + let shape = SHAPES[rng.below(SHAPES.len())]; + d = edit(&d, shape, rng); + } + d +} diff --git a/crates/kit/tests/session.rs b/crates/kit/tests/session.rs new file mode 100644 index 0000000..d1b36a7 --- /dev/null +++ b/crates/kit/tests/session.rs @@ -0,0 +1,855 @@ +//! The session against the fake agent: two machines (or apps) on one hub, a small model (an opaque title and a list). + +mod common; + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use common::{App, Notes, NotesApp, change, open, until}; +use craft_cloud_app::fake::Hub; +use craft_cloud_kit::{AgentBlobs, BlobStore, KitError, MAX_QUEUE_BYTES, Open, SPLIT_BYTES, Session}; + +#[test] +fn a_file_imported_on_one_machine_is_live_on_the_other() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "notes.txt", Notes::new("Groceries", &["eggs", "milk"])); + let mut b = open::(&laptop, "notes.txt", Notes::default()); + assert_eq!(b.model().texts(), vec!["eggs", "milk"], "the laptop built its model from the live document"); + assert_eq!(*b.model().title, "Groceries"); + assert!(a.status().read_only.is_none() && b.status().read_only.is_none()); + change(&mut a, |m| m.with_title("Shopping")).unwrap(); + change(&mut b, |m| m.with_item(1, "bread")).unwrap(); + until(&mut [&mut a, &mut b], "both edits on both", |s| { + s.iter().all(|s| *s.model().title == "Shopping" && s.model().texts() == ["eggs", "bread", "milk"]) + }); + // Unchanged parts write nothing: the same model again sends nothing. + let before = hub.updates("notes.txt"); + let same = a.model().clone(); + a.local_change(&same, &Arc::new((*same).clone())).unwrap(); + std::thread::sleep(Duration::from_millis(50)); + a.poll(); + assert_eq!(hub.updates("notes.txt"), before, "nothing changed, nothing sent"); +} + +/// A document made by a newer version of the app opens read-only: it is shown, remote changes still come, and local +/// ones (and undo) are refused; the same version and an older one are writable. +#[test] +fn a_document_from_a_newer_app_is_read_only() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut newer = open::>(&pc, "n.txt", Notes::new("v2", &["a"])); + let mut same = open::>(&laptop, "n.txt", Notes::default()); + assert!(same.status().read_only.is_none(), "the same version is writable"); + let mut older = open::>(&laptop, "n.txt", Notes::default()); + let ro = older.status().read_only.clone().unwrap(); + assert_eq!((ro.app.as_str(), ro.version), ("notes", 2)); + assert_eq!(older.model().texts(), vec!["a"], "shown"); + let refused = change(&mut older, |m| m.with_title("nope")); + assert!(matches!(refused, Err(KitError::ReadOnly { version: 2, .. })), "{refused:?}"); + assert!(matches!(older.undo(), Err(KitError::ReadOnly { .. }))); + change(&mut newer, |m| m.with_title("from v2")).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while *older.model().title != "from v2" { + newer.poll(); + same.poll(); + older.poll(); + assert!(Instant::now() < deadline, "the newer app's change came"); + std::thread::sleep(Duration::from_millis(5)); + } + // A document whose version is older than the app's opens writable (migrating it is the app's own change). + let hub2 = Hub::new(); + let pc2 = hub2.machine("pc").unwrap(); + let _v1 = open::>(&pc2, "o.txt", Notes::new("v1", &[])); + let mut v2 = open::>(&pc2, "o.txt", Notes::default()); + assert!(v2.status().read_only.is_none()); + change(&mut v2, |m| m.with_title("migrated")).unwrap(); +} + +/// A change over the split size goes as several commits, each under it: a 5 MB change (twenty 250 KiB notes) arrives +/// whole on the other machine, and every update stays under the agent's 1 MiB cap. +#[test] +fn a_five_megabyte_change_arrives_whole() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "big.txt", Notes::new("big", &[])); + let mut b = open::(&laptop, "big.txt", Notes::default()); + let before = hub.updates("big.txt"); + let notes: Vec = (0..20).map(|i| format!("{i:02}{}", "x".repeat(250 * 1024))).collect(); + change(&mut a, |m| { + let mut items = (*m.items).clone(); + let mut hint = Vec::new(); + for n in ¬es { + let id = craft_cloud_kit::ElemId::new(); + hint.push(id); + items.push((id, n.clone())); + } + Notes { items: Arc::new(items), hint: Some(hint), ..m.clone() } + }) + .unwrap(); + until(&mut [&mut a, &mut b], "the laptop has all 5 MB", |s| s.get(1).is_some_and(|b| b.model().texts() == notes)); + let sent = hub.updates("big.txt") - before; + assert!(sent >= 10, "{sent} updates for 5 MB: it was split"); + assert!(a.status().error.is_none() && b.status().error.is_none(), "{:?} {:?}", a.status().error, b.status().error); + const { assert!(SPLIT_BYTES * 2 <= craft_cloud_api::limits::MAX_UPDATE_BYTES) }; + // One undo step takes the whole change back, on both. + assert!(a.undo().unwrap()); + until(&mut [&mut a, &mut b], "the undo is on both", |s| s.iter().all(|s| s.model().items.is_empty())); +} + +/// Reconnecting: the agent drops the app; the model stays editable, changes queue, and go out once the session is +/// back by itself (none lost, none twice). +#[test] +fn a_dropped_connection_comes_back_and_sends_what_queued() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "r.txt", Notes::new("r", &[])); + let mut b = open::(&laptop, "r.txt", Notes::default()); + pc.restart(); + until(&mut [&mut a], "the drop is seen", |s| s.iter().all(|s| !s.status().connected)); + for i in 0..5 { + change(&mut a, |m| m.with_item(i, &format!("offline {i}"))).unwrap(); + } + assert_eq!(a.poll().status.queued, 5); + pc.start(); + until(&mut [&mut a, &mut b], "back, and sent", |s| { + s.get(1).is_some_and(|b| b.model().items.len() == 5) && s.iter().all(|s| s.status().connected) + }); + until(&mut [&mut a, &mut b], "acked", |s| s.first().is_some_and(|a| a.status().queued == 0)); + assert_eq!(b.model().texts(), (0..5).map(|i| format!("offline {i}")).collect::>()); + // A dropped connection while the agent keeps running. + pc.drop_connections(); + change(&mut a, |m| m.with_title("after the drop")).unwrap(); + until(&mut [&mut a, &mut b], "the title came", |s| s.get(1).is_some_and(|b| *b.model().title == "after the drop")); + let wakes = Arc::new(AtomicUsize::new(0)); + let w = wakes.clone(); + b.set_waker(Arc::new(move || { + w.fetch_add(1, Ordering::Relaxed); + })); + change(&mut a, |m| m.with_title("wake up")).unwrap(); + until(&mut [&mut a, &mut b], "woken", |_| wakes.load(Ordering::Relaxed) > 0); +} + +/// Restarting offline with unsent edits: the app's edits were taken by the agent (its journal) while the machine was +/// offline; the app quits; the agent restarts; the app opens the document again and sees its own edits, and they go +/// out when the machine is back online. +#[test] +fn an_app_restarted_offline_sees_its_unsent_edits() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "j.txt", Notes::new("j", &[])); + let mut b = open::(&laptop, "j.txt", Notes::default()); + pc.set_online(false); + change(&mut a, |m| m.with_item(0, "written offline")).unwrap(); + until(&mut [&mut a], "acked by the agent", |s| s.iter().all(|s| s.status().queued == 0)); + assert_eq!(pc.waiting("j.txt"), 1); + drop(a); + pc.restart(); + pc.start(); + let mut again = open::(&pc, "j.txt", Notes::default()); + assert_eq!(again.status().waiting, 1); + // The agent's state, then the updates it holds that the server hasn't acked yet (taken at the first polls). + until(&mut [&mut again], "its own unsent edit, from the agent", |s| s.iter().all(|s| s.model().texts() == ["written offline"])); + pc.set_online(true); + until(&mut [&mut again, &mut b], "out once online", |s| s.get(1).is_some_and(|b| b.model().texts() == ["written offline"])); +} + +/// The queue's cap: past 64 MiB waiting for the agent, local changes are refused (stalled) until it is back. +#[test] +fn past_the_queue_cap_changes_are_refused_until_the_agent_is_back() { + let hub = Hub::new(); + let pc = hub.machine("pc").unwrap(); + let mut a = open::(&pc, "q.txt", Notes::new("q", &[])); + pc.restart(); + until(&mut [&mut a], "the drop is seen", |s| s.iter().all(|s| !s.status().connected)); + // Big changes up to near the cap, then small ones (a few hundred bytes each), so the edge is met within one. + let mut i = 0usize; + let mut last_taken = 0; + loop { + let queued = a.poll().status.queued_bytes; + let block = match MAX_QUEUE_BYTES - queued.min(MAX_QUEUE_BYTES) { + left if left > 1024 * 1024 => 200 * 1024, + left if left > 8 * 1024 => 4 * 1024, + _ => 1, + }; + let title = format!("{i:08}{}", "q".repeat(block)); + let result = change(&mut a, |m| m.with_title(&title)); + if queued < MAX_QUEUE_BYTES { + assert!(result.is_ok(), "{queued} bytes queued: still taken ({result:?})"); + last_taken = queued; + } else { + assert!(matches!(result, Err(KitError::Stalled(_))), "{queued} bytes queued: refused"); + assert!(a.poll().status.stalled); + assert!(queued - MAX_QUEUE_BYTES < 1024, "refused at the first change past the cap ({queued} bytes)"); + break; + } + i += 1; + } + assert!(MAX_QUEUE_BYTES - last_taken < 1024, "taken up to one small change below the cap ({last_taken} bytes)"); + pc.start(); + until(&mut [&mut a], "drained", |s| s.iter().all(|s| s.status().queued == 0 && !s.status().stalled)); + change(&mut a, |m| m.with_title("taken again")).unwrap(); +} + +/// Review focus 1: a remote change that comes while the app holds remote changes back (a drag, an IME composition) +/// is applied at the next poll after the gesture, never inside it; the local edit isn't lost or doubled. A change +/// from a stale model is refused and writes nothing. +#[test] +fn a_remote_change_during_a_gesture_waits_for_its_end() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "g.txt", Notes::new("g", &["one"])); + let mut b = open::(&laptop, "g.txt", Notes::default()); + a.hold_remote(true); + let before = a.model().clone(); + change(&mut b, |m| m.with_item(1, "remote")).unwrap(); + let deadline = Instant::now() + Duration::from_millis(300); + while Instant::now() < deadline { + b.poll(); + assert!(!a.poll().changed, "nothing applied during the gesture"); + std::thread::sleep(Duration::from_millis(5)); + } + assert!(Arc::ptr_eq(a.model(), &before)); + let after = Arc::new(before.with_edit(0, "one, dragged")); + a.local_change(&before, &after).unwrap(); + a.hold_remote(false); + until(&mut [&mut a, &mut b], "both have both", |s| s.iter().all(|s| s.model().texts() == ["one, dragged", "remote"])); + // A stale `old` (not the session's model) writes nothing. + let stale = before.clone(); + let refused = a.local_change(&stale, &Arc::new(stale.with_title("stale"))); + assert_eq!(refused, Err(KitError::Stale)); + assert_eq!(*a.model().title, "g"); +} + +#[test] +fn presence_goes_to_the_other_machine_and_fades() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "p.txt", Notes::new("p", &[])); + let mut b = open::(&laptop, "p.txt", Notes::default()); + a.set_presence(Some(b"cursor at 3".to_vec())).unwrap(); + until(&mut [&mut a, &mut b], "the laptop sees it", |s| s.get(1).is_some_and(|b| b.peers().values().any(|p| p.bytes == b"cursor at 3"))); + assert!(b.peers().contains_key(&a.peer())); + assert!(a.peers().is_empty(), "its own isn't shown"); + assert!(a.set_presence(Some(vec![0; 4096])).is_err(), "at most 4 KiB with its header"); +} + +#[test] +fn blobs_go_through_the_agent_and_an_old_agent_says_so() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let (a, b) = (AgentBlobs(pc.client().blobs()), AgentBlobs(laptop.client().blobs())); + let chunk = a.put(b"an image").unwrap(); + assert_eq!(&*b.get(&chunk).unwrap(), b"an image"); + laptop.too_old(); + let e = b.get(&chunk).unwrap_err(); + assert!(e.to_string().contains("agent_too_old"), "{e}"); + // Not running at all: the file opens as a plain file. + pc.restart(); + let opened = Session::::open(&common::agent(&pc), "x.txt", Arc::new(Notes::default()), None).unwrap(); + assert!(matches!(opened, Open::NotLive(_))); +} + +/// A change the agent refuses for now (its disk) is never dropped: it and the changes after it go again, in order, +/// and the other machine ends with all of them. +#[test] +fn a_change_refused_for_now_goes_again_in_order() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "f.txt", Notes::new("f", &[])); + let mut b = open::(&laptop, "f.txt", Notes::default()); + pc.refuse_next("internal", 2); + for i in 0..4 { + change(&mut a, |m| m.with_item(i, &format!("item {i}"))).unwrap(); + } + until(&mut [&mut a, &mut b], "all four on the laptop", |s| s.get(1).is_some_and(|b| b.model().items.len() == 4)); + until(&mut [&mut a, &mut b], "nothing left queued", |s| s.first().is_some_and(|a| a.status().queued == 0)); + assert!(!a.status().stopped); + assert_eq!(b.model().texts(), (0..4).map(|i| format!("item {i}")).collect::>()); +} + +/// A change the agent refuses for good (it doesn't decode) stops the session where the app sees it: nothing after it +/// is sent into a hole, the model stays editable, and no change is silently stranded. +#[test] +fn a_change_refused_for_good_stops_the_session() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "g.txt", Notes::new("g", &[])); + let mut b = open::(&laptop, "g.txt", Notes::default()); + pc.refuse_next("invalid", 1); + change(&mut a, |m| m.with_item(0, "refused")).unwrap(); + until(&mut [&mut a, &mut b], "stopped", |s| s.first().is_some_and(|a| a.status().stopped)); + let error = a.status().error.clone().unwrap_or_default(); + assert!(error.contains("invalid") && error.contains("save it to the file"), "{error}"); + // Still editable; the change is kept (queued), never shown as sent. + change(&mut a, |m| m.with_item(1, "after")).unwrap(); + assert_eq!(a.model().texts(), vec!["refused", "after"]); + assert_eq!(a.poll().status.queued, 2, "both kept, neither dropped"); + std::thread::sleep(Duration::from_millis(100)); + b.poll(); + assert!(b.model().items.is_empty(), "nothing went out after the refusal"); +} + +/// A flat list of layers whose `apply_remote` fails while `fail` is set (an app's codec that can't read something). +#[derive(Clone, Debug)] +struct Flaky { + layers: Arc>, + fail: Arc, +} + +struct FlakyLayers<'a>(&'a Flaky); + +impl craft_cloud_kit::TreeSource for FlakyLayers<'_> { + fn roots(&self) -> Vec { + self.0.layers.iter().map(|(id, _)| *id).collect() + } + fn children(&self, _: craft_cloud_kit::ElemId) -> Vec { + Vec::new() + } + fn write(&self, id: craft_cloud_kit::ElemId, props: &mut craft_cloud_kit::Props<'_, '_>) -> Result<(), KitError> { + if let Some((_, name)) = self.0.layers.iter().find(|(i, _)| *i == id) { + props.set("name", name.as_str())?; + } + Ok(()) + } +} + +struct FlakyApp; + +impl craft_cloud_kit::LiveDocument for FlakyApp { + type Doc = Flaky; + const APP: &'static str = "flaky"; + const SCHEMA: u32 = 1; + fn mappings(doc: &Flaky) -> craft_cloud_kit::Mappings<'_> { + craft_cloud_kit::Mappings::new().tree("layers", FlakyLayers(doc)) + } + fn apply_remote(old: &Arc, patch: &craft_cloud_kit::RemotePatch<'_>) -> Result, KitError> { + if old.fail.load(std::sync::atomic::Ordering::Relaxed) { + return Err(KitError::App("this layer doesn't read".into())); + } + let mut new = (**old).clone(); + if let Some(tree) = patch.tree("layers") { + new.layers = Arc::new(tree.roots().into_iter().map(|id| (id, tree.props(id).and_then(|p| p.str("name")).unwrap_or_default())).collect()); + } + Ok(Arc::new(new)) + } + fn normalise(_: &mut Flaky) {} + fn refresh_derived(_: &mut Flaky) {} +} + +/// A remote change the model can't take leaves it stale: local changes are refused (never a diff from a model behind +/// the document, which would delete the other machine's new layer), and once a whole rebuild succeeds at a poll the +/// session goes on, with nothing lost. +#[test] +fn a_model_behind_the_document_takes_no_local_change_until_rebuilt() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let first = craft_cloud_kit::ElemId::new(); + let flaky = |fail: bool| Flaky { layers: Arc::new(vec![(first, "one".into())]), fail: Arc::new(std::sync::atomic::AtomicBool::new(fail)) }; + let mut a = open::(&pc, "l.txt", flaky(false)); + let mut b = open::(&laptop, "l.txt", flaky(false)); + let fail = b.model().fail.clone(); + fail.store(true, std::sync::atomic::Ordering::Relaxed); + change(&mut a, |m| Flaky { layers: Arc::new(vec![(first, "one".into()), (craft_cloud_kit::ElemId::new(), "two".into())]), ..m.clone() }).unwrap(); + until(&mut [&mut a, &mut b], "the laptop failed to take it", |s| s.get(1).is_some_and(|b| b.status().error.is_some())); + let refused = change(&mut b, |m| Flaky { layers: Arc::new(vec![(first, "renamed".into())]), ..m.clone() }); + assert_eq!(refused, Err(KitError::ModelStale)); + fail.store(false, std::sync::atomic::Ordering::Relaxed); + until(&mut [&mut a, &mut b], "rebuilt", |s| s.get(1).is_some_and(|b| b.model().layers.len() == 2)); + change(&mut b, |m| { + let mut layers = (*m.layers).clone(); + layers[0].1 = "renamed".into(); + Flaky { layers: Arc::new(layers), ..m.clone() } + }) + .unwrap(); + until(&mut [&mut a, &mut b], "both layers on both", |s| { + s.iter().all(|s| s.model().layers.iter().map(|(_, n)| n.as_str()).collect::>() == ["renamed", "two"]) + }); +} + +/// The reviewer's probe: a title and layers; the layers fail to rebuild on the laptop. +#[derive(Clone, Debug)] +struct Two { + title: Arc, + layers: Arc>, + fail: Arc, +} + +struct TwoLayers<'a>(&'a Two); + +impl craft_cloud_kit::TreeSource for TwoLayers<'_> { + fn roots(&self) -> Vec { + self.0.layers.iter().map(|(id, _)| *id).collect() + } + fn children(&self, _: craft_cloud_kit::ElemId) -> Vec { + Vec::new() + } + fn write(&self, id: craft_cloud_kit::ElemId, props: &mut craft_cloud_kit::Props<'_, '_>) -> Result<(), KitError> { + if let Some((_, name)) = self.0.layers.iter().find(|(i, _)| *i == id) { + props.set("name", name.as_str())?; + } + Ok(()) + } +} + +struct TwoApp; + +impl craft_cloud_kit::LiveDocument for TwoApp { + type Doc = Two; + const APP: &'static str = "two"; + const SCHEMA: u32 = 1; + fn mappings(doc: &Two) -> craft_cloud_kit::Mappings<'_> { + craft_cloud_kit::Mappings::new() + .opaque("title", craft_cloud_kit::Opaque { value: Some(&*doc.title), key: Some(Arc::as_ptr(&doc.title) as usize) }) + .tree("layers", TwoLayers(doc)) + } + fn apply_remote(old: &Arc, patch: &craft_cloud_kit::RemotePatch<'_>) -> Result, KitError> { + let mut new = (**old).clone(); + if let Some(title) = patch.opaque("title") { + new.title = Arc::new(String::from_utf8_lossy(&title.unwrap_or_default()).into_owned()); + } + if let Some(tree) = patch.tree("layers") { + if old.fail.load(std::sync::atomic::Ordering::Relaxed) { + return Err(KitError::App("this layer doesn't read".into())); + } + new.layers = Arc::new(tree.roots().into_iter().map(|id| (id, tree.props(id).and_then(|p| p.str("name")).unwrap_or_default())).collect()); + } + Ok(Arc::new(new)) + } + fn normalise(_: &mut Two) {} + fn refresh_derived(_: &mut Two) {} +} + +/// A failed rebuild of one part, then another part's remote change rebuilt fine (the reviewer's probe): the model +/// stays stale (only a whole rebuild clears it), so the laptop's change from it is refused and gives no save point, +/// and the pc's new layer survives on every machine. +#[test] +fn a_part_rebuilt_later_leaves_the_model_stale() { + use std::sync::atomic::Ordering; + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let first = craft_cloud_kit::ElemId::new(); + let two = |fail: bool| Two { + title: Arc::new("t".into()), + layers: Arc::new(vec![(first, "one".into())]), + fail: Arc::new(std::sync::atomic::AtomicBool::new(fail)), + }; + let mut a = open::(&pc, "two.txt", two(false)); + let mut b = open::(&laptop, "two.txt", two(false)); + let fail = b.model().fail.clone(); + fail.store(true, Ordering::Relaxed); + change(&mut a, |m| Two { layers: Arc::new(vec![(first, "one".into()), (craft_cloud_kit::ElemId::new(), "two".into())]), ..m.clone() }).unwrap(); + until(&mut [&mut a, &mut b], "the laptop failed to take it", |s| s.get(1).is_some_and(|b| b.status().error.is_some())); + change(&mut a, |m| Two { title: Arc::new("from pc".into()), ..m.clone() }).unwrap(); + until(&mut [&mut a, &mut b], "the title came", |s| s.get(1).is_some_and(|b| *b.model().title == "from pc")); + assert!(b.save_point().is_err(), "still stale"); + let renamed = change(&mut b, |m| { + let mut layers = (*m.layers).clone(); + layers[0].1 = "renamed".into(); + Two { layers: Arc::new(layers), ..m.clone() } + }); + assert_eq!(renamed, Err(KitError::ModelStale)); + assert_eq!(b.status().queued, 0, "nothing written from the stale model"); + fail.store(false, Ordering::Relaxed); + until(&mut [&mut a, &mut b], "rebuilt whole", |s| s.get(1).is_some_and(|b| b.model().layers.len() == 2)); + for _ in 0..50 { + a.poll(); + b.poll(); + std::thread::sleep(Duration::from_millis(5)); + } + let names = |s: &Session| s.model().layers.iter().map(|(_, n)| n.clone()).collect::>(); + assert_eq!((names(&a), names(&b)), (vec!["one".to_owned(), "two".to_owned()], vec!["one".to_owned(), "two".to_owned()])); +} + +/// While the agent is down, another file takes the document's path (a live one, or one never live). Coming back, +/// the session finds a different document there: it stops with `document_replaced`, keeps its model, and neither +/// merges into the other document nor imports itself over the other file. +#[test] +fn a_path_that_is_another_document_now_is_never_merged_into() { + for live in [true, false] { + let hub = Hub::new(); + let pc = hub.machine("pc").unwrap(); + let mut a = open::(&pc, "d.txt", Notes::new("mine", &["kept"])); + pc.restart(); + until(&mut [&mut a], "down", |s| s.iter().all(|s| !s.status().connected)); + let other = open_other(); + hub.replace("d.txt", live.then_some(other)); + pc.start(); + a.reconnect_now(); + until(&mut [&mut a], "replaced", |s| s.iter().all(|s| s.status().replaced)); + let status = a.poll().status; + assert!(status.stopped && !status.connected); + assert!(status.error.unwrap_or_default().contains("document_replaced")); + assert_eq!(*a.model().title, "mine", "the model is kept"); + change(&mut a, |m| m.with_title("still editable")).unwrap(); + // The other file's document has nothing of this one. + assert_eq!(hub.updates("d.txt"), 0, "live {live}: nothing was sent into it"); + let mut b = open::(&pc, "d.txt", Notes::default()); + assert_eq!(*b.model().title, if live { "other" } else { "" }, "live {live}"); + b.poll(); + } +} + +/// A snapshot of another document (title "other"). +fn open_other() -> Vec { + let mut r = craft_cloud_kit::raw::Replica::default(); + let mut schema = std::collections::HashMap::new(); + schema.insert("app".to_owned(), loro::LoroValue::from("notes")); + schema.insert("version".to_owned(), loro::LoroValue::from(1i64)); + r.doc.get_map("root").insert("schema", loro::LoroValue::from(schema)).unwrap(); + r.write::(None, &Notes::new("other", &[])).unwrap(); + r.doc.export(loro::ExportMode::Snapshot).unwrap() +} + +/// A session opened off the UI thread can be handed to it. +/// Every session writes with a fresh peer of its own (the reviewer's reuse probe, with sessions): a window with +/// changes unsent while the agent was away, and a new window opened after the agent came back that writes more, end +/// with both windows' changes on every machine, and the first window's later edits too. And however many sessions +/// have edited a document, a save record names one or two heads. +#[test] +fn every_session_has_a_peer_of_its_own_and_no_change_is_lost() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "x.txt", Notes::new("t", &["base"])); + until(&mut [&mut a], "acked", |s| s[0].status().queued == 0); + pc.restart(); + until(&mut [&mut a], "the drop is seen", |s| !s[0].status().connected); + for i in 0..3 { + change(&mut a, |m| m.with_item(0, &format!("A{i}"))).unwrap(); + } + pc.start(); + // Another window, opened before the first is back (the first open after an agent restart), writes ten. + let mut c = open::(&pc, "x.txt", Notes::default()); + assert_ne!(c.peer(), a.peer()); + for i in 0..10 { + change(&mut c, |m| m.with_item(0, &format!("C{i}"))).unwrap(); + } + until(&mut [&mut c], "C's changes are in", |s| s[0].status().queued == 0); + let mut b = open::(&laptop, "x.txt", Notes::default()); + let all = |s: &Session| { + (0..3).all(|i| s.model().texts().contains(&format!("A{i}"))) && (0..10).all(|i| s.model().texts().contains(&format!("C{i}"))) + }; + until(&mut [&mut a, &mut b, &mut c], "every change everywhere", |s| s.iter().all(|s| all(s) && s.status().queued == 0)); + assert!(!a.status().stopped && a.status().error.is_none(), "{:?}", a.status()); + change(&mut a, |m| m.with_item(0, "A-after")).unwrap(); + until(&mut [&mut a, &mut b], "a's later edit too", |s| s[1].model().texts().contains(&"A-after".to_owned())); + for round in 0..20 { + let fake = if round % 2 == 0 { &pc } else { &laptop }; + let mut s = open::(fake, "x.txt", Notes::default()); + change(&mut s, |m| m.with_item(0, &format!("round {round}"))).unwrap(); + until(&mut [&mut s], "acked", |s| s.iter().all(|s| s.status().queued == 0)); + } + until(&mut [&mut a], "every round", |s| s[0].model().texts().contains(&"round 19".to_owned())); + let point = a.save_point().unwrap(); + assert!((1..=2).contains(&point.frontiers().len()), "{:?}", point.frontiers()); +} + +#[test] +fn a_session_can_move_to_another_thread() { + fn send() {} + send::>(); +} + +fn open_with_bytes(fake: &craft_cloud_app::fake::Fake, path: &str, file: Notes, bytes: &[u8]) -> Session { + match Session::::open(&common::agent(fake), path, Arc::new(file), Some(bytes)).unwrap() { + Open::Live(s) => *s, + Open::NotLive(why) => panic!("not live: {why:?}"), + } +} + +/// Saving tells the agent (content hash and version) and marks the document; a machine that opens the file with +/// content a Craft app saved (or imported) sees nothing changed outside, one with other content does. +#[test] +fn saves_reach_the_agent_and_a_file_changed_outside_is_reported() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open_with_bytes(&pc, "s.txt", Notes::new("s", &["x"]), b"as read at import"); + assert!(!a.status().changed_outside, "the import's own bytes"); + change(&mut a, |m| m.with_title("saved")).unwrap(); + let record = save(&mut a, b"as saved"); + assert_eq!(record.hash, *blake3::hash(b"as saved").as_bytes()); + assert!(record.frontiers.iter().any(|(p, _)| *p == a.peer()), "this session's edit is its head"); + until(&mut [&mut a], "the agent has the record", |_| pc.saves().iter().any(|(h, _)| *h == craft_cloud_api::local::blobs::hex(&record.hash))); + until(&mut [&mut a], "sent", |s| s.iter().all(|s| s.status().queued == 0)); + // (The save mark may come with the agent's first updates after the state: looked at again as they come.) + let mut same = open_with_bytes(&laptop, "s.txt", Notes::default(), b"as saved"); + until(&mut [&mut same], "the save is known", |s| s.iter().all(|s| *s.model().title == "saved" && !s.status().changed_outside)); + let mut imported = open_with_bytes(&laptop, "s.txt", Notes::default(), b"as read at import"); + until(&mut [&mut imported], "the import is known", |s| s.iter().all(|s| *s.model().title == "saved" && !s.status().changed_outside)); + let mut other = open_with_bytes(&laptop, "s.txt", Notes::default(), b"edited in another program"); + until(&mut [&mut other], "caught up", |s| s.iter().all(|s| *s.model().title == "saved")); + assert!(other.status().changed_outside); + // Saving clears it. + save(&mut other, b"saved from the live document"); + assert!(!other.status().changed_outside); +} + +/// Whether the file was changed outside Craft is decided once, at the open, from the agent's records, whatever the +/// number of saves: a reader that opened the import's bytes stays "not changed" through 40 saves on another machine +/// (the reviewer's probe flipped at the 33rd), a window that opens the first of those saves' bytes sees no outside +/// edit, and one that opens other bytes does, and keeps saying so while others save, until it saves itself. +#[test] +fn changed_outside_is_decided_at_the_open_by_the_records() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let imported = b"file as read at the import".to_vec(); + let mut a = open_with_bytes(&pc, "n.txt", Notes::new("t", &["a"]), &imported); + let mut b = open_with_bytes(&laptop, "n.txt", Notes::default(), &imported); + assert!(!a.status().changed_outside && !b.status().changed_outside, "the import's own bytes"); + for i in 0..40 { + change(&mut a, |m| m.with_title(&format!("t{i}"))).unwrap(); + save(&mut a, format!("save {i}").as_bytes()); + } + until(&mut [&mut a, &mut b], "every save is known", |s| *s[1].model().title == "t39" && pc.saves().len() >= 40); + assert!(!b.status().changed_outside, "a reader is never told its Craft file was edited outside"); + let first = open_with_bytes(&laptop, "n.txt", Notes::default(), b"save 0"); + assert!(!first.status().changed_outside, "the oldest of 40 saves is still a Craft save"); + let mut edited = open_with_bytes(&laptop, "n.txt", Notes::default(), b"edited in another program"); + assert!(edited.status().changed_outside); + change(&mut a, |m| m.with_title("t40")).unwrap(); + save(&mut a, b"save 40"); + until(&mut [&mut a, &mut edited], "the next save came", |s| *s[1].model().title == "t40"); + assert!(edited.status().changed_outside, "decided at the open"); + save(&mut edited, b"saved from the live document"); + assert!(!edited.status().changed_outside); +} + +/// "Bring it in" is the user's choice, never the kit's: the session only says the file changed outside. Taken, the +/// file's content replaces the live content as one ordinary change, named "Replace with the file's content", that +/// reaches the other machine; undo brings the live edits back, everywhere. +#[test] +fn bringing_the_file_in_is_one_named_undoable_change() { + use craft_cloud_kit::REPLACE_WITH_FILE; + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open_with_bytes(&pc, "in.txt", Notes::new("live", &["kept live"]), b"imported"); + let mut b = open_with_bytes(&laptop, "in.txt", Notes::default(), b"edited in another program"); + assert!(b.status().changed_outside); + until(&mut [&mut a, &mut b], "both live", |s| s.iter().all(|s| *s.model().title == "live")); + for _ in 0..20 { + b.poll(); + } + assert_eq!(*b.model().title, "live", "nothing replaced by itself"); + b.replace_with_file(Notes::new("from the file", &["the editor's line"])).unwrap(); + assert!(!b.status().changed_outside); + assert_eq!(b.step_name(true), Some(REPLACE_WITH_FILE)); + until(&mut [&mut a, &mut b], "replaced everywhere", |s| s.iter().all(|s| s.model().texts() == ["the editor's line"])); + assert!(b.undo().unwrap()); + until(&mut [&mut a, &mut b], "the live edits back everywhere", |s| { + s.iter().all(|s| *s.model().title == "live" && s.model().texts() == ["kept live"]) + }); + assert_eq!(b.step_name(false), Some(REPLACE_WITH_FILE), "redo names it too"); +} + +/// Bringing in a file whose content is the live content already makes no step, so it names none: the next ordinary +/// edit's undo isn't called "Replace with the file's content". +#[test] +fn a_name_never_sticks_to_the_next_ordinary_edit() { + let hub = Hub::new(); + let pc = hub.machine("pc").unwrap(); + let mut a = open_with_bytes(&pc, "same.txt", Notes::new("same", &["one"]), b"imported"); + let same = (**a.model()).clone(); + a.replace_with_file(same).unwrap(); + assert_eq!(a.step_name(true), None); + change(&mut a, |m| m.with_item(1, "an ordinary edit")).unwrap(); + assert!(a.can_undo()); + assert_eq!(a.step_name(true), None, "the edit's undo has no name"); +} + +/// An app's autosave: when dirty or the file is behind, a save (with a record when it can). True if it saved. +fn autosave(s: &mut Session, dirty: &mut bool, n: &mut usize) -> bool { + if !(*dirty || s.file_behind()) { + return false; + } + let Ok(point) = s.save_point() else { return false }; + s.record(&point, format!("{:?}", point.model()).as_bytes()); + *dirty = false; + *n += 1; + true +} + +/// Two machines with the document open, both autosaving when dirty or when the file is behind: concurrent saves +/// (the newer upload wins, the other's edits not in it) and a save made offline that goes up later over the other's +/// both end with a file that has every edit, and then nobody saves again (no loop). +#[test] +fn the_file_catches_up_with_every_edit_and_saving_stops() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "f.txt", Notes::new("f", &[])); + let mut b = open::(&laptop, "f.txt", Notes::default()); + let (mut dirty_a, mut dirty_b, mut saves) = (false, false, 0usize); + let vv = |s: &Session| s.save_point().unwrap().frontiers().to_vec(); + // Settles: both poll and autosave until the file has both documents' versions and nobody saved for a while. + // (A catch-up save may wait out the backstop, `CATCH_UP_EVERY`, after the last one.) + let settle = |a: &mut Session, b: &mut Session, da: &mut bool, db: &mut bool, saves: &mut usize, what: &str| { + let deadline = Instant::now() + Duration::from_secs(20); + let mut quiet = 0; + while quiet < 60 || hub.file("f.txt").is_some_and(|f| f != vv(a) || vv(a) != vv(b)) { + a.poll(); + b.poll(); + let saved = autosave(a, da, saves) | autosave(b, db, saves); + quiet = if saved { 0 } else { quiet + 1 }; + assert!(Instant::now() < deadline, "never settled: {what} ({saves} saves)"); + std::thread::sleep(Duration::from_millis(5)); + } + }; + // Concurrent: both edit, then both save before either has the other's edit. + change(&mut a, |m| m.with_item(0, "from the pc")).unwrap(); + change(&mut b, |m| m.with_item(0, "from the laptop")).unwrap(); + autosave(&mut a, &mut true, &mut saves); + autosave(&mut b, &mut true, &mut saves); + let before = saves; + settle(&mut a, &mut b, &mut dirty_a, &mut dirty_b, &mut saves, "concurrent saves"); + assert_eq!(a.model().items.len(), 2); + assert_eq!(hub.file("f.txt"), Some(vv(&a)), "the file has every edit"); + assert_eq!(vv(&a), vv(&b)); + assert!(saves - before <= 4, "{} saves to catch up", saves - before); + // Offline: the laptop edits and saves while the pc does too; the laptop's save goes up last, without the pc's edit. + laptop.set_online(false); + until(&mut [&mut b], "offline", |s| !s[0].status().online); + change(&mut b, |m| m.with_title("laptop, offline")).unwrap(); + autosave(&mut b, &mut true, &mut saves); + change(&mut a, |m| m.with_item(0, "pc, meanwhile")).unwrap(); + autosave(&mut a, &mut true, &mut saves); + until(&mut [&mut a], "the pc's save is the file", |s| hub.file("f.txt") == Some(vv(s[0]))); + laptop.set_online(true); + let before = saves; + settle(&mut a, &mut b, &mut dirty_a, &mut dirty_b, &mut saves, "a save from offline"); + assert_eq!(*b.model().title, "laptop, offline"); + assert_eq!(a.model().items.len(), 3); + assert_eq!(hub.file("f.txt"), Some(vv(&a)), "the file has every edit"); + assert_eq!(vv(&a), vv(&b)); + assert!(saves - before <= 4, "{} saves to catch up", saves - before); + // An edit outside Craft is never written over by itself. + hub.edit_outside("f.txt"); + until(&mut [&mut a, &mut b], "heard", |s| s.iter().all(|s| !s.file_behind())); + let before = saves; + settle(&mut a, &mut b, &mut dirty_a, &mut dirty_b, &mut saves, "an outside edit"); + assert_eq!(saves, before); +} + +/// The live relay is down while the file sync goes on (the reviewer's probe): each machine saved its own edit, and +/// each file record then has the other's edit, which its live document lacks. Neither is behind: each waits for the +/// live updates, so nobody saves (was 566 saves in 3 s). Once live is back, only the side that is then ahead saves, +/// once, and the file has both edits. +#[test] +fn with_live_down_and_files_syncing_nobody_saves_in_a_loop() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "l.txt", Notes::new("l", &[])); + let mut b = open::(&laptop, "l.txt", Notes::default()); + until(&mut [&mut a, &mut b], "acked", |s| s.iter().all(|s| s.status().queued == 0)); + hub.set_live(false); + change(&mut a, |m| m.with_item(0, "pc")).unwrap(); + change(&mut b, |m| m.with_item(0, "laptop")).unwrap(); + let (mut da, mut db) = (true, true); + let (mut saves_a, mut saves_b) = (0usize, 0usize); + let t0 = Instant::now(); + while t0.elapsed() < Duration::from_secs(3) { + a.poll(); + b.poll(); + autosave(&mut a, &mut da, &mut saves_a); + autosave(&mut b, &mut db, &mut saves_b); + std::thread::sleep(Duration::from_millis(5)); + } + assert!(saves_a <= 2 && saves_b <= 2, "{saves_a} and {saves_b} saves: the edit's, and at most one more each"); + let before = saves_a + saves_b; + hub.set_live(true); + let deadline = Instant::now() + Duration::from_secs(20); + loop { + a.poll(); + b.poll(); + autosave(&mut a, &mut da, &mut saves_a); + autosave(&mut b, &mut db, &mut saves_b); + let both = |s: &Session| s.model().items.len() == 2; + if both(&a) && both(&b) && hub.file("l.txt").as_deref() == Some(a.save_point().unwrap().frontiers()) { + break; + } + assert!(Instant::now() < deadline, "never caught up"); + std::thread::sleep(Duration::from_millis(5)); + } + // Both may be ahead of the file once they have each other's edit (the same content then): at most one each. + assert!(saves_a + saves_b - before <= 2, "{} saves once live was back", saves_a + saves_b - before); +} + +/// A file ahead of the live document (another machine's save came by file sync, its live updates not yet) is never +/// written over by itself: `file_behind` says no until the live document has everything the file has. +#[test] +fn a_file_ahead_of_the_live_document_is_never_written_over() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "ahead.txt", Notes::new("a", &[])); + let mut b = open::(&laptop, "ahead.txt", Notes::default()); + until(&mut [&mut a, &mut b], "acked", |s| s.iter().all(|s| s.status().queued == 0)); + hub.set_live(false); + change(&mut a, |m| m.with_item(0, "only on the pc yet")).unwrap(); + save(&mut a, b"the pc's save"); + for _ in 0..50 { + a.poll(); + b.poll(); + std::thread::sleep(Duration::from_millis(5)); + } + assert!(hub.file("ahead.txt").is_some()); + assert!(!b.file_behind(), "the file is ahead of the laptop's live document"); + hub.set_live(true); + until(&mut [&mut a, &mut b], "the edit came live", |s| s[1].model().items.len() == 1); + assert!(!b.file_behind(), "now the same: nothing to save"); + // An edit on the laptop makes it the one ahead. + change(&mut b, |m| m.with_item(0, "the laptop's")).unwrap(); + assert!(b.file_behind()); +} + +/// A save as an app makes it: the save point, then the record (before the file is written). +fn save(s: &mut Session, bytes: &[u8]) -> craft_cloud_kit::SaveRecord { + let point = s.save_point().unwrap(); + s.record(&point, bytes) +} + +/// A save point says which model the bytes come from and the version it matches; there is none (and so no record) +/// when the document is read-only here (the reviewer's probe), stopped, or the model is behind the live document. +#[test] +fn a_save_point_is_refused_when_a_record_would_be_wrong() { + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let _newer = open::>(&pc, "r.txt", Notes::new("v2", &["a"])); + let mut older = open::>(&laptop, "r.txt", Notes::default()); + assert!(older.status().read_only.is_some()); + assert!(matches!(older.save_point(), Err(KitError::NoRecord(_))), "read-only"); + for _ in 0..20 { + older.poll(); + std::thread::sleep(Duration::from_millis(5)); + } + assert!(laptop.saves().is_empty(), "no record from a read-only document"); + // Stopped: a change refused for good. + let mut a = open::(&pc, "q.txt", Notes::new("q", &[])); + let point = a.save_point().unwrap(); + assert_eq!(**point.model(), **a.model()); + assert!(!point.frontiers().is_empty(), "the import's head"); + pc.refuse_next("invalid", 1); + change(&mut a, |m| m.with_title("refused")).unwrap(); + until(&mut [&mut a], "stopped", |s| s.iter().all(|s| s.status().stopped)); + assert!(matches!(a.save_point(), Err(KitError::NoRecord(_))), "stopped"); +} + +/// The record goes to the agent before the file is written, and one made while the agent is away is kept and goes +/// once it is back (the agent then adds it to the version it uploaded meanwhile). +#[test] +fn a_record_made_while_the_agent_is_away_goes_once_it_is_back() { + let hub = Hub::new(); + let pc = hub.machine("pc").unwrap(); + let mut a = open::(&pc, "w.txt", Notes::new("w", &[])); + let hex = |b: &[u8]| craft_cloud_api::local::blobs::hex(blake3::hash(b).as_bytes()); + save(&mut a, b"saved with the agent there"); + until(&mut [&mut a], "told at once", |_| pc.saves().iter().any(|(h, _)| *h == hex(b"saved with the agent there"))); + pc.restart(); + until(&mut [&mut a], "the agent went", |s| s.iter().all(|s| !s.status().connected)); + change(&mut a, |m| m.with_title("offline")).unwrap(); + let record = save(&mut a, b"saved while the agent was away"); + assert!(record.frontiers.iter().any(|(p, _)| *p == a.peer())); + assert!(!pc.saves().iter().any(|(h, _)| *h == hex(b"saved while the agent was away"))); + pc.start(); + until(&mut [&mut a], "the record went once the agent was back", |_| { + pc.saves().iter().any(|(h, v)| *h == hex(b"saved while the agent was away") && *v == record.frontiers) + }); +} diff --git a/crates/kit/tests/split.rs b/crates/kit/tests/split.rs new file mode 100644 index 0000000..5146fad --- /dev/null +++ b/crates/kit/tests/split.rs @@ -0,0 +1,31 @@ +//! The split at its edge: a change whose estimate is exactly [`SPLIT_BYTES`] is one commit; one byte more is two. Every +//! update stays at or under the split size. + +mod common; + +use common::{App, Notes}; +use craft_cloud_kit::raw::Replica; +use craft_cloud_kit::{OP_OVERHEAD, SPLIT_BYTES}; + +/// The estimate of adding a note is its row's ID in the list (16 bytes) and its cell (65-byte key and the text), each +/// with the per-operation overhead: two notes of `len` bytes make the split size exactly. +fn note_len_for(total: usize) -> usize { + (total - 2 * (16 + OP_OVERHEAD) - 2 * (65 + OP_OVERHEAD)) / 2 +} + +#[test] +fn a_change_at_the_split_size_is_one_commit_and_one_byte_more_is_two() { + assert_eq!(SPLIT_BYTES % 2, 0); + for (extra, commits) in [(0, 1), (1, 2)] { + let mut r = Replica::default(); + let base = Notes::new("t", &["first"]); + r.write::(None, &base).unwrap(); + let len = note_len_for(SPLIT_BYTES); + let one = base.with_item(1, &"n".repeat(len)); + let two = one.with_item(2, &"m".repeat(len + extra)); + let next = Notes { hint: Some(two.items.iter().skip(1).map(|(id, _)| *id).collect()), ..two }; + let (updates, made) = r.write::(Some(&base), &next).unwrap(); + assert_eq!(made, commits, "{extra} byte(s) over"); + assert!(updates.iter().all(|u| u.len() <= SPLIT_BYTES), "{:?}", updates.iter().map(Vec::len).collect::>()); + } +} diff --git a/crates/kit/tests/undo.rs b/crates/kit/tests/undo.rs new file mode 100644 index 0000000..b8296cd --- /dev/null +++ b/crates/kit/tests/undo.rs @@ -0,0 +1,202 @@ +//! Undo while live, two sessions on the fake agent: undo takes back only this machine's change (the other's edit to +//! the same element stays), redo puts it back, a group is one step, and a change split into several commits is one +//! step. + +mod common; +mod model; + +use std::sync::Arc; + +use common::{change, open, until}; +use craft_cloud_app::fake::Hub; +use craft_cloud_kit::{RichText, Session, UndoKind}; +use model::{Doc, Node, TestApp}; + +fn story(s: &Session) -> String { + s.model().story.text.clone() +} + +fn type_end(doc: &Doc, s: &str) -> Doc { + let len = doc.story.text.chars().count(); + let mut d = Doc { story: Arc::new(model::type_at(&doc.story, len, s)), ..doc.clone() }; + ::refresh_derived(&mut d); + d +} + +fn pair(hub: &Hub, start: Doc) -> (Session, Session, craft_cloud_app::fake::Fake, craft_cloud_app::fake::Fake) { + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let a = open::(&pc, "u.kitdemo", start); + let b = open::(&laptop, "u.kitdemo", Doc::empty()); + (a, b, pc, laptop) +} + +#[test] +fn undo_takes_back_only_this_machines_change_and_redo_puts_it_back() { + let hub = Hub::new(); + let start = Doc { story: Arc::new(RichText::plain("Note: ")), ..Doc::empty() }; + let (mut a, mut b, _pc, _laptop) = pair(&hub, start); + assert!(!a.can_undo(), "opening isn't a step"); + assert_eq!(a.undo_kind(), UndoKind::Live); + change(&mut a, |d| type_end(d, "hello")).unwrap(); + until(&mut [&mut a, &mut b], "the laptop has it", |s| s.iter().all(|s| story(s) == "Note: hello")); + // The other machine edits the same text right after. + change(&mut b, |d| type_end(d, " world")).unwrap(); + until(&mut [&mut a, &mut b], "both have both", |s| s.iter().all(|s| story(s) == "Note: hello world")); + assert!(a.undo().unwrap()); + assert_eq!(story(&a), "Note: world", "only this machine's word went, at once"); + until(&mut [&mut a, &mut b], "the undo reached the laptop", |s| s.iter().all(|s| story(s) == "Note: world")); + assert!(a.can_redo()); + assert!(a.redo().unwrap()); + until(&mut [&mut a, &mut b], "the redo too", |s| s.iter().all(|s| story(s) == "Note: hello world")); + // The laptop's own undo takes back its word only. + assert!(b.undo().unwrap()); + until(&mut [&mut a, &mut b], "the laptop's undo", |s| s.iter().all(|s| story(s) == "Note: hello")); + assert!(!b.undo().unwrap(), "nothing more of the laptop's"); +} + +/// The same element: a layer renamed here and moved there. Undo takes back the rename; the move stays. +#[test] +fn undo_after_a_remote_edit_to_the_same_element_keeps_the_remote_edit() { + let hub = Hub::new(); + let (x, y) = (craft_cloud_kit::ElemId::new(), craft_cloud_kit::ElemId::new()); + let layers = + vec![Arc::new(Node { id: x, name: "x".into(), children: Vec::new() }), Arc::new(Node { id: y, name: "y".into(), children: Vec::new() })]; + let (mut a, mut b, _pc, _laptop) = pair(&hub, Doc { layers: Arc::new(layers), ..Doc::empty() }); + change(&mut a, |d| Doc { layers: Arc::new(model::rename(&d.layers, x, "renamed")), ..d.clone() }).unwrap(); + until(&mut [&mut a, &mut b], "renamed on both", |s| s.iter().all(|s| model::find(&s.model().layers, x).is_some_and(|n| n.name == "renamed"))); + change(&mut b, |d| { + let (rest, taken) = model::remove(&d.layers, x); + Doc { layers: Arc::new(model::insert(&rest, Some(y), 0, taken.unwrap())), ..d.clone() } + }) + .unwrap(); + let under_y = |s: &Session| model::flatten(&s.model().layers).contains(&(x, Some(y))); + until(&mut [&mut a, &mut b], "moved on both", |s| s.iter().all(|s| under_y(s))); + assert!(a.undo().unwrap()); + until(&mut [&mut a, &mut b], "the rename undone, the move kept", |s| { + s.iter().all(|s| under_y(s) && model::find(&s.model().layers, x).is_some_and(|n| n.name == "x")) + }); +} + +/// Typing grouped into one step; a big change (here 1.25 MB of cells, split into several commits under the cap) is +/// one step too. (Loro's undo of one big run of text takes time growing with its square, about 10 s for 1 MB: the +/// report's concerns, so the big change here is cells.) +#[test] +fn a_group_and_a_split_change_are_one_step_each() { + let hub = Hub::new(); + let (mut a, mut b, _pc, _laptop) = pair(&hub, Doc { story: Arc::new(RichText::plain("")), ..Doc::empty() }); + a.group(|s| { + for word in ["one", " two", " three"] { + change(s, |d| type_end(d, word)).unwrap(); + } + }); + assert!(a.begin_group()); + change(&mut a, |d| type_end(d, " four")).unwrap(); + a.end_group(); + let col = craft_cloud_kit::ElemId::new(); + let big = |d: &Doc| { + let mut g = model::Grid { cols: vec![col], ..Default::default() }; + for i in 0..5 { + let row = craft_cloud_kit::ElemId::new(); + g.rows.push(row); + g.cells.insert((row, col), format!("{i}{}", "c".repeat(250 * 1024))); + } + Doc { grid: Arc::new(g), ..d.clone() } + }; + let sent = hub.updates("u.kitdemo"); + change(&mut a, big).unwrap(); + until(&mut [&mut a, &mut b], "the big change arrived", |s| s.iter().all(|s| s.model().grid.cells.len() == 5)); + assert!(hub.updates("u.kitdemo") - sent >= 3, "split into several updates"); + assert!(a.undo().unwrap(), "the big change"); + until(&mut [&mut a, &mut b], "the big change undone in one step", |s| { + s.iter().all(|s| s.model().grid.rows.is_empty() && story(s) == "one two three four") + }); + assert!(a.undo().unwrap(), "the second group"); + assert_eq!(story(&a), "one two three"); + assert!(a.undo().unwrap(), "the first group"); + assert_eq!(story(&a), ""); + until(&mut [&mut a, &mut b], "empty on both", |s| s.iter().all(|s| story(s).is_empty())); + assert!(a.redo().unwrap() && a.redo().unwrap() && a.redo().unwrap()); + until(&mut [&mut a, &mut b], "all redone", |s| s.iter().all(|s| s.model().grid.cells.len() == 5 && story(s) == "one two three four")); + assert!(a.status().error.is_none(), "{:?}", a.status().error); +} + +/// An undo asked for during a gesture (remote changes held) changes nothing under the app: the gesture's change is +/// taken (its `old` is still the session's model), and the undo runs once the gesture ends, after the held remote +/// change (the reviewer's probe). +#[test] +fn an_undo_during_a_gesture_waits_for_its_end() { + use common::{App, Notes}; + let hub = Hub::new(); + let (pc, laptop) = (hub.machine("pc").unwrap(), hub.machine("laptop").unwrap()); + let mut a = open::(&pc, "h.txt", Notes::new("t", &["x"])); + let mut b = open::(&laptop, "h.txt", Notes::default()); + change(&mut a, |m| m.with_item(1, "mine")).unwrap(); + until(&mut [&mut a, &mut b], "both", |s| s.iter().all(|s| s.model().texts().len() == 2)); + a.hold_remote(true); + let gesture_old = a.model().clone(); + change(&mut b, |m| m.with_title("remote")).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(300); + while std::time::Instant::now() < deadline { + b.poll(); + a.poll(); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(!a.undo().unwrap(), "deferred"); + assert!(Arc::ptr_eq(a.model(), &gesture_old), "nothing changed under the gesture"); + let gesture_new = Arc::new(gesture_old.with_edit(0, "dragged")); + a.local_change(&gesture_old, &gesture_new).unwrap(); + a.hold_remote(false); + assert!(a.poll().changed); + // The remote title came, and the deferred undo took back the gesture's change (the last step). + assert_eq!(*a.model().title, "remote"); + assert_eq!(a.model().texts(), vec!["x", "mine"]); + until(&mut [&mut a, &mut b], "the same on both", |s| s.iter().all(|s| *s.model().title == "remote" && s.model().texts() == ["x", "mine"])); +} + +/// The story with `paste` added at its end, as one change (plain text: no runs to move). +fn paste(s: &mut Session, paste: &str) { + change(s, |d| { + let mut n = Doc { story: Arc::new(RichText::plain(&format!("{}{paste}", d.story.text))), ..d.clone() }; + ::refresh_derived(&mut n); + n + }) + .unwrap(); +} + +/// A step that changed up to 64 KiB of text undoes and redoes quickly; one byte more, and the step is refused +/// (quickly, never a frozen app) and stays as a barrier: nothing before it can be undone. A 1 MB paste likewise. +#[test] +fn a_step_too_big_to_undo_is_refused_and_a_barrier() { + use craft_cloud_kit::{KitError, MAX_UNDO_TEXT_BYTES}; + use std::time::{Duration, Instant}; + let hub = Hub::new(); + let pc = hub.machine("pc").unwrap(); + let mut a = open::(&pc, "big.kitdemo", Doc { story: Arc::new(RichText::plain("Start.")), ..Doc::empty() }); + paste(&mut a, " small"); + // At the limit: undone and redone, quickly. + paste(&mut a, &"x".repeat(MAX_UNDO_TEXT_BYTES)); + assert!(a.can_undo() && a.undo_blocked().is_none()); + let t = Instant::now(); + assert!(a.undo().unwrap()); + assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed()); + assert_eq!(story(&a), "Start. small"); + let t = Instant::now(); + assert!(a.redo().unwrap()); + assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed()); + assert_eq!(story(&a).len(), "Start. small".len() + MAX_UNDO_TEXT_BYTES); + // One byte over: refused at once, and a barrier. + paste(&mut a, &"y".repeat(MAX_UNDO_TEXT_BYTES + 1)); + assert!(!a.can_undo()); + assert!(a.undo_blocked().unwrap_or_default().contains("too large to undo while others edit")); + for _ in 0..2 { + let t = Instant::now(); + assert_eq!(a.undo(), Err(KitError::UndoTooLarge { bytes: MAX_UNDO_TEXT_BYTES + 1 })); + assert!(t.elapsed() < Duration::from_millis(200), "{:?}", t.elapsed()); + } + assert_eq!(story(&a).len(), "Start. small".len() + 2 * MAX_UNDO_TEXT_BYTES + 1, "nothing undone"); + // 1 MB. + paste(&mut a, &"z".repeat(1024 * 1024)); + let t = Instant::now(); + assert_eq!(a.undo(), Err(KitError::UndoTooLarge { bytes: 1024 * 1024 })); + assert!(t.elapsed() < Duration::from_millis(200), "{:?}", t.elapsed()); +} diff --git a/crates/meta/Cargo.toml b/crates/meta/Cargo.toml index 5e7f4fd..318a199 100644 --- a/crates/meta/Cargo.toml +++ b/crates/meta/Cargo.toml @@ -9,6 +9,8 @@ publish = false [dependencies] craft-cloud-proto.workspace = true +# A version's save record, kept as the JSON the agent sent. +serde_json.workspace = true sqlx.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/meta/migrations/postgres/0006_save_records.sql b/crates/meta/migrations/postgres/0006_save_records.sql new file mode 100644 index 0000000..6eaccf5 --- /dev/null +++ b/crates/meta/migrations/postgres/0006_save_records.sql @@ -0,0 +1,4 @@ +-- A version's save record (when a Craft app saved it from a live document): the JSON of +-- `craft_cloud_proto::SaveRecord` (the document and its version vector then), stored as the agent sent it and never +-- read beyond that. Older versions, and versions made outside Craft, have none. +ALTER TABLE versions ADD COLUMN save TEXT; diff --git a/crates/meta/migrations/sqlite/0006_save_records.sql b/crates/meta/migrations/sqlite/0006_save_records.sql new file mode 100644 index 0000000..6eaccf5 --- /dev/null +++ b/crates/meta/migrations/sqlite/0006_save_records.sql @@ -0,0 +1,4 @@ +-- A version's save record (when a Craft app saved it from a live document): the JSON of +-- `craft_cloud_proto::SaveRecord` (the document and its version vector then), stored as the agent sent it and never +-- read beyond that. Older versions, and versions made outside Craft, have none. +ALTER TABLE versions ADD COLUMN save TEXT; diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index e167a7c..bd8911b 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -14,7 +14,7 @@ use std::collections::HashMap; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_TREE_DEPTH}; -use craft_cloud_proto::{Conflict, ConflictKind, DeviceId, Epoch, Name, NewVersion, NodeId, Op, SpaceId, VersionId}; +use craft_cloud_proto::{Conflict, ConflictKind, DeviceId, Epoch, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId}; use sqlx::Row; use crate::blobs::manifest_file_size; @@ -180,9 +180,14 @@ impl Ctx<'_> { let Some(size) = manifest_file_size(self.conn, space, &v.manifest).await? else { return Ok(Err(ConflictKind::MissingManifest { manifest: v.manifest })); }; + // The save record as the agent sent it (the server never reads what it means), if well formed. + let save = match &v.save { + Some(record) if record.is_valid() => serde_json::to_string(record).ok(), + _ => None, + }; sqlx::query( - "INSERT INTO versions (space_id, id, node, parent_version, epoch, manifest, size, mtime_ms, exec, device, created_ms, seq) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + "INSERT INTO versions (space_id, id, node, parent_version, epoch, manifest, size, mtime_ms, exec, device, created_ms, seq, save) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", ) .bind(self.space()) .bind(uuid_bytes(v.id.as_uuid())) @@ -196,6 +201,7 @@ impl Ctx<'_> { .bind(uuid_bytes(self.device.as_uuid())) .bind(self.now_ms) .bind(self.seq) + .bind(save) .execute(&mut *self.conn) .await?; Ok(Ok(())) @@ -410,4 +416,46 @@ impl Meta { w.commit().await?; Ok(CommitOutcome { seq, epoch: info.epoch }) } + + /// Adds a save record to `node`'s version `version`, which went up without one (the app's record reached the + /// agent only after the upload: it was saved while the agent was away). Appended only: a version that has a + /// record keeps it, and the same call again changes nothing. When it is added, the node is in the change feed + /// again, so the other machines learn it. `NotFound` when `node` has no such version. + pub async fn attach_save(&self, space: SpaceId, node: NodeId, version: VersionId, save: &SaveRecord) -> Result { + if !save.is_valid() { + return Err(MetaError::Invalid("a save record's frontiers are at most 256 heads, sorted and distinct")); + } + if save.doc != craft_cloud_proto::live::node_doc(space, node) { + return Err(MetaError::Invalid("a save record names the live document of its own file")); + } + let json = serde_json::to_string(save).map_err(|_| MetaError::Invalid("a save record that can't be written"))?; + let mut w = self.begin_write().await?; + let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let found = sqlx::query("SELECT save FROM versions WHERE space_id = $1 AND id = $2 AND node = $3") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(version.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *w.tx) + .await? + .ok_or(MetaError::NotFound("version"))?; + if found.try_get::, _>("save")?.is_some() { + return Ok(CommitOutcome { seq: info.seq, epoch: info.epoch }); + } + let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; + sqlx::query("UPDATE versions SET save = $1 WHERE space_id = $2 AND id = $3") + .bind(json) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(version.as_uuid())) + .execute(&mut *w.tx) + .await?; + sqlx::query("UPDATE nodes SET changed_seq = $1 WHERE space_id = $2 AND id = $3") + .bind(to_i64(seq)?) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .execute(&mut *w.tx) + .await?; + sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(CommitOutcome { seq, epoch: info.epoch }) + } } diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index ee4199c..1784df2 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -31,7 +31,7 @@ pub use live::{ Appended, DEFAULT_LIVE_BYTES_PER_SPACE, LiveDoc, LiveUpdate, MAX_LIVE_DOC_BYTES, MAX_LIVE_DOC_ROWS_PER_SPACE, MAX_LIVE_DOCS_PER_SPACE, MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, MAX_LIVE_DOCUMENTS_PER_SPACE, MAX_LIVE_PAGE, SnapshotRef, }; -pub use tree::SpaceInfo; +pub use tree::{MAX_PAGE_SAVE_BYTES, SpaceInfo}; pub use users::{DeviceAuth, DeviceRecord, Role, UserId, UserRecord, valid_username}; #[derive(Debug, thiserror::Error)] diff --git a/crates/meta/src/schema.rs b/crates/meta/src/schema.rs index 61cf3a3..e4c5220 100644 --- a/crates/meta/src/schema.rs +++ b/crates/meta/src/schema.rs @@ -11,6 +11,7 @@ const SQLITE: &[&str] = &[ include_str!("../migrations/sqlite/0003_live.sql"), include_str!("../migrations/sqlite/0004_join.sql"), include_str!("../migrations/sqlite/0005_live_documents.sql"), + include_str!("../migrations/sqlite/0006_save_records.sql"), ]; const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0001_init.sql"), @@ -18,6 +19,7 @@ const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0003_live.sql"), include_str!("../migrations/postgres/0004_join.sql"), include_str!("../migrations/postgres/0005_live_documents.sql"), + include_str!("../migrations/postgres/0006_save_records.sql"), ]; /// Arbitrary fixed key for the Postgres advisory lock that serialises migrations between servers sharing one diff --git a/crates/meta/src/tree.rs b/crates/meta/src/tree.rs index 9ab87cd..c5b30d0 100644 --- a/crates/meta/src/tree.rs +++ b/crates/meta/src/tree.rs @@ -13,9 +13,13 @@ use crate::{Meta, MetaError}; /// Node columns plus its current version (if a file). const NODE_SELECT: &str = "SELECT n.id, n.parent, n.name, n.is_dir, n.deleted, n.changed_seq, v.id AS v_id, v.manifest AS v_manifest, v.size AS v_size, \ - v.mtime_ms AS v_mtime, v.exec AS v_exec, v.device AS v_device, v.epoch AS v_epoch \ + v.mtime_ms AS v_mtime, v.exec AS v_exec, v.device AS v_device, v.epoch AS v_epoch, v.save AS v_save \ FROM nodes n LEFT JOIN versions v ON v.space_id = n.space_id AND v.id = n.version"; +/// The most bytes of save records one page of the tree or the change feed carries (it ends early otherwise): with the +/// rest of [`MAX_PAGE_NODES`] nodes, well under the 16 MiB an agent reads of a page. +pub const MAX_PAGE_SAVE_BYTES: usize = 4 * 1024 * 1024; + /// A space's epoch, current sequence number and root directory. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SpaceInfo { @@ -57,6 +61,8 @@ fn node_state(row: &AnyRow) -> Result { exec: flag(row, "v_exec")?, device: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("v_device")?)?), epoch: Epoch::from_uuid(uuid_from(&row.try_get::, _>("v_epoch")?)?), + // A record that doesn't read is as none: the file syncs as any file does. + save: row.try_get::, _>("v_save")?.and_then(|json| serde_json::from_str(&json).ok()), }) }; Ok(NodeState { @@ -129,8 +135,19 @@ impl Meta { .fetch_all(&mut *tx) .await?; tx.commit().await?; - let more = rows.len() > usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; - let kept = if more { rows.len() - 1 } else { rows.len() }; + let mut more = rows.len() > usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; + let mut kept = if more { rows.len() - 1 } else { rows.len() }; + // Save records are what can make a page big (up to about 9 KiB each): the page ends early rather than pass + // their budget, so no listing comes near what an agent reads of one. + let mut save_bytes = 0usize; + for (i, row) in rows.iter().take(kept).enumerate() { + save_bytes = save_bytes.saturating_add(row.try_get::, _>("v_save")?.map_or(0, |s| s.len())); + if save_bytes > MAX_PAGE_SAVE_BYTES && i > 0 { + kept = i; + more = true; + break; + } + } let nodes = rows.iter().take(kept).map(node_state).collect::, _>>()?; let next = match (more, kept.checked_sub(1).and_then(|i| rows.get(i)), nodes.last()) { (true, Some(row), Some(last)) => Some(PageCursor { seq: to_u64(row.try_get("changed_seq")?)?, node: last.id }), diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index e2ed4af..d875fc3 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -85,6 +85,9 @@ macro_rules! on_every_backend { on_every_backend!( users_and_devices, a_new_space_has_only_its_root, + a_version_keeps_its_save_record, + pages_of_save_records_stay_small, + a_late_save_record_is_appended_once, manifests_need_their_chunks_and_are_per_space, stored_bytes_are_counted_once_per_space_and_capped, concurrent_chunk_batches_charge_a_shared_chunk_once, @@ -160,6 +163,96 @@ async fn device(meta: &Meta, u: &UserRecord) -> DeviceId { } /// Uploads one random chunk and a manifest for it to the user's space; returns the manifest ID. +/// A save record that comes after its version: added to that version (the node is in the change feed again), once; +/// a version with a record keeps it; the same call again changes nothing; another node's version is not found. +async fn a_late_save_record_is_appended_once(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = meta.create_device(u.id, "pc", &[3u8; 32], NOW).await.unwrap(); + let parent = root(meta, &u).await; + let m = manifest(meta, &u, 4).await; + let (node, v) = (NodeId::generate(), version(m)); + meta.commit(u.space, dev, &[Op::Create { node, parent, name: name("late.kitdemo"), version: v.clone() }], NOW).await.unwrap(); + let before = meta.space_info(u.space).await.unwrap().seq; + let doc = craft_cloud_proto::live::node_doc(u.space, node); + let stranger = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::DocId::generate(), frontiers: vec![(1, 5)] }; + assert!(matches!(meta.attach_save(u.space, node, v.id, &stranger).await, Err(MetaError::Invalid(_))), "another file's document"); + let unsorted = craft_cloud_proto::SaveRecord { doc, frontiers: vec![(2, 1), (1, 5)] }; + assert!(matches!(meta.attach_save(u.space, node, v.id, &unsorted).await, Err(MetaError::Invalid(_)))); + let record = craft_cloud_proto::SaveRecord { doc, frontiers: vec![(1, 5)] }; + let added = meta.attach_save(u.space, node, v.id, &record).await.unwrap(); + assert_eq!(added.seq, before + 1); + let feed = meta.changes(u.space, before, None, MAX_PAGE_NODES).await.unwrap(); + let NodeKind::File(f) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!((f.version, f.save.clone()), (v.id, Some(record.clone())), "in the feed again, with its record"); + let other = craft_cloud_proto::SaveRecord { doc: record.doc, frontiers: vec![(2, 1)] }; + assert_eq!(meta.attach_save(u.space, node, v.id, &other).await.unwrap().seq, before + 1, "kept, nothing changed"); + assert_eq!(meta.attach_save(u.space, node, v.id, &record).await.unwrap().seq, before + 1); + let tree = full_tree(meta, u.space).await; + let NodeKind::File(f) = &find(&tree.nodes, node).kind else { panic!("a file") }; + assert_eq!(f.save, Some(record)); + let nowhere = NodeId::generate(); + let theirs = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, nowhere), frontiers: vec![(2, 1)] }; + assert!(matches!(meta.attach_save(u.space, nowhere, v.id, &theirs).await, Err(MetaError::NotFound(_)))); + let too_many = + craft_cloud_proto::SaveRecord { doc: other.doc, frontiers: (0..=craft_cloud_proto::ops::MAX_SAVE_HEADS as u64).map(|p| (p, 1)).collect() }; + assert!(matches!(meta.attach_save(u.space, node, v.id, &too_many).await, Err(MetaError::Invalid(_)))); +} + +/// Files whose versions all carry the biggest save record there can be: every page of the listing and of the change +/// feed ends before its records pass their budget, far under what an agent reads of a page, and the pages together +/// still list every file once, each with its record. +async fn pages_of_save_records_stay_small(meta: &Meta) { + use craft_cloud_meta::MAX_PAGE_SAVE_BYTES; + let u = user(meta, "ana").await; + let dev = meta.create_device(u.id, "pc", &[3u8; 32], NOW).await.unwrap(); + let parent = root(meta, &u).await; + let m = manifest(meta, &u, 4).await; + let biggest = craft_cloud_proto::SaveRecord { + doc: craft_cloud_proto::DocId::generate(), + frontiers: (0..craft_cloud_proto::ops::MAX_SAVE_HEADS as u64).map(|p| (u64::MAX - 1000 + p, u32::MAX)).collect(), + }; + let record_bytes = serde_json::to_string(&biggest).unwrap().len(); + assert!(record_bytes < 10 * 1024, "{record_bytes}"); + let files = 2 * MAX_PAGE_SAVE_BYTES / record_bytes + 10; + let mut made = std::collections::BTreeSet::new(); + for batch in 0..files.div_ceil(500) { + let ops: Vec = (0..500.min(files - batch * 500)) + .map(|i| { + let node = NodeId::generate(); + made.insert(node); + let v = NewVersion { save: Some(biggest.clone()), ..version(m) }; + Op::Create { node, parent, name: name(&format!("f{batch}-{i}.kitdemo")), version: v } + }) + .collect(); + meta.commit(u.space, dev, &ops, NOW).await.unwrap(); + } + for feed in [false, true] { + let (mut after, mut seen, mut pages) = (None, std::collections::BTreeSet::new(), 0); + loop { + let page = if feed { + meta.changes(u.space, 0, after, MAX_PAGE_NODES).await.unwrap() + } else { + meta.tree(u.space, after, MAX_PAGE_NODES).await.unwrap() + }; + pages += 1; + let json = serde_json::to_vec(&page).unwrap().len(); + assert!(json < MAX_PAGE_SAVE_BYTES + 2 * 1024 * 1024, "a page of {json} bytes"); + for n in &page.nodes { + if let NodeKind::File(f) = &n.kind { + assert_eq!(f.save.as_ref(), Some(&biggest)); + assert!(seen.insert(n.id), "listed twice"); + } + } + match page.next { + Some(next) => after = Some(next), + None => break, + } + } + assert_eq!(seen, made, "every file, once"); + assert!(pages >= 3, "cut into pages by their records: {pages}"); + } +} + async fn manifest(meta: &Meta, u: &UserRecord, len: u32) -> ManifestId { let chunk = ChunkId::from_bytes(*blake3::hash(uuid::Uuid::now_v7().as_bytes()).as_bytes()); meta.add_chunk(u.space, &chunk, u64::from(len) + 1, NOW).await.unwrap(); @@ -171,7 +264,7 @@ async fn manifest(meta: &Meta, u: &UserRecord, len: u32) -> ManifestId { } fn version(manifest: ManifestId) -> NewVersion { - NewVersion { id: VersionId::generate(), manifest, mtime_ms: NOW, exec: false } + NewVersion { id: VersionId::generate(), manifest, mtime_ms: NOW, exec: false, save: None } } async fn full_tree(meta: &Meta, space: craft_cloud_proto::SpaceId) -> craft_cloud_proto::ChangeSet { @@ -220,6 +313,35 @@ async fn users_and_devices(meta: &Meta) { assert!(meta.list_devices(ana.id).await.unwrap()[0].revoked); } +/// A version a Craft app saved keeps its save record (as stored, unread), in the listing and the change feed; one +/// without has none; one over the record limit is stored without it. +async fn a_version_keeps_its_save_record(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = meta.create_device(u.id, "pc", &[3u8; 32], NOW).await.unwrap(); + let parent = root(meta, &u).await; + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::DocId::generate(), frontiers: vec![(1, 5), (2, 3)] }; + let (saved, plain, huge) = (NodeId::generate(), NodeId::generate(), NodeId::generate()); + let op = |node: NodeId, name: &str, v: NewVersion| Op::Create { node, parent, name: craft_cloud_proto::Name::new(name).unwrap(), version: v }; + let m = manifest(meta, &u, 4).await; + let too_many = + craft_cloud_proto::SaveRecord { doc: record.doc, frontiers: (0..=craft_cloud_proto::ops::MAX_SAVE_HEADS as u64).map(|p| (p, 1)).collect() }; + let ops = [ + op(saved, "a.kitdemo", NewVersion { save: Some(record.clone()), ..version(m) }), + op(plain, "b.txt", version(m)), + op(huge, "c.kitdemo", NewVersion { save: Some(too_many), ..version(m) }), + ]; + meta.commit(u.space, dev, &ops, NOW).await.unwrap(); + for listing in [full_tree(meta, u.space).await, meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap()] { + let save = |id| match &find(&listing.nodes, id).kind { + NodeKind::File(f) => f.save.clone(), + NodeKind::Dir => None, + }; + assert_eq!(save(saved), Some(record.clone())); + assert_eq!(save(plain), None); + assert_eq!(save(huge), None, "over the limit: stored without it"); + } +} + async fn a_new_space_has_only_its_root(meta: &Meta) { let u = user(meta, "ana").await; let tree = full_tree(meta, u.space).await; @@ -1659,6 +1781,8 @@ async fn upgrading_counts_what_each_space_already_stores() { // And before join codes. sqlx::query("DROP TABLE pairings").execute(&mut conn).await.unwrap(); sqlx::query("DROP TABLE invites").execute(&mut conn).await.unwrap(); + // And before save records. + sqlx::query("ALTER TABLE versions DROP COLUMN save").execute(&mut conn).await.unwrap(); sqlx::query("DELETE FROM schema_version WHERE version >= 2").execute(&mut conn).await.unwrap(); } let meta = Meta::open(&url).await.unwrap(); @@ -1683,5 +1807,5 @@ async fn concurrent_opens_migrate_once_on_postgres() { use sqlx::Connection; let mut conn = sqlx::PgConnection::connect(&url).await.unwrap(); let versions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(versions, 5); + assert_eq!(versions, 6); } diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index 1991245..ca6f316 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -9,7 +9,7 @@ publish = false [dependencies] blake3.workspace = true -caseless.workspace = true +craft-cloud-api.workspace = true getrandom.workspace = true postcard.workspace = true serde.workspace = true diff --git a/crates/proto/src/api.rs b/crates/proto/src/api.rs index 1d96a08..6ff15c5 100644 --- a/crates/proto/src/api.rs +++ b/crates/proto/src/api.rs @@ -8,6 +8,9 @@ use crate::hash::ChunkId; use crate::ids::{DeviceId, Epoch, NodeId, SpaceId}; use crate::ops::{Conflict, Op}; +/// What apps see of these bodies lives in `craft-cloud-api` (no postcard), re-exported here. +pub use craft_cloud_api::api::{DeviceInfo, ErrorAnswer, PairingState, PairingStatus, codes}; + /// Protocol versions this build speaks. A client and server must share at least one. pub const PROTOCOL_MIN: u32 = 1; pub const PROTOCOL_MAX: u32 = 1; @@ -92,27 +95,6 @@ impl std::fmt::Debug for PairingCreated { } } -/// Where an add-a-computer code stands, in [`PairingStatus`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PairingState { - /// Not used yet, and still good. - Waiting, - /// A computer joined with it; [`PairingStatus::device_name`] names it. - Used, - /// Its time ran out unused. - Expired, -} - -/// `GET /api/v1/pairings/{id}` (device auth, the account's own pairings only). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PairingStatus { - pub state: PairingState, - /// The device that joined, once `used`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub device_name: Option, -} - /// `POST /api/v1/join` (no auth): redeem an invite or an add-a-computer code (normalised or as typed) for a new /// device; answered with a [`LoginResponse`]. `password` is required, and only taken, for an invite to an account /// that has none yet. Its `Debug` output hides the code and the password. @@ -135,18 +117,6 @@ impl std::fmt::Debug for JoinRequest { } } -/// One of the signed-in user's devices. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DeviceInfo { - pub id: DeviceId, - pub name: String, - pub created_ms: i64, - pub last_used_ms: i64, - pub revoked: bool, - /// The device making this request. - pub current: bool, -} - /// `POST /api/v1/chunks/missing` and its answer: which of `ids` this space doesn't have. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct MissingRequest { @@ -164,6 +134,17 @@ pub struct CommitRequest { pub ops: Vec, } +/// `POST /api/v1/saves`: a save record for a version that went up without one (its app's record reached the agent +/// late). Appended: a version that has one keeps it, and the same call again changes nothing. Answered with a +/// [`CommitResponse`] (the sequence number the node is in the change feed at, or the current one when nothing +/// changed); `not_found` when the node has no such version. An older server answers `not_found` for the path. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SaveAttach { + pub node: crate::NodeId, + pub version: crate::VersionId, + pub save: crate::SaveRecord, +} + /// A commit that applied. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CommitResponse { @@ -171,34 +152,6 @@ pub struct CommitResponse { pub epoch: Epoch, } -/// Stable error codes, the `error` field of [`ErrorBody`]. -pub mod codes { - /// A commit's operations clash with the current state; `conflicts` lists them. - pub const CONFLICT: &str = "conflict"; - /// Chunks must be uploaded first; `missing` lists them. - pub const MISSING_CHUNKS: &str = "missing_chunks"; - /// The client's change counter is stale or ahead of the server: list the whole tree again. - pub const RESYNC_REQUIRED: &str = "resync_required"; - pub const UNAUTHORIZED: &str = "unauthorized"; - pub const RATE_LIMITED: &str = "rate_limited"; - pub const INVALID: &str = "invalid"; - pub const NOT_FOUND: &str = "not_found"; - pub const TOO_LARGE: &str = "too_large"; - /// A request body took longer to arrive than the server allows for it (HTTP 408); send it again. - pub const TIMEOUT: &str = "timeout"; - pub const USERNAME_TAKEN: &str = "username_taken"; - /// A chunk or manifest would take the account over the bytes the server lets it store (HTTP 507); `quota` says - /// how much it stores and may store. - pub const QUOTA_EXCEEDED: &str = "quota_exceeded"; - /// `join`: the code is wrong, expired or used (never which). - pub const INVALID_CODE: &str = "invalid_code"; - /// `join`: an invite to an account without a password needs one; send it again with `password`. - pub const PASSWORD_REQUIRED: &str = "password_required"; - /// `join`: the new password breaks the server's rules (the message says which). - pub const WEAK_PASSWORD: &str = "weak_password"; - pub const INTERNAL: &str = "internal"; -} - /// How much an account stores on the server and may store, in bytes, with `quota_exceeded`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct QuotaUse { diff --git a/crates/proto/src/lib.rs b/crates/proto/src/lib.rs index 52b5c36..ae04398 100644 --- a/crates/proto/src/lib.rs +++ b/crates/proto/src/lib.rs @@ -1,25 +1,24 @@ //! Craft Cloud protocol types, shared by the server and the sync agent. //! //! Pure: no I/O. The only clock read is in the `generate()` constructors of the ID types (UUIDv7). The live protocol -//! ([`live`], agent ↔ server) and the agent's local API ([`local`], apps ↔ agent) live here too, so the server, the +//! ([`live`], agent ↔ server) lives here. What apps need too (IDs, names, limits, the agent's local API [`local`]) +//! lives in `craft-cloud-api`, which has no postcard, and is re-exported here under the same paths, so the server, the //! agent and `craft-cloud-app` share one definition. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod api; pub mod conflict; pub mod hash; -pub mod ids; pub mod join; -pub mod limits; pub mod live; -pub mod local; pub mod manifest; -pub mod name; pub mod ops; +pub use craft_cloud_api::{ids, limits, local, name}; + pub use conflict::{ConflictStamp, conflict_name}; pub use hash::{BlobKind, ChunkId, FileHash, HexIdError, ManifestId}; pub use ids::{DeviceId, DocId, Epoch, NodeId, SpaceId, VersionId}; pub use manifest::{ChunkRef, ChunkerParamsId, Manifest, ManifestError}; pub use name::{FoldKey, Name, NameError, UNICODE_VERSION}; -pub use ops::{ChangeSet, Conflict, ConflictKind, FileState, NewVersion, NodeKind, NodeState, Op, PageCursor}; +pub use ops::{ChangeSet, Conflict, ConflictKind, FileState, NewVersion, NodeKind, NodeState, Op, PageCursor, SaveRecord}; diff --git a/crates/proto/src/live.rs b/crates/proto/src/live.rs index c73a478..aa8e9ef 100644 --- a/crates/proto/src/live.rs +++ b/crates/proto/src/live.rs @@ -67,17 +67,14 @@ pub const LIVE_PROTOCOL_MAX: u32 = 2; /// carries recents only. pub const LIVE_DOCUMENTS_PROTOCOL: u32 = 2; -/// Largest update, in bytes. -pub const MAX_UPDATE_BYTES: usize = 1024 * 1024; +/// Largest update, in bytes; largest snapshot of a document (an import, or a compaction), in bytes (it travels through +/// the chunk API, never in a frame); largest presence message (Loro's ephemeral state: a cursor, a selection, a machine +/// name), in bytes. Apps see them too, so they live in `craft-cloud-api`. +pub use craft_cloud_api::limits::{MAX_PRESENCE_BYTES, MAX_SNAPSHOT_BYTES, MAX_UPDATE_BYTES}; /// Most documents one device may be subscribed to at once. pub const MAX_SUBSCRIPTIONS: usize = 256; /// Largest encoded frame: an update and its envelope (IDs, numbers, an error message). pub const MAX_FRAME_BYTES: usize = MAX_UPDATE_BYTES + 4096; -/// Largest snapshot of a document (an import, or a compaction), in bytes (it travels through the chunk API, never in -/// a frame). -pub const MAX_SNAPSHOT_BYTES: usize = crate::limits::MAX_MANIFEST_BYTES; -/// Largest presence message (Loro's ephemeral state: a cursor, a selection, a machine name), in bytes. -pub const MAX_PRESENCE_BYTES: usize = 4096; /// Most presence messages one device may send a second for one document (more are dropped, never stored anyway). pub const PRESENCE_PER_SEC: u32 = 20; /// A document's log above its snapshot is compacted once it holds more than this many updates… diff --git a/crates/proto/src/ops.rs b/crates/proto/src/ops.rs index 8a340c2..55dbe28 100644 --- a/crates/proto/src/ops.rs +++ b/crates/proto/src/ops.rs @@ -10,6 +10,8 @@ use crate::hash::ManifestId; use crate::ids::{DeviceId, Epoch, NodeId, VersionId}; use crate::name::Name; +pub use craft_cloud_api::local::{MAX_SAVE_HEADS, SaveRecord}; + /// A new file version. Its size comes from the manifest, which must already be uploaded. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct NewVersion { @@ -18,6 +20,10 @@ pub struct NewVersion { /// Modification time on the uploading device, milliseconds since the Unix epoch. Display only. pub mtime_ms: i64, pub exec: bool, + /// A Craft app saved this content from a live document: which, and its version then. Appended: an older agent + /// sends none, an older server ignores it. One that doesn't read is none ([`SaveRecord::lenient`]). + #[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "SaveRecord::lenient")] + pub save: Option, } /// One operation in a batch commit. Node IDs are made by the client (UUIDv7). @@ -105,6 +111,11 @@ pub struct FileState { pub device: DeviceId, /// The space epoch the version was committed in. pub epoch: Epoch, + /// The version's save record, when a Craft app saved it from a live document (see [`NewVersion::save`]). + /// Appended: an older server sends none, an older agent ignores it. One that doesn't read (an earlier build's + /// form, a damaged one) is none, and the file state reads ([`SaveRecord::lenient`]). + #[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "SaveRecord::lenient")] + pub save: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -178,6 +189,7 @@ mod tests { manifest: crate::ManifestId::from_bytes([0xff; 32]), mtime_ms: i64::MIN, exec: false, + save: None, }; let create = Op::Create { node: node(u128::MAX), parent: node(u128::MAX), name, version }; let largest = [mv, create].into_iter().max_by_key(|op| serde_json::to_vec(op).unwrap().len()).unwrap(); @@ -246,6 +258,7 @@ mod tests { exec: true, device: DeviceId::from_uuid(Uuid::from_u128(4)), epoch: Epoch::from_uuid(Uuid::from_u128(6)), + save: None, }), }; let json = serde_json::to_value(&file).unwrap(); @@ -267,4 +280,69 @@ mod tests { fn op_node_is_the_acted_on_node() { assert_eq!(Op::Delete { node: node(5), expected: None }.node(), node(5)); } + + /// A save record is appended: a version without one has the JSON it always had (an older peer reads it), one + /// with it reads back, and an older peer's JSON (no field) reads as none. + #[test] + fn save_records_are_appended() { + let plain = NewVersion { + id: VersionId::from_uuid(Uuid::from_u128(1)), + manifest: ManifestId::from_bytes([1; 32]), + mtime_ms: 1, + exec: false, + save: None, + }; + let json = serde_json::to_value(&plain).unwrap(); + assert!(json.get("save").is_none(), "{json}"); + let doc = crate::DocId::from_uuid(Uuid::from_u128(9)); + let saved = NewVersion { save: Some(SaveRecord { doc, frontiers: vec![(1, 5), (2, 3)] }), ..plain.clone() }; + let json = serde_json::to_value(&saved).unwrap(); + assert_eq!(serde_json::from_value::(json).unwrap(), saved); + let older: NewVersion = serde_json::from_value( + serde_json::json!({ "id": Uuid::from_u128(1), "manifest": ManifestId::from_bytes([1; 32]), "mtime_ms": 1, "exec": false }), + ) + .unwrap(); + assert_eq!(older, plain); + // An agent from before save records reads a file state that has one (the field is unknown to it). + #[derive(Deserialize)] + #[allow(dead_code)] + struct OlderFileState { + version: VersionId, + manifest: ManifestId, + size: u64, + mtime_ms: i64, + exec: bool, + device: DeviceId, + epoch: Epoch, + } + let state = FileState { + version: plain.id, + manifest: plain.manifest, + size: 1, + mtime_ms: 1, + exec: false, + device: DeviceId::from_uuid(Uuid::from_u128(2)), + epoch: Epoch::from_uuid(Uuid::from_u128(3)), + save: saved.save.clone(), + }; + let json = serde_json::to_value(&state).unwrap(); + assert!(json.get("save").is_some()); + assert!(serde_json::from_value::(json).is_ok()); + // A record from before frontiers (a version vector), a damaged one or one not well formed is no record, and the + // file state and new version holding it still read (the reviewer's probe: the whole file state failed). + let vector = serde_json::json!({ "doc": doc, "version": [[1, 5]] }); + assert!(serde_json::from_value::(vector.clone()).is_err()); + for odd in [vector, serde_json::json!("junk"), serde_json::json!({ "doc": doc, "frontiers": [[2, 1], [1, 5]] })] { + let mut j = serde_json::to_value(&state).unwrap(); + j["save"] = odd.clone(); + let read = serde_json::from_value::(j).unwrap_or_else(|e| panic!("{odd}: {e}")); + assert_eq!(read, FileState { save: None, ..state.clone() }); + let mut j = serde_json::to_value(&saved).unwrap(); + j["save"] = odd; + assert_eq!(serde_json::from_value::(j).unwrap(), plain); + } + let mut j = serde_json::to_value(&state).unwrap(); + j["save"] = serde_json::Value::Null; + assert_eq!(serde_json::from_value::(j).unwrap().save, None); + } } diff --git a/crates/sync/src/engine/apply.rs b/crates/sync/src/engine/apply.rs index e58f401..9a78bb4 100644 --- a/crates/sync/src/engine/apply.rs +++ b/crates/sync/src/engine/apply.rs @@ -12,6 +12,16 @@ use crate::transport::Transport; use super::{Engine, Local, Mode, OTHER_REASON, Report, SyncError, io_err, per_node}; +/// Who wins a file both sides saved from its live document. +enum LiveWinner { + /// The server's version (its save includes ours). + Server, + /// Ours, uploaded over it. + Here, + /// Not known yet which is newer: neither is taken this pass. + Wait, +} + impl Engine { pub(super) async fn apply_remote(&mut self, mode: Mode, report: &mut Report) -> Result<(), SyncError> { let root = self.state.root.ok_or_else(|| SyncError::Corrupt("no root".into()))?; @@ -303,6 +313,52 @@ impl Engine { self.state.base.insert(id, new_entry); return Ok(()); } + // Both saved by Craft apps from the file's live document: no conflict copy (see `SaveRecords`). + if let Some(file) = &remote.file + && let Some((winner, checked)) = self.live_winner(id, file, &bpath)? + { + if matches!(winner, LiveWinner::Wait) { + // The base stays as it is, and the push leaves the file too: the next pass decides again. + report.skipped.push((bpath.to_string(), format!("{}; tried again next pass", super::SAVE_ORDER_WAITS))); + return Ok(()); + } + if moved { + self.settle_occupants(id, &rpath, Some(&bpath), None, report).await?; + self.relocate(&bpath, &rpath, report)?; + } + match winner { + LiveWinner::Server => { + // Its record includes ours: everything here is in it. Replaces exactly what was hashed, + // stat'ed before the hash (a rename keeps the file's ID and time); a save that landed since + // keeps the old base, and the next pass decides again. + if let Some((meta, hash)) = self.download(id, file, &rpath, checked, report).await? { + new_entry.kind = self.base_file(file, meta, hash); + } + } + LiveWinner::Here => { + // Ours goes up over theirs: the base takes the server's version, so the push sends ours + // as its next version (the time never matches, so it is always seen as changed). + let manifest = self.fetch_manifest(&file.manifest).await?; + let local_id = match self.stat(&rpath)? { + Some(FsKind::File(m)) => m.id, + _ => None, + }; + new_entry.kind = BaseKind::File { + version: file.version, + manifest: file.manifest, + size: file.size, + mtime_ms: i64::MIN, + exec: file.exec, + local_id, + hash: *manifest.file_hash(), + seen_ms: self.fs.now_ms(), + }; + } + LiveWinner::Wait => {} + } + self.state.base.insert(id, new_entry); + return Ok(()); + } // Both changed: keep ours as a conflict copy, take theirs. self.settle_occupants(id, &rpath, Some(&bpath), None, report).await?; self.move_aside(&bpath, report)?; @@ -986,6 +1042,60 @@ impl Engine { } /// Whether the file at `at` holds exactly the content the base recorded (hashed; an unreadable file doesn't). + /// Which side wins when `id`'s file changed here and on the server and both versions carry save records of one + /// live document; `None` when either lacks one (a conflict, as always). + /// With what the file looked like before it was hashed: a download may only replace exactly that (a save that + /// lands after the hash keeps the old base, and the next pass decides again, by its own record or as a conflict). + /// + /// When the agent can't tell yet which save is newer, the file waits ([`LiveWinner::Wait`]); after + /// `save_order_wait_ms` of that, `None` (a conflict copy). A winner is never guessed. + fn live_winner(&mut self, id: NodeId, file: &FileState, at: &RelPath) -> Result)>, SyncError> { + let checked = self.stat(at)?; + let hash = self.content_hash(at); + // Our content is a Craft save whose record is on its way: never a conflict copy, it waits for the record. + if let Some(hash) = &hash + && self.save_pending(id, hash) + { + return Ok(self.wait(id).map(|w| (w, checked))); + } + // A pair that isn't (or no longer is) two Craft saves starts no wait, and ends one under way. + let ours = hash.and_then(|h| self.save_record(id, &h)); + let pair = match (file.save.clone(), ours) { + (Some(theirs), Some(ours)) if theirs.doc == ours.doc => Some((theirs, ours)), + _ => None, + }; + let Some((theirs, ours)) = pair else { + self.unknown_since.remove(&id); + return Ok(None); + }; + let winner = match self.save_includes(&theirs, &ours) { + Some(true) => LiveWinner::Server, + Some(false) => LiveWinner::Here, + None => match self.wait(id) { + Some(wait) => wait, + None => return Ok(None), + }, + }; + if !matches!(winner, LiveWinner::Wait) { + self.unknown_since.remove(&id); + } + Ok(Some((winner, checked))) + } + + /// `id`'s file waits this pass ([`LiveWinner::Wait`]), unless it has waited `save_order_wait_ms` already: then + /// `None`, and it is handled as any two edits are (a conflict copy). + fn wait(&mut self, id: NodeId) -> Option { + // The machine's clock while awake (`Instant`), so a sleep through the wait doesn't end it. + let now = std::time::Instant::now(); + let since = *self.unknown_since.entry(id).or_insert(now); + if now.saturating_duration_since(since).as_millis() >= u128::try_from(self.config.save_order_wait_ms.max(0)).unwrap_or(0) { + self.unknown_since.remove(&id); + return None; + } + self.deferred.insert(id); + Some(LiveWinner::Wait) + } + fn holds_base_content(&self, entry: &BaseEntry, at: &RelPath) -> bool { match &entry.kind { BaseKind::File { hash, .. } => self.content_hash(at) == Some(*hash), diff --git a/crates/sync/src/engine/mod.rs b/crates/sync/src/engine/mod.rs index f9fc5fb..55f2335 100644 --- a/crates/sync/src/engine/mod.rs +++ b/crates/sync/src/engine/mod.rs @@ -48,6 +48,10 @@ pub const NOT_NFC_REASON: &str = /// Why a changed file that may still be being written ([`Fs::still_changing`]) is skipped. Every skip for it starts /// with these words, so a caller can tell that a pass should look again once the file had time to settle. pub const STILL_CHANGING: &str = "still changing"; +/// Why a file both sides saved from its live document waits (see [`SaveRecords::includes`]). +pub const SAVE_ORDER_WAITS: &str = "two Craft saves wait to learn which is newer"; +/// Why a file a Craft app is saving waits (see [`SaveRecords::pending`]). +pub const SAVE_RECORD_WAITS: &str = "a Craft save waits for its record"; /// What the report says about a download kept because the file it replaces is in use or may not be replaced. const KEPT_WAITS: &str = "the server's version is downloaded and kept, and lands once it can"; /// Most chunk IDs asked about in one missing-chunk query. @@ -124,6 +128,11 @@ pub struct Config { /// chunk, [`MAX_CHUNK_BYTES`](craft_cloud_proto::limits::MAX_CHUNK_BYTES): with less room than that, no file /// of consequence fits. Tests lower it. pub quota_floor: u64, + /// How long two Craft saves of one file may wait for the agent to learn which is newer ([`SaveRecords::includes`]) + /// before they are a conflict copy, as any two edits are (10 minutes, counted on a monotonic clock: on Linux and + /// macOS that is time awake, so a laptop that sleeps through it doesn't end it early; on Windows std's clock may + /// count sleep too). Tests lower it. + pub save_order_wait_ms: i64, } impl Config { @@ -138,6 +147,7 @@ impl Config { racy_window_ms: 2_000, max_file_bytes: MAX_FILE_BYTES, quota_floor: u64::from(MAX_CHUNK_BYTES), + save_order_wait_ms: 10 * 60 * 1000, } } @@ -346,6 +356,42 @@ pub struct Engine { /// bigger than the room left is skipped before it is read (see [`Engine::upload`]). Forgotten at the start of /// every pass, since the admin may have raised the cap or the room may have grown. quota: Option<(u64, u64)>, + /// What Craft apps saved from live documents ([`Engine::set_save_records`]). + saves: Option>, + /// Versions a late save record was sent for this run (see [`Engine::attach_late_saves`]). + attached: BTreeSet<(NodeId, craft_cloud_proto::VersionId)>, + /// Files both sides saved from their live document where it isn't known yet which save is newer: neither side is + /// taken this pass (the push leaves them too). Cleared at the start of every pass. + deferred: BTreeSet, + /// Since when each such file has waited, across passes (a monotonic clock: on Linux and macOS it stops while the + /// machine sleeps; on Windows it may not). + unknown_since: HashMap, +} + +/// What Craft apps saved from live documents, as the agent learnt it from them (`craft_cloud_proto::SaveRecord`). +/// +/// - A new version of a file whose content is one an app saved carries the save record. +/// - When a file changed here and on the server, and both this content and the server's version carry a record of +/// the same live document, there is no conflict copy: the live document holds both sides' edits. The server's +/// version wins if its record includes this one's (it was saved later, from a document with everything this one +/// had); otherwise this one goes up over it (it is the newer upload, and the next save writes everything anyway). +/// - Whether one record includes another is the agent's to say, from its copy of the document ([`Self::includes`]). +/// When it can't tell yet (it lacks the operations), the pair waits: neither side is taken this pass. Still unknown +/// after [`Config::save_order_wait_ms`], it is a conflict copy as always. A winner is never guessed. +/// - Without a record on either side (a file edited outside Craft), it is a conflict as always. +pub trait SaveRecords: Send + Sync { + /// The record of the save that wrote content `hash` to `node`'s file, if an app saved exactly that. + fn record(&self, node: NodeId, hash: &FileHash) -> Option; + /// Whether the live document as `a` names it (its frontiers) holds every operation `b`'s does; `None` while that + /// can't be told (the operations aren't here yet). + fn includes(&self, a: &craft_cloud_proto::SaveRecord, b: &craft_cloud_proto::SaveRecord) -> Option; + /// Whether an app announced a save of exactly content `hash` to `node`'s file whose record hasn't come yet (an app + /// tells the agent before it writes, and sends the record after the changes it covers). Such content is never + /// taken for an edit outside Craft: it isn't uploaded without its record, nor made a conflict copy, but waits. + fn pending(&self, node: NodeId, hash: &FileHash) -> bool { + let _ = (node, hash); + false + } } /// Which deletions the mass-delete guard lets through this pass. @@ -392,9 +438,38 @@ impl Engine { checkpoint: None, declined_dir_ids: BTreeSet::new(), quota: None, + saves: None, + attached: BTreeSet::new(), + deferred: BTreeSet::new(), + unknown_since: HashMap::new(), }) } + /// Where the engine learns what Craft apps saved from live documents: a version uploaded with content an app + /// saved carries its record, and two versions of a live-linked file that both do never make a conflict copy + /// (see [`SaveRecords`]). + pub fn set_save_records(&mut self, saves: std::sync::Arc) { + self.saves = Some(saves); + } + + /// The save record of `node`'s content `hash`, if well formed. + pub(crate) fn save_record(&self, node: NodeId, hash: &FileHash) -> Option { + self.saves.as_ref()?.record(node, hash).filter(craft_cloud_proto::SaveRecord::is_valid) + } + + /// Whether a Craft save of `node`'s content `hash` is on its way: announced, with no record yet. + pub(crate) fn save_pending(&self, node: NodeId, hash: &FileHash) -> bool { + self.save_record(node, hash).is_none() && self.saves.as_ref().is_some_and(|s| s.pending(node, hash)) + } + + /// Whether save `a` includes save `b`, as the agent can tell; `None` while it can't. + pub(crate) fn save_includes(&self, a: &craft_cloud_proto::SaveRecord, b: &craft_cloud_proto::SaveRecord) -> Option { + if a == b { + return Some(true); + } + self.saves.as_ref()?.includes(a, b) + } + /// Saves the state at safe points during a pass, so a crash loses less than the whole pass: once the server's /// changes are applied here, before a commit that deletes files (recording them in [`State::sent_deletes`]), /// and after every commit is recorded. The state is consistent at each of them, as it is when a pass fails. An @@ -457,6 +532,7 @@ impl Engine { // Anything may have changed in the folder since the last pass. self.listings.get_mut().unwrap_or_else(PoisonError::into_inner).clear(); self.quota = None; + self.deferred.clear(); // A landing a crash cut short is put right before anything looks at the folder. Until it is, the pass fails // (a filesystem error, so it is retried): a scan could take the moved file for a new one or the target for // deleted. The error names the landing's target. @@ -479,9 +555,41 @@ impl Engine { self.push(&mut report).await?; self.approval = Approval::None; self.release.clear(); + self.attach_late_saves().await; Ok(report) } + /// Save records that came after their version went up (an app saved while the agent was away, and told it once + /// back): each is added to its version on the server (appended; the same again changes nothing), so the version + /// counts as a Craft save everywhere. Only a file whose server version is the one this machine agreed on, with the + /// content the record names, and none yet. Sent once per version per run; an older server (`NotFound`) is let be, + /// anything else tried again next pass. + async fn attach_late_saves(&mut self) { + if self.saves.is_none() { + return; + } + let mut late = Vec::new(); + for (id, entry) in self.state.base.iter() { + let BaseKind::File { version, hash, .. } = &entry.kind else { continue }; + let Some(remote) = self.state.remote.get(id).and_then(|r| r.file.as_ref()) else { continue }; + if remote.version != *version || remote.save.is_some() || self.attached.contains(&(*id, *version)) { + continue; + } + if let Some(record) = self.save_record(*id, hash) { + late.push((*id, *version, record)); + } + } + for (node, version, record) in late { + self.attached.insert((node, version)); + match self.transport.attach_save(node, version, &record).await { + Ok(()) | Err(TransportError::NotFound) => {} + Err(_) => { + self.attached.remove(&(node, version)); + } + } + } + } + // ---------------------------------------------------------------- pull async fn pull(&mut self) -> Result { diff --git a/crates/sync/src/engine/push.rs b/crates/sync/src/engine/push.rs index 55033cc..470e36f 100644 --- a/crates/sync/src/engine/push.rs +++ b/crates/sync/src/engine/push.rs @@ -5,7 +5,20 @@ use std::io; use bytes::Bytes; use craft_cloud_chunker::{FASTCDC_V1, chunk_reader}; -use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_TREE_DEPTH}; +use craft_cloud_proto::limits::{MAX_BATCH_JSON_BYTES, MAX_BATCH_OPS, MAX_TREE_DEPTH}; + +/// What an operation adds to a commit body, at most: a longest operation without a save record is at most 1/1000 of +/// the body limit (`craft_cloud_proto::ops` tests that a full batch of them fits), and a record adds its document and +/// under 48 bytes per head of its frontiers. Without records a batch is the full [`MAX_BATCH_OPS`]. +fn op_weight(op: &Op) -> usize { + let save = match op { + Op::Create { version, .. } | Op::Update { version, .. } => { + version.save.as_ref().map_or(0, |s| s.frontiers.len().saturating_mul(48).saturating_add(64)) + } + _ => 0, + }; + (MAX_BATCH_JSON_BYTES / MAX_BATCH_OPS).saturating_add(save) +} use craft_cloud_proto::{ ChunkId, ChunkRef, ChunkerParamsId, Conflict, Epoch, FileHash, FileState, FoldKey, Manifest, Name, NewVersion, NodeId, Op, VersionId, }; @@ -15,7 +28,7 @@ use crate::path::RelPath; use crate::state::{BaseEntry, BaseKind, HeldLink, RemoteNode}; use crate::transport::{Transport, TransportError}; -use super::{Effect, Engine, MISSING_QUERY_IDS, Report, STILL_CHANGING, SyncError, per_node}; +use super::{Effect, Engine, MISSING_QUERY_IDS, Report, SAVE_RECORD_WAITS, STILL_CHANGING, SyncError, per_node}; /// Why a file of `size` bytes is skipped when files may be at most `limit` bytes. fn too_big(size: u64, limit: u64) -> String { @@ -366,11 +379,23 @@ impl Engine { report.skipped.push((path.to_string(), format!("{STILL_CHANGING}; synced once it stops"))); continue; } + // Two Craft saves waiting to learn which is newer (see `SaveRecords::includes`): not sent over the other. + if self.deferred.contains(id) { + continue; + } let Some(BaseEntry { kind: BaseKind::File { version: expected, exec, size: base_size, .. }, .. }) = self.state.base.get(id).cloned() else { continue; }; let Some((version, size, meta, hash)) = self.upload(path, exec, base_size, report).await? else { continue }; + // Content a Craft app is saving whose record hasn't come: never sent without it (the other machines would + // take it for an edit outside Craft). The next pass, once the record is here, sends it with it. + if self.save_pending(*id, &hash) { + report.skipped.push((path.to_string(), format!("{SAVE_RECORD_WAITS}; tried again next pass"))); + continue; + } + // Content a Craft app saved from the file's live document carries its record. + let version = NewVersion { save: self.save_record(*id, &hash), ..version }; ops.push((Op::Update { node: *id, expected, version: version.clone() }, Effect::Update { id: *id, version, size, meta, hash })); } // New files. @@ -555,7 +580,7 @@ impl Engine { } report.uploaded.push(path.clone()); let exec = if self.fs.has_exec_bit() { before.exec } else { base_exec }; - let version = NewVersion { id: VersionId::generate(), manifest: manifest.id(), mtime_ms: before.mtime_ms, exec }; + let version = NewVersion { id: VersionId::generate(), manifest: manifest.id(), mtime_ms: before.mtime_ms, exec, save: None }; Ok(Some((version, summary.size, before, FileHash::from_bytes(summary.file_hash)))) } @@ -598,7 +623,17 @@ impl Engine { let mut queue: std::collections::VecDeque<(Op, Effect)> = ops.into(); let mut retried: BTreeSet = BTreeSet::new(); while !queue.is_empty() { - let take = queue.len().min(MAX_BATCH_OPS); + // At most the batch limit, and within the body limit (save records make operations bigger). + let mut take = 0usize; + let mut bytes = 0usize; + for (op, _) in queue.iter().take(MAX_BATCH_OPS) { + let weight = op_weight(op); + if take > 0 && bytes.saturating_add(weight) > MAX_BATCH_JSON_BYTES { + break; + } + bytes = bytes.saturating_add(weight); + take += 1; + } let mut batch: Vec<(Op, Effect)> = queue.drain(..take).collect(); loop { if batch.is_empty() { @@ -778,6 +813,7 @@ impl Engine { exec: version.exec, device: self.config.device, epoch, + save: version.save.clone(), }; self.state.remote.insert(id, RemoteNode { parent: Some(parent), name: Some(name.clone()), file: Some(file.clone()) }); self.state.base.insert(id, BaseEntry { parent: Some(parent), name: Some(name), kind: self.base_file(&file, meta, hash) }); @@ -792,6 +828,7 @@ impl Engine { exec: version.exec, device: self.config.device, epoch, + save: version.save.clone(), }; if let Some(r) = self.state.remote.get_mut(&id) { r.file = Some(file.clone()); diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index e46939c..c6c5945 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -12,7 +12,10 @@ pub mod path; pub mod state; pub mod transport; -pub use engine::{Config, DEFAULT_IGNORES, Engine, NOT_NFC_REASON, OTHER_REASON, Report, STILL_CHANGING, SyncError}; +pub use engine::{ + Config, DEFAULT_IGNORES, Engine, NOT_NFC_REASON, OTHER_REASON, Report, SAVE_ORDER_WAITS, SAVE_RECORD_WAITS, STILL_CHANGING, SaveRecords, + SyncError, +}; pub use fs::{FileMeta, Fs, FsEntry, FsKind, LandError, MemFs, ReadOnly, is_read_only, read_only_error, rel}; pub use path::RelPath; pub use state::{BaseEntry, BaseKind, HeldLink, RemoteNode, State}; diff --git a/crates/sync/src/transport.rs b/crates/sync/src/transport.rs index ed168c0..f5d6314 100644 --- a/crates/sync/src/transport.rs +++ b/crates/sync/src/transport.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use bytes::Bytes; use craft_cloud_proto::api::{CommitResponse, SpaceResponse}; -use craft_cloud_proto::{ChangeSet, ChunkId, Conflict, ManifestId, Op, PageCursor}; +use craft_cloud_proto::{ChangeSet, ChunkId, Conflict, ManifestId, NodeId, Op, PageCursor, SaveRecord, VersionId}; #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum TransportError { @@ -46,4 +46,10 @@ pub trait Transport: Send + Sync { async fn put_manifest(&self, id: &ManifestId, data: Bytes) -> Result<(), TransportError>; async fn get_manifest(&self, id: &ManifestId) -> Result; async fn commit(&self, ops: &[Op]) -> Result; + /// Adds a save record to `node`'s version `version`, which went up without one (`POST /saves`, appended and + /// idempotent). A server from before late records: `NotFound`, as this default says. + async fn attach_save(&self, node: NodeId, version: VersionId, save: &SaveRecord) -> Result<(), TransportError> { + let _ = (node, version, save); + Err(TransportError::NotFound) + } } diff --git a/crates/sync/tests/common/mod.rs b/crates/sync/tests/common/mod.rs index 320595c..68877bb 100644 --- a/crates/sync/tests/common/mod.rs +++ b/crates/sync/tests/common/mod.rs @@ -231,6 +231,16 @@ impl Transport for Local { } Ok(CommitResponse { seq: out.seq, epoch: out.epoch }) } + + async fn attach_save( + &self, + node: craft_cloud_proto::NodeId, + version: craft_cloud_proto::VersionId, + save: &craft_cloud_proto::SaveRecord, + ) -> Result<(), TransportError> { + self.faults.check()?; + self.meta.attach_save(self.space, node, version, save).await.map(|_| ()).map_err(meta_err) + } } pub const STAMP: ConflictStamp = ConflictStamp { year: 2026, month: 10, day: 11, hour: 14, minute: 3, utc: false }; diff --git a/crates/sync/tests/common/sim.rs b/crates/sync/tests/common/sim.rs index 412de0b..2db05a0 100644 --- a/crates/sync/tests/common/sim.rs +++ b/crates/sync/tests/common/sim.rs @@ -174,8 +174,13 @@ pub struct Linked { /// Replaces a tracked file or folder on `m` with a link, keeping what it held. fn make_link(rng: &mut Rng, m: &mut Machine) -> Option { let index = m.state().base_index(); - let candidates: Vec = - index.keys().filter(|p| !p.is_root() && matches!(m.fs().stat(p), Ok(Some(FsKind::File(_) | FsKind::Dir { .. })))).cloned().collect(); + // (Never a live document's file: only its app touches that, see `act`.) + let candidates: Vec = index + .keys() + .filter(|p| !p.is_root() && !p.file_name().is_some_and(|n| n.as_str().ends_with(".kitdemo"))) + .filter(|p| matches!(m.fs().stat(p), Ok(Some(FsKind::File(_) | FsKind::Dir { .. })))) + .cloned() + .collect(); let at = rng.pick(&candidates)?.clone(); let below = |p: &RelPath| p.names().get(at.depth()..).map(<[Name]>::to_vec).unwrap_or_default(); let was_dir = matches!(m.fs().stat(&at), Ok(Some(FsKind::Dir { .. }))); @@ -328,7 +333,9 @@ pub fn act( // It counts as destroyed only once the user's write or delete went through. let old = |m: &Machine, p: &RelPath| m.fs().files().get(p).map(|d| FileHash::from_bytes(*blake3::hash(d).as_bytes())); let dirs: Vec = m.fs().dirs(); - let files = files_of(m); + // A live document's file (`*.kitdemo`, the network simulation's Craft saves) is only ever saved by its app: the + // user doesn't edit, delete or rename it outside it. + let files: Vec = files_of(m).into_iter().filter(|f| !f.file_name().is_some_and(|n| n.as_str().ends_with(".kitdemo"))).collect(); match rng.below(11) { 0 | 1 => { let dir = if rng.below(3) == 0 { RelPath::root() } else { rel(rng.pick(DIRS).copied().unwrap_or("x")) }; @@ -761,6 +768,11 @@ pub async fn settle_and_check(seed: u64, server: &Server, ms: &mut eprintln!("TRACE settle m{i} sync: {report:?}\n files {}\n dirs {:?}", listing(m), m.fs().dirs()); } changed |= report.changed() || !report.pending_deletes.is_empty() || !report.pending_local_deletes.is_empty(); + // Two Craft saves waiting to learn which is newer are asked about again next pass: not settled yet. + changed |= report + .skipped + .iter() + .any(|(_, why)| why.starts_with(craft_cloud_sync::SAVE_ORDER_WAITS) || why.starts_with(craft_cloud_sync::SAVE_RECORD_WAITS)); if let Some(s) = skipped.get_mut(i) { *s = report.skipped.iter().filter(|(_, why)| is_case_clash(why)).map(|(p, _)| rel(p)).collect(); } diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index 8cab8c2..a8570f7 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3099,7 +3099,7 @@ async fn a_pass_costs_about_the_size_of_the_folder_not_its_square() { let folder = NodeId::generate(); let mut ops = vec![Op::Mkdir { node: folder, parent: root, name: Name::new(&format!("f{f}")).unwrap() }]; for i in 0..PER_FOLDER { - let version = NewVersion { id: VersionId::generate(), manifest: manifest.id(), mtime_ms: 1, exec: false }; + let version = NewVersion { id: VersionId::generate(), manifest: manifest.id(), mtime_ms: 1, exec: false, save: None }; ops.push(Op::Create { node: NodeId::generate(), parent: folder, name: Name::new(&format!("{i}.txt")).unwrap(), version }); if ops.len() == 1000 { server.meta.commit(ana.space, device, &std::mem::take(&mut ops), 0).await.unwrap(); diff --git a/crates/sync/tests/live_saves.rs b/crates/sync/tests/live_saves.rs new file mode 100644 index 0000000..8c798da --- /dev/null +++ b/crates/sync/tests/live_saves.rs @@ -0,0 +1,324 @@ +//! Files saved by Craft apps from live documents: two machines saving one live-linked file at once never make a +//! conflict copy (the version whose save includes the other's wins, else the newer upload), the versions carry their +//! records to the server, and a file edited outside Craft still gets its conflict copy. + +mod common; + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use common::{Machine, Server, machine, path}; +use craft_cloud_proto::{DocId, FileHash, NodeId, SaveRecord}; +use craft_cloud_sync::{Fs, SaveRecords, Transport}; + +/// What each machine's apps saved: (node, content hash) to record. Here a record's frontiers stand for a version +/// vector, `(peer, end)`, and one includes another as vectors do (the agent asks its copy of the document; the tests +/// need no Loro). With `unknown` set, nothing can be told. +#[derive(Default, Clone)] +struct Book(Arc>>, Arc, Arc>>); + +impl SaveRecords for Book { + fn record(&self, node: NodeId, hash: &FileHash) -> Option { + self.0.lock().unwrap().get(&(node, *hash)).cloned() + } + + fn pending(&self, node: NodeId, hash: &FileHash) -> bool { + self.2.lock().unwrap().contains(&(node, *hash)) + } + + fn includes(&self, a: &SaveRecord, b: &SaveRecord) -> Option { + if self.1.load(std::sync::atomic::Ordering::Relaxed) { + return None; + } + Some(b.frontiers.iter().all(|(peer, end)| a.frontiers.iter().any(|(p, e)| p == peer && e >= end))) + } +} + +impl Book { + /// The app is saving `data` to `p` and wrote the file; its record hasn't reached the agent yet (announced only). + fn saving(&self, m: &mut Machine, p: &str, data: &[u8]) -> (NodeId, FileHash) { + m.fs_mut().write(&path(p), data, false, 0).unwrap(); + let key = (node(m, p), FileHash::from_bytes(*blake3::hash(data).as_bytes())); + self.2.lock().unwrap().push(key); + key + } + + /// The record of an announced save comes. + fn record_comes(&self, key: (NodeId, FileHash), doc: DocId, version: &[(u64, u32)]) { + self.2.lock().unwrap().retain(|k| *k != key); + self.0.lock().unwrap().insert(key, SaveRecord { doc, frontiers: version.to_vec() }); + } + + /// The app saved `data` to `p` from the document at `version`. + fn save(&self, m: &mut Machine, p: &str, data: &[u8], doc: DocId, version: &[(u64, u32)]) { + m.fs_mut().write(&path(p), data, false, 0).unwrap(); + let node = node(m, p); + self.0.lock().unwrap().insert((node, FileHash::from_bytes(*blake3::hash(data).as_bytes())), SaveRecord { doc, frontiers: version.to_vec() }); + } +} + +fn node(m: &Machine, p: &str) -> NodeId { + let want = path(p); + m.state().base.keys().copied().find(|id| m.state().base_path(*id).as_ref() == Some(&want)).unwrap() +} + +fn files(m: &Machine) -> Vec<(String, Vec)> { + m.fs().files().into_iter().map(|(p, d)| (p.to_string(), d)).collect() +} + +/// Two machines with save books, sharing `doc.kitdemo` (live-linked) at `v1`. +async fn two(server: &Server) -> (Machine, Machine, Book, Book, DocId) { + let ana = server.user("ana").await; + let (mut a, mut b) = (machine(server, &ana, "laptop").await, machine(server, &ana, "desktop").await); + let (book_a, book_b) = (Book::default(), Book::default()); + a.set_save_records(Arc::new(book_a.clone())); + b.set_save_records(Arc::new(book_b.clone())); + a.fs_mut().write(&path("doc.kitdemo"), b"v1", false, 0).unwrap(); + a.sync_once().await.unwrap(); + b.sync_once().await.unwrap(); + // The file's live document, as the agent names it. + let doc = craft_cloud_proto::live::node_doc(a.state().space.unwrap(), node(&a, "doc.kitdemo")); + (a, b, book_a, book_b, doc) +} + +async fn rounds(a: &mut Machine, b: &mut Machine) { + for _ in 0..4 { + a.sync_once().await.unwrap(); + b.sync_once().await.unwrap(); + } +} + +/// The desktop's save includes everything the laptop's had (it was made later from the live document): the laptop +/// takes the desktop's version, and there is no conflict copy. +#[tokio::test] +async fn a_save_that_includes_the_other_wins_without_a_conflict_copy() { + let server = Server::new().await; + let (mut a, mut b, book_a, book_b, doc) = two(&server).await; + book_a.save(&mut a, "doc.kitdemo", b"laptop's save", doc, &[(1, 5)]); + book_b.save(&mut b, "doc.kitdemo", b"desktop's later save", doc, &[(1, 5), (2, 3)]); + b.sync_once().await.unwrap(); + rounds(&mut a, &mut b).await; + let want = vec![("doc.kitdemo".to_owned(), b"desktop's later save".to_vec())]; + assert_eq!(files(&a), want); + assert_eq!(files(&b), want); +} + +/// Neither save includes the other (both machines edited the live document offline, say): the newer upload wins +/// (it overwrites, no conflict copy; the live document has both sides' edits, and the next save writes them all). +#[tokio::test] +async fn concurrent_saves_take_the_newer_upload_without_a_conflict_copy() { + let server = Server::new().await; + let (mut a, mut b, book_a, book_b, doc) = two(&server).await; + book_a.save(&mut a, "doc.kitdemo", b"laptop's save", doc, &[(1, 6)]); + book_b.save(&mut b, "doc.kitdemo", b"desktop's save", doc, &[(1, 5), (2, 3)]); + b.sync_once().await.unwrap(); + rounds(&mut a, &mut b).await; + let want = vec![("doc.kitdemo".to_owned(), b"laptop's save".to_vec())]; + assert_eq!(files(&a), want, "the laptop's went up last"); + assert_eq!(files(&b), want); + // The version on the server carries the laptop's record. + let listing = a.transport().tree(None).await.unwrap(); + let file = listing.nodes.into_iter().find_map(|n| match n.kind { + craft_cloud_proto::NodeKind::File(f) if !n.deleted => Some(f), + _ => None, + }); + assert_eq!(file.and_then(|f| f.save), Some(SaveRecord { doc, frontiers: vec![(1, 6)] })); +} + +/// Autosave on both machines, round after round: never a conflict copy, and both end on the same content. +#[tokio::test] +async fn autosaves_on_both_machines_never_make_a_conflict_copy() { + let server = Server::new().await; + let (mut a, mut b, book_a, book_b, doc) = two(&server).await; + for round in 0..6u32 { + // Each machine's live document has the other's earlier edits and its own new one. + book_a.save(&mut a, "doc.kitdemo", format!("laptop {round}").as_bytes(), doc, &[(1, 10 + round), (2, 10 + round)]); + book_b.save(&mut b, "doc.kitdemo", format!("desktop {round}").as_bytes(), doc, &[(1, 10 + round), (2, 11 + round)]); + a.sync_once().await.unwrap(); + b.sync_once().await.unwrap(); + } + rounds(&mut a, &mut b).await; + assert_eq!(files(&a), files(&b)); + assert_eq!(files(&a).len(), 1, "no conflict copy: {:?}", files(&a)); +} + +/// One side edited outside Craft (no record): a conflict copy, as always. +#[tokio::test] +async fn a_file_edited_outside_craft_still_gets_a_conflict_copy() { + let server = Server::new().await; + let (mut a, mut b, book_a, _book_b, doc) = two(&server).await; + book_a.save(&mut a, "doc.kitdemo", b"laptop's save", doc, &[(1, 5)]); + b.fs_mut().write(&path("doc.kitdemo"), b"edited in a text editor", false, 0).unwrap(); + b.sync_once().await.unwrap(); + rounds(&mut a, &mut b).await; + let all = files(&a); + assert_eq!(all, files(&b)); + assert_eq!(all.len(), 2, "{all:?}"); + assert!(all.iter().any(|(p, _)| p.contains("(conflict")), "{all:?}"); + assert!(all.iter().any(|(_, d)| d == b"laptop's save") && all.iter().any(|(_, d)| d == b"edited in a text editor")); +} + +/// Saves of two different documents (a copy of the file, say) are not one document's: a conflict copy. +#[tokio::test] +async fn saves_of_different_documents_are_a_conflict() { + let server = Server::new().await; + let (mut a, mut b, book_a, book_b, doc) = two(&server).await; + book_a.save(&mut a, "doc.kitdemo", b"from one document", doc, &[(1, 5)]); + book_b.save(&mut b, "doc.kitdemo", b"from another", DocId::generate(), &[(1, 5), (2, 1)]); + b.sync_once().await.unwrap(); + rounds(&mut a, &mut b).await; + assert_eq!(files(&a).len(), 2, "{:?}", files(&a)); +} + +/// The server's save includes the laptop's, so the laptop's file is to be replaced; but another program saves the +/// file while the engine reads it (at each of its reads in turn). That save is never replaced unseen: it ends as a +/// file on both machines (a conflict copy, having no record). +#[tokio::test] +async fn a_save_landing_while_the_server_wins_is_kept() { + use common::{SaveDuring, SaveOn, machine_on}; + for n in 1..=5 { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine_on(&server, &ana, "laptop", SaveDuring::default(), 50, |_| {}).await; + let mut b = machine(&server, &ana, "desktop").await; + let (book_a, book_b) = (Book::default(), Book::default()); + a.set_save_records(Arc::new(book_a.clone())); + b.set_save_records(Arc::new(book_b.clone())); + a.fs().write(&path("doc.kitdemo"), b"v1"); + a.sync_once().await.unwrap(); + b.sync_once().await.unwrap(); + let doc = DocId::generate(); + // The laptop's save, and the desktop's later one that includes it. + let laptop = b"laptop's save".to_vec(); + a.fs().write(&path("doc.kitdemo"), &laptop); + let at = a.state().base.keys().copied().find(|id| a.state().base_path(*id).as_ref() == Some(&path("doc.kitdemo"))).unwrap(); + book_a.0.lock().unwrap().insert((at, FileHash::from_bytes(*blake3::hash(&laptop).as_bytes())), SaveRecord { doc, frontiers: vec![(1, 5)] }); + book_b.save(&mut b, "doc.kitdemo", b"desktop's later save", doc, &[(1, 5), (2, 3)]); + b.sync_once().await.unwrap(); + a.fs().arm(path("doc.kitdemo"), b"another program's save, meanwhile", SaveOn::Open(n)); + for _ in 0..5 { + a.sync_once().await.unwrap(); + b.sync_once().await.unwrap(); + } + let on_a: Vec> = a.fs().files().into_values().collect(); + let on_b: Vec> = b.fs().files().into_values().collect(); + if a.fs().fired() { + for files in [&on_a, &on_b] { + assert!(files.iter().any(|d| d == b"another program's save, meanwhile"), "read {n}: {files:?}"); + } + } + // (A save landing after the download replaced the file is an ordinary later edit: it goes up over the + // desktop's version, which stays in the server's history.) + assert_eq!(on_a.len(), on_b.len(), "read {n}"); + } +} + +/// The server's file version of `doc.kitdemo`, as `m` lists it. +async fn server_file(m: &Machine) -> craft_cloud_proto::FileState { + let listing = m.transport().tree(None).await.unwrap(); + listing + .nodes + .into_iter() + .find_map(|n| match n.kind { + craft_cloud_proto::NodeKind::File(f) if !n.deleted => Some(f), + _ => None, + }) + .unwrap() +} + +/// A record that reaches the agent after its file went up (the app saved while the agent was away): the next pass adds +/// it to that version on the server, once; the other machine learns it from the change feed; a version that has a +/// record keeps it; and the call again changes nothing. +#[tokio::test] +async fn a_record_that_comes_late_is_added_to_its_version() { + let server = Server::new().await; + let (mut a, mut b, book_a, _book_b, doc) = two(&server).await; + let saved = b"saved while the agent was away".to_vec(); + a.fs_mut().write(&path("doc.kitdemo"), &saved, false, 0).unwrap(); + a.sync_once().await.unwrap(); + let up = server_file(&a).await; + assert_eq!(up.save, None, "it went up without its record"); + // The record comes. + let record = SaveRecord { doc, frontiers: vec![(1, 7)] }; + book_a.0.lock().unwrap().insert((node(&a, "doc.kitdemo"), FileHash::from_bytes(*blake3::hash(&saved).as_bytes())), record.clone()); + a.sync_once().await.unwrap(); + let late = server_file(&a).await; + assert_eq!((late.version, late.save.clone()), (up.version, Some(record.clone())), "added to the same version"); + b.sync_once().await.unwrap(); + let on_b = b.state().remote.values().find_map(|n| n.file.clone()).unwrap(); + assert_eq!(on_b.save, Some(record.clone()), "the other machine learns it"); + // Appended only, and the same again changes nothing. + let other = SaveRecord { doc, frontiers: vec![(9, 9)] }; + let seq = a.transport().space().await.unwrap().seq; + a.transport().attach_save(node(&a, "doc.kitdemo"), up.version, &other).await.unwrap(); + a.transport().attach_save(node(&a, "doc.kitdemo"), up.version, &record).await.unwrap(); + assert_eq!(server_file(&a).await.save, Some(record)); + assert_eq!(a.transport().space().await.unwrap().seq, seq, "nothing changed"); + assert!(a.transport().attach_save(NodeId::generate(), up.version, &other).await.is_err(), "no such version there"); +} + +/// When the agent can't tell which of two Craft saves is newer, neither is taken: no conflict copy, nothing goes up +/// over the other, and both files stay as they are; once it can tell, the newer wins as always. Past the wait, it is +/// a conflict copy, as any two edits are (never a guessed winner). +#[tokio::test] +async fn saves_whose_order_is_unknown_wait_then_are_a_conflict() { + use std::sync::atomic::Ordering; + let server = Server::new().await; + let (mut a, mut b, book_a, book_b, doc) = two(&server).await; + book_a.1.store(true, Ordering::Relaxed); + book_b.1.store(true, Ordering::Relaxed); + book_a.save(&mut a, "doc.kitdemo", b"laptop's save", doc, &[(1, 5)]); + book_b.save(&mut b, "doc.kitdemo", b"desktop's later save", doc, &[(1, 5), (2, 3)]); + b.sync_once().await.unwrap(); + rounds(&mut a, &mut b).await; + assert_eq!(files(&a), vec![("doc.kitdemo".to_owned(), b"laptop's save".to_vec())], "waits: kept as it is"); + assert_eq!(files(&b), vec![("doc.kitdemo".to_owned(), b"desktop's later save".to_vec())]); + assert_eq!(server_file(&a).await.save.map(|s| s.frontiers), Some(vec![(1, 5), (2, 3)]), "nothing went up over it"); + // Known now: the desktop's includes the laptop's. + book_a.1.store(false, Ordering::Relaxed); + rounds(&mut a, &mut b).await; + let want = vec![("doc.kitdemo".to_owned(), b"desktop's later save".to_vec())]; + assert_eq!((files(&a), files(&b)), (want.clone(), want)); + // Never known, on a machine that waits no longer: a conflict copy, as always. + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let mut c = common::machine_config(&server, &ana, "tablet", craft_cloud_sync::MemFs::new(), 50, |c| c.save_order_wait_ms = 0).await; + let book_c = Book::default(); + c.set_save_records(Arc::new(book_c.clone())); + c.sync_once().await.unwrap(); + book_c.1.store(true, Ordering::Relaxed); + book_c.save(&mut c, "doc.kitdemo", b"tablet's save", doc, &[(3, 1)]); + book_b.save(&mut b, "doc.kitdemo", b"desktop's next save", doc, &[(1, 5), (2, 4)]); + b.sync_once().await.unwrap(); + for _ in 0..3 { + c.sync_once().await.unwrap(); + b.sync_once().await.unwrap(); + } + assert_eq!(files(&c).len(), 2, "a conflict copy: {:?}", files(&c)); +} + +/// A Craft save whose record is slow to reach the agent (announced, the file written, the record late): it isn't +/// uploaded without its record, and when the other machine's save comes meanwhile there is no conflict copy; both +/// wait, and once the record is in, the saves resolve as any two Craft saves do. (Before: the save went up as an edit +/// outside Craft, or met the other's as a conflict copy, the flake the busy-machine run showed.) +#[tokio::test] +async fn a_save_whose_record_is_late_waits_for_it_and_never_conflicts() { + let server = Server::new().await; + let (mut a, mut b, book_a, book_b, doc) = two(&server).await; + let before = server_file(&a).await; + let late = book_a.saving(&mut a, "doc.kitdemo", b"laptop's save, record on its way"); + a.sync_once().await.unwrap(); + let up = server_file(&a).await; + assert_eq!(up.version, before.version, "nothing went up yet (not even without its record)"); + // The desktop's save meanwhile. + book_b.save(&mut b, "doc.kitdemo", b"desktop's save", doc, &[(2, 3)]); + b.sync_once().await.unwrap(); + rounds(&mut a, &mut b).await; + assert_eq!(files(&a), vec![("doc.kitdemo".to_owned(), b"laptop's save, record on its way".to_vec())], "waits, kept as it is"); + assert_ne!(server_file(&a).await.version, up.version, "the desktop's went up"); + // The record comes: the laptop's save includes the desktop's (it had its edits live), so it goes up over it. + book_a.record_comes(late, doc, &[(1, 6), (2, 3)]); + rounds(&mut a, &mut b).await; + let want = vec![("doc.kitdemo".to_owned(), b"laptop's save, record on its way".to_vec())]; + assert_eq!((files(&a), files(&b)), (want.clone(), want), "no conflict copy"); + assert_eq!(server_file(&a).await.save.map(|s| s.frontiers), Some(vec![(1, 6), (2, 3)]), "with its record"); +} diff --git a/docs/design/live.md b/docs/design/live.md index 31b5251..75e9a4e 100644 --- a/docs/design/live.md +++ b/docs/design/live.md @@ -66,14 +66,32 @@ per app connection for live sessions. `run` serves it while signed out too. Call kept); - a binary presence from the app (at most 4 KiB, at most 20 a second per document) goes to the document's other machines and apps; `close {doc}`: the app is done with it (the agent keeps it); + - `saved {doc, hash, frontiers, version}` (appended): the app saved the document's file; `hash` is the BLAKE3 hash + of the bytes it wrote (hex), `frontiers` Loro's (hex, kept for the app's own use) and `version` the document's + version vector as `[peer, end counter]` pairs, at most 4,096. The agent keeps the newest 512 in + `live//saves.json` (atomically) and answers nothing; an `error` only when it doesn't look like a save. An + agent from before save records answers `error` (not a request it knows) and nothing else changes: the file syncs + as any file; - a binary frame without a header, or of a kind an app doesn't send, ends the connection; a text message that isn't a request is answered `error`. An app that doesn't take a message within 10 s is dropped (so is one whose messages queue up past 1,024 in the agent); its documents stay with the agent. An app that goes away mid-import leaves nothing on the server (the import goes up only once whole); the next app to open the file is asked. +- `PUT /v1/blobs` (the body: 1 byte to 64 MiB of binary content an app keeps out of a live document) → `{hash, len}` + once the bytes are on the agent's disk (`live//blobs/`, with a `.pending` mark written first); + `hash` is the BLAKE3 hash of the manifest the bytes cut into with `fastcdc-v1`, so the same bytes have the same + name everywhere. The bytes go up through the chunk API (the chunks the server lacks, then the manifest, the + account's storage cap applying) at once when the server can be reached, and otherwise with the next put or every + 30 s; the mark goes once they are up. `GET /v1/blobs/{hash}` → the bytes: from the agent's disk (checked against + the name on every read; a damaged copy is dropped and fetched again), or downloaded (every chunk checked, the whole + against the manifest) and kept. `blob_not_found` when the account has no such blob, `offline` when it isn't here + and the server can't be reached. An agent from before blobs answers both with the local API's `not_found` for an + unknown call, which `craft-cloud-app` reports as `agent_too_old` (blob calls only: everything else works as + before). + Errors use stable codes (`forbidden`, `wrong_password`, `already_signed_in`, `not_signed_in`, `offline`, `needs_you`, and for joining `invalid_code`, `password_required`, `weak_password`, `untrusted_certificate`, -`cannot_remove_self`, `autostart_failed`, `folder_has_files`). Every call has the same `Host`, `Origin` and token checks. Each request is answered before the next is read; a client that doesn't take a message within 10 s is +`cannot_remove_self`, `autostart_failed`, `folder_has_files`, and for blobs `blob_not_found`). Every call has the same `Host`, `Origin` and token checks. Each request is answered before the next is read; a client that doesn't take a message within 10 s is disconnected, and the crate reconnects. **Server live endpoint** (agent ↔ server): `GET /api/v1/live` upgraded to a WebSocket, signed in with the device @@ -186,4 +204,38 @@ for opaque updates; variants are only ever appended, anything new is negotiated alone. - Loro failing on a row of a catch-up taken as a batch: the catch-up is taken again one row at a time, so only the row at fault is dropped (and said). A second failure in a run stops the document (`status.stopped`) until the agent - restarts. + restarts. A batch goes to Loro in groups of at most 16 KiB of updates (Loro's batch import of much text at once + takes time growing with its square). + +**Save records** (the file sync and live documents): each file version a Craft app saves records the live +document's version it was made from. +- The app tells its agent (`saved`, above); the sync engine's new version of a file whose content an app saved + carries the record, `NewVersion.save = {doc, version}` (appended: an older agent sends none, an older server ignores + it). The server keeps it with the version as the JSON it came as (`versions.save`, migration 0006; a record of more + than 4,096 entries is kept as none) and lists it as `FileState.save` (an older agent ignores it). The server never + reads what it means, and never parses Loro. +- When a file changed here and on the server, and this content and the server's version both carry a record of the + same live document, the engine makes no conflict copy: the server's version wins if its record's version includes + this one's (it was saved later from a document that had everything this one had: it is downloaded over the file); + otherwise this one goes up over it (the newer upload; the live document holds both sides' edits, so the next save + writes them all). A version without a record (edited outside Craft), or records of two documents, is a conflict as + always, with its copy. +- A file whose bytes are no save (or the import) of its document is reported by the app kit on open + (`changed_outside`): the app may bring the file in as an ordinary local change, or keep the live content. The kit + keeps the newest 32 saves' hashes in the document's `$saves` map for that. + +**The app kit** (`craft-cloud-kit`, what apps link; see `docs/kit.md`): the content's layout in Loro. +- The root map `root` holds `schema: {app, version}`, written at import. A document whose app differs, whose version + is newer than the app's, or that has none opens read-only. +- Each part of an app's model has a key: a tree `k` (`LoroTree`, fractional index on, each node's meta map holding + the element's ID under `$id` and its properties), a rich text `k` (`LoroText`: character attributes as marks + expanding after, paragraph attributes as marks `¶` on each paragraph's `\n` never expanding, the last + paragraph's in the root map `k.last`), a table (`k.rows`, `k.cols`: movable lists of IDs; `k.cells`: a map from + `row:column` to a value), a timeline (`k.tracks`: a movable list; `k.track`: track properties; `k.items`: item ID + to `{$track, $start, $len, …}`; `k.keys`: keyframe ID to `{$item, $at, $value}`), an opaque value (the root map + `$opaque`, key `k`, bytes). IDs are 16 bytes (little-endian `u128`), or 32 hex digits as map keys. Binary content + is never in Loro: it is a `{$chunk, $len}` value naming a blob. +- A local change is written as commits of at most 512 KiB each (estimated: payload and 32 bytes per operation), each + exported at once; an export still over 512 KiB is cut by counter range. One change is one undo step. +- Presence an app sets is sent as `[u64 peer, big-endian][the app's bytes]`, again every 3 s, and dropped 10 s after + its peer goes quiet. diff --git a/docs/kit.md b/docs/kit.md new file mode 100644 index 0000000..91b2abc --- /dev/null +++ b/docs/kit.md @@ -0,0 +1,240 @@ +# The app kit: live documents in a Craft app + +`craft-cloud-kit` makes a Craft app's documents live: the same file open on two machines (or in two apps on one) +shows every edit on the other as it is made, works offline, and never makes conflict copies of its own saves. The +app's whole change is one adapter. Everything else (talking to the Craft Cloud agent, the Loro document, +reconnecting, binary content, undo while live) is in the kit. `craft-cloud-egui` adds Craft Cloud's piece of the +window: the account line, the shared recent files and the document's badge. + +The demo app in this repository (`apps/craft-cloud-kit-demo`, never packaged) is a complete example: a rich-text +story and a layer tree; its adapter is about 150 lines of `model.rs`, its hook one function of `ui.rs`. The +snippets below are from it. + +## Adding it + +```toml +[dependencies] +craft-cloud-kit = { git = "https://github.com/itrycodinganditsbad/craft-cloud", tag = "v0.1.0" } +craft-cloud-egui = { git = "https://github.com/itrycodinganditsbad/craft-cloud", tag = "v0.1.0" } + +[dev-dependencies] +# The fake agent, for the adapter's tests. +craft-cloud-kit = { git = "https://github.com/itrycodinganditsbad/craft-cloud", tag = "v0.1.0", features = ["fake"] } +``` + +The kit brings Loro (and Loro's own crates) and nothing a Craft app doesn't already lock: `cargo xtask suite-deps +` checks it. There is no async runtime: the kit is blocking where it waits (opening) and +never blocks where it is called each frame. + +## The adapter + +The app implements [`LiveDocument`] for its model, the `Arc`-shared document its commands already clone +copy-on-write: + +```rust +impl LiveDocument for DemoApp { + type Doc = DemoDoc; + const APP: &'static str = "kitdemo"; // as in recents + const SCHEMA: u32 = 1; // the layout the mappings write + + fn mappings(doc: &DemoDoc) -> Mappings<'_> { + Mappings::new().tree("layers", LayersSrc::new(&doc.layers)).text("story", StorySrc(&doc.story)) + } + + fn apply_remote(old: &Arc, patch: &RemotePatch<'_>) -> Result, KitError> { + let mut new = (**old).clone(); + if let Some(tree) = patch.tree("layers") { + new.layers = Arc::new(build(&tree, tree.roots())); // or only the touched nodes + } + if let Some(story) = patch.text("story") { + new.story = Arc::new(story); + } + Ok(Arc::new(new)) + } + + fn normalise(doc: &mut DemoDoc) { /* name a layer two machines both cleared */ } + fn refresh_derived(doc: &mut DemoDoc) { doc.paragraphs = doc.story.paragraphs.len(); } +} +``` + +- **`mappings`** names the parts of the model the live document holds, each by a key, each a shape: a **tree** + ([`TreeSource`]: layers, objects, bins), **rich text** ([`TextSource`]: a story; [`RichText`] is one), a **table** + ([`TableSource`]: rows, columns and cells; one column makes keyed records), a **timeline** ([`TimelineSource`]: + tracks, items, keyframes) or an **opaque value** ([`OpaqueSource`], last writer wins; [`Opaque`] wraps an + [`OpaqueCodec`]). Everything else (view state, selection, caches) stays the app's own. +- The sources are the app's own types read in place. Give every source its `key` (the `Arc`'s address) and the tree + its `subtree_key`: an unchanged part or subtree is skipped without being compared, so a keystroke in a 1 MB story + costs well under a millisecond with 5,000 layers beside it. A table can say which cells changed + (`changed_cells`). +- **IDs** come from the app (`ElemId`, a `u128`): the kit never makes them from positions. A model that addresses + by index (paragraphs, table cells) adds IDs first; `ElemId::new()` makes a random one. +- **`apply_remote`** gets the model before a remote change and what it touched, part by part (`None`: untouched, + keep it). Rebuild only that: the views say what changed (`touch()`), and read the part as it is now. A rich text + comes whole, patched from the old one. +- **`normalise`** is required: it repairs what a merge can break (two layers with one name, a cell whose row went, a + reference to a deleted element), so the result is a model the app could have made itself. The kit writes back what + it changed (no undo step). It runs after every remote change, then **`refresh_derived`** recomputes what is + derived and never synced, before the app gets the model. Every machine normalises the same merge and writes its + repair, so a repair must be deterministic: the same model in, the same repair out, with any new element's ID and + value derived from what it repairs (never `ElemId::new()` or the clock), so two machines' repairs are one. +- Property values ([`Value`]) are numbers, strings, bytes and chunk references, at most 256 KiB each. Binary + content never goes into the document: put it in a [`BlobStore`] ([`AgentBlobs`] is the agent's) and keep the + [`ChunkRef`]. + +## Opening + +```rust +match Session::::open(&Agent::local(), "Notes/story.kitdemo", file_model, Some(&file_bytes))? { + Open::Live(session) => { /* the model is session.model() */ } + Open::NotLive(why) => { /* a plain file: open and save it as always, with the app's own undo */ } +} +``` + +The path is relative to the synced folder. A file that has never been live is imported from the app's model (the +one it read from the file); otherwise the model is built from the live document, which may be newer than the file. +Opening waits for the agent (up to 30 s, when it must sync the file first): call it where the app opens files, or on +a thread (a `Session` is `Send`). A document made by a newer version of the app opens **read-only** +(`status().read_only`: show "update the app"); local changes and undo are refused. + +## The hook + +At the end of each top-level command, with the model before and after it: + +```rust +if !Arc::ptr_eq(&before, &self.doc) { + self.session.local_change(&before, &self.doc)?; // into Loro, out through the agent; one undo step +} +``` + +- `before` must be the session's model ([`KitError::Stale`] otherwise: a remote change came in between; make the + change again from `session.model()`). +- A change is cut into commits under 512 KiB (the agent takes updates of up to 1 MiB): a 5 MB paste goes, and + undoes, whole. +- **A gesture** (a drag, an IME composition) whose `before` must stay put until it ends: `session.hold_remote(true)` + while it runs; remote changes wait and come at the first poll after. +- Typing: `session.group(|s| …)` (or `begin_group`/`end_group`) makes several changes one undo step. + +## Each frame + +```rust +let polled = self.session.poll(); // never blocks +if polled.changed { self.doc = self.session.model().clone(); } +craft_cloud_egui::badge(ui, &polled.status); // Live, Waiting 3, Read-only, Offline +``` + +`session.set_waker(Arc::new(move || ctx.request_repaint()))` wakes the window when something comes from the agent. +When the agent goes away the model stays editable: changes queue in memory (at most 64 MiB; past that +[`KitError::Stalled`]) and go out once the session is back, which it finds by itself with a growing pause, off the +UI thread (`reconnect_now()` skips the pause). + +## Undo while live + +While a document is live, undo and redo go through the session: `session.undo()`, `session.redo()`, +`can_undo()`. They take back only this machine's changes (another machine's edit to the same text stays), and the +result comes back to the model as a remote change does (take `session.model()`). The app's own snapshot undo would +silently revert other people's edits: use it only while the document isn't live. `session.undo_kind()` says which +applies. + +## Saving + +Save and autosave work as before, in this order: + +```rust +// 1. Where the save starts: the model to write and the live document's frontiers it matches. +let point = self.session.save_point(); +// 2. The bytes, from that model (or, with no save point, from the app's own model: no record then). +let bytes = to_file(point.as_ref().map_or(&self.model, |p| p.model())); +// 3. The record (content hash and frontiers) to the agent, before the file changes on disk. +if let Ok(point) = &point { + self.session.record(point, &bytes); +} +// 4. The write. +write_atomic(&path, &bytes)?; +``` + +`save_point()` refuses (`KitError::NoRecord`) when a record would be wrong: the document is read-only here (a newer +app's), the session is stalled or stopped, or the model is behind the live document. The app saves its own model +then, and the file syncs as any file does. + +A record names the live document's frontiers (its heads, one or two operation IDs however many machines and sessions +edited it), never a version vector. `record` first announces the save to the agent at once (its content's hash), +before the write: until the record itself comes, the agent's sync takes that content for a Craft save on its way, and +neither uploads it without its record nor makes a conflict copy of it, but waits. A save made while the app has no +agent connection (reconnecting, the agent restarting) can't be announced; the agent waits for those too, for any +change to a file whose document an app had open within the last 60 s (it keeps that on disk), and the session +announces and records them when it is back. Each content is waited for 10 minutes in all; a record that never comes +leaves the version as made outside Craft, as before. (The announcement isn't acknowledged: an agent stalled for longer +than its sync's settle window before it reads the announcement can still miss it.) The session sends the record on the same stream as its changes, after every +change it covers; the agent keeps it once all those operations are in its copy (a record that comes first waits for them, and +is dropped if the window closes before they come: that version then has no record, and its edits are in the file, so +nothing is lost). The record goes before the write so the agent knows the content before its sync pass sees it; while +the agent is away the session keeps the record and tells it once back, and the agent adds it to the version it +uploaded meanwhile (a server call of its own, appended once). Every session writes with a fresh random Loro peer of +its own; nothing is ever written with a peer another session may have had. + +The agent records it with the version it uploads, so two machines saving one live file never make a conflict copy: +the save that includes the other's wins, or else the newer upload. Which includes which the agent asks its own copy +of the document; while it can't tell (it lacks the operations), neither side is taken and the next pass asks again, +and after 10 minutes still unknown it is a conflict copy, as for any two edits. A winner is never guessed. The losing +version is no conflict copy, but it stays in the server's history, and nothing of it is lost: the live document has +both sides' edits. + +The live document is the truth and the file a copy of it, which may be late. To have it catch up on its own, +autosave when the model is dirty **or** when `session.file_behind()` says the live document is strictly ahead of the +file (another machine's edits came live, or this machine's after another machine's save won over this one's): + +```rust +if autosave_due && (self.dirty || self.session.file_behind()) { + self.save(); // the order above +} +``` + +The session knows the record of the file as it is on disk from the agent (it looks at the file of every open +document each second), and from its own last save. `file_behind()` is true only when everything the file was saved +from is in the live document and the live document has more. When the file has anything this live document lacks +(another machine's save that came by file sync before its live updates, a save made apart), it is false: this side +waits for the live updates, so a file ahead of the live document is never written over by itself, and two machines +whose live connection is down while files still sync never save back and forth. It is false too when no record of the +file is known (an edit outside Craft; an older agent), when a save couldn't carry a record, and for 5 s after a save +it asked for (a backstop). Once every window's file has the live document's frontiers, nobody saves again. When no +app has the document open, nobody writes the file: it stays behind until the next open, which shows the live document +and saves when it is ahead. + +A file edited outside Craft has no record: it gets a conflict copy as always, and the kit says so on open +(`status().changed_outside`). It is decided once, at the open, from the bytes the app read (`Session::open`'s +`file_bytes`): the agent looks their hash up among this machine's records and the record of the file's version on the +server, so any Craft save, however many came after it, is never taken for an outside edit. A Craft save made without +a record (read-only, stopped, a record that never reached the agent) reads as changed outside too; its edits are in +the file. + +Bringing the file in is the user's choice, never the kit's: the app shows `changed_outside` and offers it, and on a +yes calls `session.replace_with_file(file_model)`. The file's content then replaces the live content as one ordinary +change on every machine, one undo step named "Replace with the file's content" (`step_name`, for the Undo label). Say +plainly in the prompt that it **replaces the live edits made since the file was written**, and that Undo brings them +back. Saving clears `changed_outside`. + +## Presence + +`session.set_presence(Some(bytes))` shows this app's cursor or selection to the others (the bytes are the app's, +at most 4 KiB less 8); `session.peers()` is theirs, dropped 10 s after they go quiet. Drawing them is the app's. + +## Craft Cloud's piece of the window + +`craft_cloud_egui::CloudWatch::start(app, connect, wake)` keeps the agent's state and the app's recents current off +the UI thread; `account_line(ui, &watch.state())` and `recents(ui, watch.entries(), now)` draw them (a click returns +the path); `watch.add_recent(path, saved)` puts a file on every machine's list. + +## Testing the adapter + +With the `fake` feature, `craft_cloud_kit::testing::{Hub, Fake}` is an agent in the test's own process: machines of +one account on a shared hub, with `set_online`, `restart` and `start`, `drop_connections`, and an old-agent mode. +Two sessions on two fakes test merges, undo and reconnects with no server; the demo's tests drive two windows that +way with kittest. + +## Costs to know + +- Loro's undo of one big run of inserted text takes time growing with its square (about 10 s for a 1 MB paste). + Undo of ordinary edits is instant. +- A remote change to a rich text is patched into the old text; a tree's structure change reads the tree's + structure again (cheap per node, 5,000 nodes in a few milliseconds); a table or timeline is read back by the app as + much as it wants. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b166b7c..49fc5b4 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -8,6 +8,7 @@ mod icons; mod licenses; mod notices; mod package; +mod suite_deps; mod version; use std::path::{Path, PathBuf}; @@ -24,6 +25,9 @@ commands: (zipped) or the Linux tarball version [set X.Y.Z[-pre]] print the workspace version, or set it (Cargo.toml + Cargo.lock) + suite-deps ... + the crates craft-cloud-kit and craft-cloud-egui pull in that none of the given lock files (the + Craft apps') has; fails if any but Loro's own (not part of ci) "; fn main() -> ExitCode { @@ -35,6 +39,7 @@ fn main() -> ExitCode { Some("icons") => icons::run(&root(), &rest), Some("package") => package::run(&root()), Some("version") => version::run(&root(), &rest), + Some("suite-deps") => suite_deps::run(&rest), Some("-h" | "--help" | "help") | None => { print!("{USAGE}"); Ok(()) diff --git a/xtask/src/suite_deps.rs b/xtask/src/suite_deps.rs new file mode 100644 index 0000000..b89ab7b --- /dev/null +++ b/xtask/src/suite_deps.rs @@ -0,0 +1,103 @@ +//! `cargo xtask suite-deps `: the crates that what the Craft apps link (`craft-cloud-kit`, +//! `craft-cloud-egui` and what they pull in, normal dependencies only) brings that none of the given lock files has. +//! The Craft apps take only crates another Craft app already locks, with Loro's own crates the one exception, so the +//! list is split into the crates that come only through Loro and the rest; the command fails when the rest isn't empty. +//! CI doesn't run it (the apps' lock files aren't in this repository): run it with their paths, as +//! `cargo xtask suite-deps ../*craft/Cargo.lock`. + +use std::collections::BTreeSet; + +/// The crates the Craft apps link. +const APP_CRATES: &[&str] = &["craft-cloud-kit", "craft-cloud-egui", "craft-cloud-app"]; +/// The crate whose own dependencies are the accepted exception. +const LORO: &str = "loro"; + +pub fn run(args: &[&str]) -> Result<(), String> { + if args.is_empty() { + return Err("usage: cargo xtask suite-deps … (the Craft apps' lock files)".into()); + } + let mut known = BTreeSet::new(); + for path in args { + let text = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + known.extend(lock_names(&text)); + } + let loro = tree(LORO)?.unwrap_or_default(); + let mut through_loro = BTreeSet::new(); + let mut other = BTreeSet::new(); + for app in APP_CRATES { + let Some(deps) = tree(app)? else { + println!("{app}: not in the workspace"); + continue; + }; + for (name, version) in deps { + if known.contains(&name) { + continue; + } + let line = format!("{name} {version}"); + let list = if loro.iter().any(|(n, _)| *n == name) { &mut through_loro } else { &mut other }; + list.insert(line); + } + } + println!("suite-deps: {} lock files, {} crate names in them", args.len(), known.len()); + println!("crates of {} that no lock has:", APP_CRATES.join(" and ")); + println!(" through Loro (the accepted exception): {}", if through_loro.is_empty() { "none".to_owned() } else { String::new() }); + for line in &through_loro { + println!(" {line}"); + } + println!(" others: {}", if other.is_empty() { "none" } else { "" }); + for line in &other { + println!(" {line}"); + } + if other.is_empty() { Ok(()) } else { Err(format!("{} crates no Craft app locks (see above)", other.len())) } +} + +/// The package names a `Cargo.lock` lists. +fn lock_names(text: &str) -> BTreeSet { + text.lines().filter_map(|line| line.strip_prefix("name = \"")).filter_map(|rest| rest.strip_suffix('"')).map(str::to_owned).collect() +} + +/// `package`'s normal dependencies (itself included) as (name, version), without the workspace's own crates; `None` +/// when the build has no such package. +fn tree(package: &str) -> Result>, String> { + let mut cmd = crate::cargo(); + cmd.args(["tree", "--locked", "-p", package, "-e", "normal", "--target", "all", "--prefix", "none", "--format", "{p}", "--color", "never"]); + let output = cmd.output().map_err(|e| format!("cargo tree: failed to spawn: {e}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + if stderr.contains("did not match any packages") { + return Ok(None); + } + return Err(format!("cargo tree -p {package}: exited with {}: {}", output.status, stderr.trim())); + } + Ok(Some(parse_tree(&String::from_utf8_lossy(&output.stdout)))) +} + +/// `cargo tree --format {p}` lines as (name, version), skipping path crates (the workspace's own: `(/…)` after the +/// version) and the `(*)` of repeats. +fn parse_tree(text: &str) -> BTreeSet<(String, String)> { + let mut out = BTreeSet::new(); + for line in text.lines() { + let mut words = line.split_whitespace(); + let (Some(name), Some(version)) = (words.next(), words.next()) else { continue }; + let rest: Vec<&str> = words.collect(); + if rest.iter().any(|w| w.starts_with("(/") || w.starts_with("(C:") || w.starts_with("(\\")) { + continue; + } + out.insert((name.to_owned(), version.trim_start_matches('v').to_owned())); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn locks_and_trees_parse() { + let lock = "[[package]]\nname = \"egui\"\nversion = \"0.36.2\"\n\n[[package]]\nname = \"serde\"\n"; + assert_eq!(lock_names(lock), BTreeSet::from(["egui".to_owned(), "serde".to_owned()])); + let tree = "craft-cloud-kit v0.1.0 (/home/x/crates/kit)\nloro v1.16.2\nserde v1.0.228\nserde v1.0.228 (*)\n"; + let parsed = parse_tree(tree); + assert_eq!(parsed, BTreeSet::from([("loro".to_owned(), "1.16.2".to_owned()), ("serde".to_owned(), "1.0.228".to_owned())])); + } +}