From 9d125a77523888669638452c958ce82dc46e475b Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 16:20:13 +0800 Subject: [PATCH 01/11] feat: presence via ESET, complete collection values in live queries, jittered reconnect backoff --- CHANGELOG.md | 57 +++++++ Cargo.lock | 8 +- Cargo.toml | 2 +- core-engine/src/cmd.rs | 11 ++ core-engine/src/store.rs | 181 +++++++++++++++++++-- docs/roadmap.md | 48 ++---- docs/server/commands.md | 33 +++- docs/server/troubleshooting.md | 8 +- sdks/recached-react/package.json | 2 +- sdks/recached-vue/package.json | 2 +- server-native/src/main.rs | 231 ++++++++++++++++++++++++++- sync-client/src/lib.rs | 128 +++++++++++++-- sync-client/src/tests.rs | 260 ++++++++++++++++++++++++++++++- wasm-edge/package.json | 2 +- 14 files changed, 889 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55753c8..65aa2ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,63 @@ All notable changes to Recached are documented here. --- +## [0.2.2] — Unreleased + +### Added + +- **`ESET` — connection-scoped keys for presence.** A key written with `ESET` lives exactly as long + as the connection that wrote it; when that connection closes the server deletes it and pushes the + deletion to live queries. Presence, cursors and "who is online" previously had to be hand-rolled + with `SETEX` plus a heartbeat, which leaves ghost entries for the length of the TTL whenever a tab + closes. + + Ownership transfers on each write, which is what makes multiple tabs behave: two tabs both setting + `presence:user:42` leave the **later** one as owner, so closing the first does not mark the user + offline. Replicas receive the write as a plain `SET` — they have no connection to scope a lifetime + to, and the owning server broadcasts the deletion. + +### Changed + +- **Live queries now carry collection values.** `qstate` and `keychange` previously delivered only a + *type name* for hashes, lists, sets, sorted sets and JSON, so every subscriber had to follow up with + `HGETALL`/`LRANGE`/`JGET` — a network round-trip in a system whose premise is that reads are local. + Collections now arrive **type-tagged and complete**: + + ```text + hash → ["hash", field, value, ...] fields sorted + list → ["list", element, ...] head to tail + set → ["set", member, ...] + zset → ["zset", member, score, ...] ascending score + json → ["json", document] + ``` + + The tag is required for the payload to be unambiguous — a four-element array would otherwise be + indistinguishable between a list of four items and a hash of two pairs. Ordering is deterministic + so two clients build identical local state, and each notification carries the complete value, which + is what allows a removed member to propagate. + + ::: warning Wire-format change + A client older than 0.2.2 does not understand the tagged shape and will ignore collection values + from live queries. Server and SDKs are released in lockstep at the same version — run matching + versions. + ::: + +- **Reconnect backoff is jittered.** Delays now land in `[nominal/2, nominal]` instead of an exact + `500ms × 2^attempts`. Without jitter every client disconnected by the same event computes an + identical schedule and reconnects in lockstep — a thundering herd that can keep a recovering server + down. The jitter source is seeded from the client id rather than a system RNG, so `sync-client` + stays dependency-free and I/O-free and the sequence remains reproducible in tests. + +### Fixed + +- **`ESET` would not have replicated.** `is_write_command` is a `matches!` list, which — unlike a + `match` — has no exhaustiveness check, so a newly added command silently defaults to "not a write" + and never reaches replicas, the AOF, or live queries. Caught while wiring `ESET`; a cross-check test + now asserts that every command reporting written keys is also classified as a write, so the next + addition cannot repeat it. + +--- + ## [0.2.1] — 2026-07-19 ### Fixed — Browser SDK (critical) diff --git a/Cargo.lock b/Cargo.lock index b815a11..7be72c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "core-engine" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dashmap", "indexmap", @@ -1066,7 +1066,7 @@ dependencies = [ [[package]] name = "server-native" -version = "0.2.1" +version = "0.2.2" dependencies = [ "base64", "core-engine", @@ -1203,7 +1203,7 @@ dependencies = [ [[package]] name = "sync-client" -version = "0.2.1" +version = "0.2.2" dependencies = [ "core-engine", ] @@ -1589,7 +1589,7 @@ checksum = "60238e5b4b1b295701d6f9a66d2a126fe19990348f5fb9dae3b623a370119d94" [[package]] name = "wasm-edge" -version = "0.2.1" +version = "0.2.2" dependencies = [ "core-engine", "getrandom 0.3.4", diff --git a/Cargo.toml b/Cargo.toml index 04a5135..d659559 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ resolver = "2" # ── Single source of truth for all crate versions ──────────────────────────── # Members inherit with: version.workspace = true / edition.workspace = true [workspace.package] -version = "0.2.1" +version = "0.2.2" edition = "2024" license = "Apache-2.0" authors = ["ThinkGrid Labs"] diff --git a/core-engine/src/cmd.rs b/core-engine/src/cmd.rs index 0877954..621ee2a 100644 --- a/core-engine/src/cmd.rs +++ b/core-engine/src/cmd.rs @@ -50,6 +50,10 @@ pub enum Command { // ── Strings ────────────────────────────────────────────────────────────── Set(String, String, SetOptions), Get(String), + /// ESET — a SET whose key is owned by the connection that wrote it. + /// The engine stores it like any string; the *server* deletes it when that + /// connection closes. Presence, cursors, "who is online". + ESet(String, String), Del(Vec), Unlink(Vec), Append(String, String), @@ -308,6 +312,13 @@ impl Command { need!(2); Ok(Command::Get(extract_key(&arr[1])?)) } + "ESET" => { + need!(3); + Ok(Command::ESet( + extract_key(&arr[1])?, + extract_string(&arr[2]).unwrap_or_default(), + )) + } "DEL" => { need!(2); Ok(Command::Del(extract_keys(&arr[1..])?)) diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 9902865..6731327 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -614,19 +614,70 @@ impl KeyValueStore { /// Strings are returned as bulk strings. Complex types return nil — the /// watcher must use a type-specific command (HGETALL, LRANGE, etc.) to /// fetch the full value. Deleted or expired keys also return nil. + /// The current value of `key`, as delivered to live-query subscribers. + /// + /// Strings come back as a bulk string and a missing or expired key as nil. + /// Collections come back **type-tagged**: an array whose first element names + /// the type, followed by its contents. + /// + /// ```text + /// hash → ["hash", field, value, ...] (HGETALL order) + /// list → ["list", element, ...] (head to tail) + /// set → ["set", member, ...] + /// zset → ["zset", member, score, ...] (ascending score) + /// json → ["json", serialized-document] + /// ``` + /// + /// The tag is what makes the payload unambiguous — a four-element array is + /// otherwise indistinguishable between a list of four items and a hash of + /// two pairs. Earlier versions sent only the type name, which forced every + /// subscriber into a follow-up `HGETALL`/`LRANGE` round-trip: exactly the + /// network hop local reads exist to avoid. pub fn get_current(&self, key: &str) -> Value { + fn tagged(tag: &str, mut items: Vec) -> Value { + let mut out = Vec::with_capacity(items.len() + 1); + out.push(Value::BulkString(Some(tag.as_bytes().to_vec()))); + out.append(&mut items); + Value::Array(Some(out)) + } + fn bulk(s: &str) -> Value { + Value::BulkString(Some(s.as_bytes().to_vec())) + } + let now = now_ms(); match self.data.get(key) { None => Value::BulkString(None), Some(e) if e.is_expired(now) => Value::BulkString(None), Some(e) => match &e.value { EntryValue::Str(s) => Value::BulkString(Some(s.clone().into_bytes())), - EntryValue::Hash(_) => Value::SimpleString("hash".to_string()), - EntryValue::List(_) => Value::SimpleString("list".to_string()), - EntryValue::Set(_) => Value::SimpleString("set".to_string()), - EntryValue::ZSet(_) => Value::SimpleString("zset".to_string()), + EntryValue::Hash(m) => { + let mut fields: Vec<(&String, &String)> = m.iter().collect(); + fields.sort_by(|a, b| a.0.cmp(b.0)); + let items = fields + .into_iter() + .flat_map(|(f, v)| [bulk(f), bulk(v)]) + .collect(); + tagged("hash", items) + } + EntryValue::List(l) => tagged("list", l.iter().map(|v| bulk(v)).collect()), + EntryValue::Set(st) => tagged("set", st.iter().map(|m| bulk(m)).collect()), + EntryValue::ZSet(z) => { + let mut pairs: Vec<(&String, &f64)> = z.scores.iter().collect(); + pairs.sort_by(|a, b| { + a.1.partial_cmp(b.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.cmp(b.0)) + }); + let items = pairs + .into_iter() + .flat_map(|(m, sc)| [bulk(m), bulk(&format_score(*sc))]) + .collect(); + tagged("zset", items) + } + // Attempt state is transient and server-side; clients have no + // use for it and it must not leak into a browser replica. EntryValue::RateLimiter(_) => Value::SimpleString("ratelimit".to_string()), - EntryValue::Json(_) => Value::SimpleString("json".to_string()), + EntryValue::Json(doc) => tagged("json", vec![bulk(&doc.to_string())]), }, } } @@ -801,6 +852,13 @@ impl KeyValueStore { pub fn execute(&self, cmd: Command) -> Value { match cmd { + // Ephemeral keys are ordinary strings to the engine — their lifetime + // is enforced by the server, which is the layer that knows about + // connections. Keeping the engine unaware keeps it I/O-free. + Command::ESet(key, value) => { + self.execute(Command::Set(key, value, crate::cmd::SetOptions::default())) + } + // ── Core ───────────────────────────────────────────────────────── Command::Ping(msg) => match msg { Some(m) => Value::BulkString(Some(m.into_bytes())), @@ -4432,23 +4490,72 @@ mod capacity_tests { } #[test] - fn get_current_returns_type_markers_for_collections() { + fn get_current_returns_type_tagged_collection_values() { + // Live-query subscribers get the actual contents, tagged with the type + // so the payload is unambiguous. Previously only the type name was sent, + // which forced a follow-up HGETALL/LRANGE — a network round-trip in a + // system whose whole premise is local reads. let s = KeyValueStore::new(); s.execute(Command::HSet("h".into(), vec![("f".into(), "v".into())])); - s.execute(Command::LPush("l".into(), vec!["a".into()])); - s.execute(Command::SAdd("st".into(), vec!["a".into()])); + s.execute(Command::RPush("l".into(), vec!["a".into(), "b".into()])); + s.execute(Command::SAdd("st".into(), vec!["m".into()])); s.execute(Command::ZAdd( "z".into(), Default::default(), - vec![(1.0, "a".into())], + vec![(1.5, "alice".into())], )); s.execute(Command::JSet("j".into(), "$".into(), "{\"a\":1}".into())); - assert_eq!(s.get_current("h"), Value::SimpleString("hash".into())); - assert_eq!(s.get_current("l"), Value::SimpleString("list".into())); - assert_eq!(s.get_current("st"), Value::SimpleString("set".into())); - assert_eq!(s.get_current("z"), Value::SimpleString("zset".into())); - assert_eq!(s.get_current("j"), Value::SimpleString("json".into())); + fn parts(v: Value) -> Vec { + match v { + Value::Array(Some(items)) => items + .iter() + .map(|i| match i { + Value::BulkString(Some(b)) => String::from_utf8_lossy(b).into_owned(), + other => format!("{other:?}"), + }) + .collect(), + other => panic!("expected a tagged array, got {other:?}"), + } + } + + assert_eq!(parts(s.get_current("h")), vec!["hash", "f", "v"]); + assert_eq!(parts(s.get_current("l")), vec!["list", "a", "b"]); + assert_eq!(parts(s.get_current("st")), vec!["set", "m"]); + assert_eq!(parts(s.get_current("z")), vec!["zset", "alice", "1.5"]); + assert_eq!(parts(s.get_current("j")), vec!["json", "{\"a\":1}"]); + } + + #[test] + fn get_current_orders_collections_deterministically() { + // Two clients receiving the same key must build identical local state, + // so ordering cannot depend on hash iteration order. + let s = KeyValueStore::new(); + s.execute(Command::HSet( + "h".into(), + vec![("b".into(), "2".into()), ("a".into(), "1".into())], + )); + s.execute(Command::ZAdd( + "z".into(), + Default::default(), + vec![(9.0, "high".into()), (1.0, "low".into())], + )); + for _ in 0..5 { + match s.get_current("h") { + Value::Array(Some(items)) => { + // Fields sorted: a before b. + assert_eq!(items[1], Value::BulkString(Some(b"a".to_vec()))); + } + other => panic!("{other:?}"), + } + match s.get_current("z") { + Value::Array(Some(items)) => { + // Ascending score: low before high. + assert_eq!(items[1], Value::BulkString(Some(b"low".to_vec()))); + } + other => panic!("{other:?}"), + } + } } // ── sweep_expired ───────────────────────────────────────────────────────── @@ -5214,3 +5321,49 @@ mod critical_path_tests { ); } } + +#[cfg(test)] +mod ephemeral_tests { + use super::*; + use crate::cmd::Command; + + #[test] + fn eset_stores_a_value_like_set() { + // To the engine an ephemeral key is an ordinary string — lifetime is + // enforced by the server, which is the layer that knows about + // connections. Keeping the engine unaware is what keeps it I/O-free + // and identical between native and wasm builds. + let s = KeyValueStore::new(); + assert_eq!( + s.execute(Command::ESet("presence:1".into(), "online".into())), + Value::SimpleString("OK".into()) + ); + assert_eq!( + s.execute(Command::Get("presence:1".into())), + Value::BulkString(Some(b"online".to_vec())) + ); + assert_eq!( + s.execute(Command::Type("presence:1".into())), + Value::SimpleString("string".into()) + ); + // No TTL — the engine must not invent one. + assert_eq!( + s.execute(Command::Ttl("presence:1".into())), + Value::Integer(-1) + ); + } + + #[test] + fn eset_overwrites_and_is_visible_to_reads_and_live_queries() { + let s = KeyValueStore::new(); + s.execute(Command::ESet("presence:1".into(), "first".into())); + s.execute(Command::ESet("presence:1".into(), "second".into())); + assert_eq!( + s.get_current("presence:1"), + Value::BulkString(Some(b"second".to_vec())) + ); + // Pattern matching picks it up like any other key. + let matched = s.matching_key_values("presence:*", 10); + assert_eq!(matched.len(), 1); + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 1de72e5..8525257 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -5,19 +5,21 @@ Recached competes on **where the data can live** — the same engine on the serv Numbered items are stable identifiers, not an ordering — code and changelog entries reference them (e.g. "roadmap #6"), so they are never renumbered. **Near-term priorities** are called out below. -Items 1–5 have shipped: rate-limiting commands, scoped sync with per-client auth, live queries, the native JSON type, and offline-first writes with merge semantics. Their numbering is retained below so existing references stay valid — see the [changelog](https://github.com/thinkgrid-labs/recached/blob/main/CHANGELOG.md) for what landed in each. +Items 1–5 and 13 have shipped: rate-limiting commands, scoped sync with per-client auth, live queries, the native JSON type, and offline-first writes with merge semantics. Their numbering is retained below so existing references stay valid — see the [changelog](https://github.com/thinkgrid-labs/recached/blob/main/CHANGELOG.md) for what landed in each. ## Near-term -The next three things worth doing, in order: +The three items previously listed here shipped in **0.2.2**: reconnect backoff jitter, presence via +connection-scoped keys (`ESET`), and live queries carrying complete collection values. See the +[changelog](https://github.com/thinkgrid-labs/recached/blob/main/CHANGELOG.md). -1. **[Reconnect backoff jitter](#reliability)** — a few lines; prevents a thundering - herd every time the server restarts. -2. **[Presence — connection-scoped keys](#_13-presence-—-connection-scoped-keys)** — the docs already - sell presence as a use case; the primitive does not exist yet. -3. **[Live queries carry collection values](#live-queries)** — removes a - round-trip that currently contradicts the local-read premise. +Next up, in order: +1. **[Exactly-once across a server restart](#reliability)** — the last documented caveat on the + delivery guarantee. +2. **[Atomic WAL compaction](#reliability)** — removes a browser-side data-loss window. +3. **[Capacity and sync metrics](#operability)** — operators currently cannot see memory, key count, + eviction rate, or replication lag. --- ## 6. Mobile SDKs — React Native, Flutter, Kotlin, Swift @@ -38,25 +40,6 @@ A `wasm32-wasip1` build of `wasm-edge` for Cloudflare Workers and Deno Deploy, r `core-engine` is already `wasm32`-compatible; the work is adapting the WebSocket and persistence layers to WASI. Last on the list because the platform fights the model — Workers cannot hold persistent WebSockets outside Durable Objects — and edge platforms ship native KV stores. -## 13. Presence — connection-scoped keys - -Keys whose lifetime is tied to the connection that set them. The server drops them when the socket -closes, and the deletion fans out through live queries like any other change. - -```bash -ESET presence:room:7:user:42 "typing" # lives exactly as long as this connection -``` - -Today presence has to be hand-rolled with `SETEX` plus a heartbeat, which leaves ghost entries for -the length of the TTL whenever a tab closes — the classic "still shows as online" bug. Every piece -needed already exists: the WebSocket handler knows when a connection ends, and `QSUB` already -broadcasts deletions. - -Worth doing early because presence, live dashboards, and collaborative cursors are the use cases the -architecture is uniquely suited to, and they are already named in the -[use cases](/guide/use-cases) page. - ---- ## AI-era features @@ -100,12 +83,6 @@ independent — pick them off in any order. ### Reliability -**Reconnect backoff jitter.** `on_close()` computes `500ms × 2^attempts` with no randomisation, so -every client disconnected by a server restart reconnects on an identical schedule. At scale that is a -thundering herd that can keep knocking the server over as it comes back up. Full or decorrelated -jitter is a few lines, and the existing backoff test only needs to assert a range instead of an exact -value. - **Exactly-once across a server restart.** `DEDUP` high-water marks live in server memory and are swept after 24 h idle, so a restart inside the acknowledgement window can admit one duplicate — a caveat currently documented rather than fixed. Persisting the marks alongside the snapshot would make @@ -121,11 +98,6 @@ deliberately instead of losing writes without a signal. ### Live queries -**Live queries carry collection values.** Collection types currently arrive as *type markers*, so a -subscriber must follow every change with a typed re-read (`HGETALL`, `LRANGE`). That round-trip is -precisely what the local-read model exists to remove, so the promise only fully holds for strings -today. Sending the value — or a delta — inline closes the gap. - **`FLUSHDB` should emit per-key diffs.** Subscribers currently miss a mass deletion entirely. ### Performance diff --git a/docs/server/commands.md b/docs/server/commands.md index 6cfbcdc..d32ec02 100644 --- a/docs/server/commands.md +++ b/docs/server/commands.md @@ -22,6 +22,7 @@ The most common data type. Values are always stored as byte strings; numeric ope | `GETSET key value` | Sets the key to a new value and returns the old value atomically. Deprecated in Redis 6.2 — prefer `SET key value GET`. | | `MGET key [key ...]` | Returns the values of multiple keys. Keys that do not exist return nil. | | `MSET key value [key value ...]` | Sets multiple keys to their respective values in a single atomic operation. | +| `ESET key value` | **Ephemeral set.** Stores a string like `SET`, but the key's lifetime is bound to the connection that wrote it — when that connection closes, the server deletes the key and the deletion is pushed to live queries. Writing the same key again transfers ownership to the newest connection, so a second browser tab keeps presence alive when the first closes. Intended for presence, cursors, and "who is online"; use `SET` for anything that should outlive a connection. | | `SETNX key value` | Set a key only if it does not exist. Returns 1 if set, 0 if the key already existed. | | `SETEX key seconds value` | Set a key with an integer-second expiry. Equivalent to `SET key value EX seconds`. | | `PSETEX key milliseconds value` | Set a key with a millisecond-precision expiry. | @@ -197,7 +198,7 @@ JGET doc:42 # {"meta":{"views":17},"title":"Final"} The browser SDK exposes the same commands as [`jset` / `jget` / `jmerge`](/browser/api-reference#json-documents) with `JSON.stringify`/`parse` handled for you — a `JMERGE` from any client updates every connected browser's local document. -In live queries (`QSUB` / `useKeys`), JSON keys appear with a `json` type marker rather than the document — subscribe for change signals and read the document with `JGET`/`jget`. +In live queries (`QSUB` / `useKeys`), JSON keys arrive as `["json", document]` — the document travels with the notification, so no follow-up `JGET` is needed. --- @@ -265,7 +266,7 @@ A live query delivers the current state of every key matching a glob pattern, th | Command | Description | |---|---| -| `QSUB pattern` | Subscribe. The reply is `["qstate", pattern, key, value, ...]` — the current state of every live key matching the pattern as flat pairs (strings in full; collection types as their type name, fetch them with a typed read). Afterwards, every mutation to a matching key — including keys created later — arrives as a `["keychange", key, value]` push; deletions arrive with a nil value. Initial state is capped at 10 000 keys. Up to 64 live queries per connection. | +| `QSUB pattern` | Subscribe. The reply is `["qstate", pattern, key, value, ...]` — the current state of every live key matching the pattern as flat pairs. Afterwards, every mutation to a matching key — including keys created later — arrives as a `["keychange", key, value]` push; deletions arrive with a nil value. Initial state is capped at 10 000 keys. Up to 64 live queries per connection. | | `QUNSUB [pattern]` | Drop one live query, or all of them without an argument. | ```bash @@ -280,7 +281,33 @@ QSUB cart:42:* Under strict sync scoping, `QSUB` patterns must sit inside the connection's granted scopes — a grant of `cart:42:*` covers `QSUB cart:42:*` and narrower prefix patterns. Live-query pushes never interfere with `WATCH` transactions (they travel on a separate internal channel). -Two current limitations: `FLUSHDB` does not emit per-key diffs to live queries, and collection-type values arrive as type markers rather than full values — subscribe plus a typed re-read (`HGETALL`, `LRANGE`) on change. +### Value shapes + +Strings arrive as a bulk string and deletions as nil. Collections arrive **type-tagged** — an array +whose first element names the type — so a subscriber can rebuild the value without a follow-up read: + +```text +hash → ["hash", field, value, ...] fields sorted +list → ["list", element, ...] head to tail +set → ["set", member, ...] +zset → ["zset", member, score, ...] ascending score +json → ["json", document] +``` + +The tag is what makes the payload unambiguous: a four-element array would otherwise be +indistinguishable between a list of four items and a hash of two pairs. Ordering is deterministic, so +two clients receiving the same notification build identical local state. + +Each notification carries the **complete** current value, so the receiver replaces the key rather than +merging — which is what allows a removed member to propagate. + +Remaining limitation: `FLUSHDB` does not emit per-key diffs to live queries. + +::: warning Changed in 0.2.2 +Before 0.2.2 collections arrived as a bare type name (`"hash"`), and subscribers had to follow up with +`HGETALL`/`LRANGE`. Server and SDK are released in lockstep — run matching versions, since a 0.2.1 +client ignores the new shape. +::: --- diff --git a/docs/server/troubleshooting.md b/docs/server/troubleshooting.md index 339a4a8..027e1df 100644 --- a/docs/server/troubleshooting.md +++ b/docs/server/troubleshooting.md @@ -151,9 +151,11 @@ See [Offline & Reconnection](/browser/offline). A live query's initial state is capped at **10,000 keys**. Beyond that the snapshot is truncated. Narrow the pattern. -Also note two documented limits of live queries: `FLUSHDB` does not emit per-key diffs, and -collection values arrive as type markers rather than full values — subscribe, then re-read with -`HGETALL` / `LRANGE` on change. +Also note a remaining limit: `FLUSHDB` does not emit per-key diffs. + +If collection values arrive as a bare type name (`"hash"`) rather than their contents, the server and +SDK are on different versions — collection values ship complete from 0.2.2 onward, and the two are +released in lockstep. ### Too many live queries diff --git a/sdks/recached-react/package.json b/sdks/recached-react/package.json index 74fbcf3..ed5b6ff 100644 --- a/sdks/recached-react/package.json +++ b/sdks/recached-react/package.json @@ -1,6 +1,6 @@ { "name": "@recached/react", - "version": "0.2.1", + "version": "0.2.2", "description": "Official React hooks for Recached \u2014 zero-latency reactive cache", "type": "module", "main": "./dist/index.js", diff --git a/sdks/recached-vue/package.json b/sdks/recached-vue/package.json index 4f4ffa8..3a71e22 100644 --- a/sdks/recached-vue/package.json +++ b/sdks/recached-vue/package.json @@ -1,6 +1,6 @@ { "name": "@recached/vue", - "version": "0.2.1", + "version": "0.2.2", "description": "Official Vue 3 composables for Recached \u2014 zero-latency reactive cache", "type": "module", "main": "./dist/index.js", diff --git a/server-native/src/main.rs b/server-native/src/main.rs index c43db87..3b3d6f3 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -59,6 +59,7 @@ fn command_name(cmd: &Command) -> &'static str { Command::Ping(_) => "ping", Command::Auth(_) => "auth", Command::Get(_) => "get", + Command::ESet(_, _) => "eset", Command::Set(_, _, _) => "set", Command::Del(_) => "del", Command::Unlink(_) => "unlink", @@ -605,6 +606,40 @@ struct ServerState { /// client suffices — no seen-set. In-memory only: a server restart /// reopens the (already narrow) duplicate window, which is documented. dedup: std::sync::Mutex>, + /// Ephemeral (`ESET`) keys → the connection that currently owns them. + /// + /// Ownership transfers on each `ESET`, which is what makes multiple tabs + /// work: two tabs both setting `presence:user:42` leave the *later* one as + /// owner, so the first tab closing does not mark the user offline. Only the + /// owning connection's close deletes the key. + ephemeral: std::sync::Mutex>, +} + +impl ServerState { + /// Record `conn_id` as the owner of an ephemeral key, replacing any + /// previous owner. + fn claim_ephemeral(&self, key: &str, conn_id: u64) { + if let Ok(mut map) = self.ephemeral.lock() { + map.insert(key.to_string(), conn_id); + } + } + + /// Keys still owned by `conn_id`, removed from the registry. Called once + /// when a connection closes. + fn take_ephemeral_for(&self, conn_id: u64) -> Vec { + let Ok(mut map) = self.ephemeral.lock() else { + return Vec::new(); + }; + let owned: Vec = map + .iter() + .filter(|(_, id)| **id == conn_id) + .map(|(k, _)| k.clone()) + .collect(); + for k in &owned { + map.remove(k); + } + owned + } } /// Sweep dedup client entries idle longer than this once the map is large. @@ -697,6 +732,7 @@ fn is_write_command(cmd: &Command) -> bool { matches!( cmd, Command::Set(..) + | Command::ESet(..) | Command::Del(..) | Command::Unlink(..) | Command::Append(..) @@ -1147,7 +1183,8 @@ fn command_scope(cmd: &Command) -> CommandScope { | Command::LastSave | Command::ReplicaOfNoOne => CommandScope::Admin, - Command::Set(k, _, _) + Command::ESet(k, _) + | Command::Set(k, _, _) | Command::Get(k) | Command::Append(k, _) | Command::Strlen(k) @@ -1474,7 +1511,8 @@ type WatchRegistry = Arc; /// already confirmed a mutation occurred. fn primary_keys(cmd: &Command) -> Vec { match cmd { - Command::Set(k, _, _) + Command::ESet(k, _) + | Command::Set(k, _, _) | Command::Append(k, _) | Command::GetSet(k, _) | Command::SetNx(k, _) @@ -1710,6 +1748,9 @@ fn resp_push(parts: &[&str]) -> String { /// if the command mutated nothing (read-only or conditional-and-failed). fn broadcast_for(cmd: &Command, response: &Value) -> Option { match cmd { + // Replays as SET: a replica has no connection to scope the lifetime to, + // and the owning server broadcasts the DEL when the connection closes. + Command::ESet(k, v) => Some(resp_push(&["SET", k, v])), Command::Set(k, v, opts) => { // Without GET: nil response means NX/XX condition failed — don't broadcast. // With GET: nil means key didn't exist before, but SET still happened. @@ -2343,6 +2384,7 @@ async fn main() -> Result<(), Box> { replicas: Arc::clone(&replicas), is_replica: std::sync::atomic::AtomicBool::new(is_replica_start), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); // ── autosave ────────────────────────────────────────────────────────── @@ -3446,6 +3488,15 @@ async fn handle_ws( } _ => {} } + // Ephemeral keys are owned by the connection that wrote + // them until another claims them; the close handler deletes + // whatever is still ours. Claimed outside the write-effects + // branch below, which only runs when a peer, replica, AOF or + // watcher is present — ownership must be recorded even on a + // standalone server with no listeners. + if let Command::ESet(ref k, _) = cmd { + state.claim_ephemeral(k, conn_id); + } let response = if is_write_command(&cmd) && write_effects_armed(&tx, &state, &watch_registry) { @@ -3551,6 +3602,25 @@ async fn handle_ws( watch_registry.sync_len(®); } unregister_all_qsubs(&watch_registry, conn_id, &mut qsub_patterns).await; + + // Delete ephemeral keys this connection still owns and fan the deletions + // out, so every subscriber sees the peer go away immediately rather than + // waiting for a heartbeat TTL to lapse. + let expired = state.take_ephemeral_for(conn_id); + if !expired.is_empty() { + let del = Command::Del(expired); + let response = store.execute(del.clone()); + apply_write_effects( + &del, + &response, + &tx, + conn_id, + &state, + &watch_registry, + &store, + ) + .await; + } } #[cfg(test)] @@ -3606,6 +3676,7 @@ mod tests { replicas: ReplHub::new(), is_replica: AtomicBool::new(start_as_replica), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -4102,6 +4173,7 @@ mod tests { replicas: ReplHub::new(), is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); // Simulate writes captured by AOF @@ -4176,6 +4248,108 @@ mod tests { assert!(!srv.state.is_replica()); } + // ── Presence: connection-scoped keys ────────────────────────────────────── + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn eset_key_is_deleted_when_its_connection_closes() { + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "presence:*"]).await; + + { + let mut presence = WsClient::connect(srv.tcp_addr).await; + assert_eq!( + presence.cmd(&["ESET", "presence:user:42", "online"]).await, + Value::SimpleString("OK".into()) + ); + // Visible to everyone while the connection is open. + assert_eq!( + srv.store.execute(Command::Get("presence:user:42".into())), + Value::BulkString(Some(b"online".to_vec())) + ); + } // connection dropped here + + // The key goes away on its own — no heartbeat, no TTL to wait out. + for _ in 0..40 { + if srv.store.execute(Command::Get("presence:user:42".into())) == Value::BulkString(None) + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + } + panic!("ephemeral key outlived its connection"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn eset_deletion_is_broadcast_to_live_queries() { + // Presence is only useful if peers are *told*; polling for the absence + // of a key is the thing this replaces. + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "presence:*"]).await; + + { + let mut presence = WsClient::connect(srv.tcp_addr).await; + presence.cmd(&["ESET", "presence:user:7", "online"]).await; + // Drain the set notification. + let _ = watcher.recv_push(1000).await; + } + + let push = watcher + .recv_push(3000) + .await + .expect("expected a keychange for the departing peer"); + assert!(push.contains("presence:user:7"), "unexpected push: {push}"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_second_tab_keeps_presence_alive_when_the_first_closes() { + // The multi-tab case. Ownership transfers to the most recent writer, so + // closing an older tab must not mark the user offline. + let srv = spawn_ws_server().await; + + let mut tab_b = WsClient::connect(srv.tcp_addr).await; + { + let mut tab_a = WsClient::connect(srv.tcp_addr).await; + tab_a.cmd(&["ESET", "presence:user:9", "online"]).await; + // Tab B claims the same key — it is now the owner. + tab_b.cmd(&["ESET", "presence:user:9", "online"]).await; + } // tab A closes + + // Give the close handler time to run, then confirm the key survived. + tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; + assert_eq!( + srv.store.execute(Command::Get("presence:user:9".into())), + Value::BulkString(Some(b"online".to_vec())), + "closing an older tab must not clear presence held by a newer one" + ); + + drop(tab_b); + for _ in 0..40 { + if srv.store.execute(Command::Get("presence:user:9".into())) == Value::BulkString(None) + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + } + panic!("key outlived its last owner"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn plain_set_is_not_ephemeral() { + // Only ESET opts into connection-scoped lifetime; SET must be unaffected. + let srv = spawn_ws_server().await; + { + let mut c = WsClient::connect(srv.tcp_addr).await; + c.cmd(&["SET", "durable:key", "value"]).await; + } + tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; + assert_eq!( + srv.store.execute(Command::Get("durable:key".into())), + Value::BulkString(Some(b"value".to_vec())) + ); + } + #[tokio::test] async fn integration_replica_receives_write() { // Spawn primary with a separate replication listener on a random port @@ -4230,6 +4404,7 @@ mod tests { replicas: Arc::clone(&repl_registry), is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -4277,6 +4452,7 @@ mod tests { replicas: ReplHub::new(), is_replica: AtomicBool::new(true), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); let rs = Arc::clone(&replica_store); let rst = Arc::clone(&replica_state); @@ -4349,6 +4525,7 @@ mod tests { replicas: ReplHub::new(), is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -4466,6 +4643,7 @@ mod tests { replicas: ReplHub::new(), is_replica: AtomicBool::new(true), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); let replica_store = Arc::new(KeyValueStore::new()); let rs = Arc::clone(&replica_store); @@ -4512,6 +4690,7 @@ mod tests { replicas: ReplHub::new(), is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -5043,6 +5222,7 @@ mod tests { Expect::Keys(&["k"]), ), (Command::Get("k".into()), Expect::Keys(&["k"])), + (Command::ESet("k".into(), "v".into()), Expect::Keys(&["k"])), (Command::Del(vec!["k".into()]), Expect::Keys(&["k"])), (Command::Unlink(vec!["k".into()]), Expect::Keys(&["k"])), ( @@ -5288,6 +5468,53 @@ mod tests { } } + #[test] + fn every_key_writing_command_is_classified_as_a_write() { + // `is_write_command` is a `matches!` list, which — unlike a `match` — + // has no exhaustiveness check: a new variant silently defaults to "not + // a write" and is then never replicated, logged to AOF, or broadcast. + // `primary_keys` reports the keys a command *writes*, so anything it + // names must also be classified as a write. This cross-check is what + // makes the missing entry impossible to ship. + for (cmd, _) in all_commands() { + if !primary_keys(&cmd).is_empty() { + assert!( + is_write_command(&cmd), + "{cmd:?} writes keys but is_write_command() says otherwise — \ + it would never reach replicas, the AOF, or live queries" + ); + } + } + } + + #[test] + fn eset_is_a_write_and_reports_its_key() { + let cmd = Command::ESet("presence:1".into(), "on".into()); + assert!(is_write_command(&cmd)); + assert_eq!(primary_keys(&cmd), vec!["presence:1".to_string()]); + assert_eq!(command_name(&cmd), "eset"); + // Scoped connections must not be able to write presence keys outside + // their grant. + assert!(matches!(command_scope(&cmd), CommandScope::Keys(_))); + } + + #[test] + fn eset_replays_to_replicas_as_a_plain_set() { + // A replica has no connection to scope the lifetime to, so it stores an + // ordinary key; the owning server broadcasts the DEL on disconnect. + let frame = broadcast_for( + &Command::ESet("presence:1".into(), "on".into()), + &Value::SimpleString("OK".into()), + ) + .expect("ESET must broadcast"); + assert!(frame.contains("SET"), "{frame}"); + assert!(frame.contains("presence:1")); + assert!( + !frame.contains("ESET"), + "replica should receive SET, not ESET" + ); + } + #[test] fn every_command_has_a_metrics_label() { for (cmd, _) in all_commands() { diff --git a/sync-client/src/lib.rs b/sync-client/src/lib.rs index 25247a5..61d48e7 100644 --- a/sync-client/src/lib.rs +++ b/sync-client/src/lib.rs @@ -29,6 +29,16 @@ pub const MAX_PENDING_WRITES: usize = 10_000; pub const BACKOFF_BASE_MS: u32 = 500; pub const BACKOFF_CAP_MS: u32 = 30_000; +/// FNV-1a over the client id — a stable, non-zero seed per client. +fn seed_from(client_id: &str) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in client_id.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h | 1 +} + /// What an incoming frame turned out to be, and what the adapter must do. #[derive(Debug, PartialEq)] pub enum Incoming { @@ -83,10 +93,16 @@ pub struct SyncClient { sync_scopes_csv: Option, live_queries: Vec, attempts: u32, + /// Jitter source for reconnect backoff. Seeded from `client_id` rather than + /// a system RNG so this crate stays dependency-free and I/O-free — and so + /// the sequence is reproducible in tests. Different clients get different + /// sequences, which is the only property that matters here. + jitter: u64, } impl SyncClient { pub fn new(store: Arc, client_id: String) -> Self { + let jitter = seed_from(&client_id); Self { store, outbox: VecDeque::new(), @@ -99,6 +115,7 @@ impl SyncClient { sync_scopes_csv: None, live_queries: Vec::new(), attempts: 0, + jitter, } } @@ -228,12 +245,35 @@ impl SyncClient { /// The socket closed: returns the delay before the next reconnect /// attempt (exponential backoff, capped). + /// Delay before the next reconnect attempt, in milliseconds. + /// + /// Exponential with a cap, then **jittered into `[delay/2, delay]`**. Without + /// jitter every client disconnected by the same event computes an identical + /// schedule and reconnects in lockstep — a thundering herd that can keep a + /// recovering server down. Halving the floor keeps the growth shape intact + /// while spreading arrivals across the window. pub fn on_close(&mut self) -> u32 { let delay = BACKOFF_BASE_MS .saturating_mul(1u32 << self.attempts.min(6)) .min(BACKOFF_CAP_MS); self.attempts = self.attempts.saturating_add(1); - delay + + let half = delay / 2; + if half == 0 { + return delay; + } + half + (self.next_jitter() % (half as u64 + 1)) as u32 + } + + /// xorshift64* — small, dependency-free, and good enough for spreading + /// reconnects. Not for anything security-sensitive. + fn next_jitter(&mut self) -> u64 { + let mut x = self.jitter; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.jitter = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) } // ── writes ──────────────────────────────────────────────────────────── @@ -402,8 +442,71 @@ impl SyncClient { Value::BulkString(None) => { self.store.execute(Command::Del(vec![key])); } - // Collection/JSON type marker: the value is not carried; regular - // mutation pushes keep those in sync — nothing to apply here. + // Type-tagged collection: ["hash"|"list"|"set"|"zset"|"json", ...]. + Value::Array(Some(items)) => self.apply_tagged(key, items), + // Anything else (e.g. the "ratelimit" marker) carries no client- + // usable value. + _ => {} + } + } + + /// Apply a type-tagged collection value from a keychange or qstate frame. + /// + /// The key is replaced wholesale rather than merged: the frame carries the + /// complete current value, so rebuilding is both simpler and correct when a + /// member was removed. Anything unrecognised is ignored rather than guessed + /// at, so a newer server cannot corrupt an older client's local copy. + fn apply_tagged(&self, key: String, items: &[Value]) { + fn text(v: &Value) -> Option { + match v { + Value::BulkString(Some(b)) => Some(String::from_utf8_lossy(b).into_owned()), + Value::SimpleString(s) => Some(s.clone()), + _ => None, + } + } + let Some(tag) = items.first().and_then(text) else { + return; + }; + let rest: Vec = items[1..].iter().filter_map(text).collect(); + + // Clear first so removed members do not linger. + self.store.execute(Command::Del(vec![key.clone()])); + match tag.as_str() { + "hash" => { + let pairs: Vec<(String, String)> = rest + .chunks_exact(2) + .map(|c| (c[0].clone(), c[1].clone())) + .collect(); + if !pairs.is_empty() { + self.store.execute(Command::HSet(key, pairs)); + } + } + "list" => { + if !rest.is_empty() { + self.store.execute(Command::RPush(key, rest)); + } + } + "set" => { + if !rest.is_empty() { + self.store.execute(Command::SAdd(key, rest)); + } + } + "zset" => { + let members: Vec<(f64, String)> = rest + .chunks_exact(2) + .filter_map(|c| c[1].parse::().ok().map(|sc| (sc, c[0].clone()))) + .collect(); + if !members.is_empty() { + self.store + .execute(Command::ZAdd(key, Default::default(), members)); + } + } + "json" => { + if let Some(doc) = rest.first() { + self.store + .execute(Command::JSet(key, "$".to_string(), doc.clone())); + } + } _ => {} } } @@ -413,12 +516,19 @@ impl SyncClient { let Value::BulkString(Some(k)) = &pair[0] else { continue; }; - if let Value::BulkString(Some(v)) = &pair[1] { - self.store.execute(Command::Set( - String::from_utf8_lossy(k).into_owned(), - String::from_utf8_lossy(v).into_owned(), - Default::default(), - )); + let key = String::from_utf8_lossy(k).into_owned(); + match &pair[1] { + Value::BulkString(Some(v)) => { + self.store.execute(Command::Set( + key, + String::from_utf8_lossy(v).into_owned(), + Default::default(), + )); + } + // Collections arrive type-tagged, so the initial state of a + // live query is complete — no follow-up typed read needed. + Value::Array(Some(inner)) => self.apply_tagged(key, inner), + _ => {} } } } diff --git a/sync-client/src/tests.rs b/sync-client/src/tests.rs index ee6d49a..55c086a 100644 --- a/sync-client/src/tests.rs +++ b/sync-client/src/tests.rs @@ -94,17 +94,74 @@ fn session_commands_sent_while_open_occupy_reply_slots() { #[test] fn backoff_doubles_and_caps() { + // Jitter puts each delay in [nominal/2, nominal]; assert the band rather + // than an exact value. + fn band(delay: u32, nominal: u32) { + assert!( + delay >= nominal / 2 && delay <= nominal, + "delay {delay} outside [{}, {nominal}]", + nominal / 2 + ); + } + let mut c = client(); - assert_eq!(c.on_close(), 500); - assert_eq!(c.on_close(), 1000); - assert_eq!(c.on_close(), 2000); + band(c.on_close(), 500); + band(c.on_close(), 1000); + band(c.on_close(), 2000); for _ in 0..10 { c.on_close(); } - assert_eq!(c.on_close(), 30_000); + band(c.on_close(), 30_000); // A successful open resets the schedule. c.on_open(); - assert_eq!(c.on_close(), 500); + band(c.on_close(), 500); +} + +#[test] +fn backoff_is_jittered_across_clients() { + // The point of jitter: two clients disconnected by the same event must not + // compute the same schedule, or they reconnect in lockstep. + let mut a = SyncClient::new(Arc::new(KeyValueStore::new()), "client-a".to_string()); + let mut b = SyncClient::new(Arc::new(KeyValueStore::new()), "client-b".to_string()); + + let seq_a: Vec = (0..6).map(|_| a.on_close()).collect(); + let seq_b: Vec = (0..6).map(|_| b.on_close()).collect(); + assert_ne!(seq_a, seq_b, "distinct clients must not share a schedule"); +} + +#[test] +fn backoff_never_returns_zero_or_exceeds_the_cap() { + // A zero delay would busy-loop reconnects; exceeding the cap would strand + // a client for longer than intended. + let mut c = client(); + for _ in 0..40 { + let d = c.on_close(); + assert!(d > 0, "backoff must never be zero"); + assert!(d <= BACKOFF_CAP_MS, "backoff {d} exceeded the cap"); + } +} + +#[test] +fn jitter_spreads_reconnects_over_the_window() { + // With many clients the delays should occupy a range, not cluster on one + // value — that is the property that prevents the herd. + let delays: Vec = (0..50) + .map(|i| { + let mut c = SyncClient::new(Arc::new(KeyValueStore::new()), format!("client-{i}")); + for _ in 0..3 { + c.on_close(); + } + c.on_close() // nominal 4000ms + }) + .collect(); + + let unique: std::collections::HashSet = delays.iter().copied().collect(); + assert!( + unique.len() > 20, + "expected a spread of delays, got {} distinct values", + unique.len() + ); + assert!(delays.iter().all(|d| *d >= 2000 && *d <= 4000)); } // ── acknowledgment & retirement ─────────────────────────────────────────────── @@ -397,8 +454,9 @@ fn on_open_resets_attempt_count_and_inflight() { c.enqueue_write(&to_resp(&["SET", "k", "v"]), true, true); c.on_open(); - // Backoff restarts from the floor after a successful open. - assert_eq!(c.on_close(), 500); + // Backoff restarts from the floor after a successful open (jittered). + let d = c.on_close(); + assert!((250..=500).contains(&d), "expected a reset floor, got {d}"); } #[test] @@ -449,3 +507,191 @@ fn epoch_is_readable_and_settable() { c.set_epoch(7); assert_eq!(c.epoch(), 7); } + +// ── Type-tagged collection values ───────────────────────────────────────────── +// A live query used to deliver only a type name for collections, forcing the +// subscriber into a follow-up HGETALL/LRANGE — a network round-trip in a system +// built on local reads. Values now arrive tagged and complete. + +/// Build the keychange frame a server sends for `key`, using the real +/// `get_current` encoding rather than a hand-written fixture. +fn keychange_frame(source: &KeyValueStore, key: &str) -> Vec { + vec![ + Value::BulkString(Some(b"keychange".to_vec())), + Value::BulkString(Some(key.as_bytes().to_vec())), + source.get_current(key), + ] +} + +/// keychange/qstate frames are plain RESP arrays, matching the wire format +/// the existing tests use. +fn push(items: Vec) -> String { + String::from_utf8_lossy(&Value::Array(Some(items)).serialize()).into_owned() +} + +#[test] +fn keychange_rebuilds_a_hash_without_a_re_read() { + let source = KeyValueStore::new(); + source.execute(Command::HSet( + "cart:42".into(), + vec![("item".into(), "book".into()), ("qty".into(), "2".into())], + )); + + let mut c = client(); + c.handle_frame(&push(keychange_frame(&source, "cart:42"))); + + assert_eq!( + c.store() + .execute(Command::HGet("cart:42".into(), "item".into())), + bulk("book") + ); + assert_eq!( + c.store() + .execute(Command::HGet("cart:42".into(), "qty".into())), + bulk("2") + ); +} + +#[test] +fn keychange_rebuilds_lists_in_order() { + let source = KeyValueStore::new(); + source.execute(Command::RPush( + "queue".into(), + vec!["first".into(), "second".into(), "third".into()], + )); + + let mut c = client(); + c.handle_frame(&push(keychange_frame(&source, "queue"))); + + assert_eq!( + c.store().execute(Command::LRange("queue".into(), 0, -1)), + Value::Array(Some(vec![bulk("first"), bulk("second"), bulk("third")])), + "list order must survive the round trip" + ); +} + +#[test] +fn keychange_rebuilds_sets_and_zsets() { + let source = KeyValueStore::new(); + source.execute(Command::SAdd( + "tags".into(), + vec!["red".into(), "blue".into()], + )); + source.execute(Command::ZAdd( + "board".into(), + Default::default(), + vec![(10.0, "alice".into()), (5.0, "bob".into())], + )); + + let mut c = client(); + c.handle_frame(&push(keychange_frame(&source, "tags"))); + c.handle_frame(&push(keychange_frame(&source, "board"))); + + assert_eq!( + c.store().execute(Command::SCard("tags".into())), + Value::Integer(2) + ); + assert_eq!( + c.store() + .execute(Command::ZScore("board".into(), "alice".into())), + bulk("10") + ); +} + +#[test] +fn keychange_rebuilds_json_documents() { + let source = KeyValueStore::new(); + source.execute(Command::JSet( + "doc".into(), + "$".into(), + "{\"a\":1}".to_string(), + )); + + let mut c = client(); + c.handle_frame(&push(keychange_frame(&source, "doc"))); + + assert_eq!( + c.store().execute(Command::JGet("doc".into(), None)), + bulk("{\"a\":1}") + ); +} + +#[test] +fn a_removed_member_disappears_from_the_local_copy() { + // The frame carries the complete value, so the key is rebuilt rather than + // merged — otherwise a removal would never propagate. + let source = KeyValueStore::new(); + source.execute(Command::SAdd( + "tags".into(), + vec!["red".into(), "blue".into()], + )); + + let mut c = client(); + c.handle_frame(&push(keychange_frame(&source, "tags"))); + assert_eq!( + c.store().execute(Command::SCard("tags".into())), + Value::Integer(2) + ); + + source.execute(Command::SRem("tags".into(), vec!["red".into()])); + c.handle_frame(&push(keychange_frame(&source, "tags"))); + + assert_eq!( + c.store() + .execute(Command::SIsMember("tags".into(), "red".into())), + Value::Integer(0), + "a member removed on the server must not linger locally" + ); + assert_eq!( + c.store().execute(Command::SCard("tags".into())), + Value::Integer(1) + ); +} + +#[test] +fn qstate_delivers_complete_collections_on_subscribe() { + // The initial state of a live query is now usable immediately. + let source = KeyValueStore::new(); + source.execute(Command::HSet( + "cart:1".into(), + vec![("item".into(), "pen".into())], + )); + source.execute(Command::RPush("cart:2".into(), vec!["a".into()])); + + let mut c = client(); + let frame = push(vec![ + Value::BulkString(Some(b"qstate".to_vec())), + Value::BulkString(Some(b"cart:*".to_vec())), + Value::BulkString(Some(b"cart:1".to_vec())), + source.get_current("cart:1"), + Value::BulkString(Some(b"cart:2".to_vec())), + source.get_current("cart:2"), + ]); + c.handle_frame(&frame); + + assert_eq!( + c.store() + .execute(Command::HGet("cart:1".into(), "item".into())), + bulk("pen") + ); + assert_eq!( + c.store().execute(Command::LLen("cart:2".into())), + Value::Integer(1) + ); +} + +#[test] +fn an_unknown_type_tag_is_ignored_rather_than_guessed() { + // Forward compatibility: a newer server sending a type this client does not + // know must not corrupt the local copy. + let mut c = client(); + c.handle_frame(&push(vec![ + Value::BulkString(Some(b"keychange".to_vec())), + Value::BulkString(Some(b"k".to_vec())), + Value::Array(Some(vec![ + Value::BulkString(Some(b"futuretype".to_vec())), + Value::BulkString(Some(b"payload".to_vec())), + ])), + ])); + assert_eq!(get(&c, "k"), Value::BulkString(None)); +} diff --git a/wasm-edge/package.json b/wasm-edge/package.json index b4fb02c..4bc3c9b 100644 --- a/wasm-edge/package.json +++ b/wasm-edge/package.json @@ -1,7 +1,7 @@ { "name": "recached-edge", "description": "Browser and edge WebAssembly client for Recached \u2014 zero-latency local cache with automatic server sync", - "version": "0.2.1", + "version": "0.2.2", "type": "module", "main": "sdk.js", "module": "sdk.js", From a535c1e6c4c30bc0813b29254e1940bfe4e87b0a Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 16:40:24 +0800 Subject: [PATCH 02/11] feat: durable exactly-once marks, atomic WAL compaction, capacity metrics --- CHANGELOG.md | 24 +++++ core-engine/src/store.rs | 106 ++++++++++++++++++- docs/roadmap.md | 37 ++----- docs/server/operations.md | 40 ++++--- server-native/src/main.rs | 216 ++++++++++++++++++++++++++++++++++++++ wasm-edge/src/lib.rs | 126 +++++++++++++++++++++- 6 files changed, 495 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65aa2ac..e6b503c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ All notable changes to Recached are documented here. offline. Replicas receive the write as a plain `SET` — they have no connection to scope a lifetime to, and the owning server broadcasts the deletion. +- **Capacity and sync metrics.** Seven new series, sampled every 5 seconds because capacity is a + level rather than an event: `recached_memory_bytes`, `recached_keys`, `recached_evictions_total`, + `recached_replicas_connected`, `recached_live_queries`, `recached_watched_keys`, and + `recached_dedup_clients_tracked`. Previously only traffic was exported, so an operator could not + answer "am I near the cap?" or "is eviction thrashing?" from a dashboard. Replication *lag* and + browser outbox depth remain unexported — see [Operations](docs/server/operations.md). + ### Changed - **Live queries now carry collection values.** `qstate` and `keychange` previously delivered only a @@ -53,6 +60,23 @@ All notable changes to Recached are documented here. ### Fixed +- **Exactly-once delivery now survives a server restart.** Dedup high-water marks were held only in + memory, so a restart inside the acknowledgement window let a client's replayed write apply twice — + the last standing caveat on the guarantee. Marks are now persisted to a `.dedup` sidecar beside the + snapshot, written atomically and only when a mark advances, and restored before the server accepts + connections. + + The map is one `u64` per client, so it is flushed on a 1-second timer as well as with each + snapshot: the residual window on an unclean shutdown is bounded by that interval rather than by the + snapshot cadence. A missing or corrupt sidecar is logged and ignored rather than being fatal — + losing the bookkeeping is bad, refusing to boot is worse. + +- **WAL compaction could destroy the browser's persisted cache.** Compaction cleared the write-ahead + log in one IndexedDB transaction and wrote the replacement snapshot in later ones, so an + interruption between them left an empty WAL and no snapshot. The existing code comment anticipated + this window; it is now closed by doing the clear and the rewrite in a **single transaction**, which + IndexedDB commits or rolls back as a unit. + - **`ESET` would not have replicated.** `is_write_command` is a `matches!` list, which — unlike a `match` — has no exhaustiveness check, so a newly added command silently defaults to "not a write" and never reaches replicas, the AOF, or live queries. Caught while wiring `ESET`; a cross-check test diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 6731327..9964688 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -510,6 +510,9 @@ pub struct KeyValueStore { max_memory_bytes: Option, eviction_policy: EvictionPolicy, dirty: Arc, + /// Total keys evicted since start. Exported as a metric: without it an + /// operator cannot tell a healthy cache from one thrashing at its cap. + evicted: Arc, } impl Default for KeyValueStore { @@ -526,6 +529,7 @@ impl KeyValueStore { max_memory_bytes: None, eviction_policy: EvictionPolicy::NoEviction, dirty: Arc::new(AtomicU64::new(0)), + evicted: Arc::new(AtomicU64::new(0)), } } @@ -536,6 +540,7 @@ impl KeyValueStore { max_memory_bytes: None, eviction_policy: EvictionPolicy::NoEviction, dirty: Arc::new(AtomicU64::new(0)), + evicted: Arc::new(AtomicU64::new(0)), } } @@ -550,6 +555,7 @@ impl KeyValueStore { max_memory_bytes, eviction_policy, dirty: Arc::new(AtomicU64::new(0)), + evicted: Arc::new(AtomicU64::new(0)), } } @@ -714,6 +720,20 @@ impl KeyValueStore { /// Evict a single entry per the configured policy. Returns the number of /// bytes freed (`Some`), or `None` if nothing could be evicted. + /// Keys evicted since start. + pub fn evicted_count(&self) -> u64 { + self.evicted.load(Ordering::Relaxed) + } + + /// Live key count, excluding expired entries awaiting sweep. + pub fn key_count(&self) -> usize { + let now = now_ms(); + self.data + .iter() + .filter(|r| !r.value().is_expired(now)) + .count() + } + fn evict_one(&self, now: u64) -> Option { const SAMPLE: usize = 10; let mut rng = rand::rng(); @@ -771,11 +791,12 @@ impl KeyValueStore { let key = chosen?; // Treat a lost race (key already gone) as a successful eviction that // freed nothing, so callers don't spin. - Some( - self.data - .remove(&key) - .map_or(0, |(k, e)| entry_size(&k, &e)), - ) + let freed = self + .data + .remove(&key) + .map_or(0, |(k, e)| entry_size(&k, &e)); + self.evicted.fetch_add(1, Ordering::Relaxed); + Some(freed) } pub fn snapshot(&self) -> Vec { @@ -5367,3 +5388,78 @@ mod ephemeral_tests { assert_eq!(matched.len(), 1); } } + +#[cfg(test)] +mod metrics_tests { + use super::*; + use crate::cmd::{Command, SetOptions}; + + #[test] + fn key_count_excludes_expired_entries() { + // The metric must report what a client can actually read, not what is + // still sitting in the map awaiting sweep — otherwise a dashboard shows + // a keyspace that is not there. + let s = KeyValueStore::new(); + s.execute(Command::Set( + "live".into(), + "v".into(), + SetOptions::default(), + )); + s.execute(Command::Set( + "dead".into(), + "v".into(), + SetOptions { + expiry: Some(crate::cmd::SetExpiry::Px(1)), + ..Default::default() + }, + )); + std::thread::sleep(std::time::Duration::from_millis(15)); + assert_eq!(s.key_count(), 1); + } + + #[test] + fn eviction_counter_starts_at_zero_and_counts_each_eviction() { + let s = KeyValueStore::with_config(Some(2), None, EvictionPolicy::AllKeysRandom); + assert_eq!(s.evicted_count(), 0); + + for i in 0..5 { + s.execute(Command::Set( + format!("k{i}"), + "v".into(), + SetOptions::default(), + )); + } + // Cap of 2 with 5 inserts means 3 evictions were required. + assert_eq!(s.key_count(), 2); + assert_eq!( + s.evicted_count(), + 3, + "eviction rate is the signal that a cache is thrashing at its cap" + ); + } + + #[test] + fn eviction_counter_stays_zero_without_pressure() { + let s = KeyValueStore::new(); + for i in 0..10 { + s.execute(Command::Set( + format!("k{i}"), + "v".into(), + SetOptions::default(), + )); + } + assert_eq!(s.evicted_count(), 0, "no cap configured, nothing to evict"); + } + + #[test] + fn memory_estimate_tracks_the_stored_data() { + let s = KeyValueStore::new(); + let empty = s.approximate_memory_bytes(); + s.execute(Command::Set( + "k".into(), + "x".repeat(4096), + SetOptions::default(), + )); + assert!(s.approximate_memory_bytes() >= empty + 4096); + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 8525257..356d746 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,24 +2,16 @@ Recached competes on **where the data can live** — the same engine on the server and in the browser, with sync in between. The [benchmarks](/guide/benchmarks) show this costs nothing in raw speed. -Numbered items are stable identifiers, not an ordering — code and changelog entries reference them -(e.g. "roadmap #6"), so they are never renumbered. **Near-term priorities** are called out below. - -Items 1–5 and 13 have shipped: rate-limiting commands, scoped sync with per-client auth, live queries, the native JSON type, and offline-first writes with merge semantics. Their numbering is retained below so existing references stay valid — see the [changelog](https://github.com/thinkgrid-labs/recached/blob/main/CHANGELOG.md) for what landed in each. - ## Near-term -The three items previously listed here shipped in **0.2.2**: reconnect backoff jitter, presence via -connection-scoped keys (`ESET`), and live queries carrying complete collection values. See the -[changelog](https://github.com/thinkgrid-labs/recached/blob/main/CHANGELOG.md). - -Next up, in order: +The next three things worth doing, in order: -1. **[Exactly-once across a server restart](#reliability)** — the last documented caveat on the - delivery guarantee. -2. **[Atomic WAL compaction](#reliability)** — removes a browser-side data-loss window. -3. **[Capacity and sync metrics](#operability)** — operators currently cannot see memory, key count, - eviction rate, or replication lag. +1. **[Byte-slice command arguments](#performance)** — the main remaining lever on unpipelined + latency, and the likeliest explanation for `HSET` trailing Redis. +2. **[Surface outbox overflow](#reliability)** — the client silently drops the oldest write past + 10 000; applications get no signal. +3. **[Live queries carry `FLUSHDB` diffs](#live-queries)** — subscribers currently miss a mass + deletion entirely. --- ## 6. Mobile SDKs — React Native, Flutter, Kotlin, Swift @@ -83,15 +75,6 @@ independent — pick them off in any order. ### Reliability -**Exactly-once across a server restart.** `DEDUP` high-water marks live in server memory and are -swept after 24 h idle, so a restart inside the acknowledgement window can admit one duplicate — a -caveat currently documented rather than fixed. Persisting the marks alongside the snapshot would make -the exactly-once guarantee unconditional. - -**Atomic WAL compaction.** Browser-side compaction clears the write-ahead log and then writes the -replacement snapshot. An interruption between the two loses the persisted cache. Writing to a new key -range and swapping removes the window entirely. - **Surface outbox overflow.** The client outbox holds 10 000 writes and silently evicts the oldest past that. An `onOutboxFull` callback, or a `pendingWrites()` accessor, lets an application degrade deliberately instead of losing writes without a signal. @@ -113,10 +96,8 @@ Redis while *beating* it pipelined. ### Operability -**Capacity and sync metrics.** Six series are exported today, all traffic. Nothing reports memory -use, key count, eviction rate, replication lag, or outbox depth — so an operator cannot answer "am I -near the cap?" or "is my replica behind?" from a dashboard. See -[Operations](/server/operations#what-is-not-exported-yet). +**Replication lag.** The number of connected replicas is exported, but not how far behind each one +is — the signal that actually matters before a failover. **Make the compiled-in limits configurable.** The 10 000-write outbox, 64 live queries per connection, and eviction's fixed 10-key sample are all constants. Redis exposes `maxmemory-samples` diff --git a/docs/server/operations.md b/docs/server/operations.md index b1a0365..6daf51c 100644 --- a/docs/server/operations.md +++ b/docs/server/operations.md @@ -35,22 +35,24 @@ real. | `recached_keyspace_hits_total` | counter | — | Reads that found a live key. | | `recached_keyspace_misses_total` | counter | — | Reads that found nothing or an expired key. | -### What is not exported yet +### Capacity and sync -Be aware of these before you build a dashboard expecting them: +Sampled every 5 seconds, because these are levels rather than events. -- **No memory metric.** Nothing reports bytes used or how close you are to `RECACHED_MAX_MEMORY`. - Monitor process RSS from your container runtime or node exporter instead. -- **No key-count metric.** Nothing reports keyspace size against `RECACHED_MAX_KEYS`. `DBSIZE` gives - it on demand, but no scrape collects it. -- **No eviction counter.** You cannot currently see whether eviction is running or how hard. -- **No replication metrics.** No lag, no replica connection state, no failover events. -- **No sync-layer metrics.** Live-query counts, outbox depth, and `DEDUP` duplicate rates are not - exported. +| Metric | Type | Meaning | +|---|---|---| +| `recached_memory_bytes` | gauge | Approximate heap used by stored data. Compare against `RECACHED_MAX_MEMORY`. | +| `recached_keys` | gauge | Live keys, excluding expired entries awaiting sweep. Compare against `RECACHED_MAX_KEYS`. | +| `recached_evictions_total` | counter | Keys evicted since start. A rising rate means the cache is working at its cap. | +| `recached_replicas_connected` | gauge | Replicas currently attached to this primary. | +| `recached_live_queries` | gauge | Registered `QSUB` patterns across all connections. | +| `recached_watched_keys` | gauge | Keys under `WATCH`. | +| `recached_dedup_clients_tracked` | gauge | Clients with exactly-once bookkeeping in memory. | + +### What is still not exported -The practical consequence: **Recached tells you about traffic, not about capacity or sync health.** -Until those land, back the traffic metrics with process-level monitoring (RSS, CPU, FD count) and -treat replication and sync as things you verify by probing, not by scraping. +- **Replication lag.** Replica connection count is exported, but not how far behind each one is. +- **Client outbox depth.** That state lives in the browser, not the server. ## Useful queries @@ -83,7 +85,9 @@ Thresholds are starting points — tune to your traffic. | Connection saturation | `recached_connections_active` > 80% of `RECACHED_MAX_CONNECTIONS` | New connections are rejected once the semaphore is exhausted — this fails hard, not gracefully. | | Hit ratio collapse | hit ratio drops sharply vs baseline | Keys expiring faster than expected, an eviction storm, or a cold restart. | | Traffic flatline | `rate(recached_commands_total[5m]) == 0` while clients are up | The process is alive enough to scrape but not serving. | -| Process memory | RSS > 80% of the container limit | There is no built-in memory metric; this is your only capacity signal. | +| Memory pressure | `recached_memory_bytes` > 80% of `RECACHED_MAX_MEMORY` | Eviction is about to start, or already has. | +| Eviction churn | `rate(recached_evictions_total[5m])` climbing | The working set no longer fits; results will start missing. | +| Replica lost | `recached_replicas_connected` drops | Failover risk — the standby is no longer following. | ## Health checking @@ -144,8 +148,12 @@ redis-cli -p 6379 LASTSAVE # timestamp advances when the save lands cp /var/lib/recached/dump.msgpack /backups/dump-$(date +%F).msgpack ``` -To restore, stop the server, put the snapshot at `RECACHED_SAVE_PATH`, and start it — the snapshot -loads at boot. There is **no import path from a Redis RDB file**; the formats are unrelated. +A sidecar file sits next to the snapshot with a `.dedup` extension, holding exactly-once high-water +marks. Back it up with the snapshot: without it a restarted server can re-apply a write a client +replays. Losing it is not fatal — the server starts normally and rebuilds the marks. + +To restore, stop the server, put the snapshot (and its `.dedup` sidecar) at `RECACHED_SAVE_PATH`, and +start it — both load at boot. There is **no import path from a Redis RDB file**; the formats are unrelated. If AOF is enabled, the AOF replays on top of the snapshot. Losing the AOF while keeping the snapshot costs you every write since the last save. diff --git a/server-native/src/main.rs b/server-native/src/main.rs index 3b3d6f3..f9cd89d 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -613,6 +613,8 @@ struct ServerState { /// owner, so the first tab closing does not mark the user offline. Only the /// owning connection's close deletes the key. ephemeral: std::sync::Mutex>, + /// Set when a dedup high-water mark advances; cleared once persisted. + dedup_dirty: std::sync::atomic::AtomicBool, } impl ServerState { @@ -682,11 +684,13 @@ impl ServerState { true } else { *hwm = id; + self.dedup_dirty.store(true, Ordering::Relaxed); false } } None => { map.insert(client.to_string(), (id, now)); + self.dedup_dirty.store(true, Ordering::Relaxed); false } } @@ -715,8 +719,66 @@ impl ServerState { self.replicas.count.store(reg.len(), Ordering::Relaxed); } + /// Path of the dedup sidecar, alongside the snapshot. + fn dedup_path(&self) -> std::path::PathBuf { + self.snap.path.with_extension("dedup") + } + + /// Persist dedup high-water marks so exactly-once delivery survives a + /// restart. Written atomically (temp + rename) and only when a mark has + /// advanced. The map is one `u64` per client, so this stays small enough to + /// flush far more often than the snapshot. + async fn persist_dedup(&self) { + if !self.dedup_dirty.swap(false, Ordering::Relaxed) { + return; + } + let marks: Vec<(String, u64)> = match self.dedup.lock() { + Ok(map) => map.iter().map(|(c, (hwm, _))| (c.clone(), *hwm)).collect(), + Err(_) => return, + }; + let path = self.dedup_path(); + let tmp = path.with_extension("dedup.tmp"); + match rmp_serde::to_vec(&marks) { + Err(e) => warn!("Dedup serialize failed: {}", e), + Ok(bytes) => match tokio::fs::write(&tmp, &bytes).await { + Err(e) => warn!("Dedup write failed: {}", e), + Ok(()) => { + if let Err(e) = tokio::fs::rename(&tmp, &path).await { + warn!("Dedup rename failed: {}", e); + } + } + }, + } + } + + /// Restore dedup marks at boot. `seen` timestamps are not persisted — they + /// only drive idle sweeping, so restored entries start their idle clock now. + async fn load_dedup(&self) { + let path = self.dedup_path(); + let Ok(bytes) = tokio::fs::read(&path).await else { + return; + }; + match rmp_serde::from_slice::>(&bytes) { + Err(e) => warn!("Dedup sidecar unreadable ({}), ignoring: {:?}", e, path), + Ok(marks) => { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + if let Ok(mut map) = self.dedup.lock() { + let count = marks.len(); + for (client, hwm) in marks { + map.insert(client, (hwm, now)); + } + info!("Restored {} dedup high-water mark(s)", count); + } + } + } + } + /// Save snapshot, reset the dirty counter, then truncate AOF (snapshot subsumes the log). async fn save(&self, store: &KeyValueStore) { + self.persist_dedup().await; save_snapshot(store, &self.snap).await; store.reset_dirty(); if let Some(aof) = &self.aof { @@ -2385,8 +2447,30 @@ async fn main() -> Result<(), Box> { is_replica: std::sync::atomic::AtomicBool::new(is_replica_start), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); + // Restore exactly-once bookkeeping before accepting connections, so a + // client replaying an unacknowledged write after a restart is recognised + // rather than applied twice. + state.load_dedup().await; + + // ── Dedup flush ─────────────────────────────────────────────────────── + // The map is one u64 per client, so it can be persisted far more often than + // the snapshot. This bounds the duplicate window on an unclean shutdown to + // roughly this interval rather than to the snapshot cadence. + { + let state_dedup = Arc::clone(&state); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(1)); + ticker.tick().await; + loop { + ticker.tick().await; + state_dedup.persist_dedup().await; + } + }); + } + // ── autosave ────────────────────────────────────────────────────────── if !save_conditions.is_empty() { let store_snap = Arc::clone(&store); @@ -2478,6 +2562,34 @@ async fn main() -> Result<(), Box> { // ── watch registry ──────────────────────────────────────────────────── let watch_registry: WatchRegistry = WatchHub::new(); + // ── Capacity & sync metrics ─────────────────────────────────────────── + // Traffic counters are event-driven, but capacity is a level, not an event: + // memory, key count and eviction rate have to be sampled. Without these an + // operator cannot answer "am I near the cap?" or "is eviction thrashing?" + // from a dashboard — see docs/server/operations.md. + { + let store_m = Arc::clone(&store); + let state_m = Arc::clone(&state); + let registry_m = watch_registry.clone(); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(5)); + loop { + ticker.tick().await; + gauge!("recached_memory_bytes").set(store_m.approximate_memory_bytes() as f64); + gauge!("recached_keys").set(store_m.key_count() as f64); + counter!("recached_evictions_total").absolute(store_m.evicted_count()); + gauge!("recached_replicas_connected") + .set(state_m.replicas.count.load(Ordering::Relaxed) as f64); + gauge!("recached_live_queries") + .set(registry_m.watched_patterns.load(Ordering::Relaxed) as f64); + gauge!("recached_watched_keys") + .set(registry_m.watched_keys.load(Ordering::Relaxed) as f64); + gauge!("recached_dedup_clients_tracked") + .set(state_m.dedup.lock().map(|m| m.len()).unwrap_or(0) as f64); + } + }); + } + // ── connection limiter ──────────────────────────────────────────────── let max_connections = std::env::var("RECACHED_MAX_CONNECTIONS") .ok() @@ -3677,6 +3789,7 @@ mod tests { is_replica: AtomicBool::new(start_as_replica), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -4174,6 +4287,7 @@ mod tests { is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); // Simulate writes captured by AOF @@ -4248,6 +4362,103 @@ mod tests { assert!(!srv.state.is_replica()); } + // ── Exactly-once across a restart ───────────────────────────────────────── + + /// Build a `ServerState` whose snapshot path (and therefore dedup sidecar) + /// is `path` — the same file a restarted process would find. + fn state_with_snapshot_path(path: PathBuf) -> Arc { + Arc::new(ServerState { + snap: Arc::new(SnapshotConfig { + path, + last_save: AtomicI64::new(now_unix_secs()), + }), + aof: None, + replicas: ReplHub::new(), + is_replica: AtomicBool::new(false), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }) + } + + #[tokio::test] + async fn dedup_marks_survive_a_restart() { + let snap = tmp_path("dedup_restart.rdb"); + let _ = std::fs::remove_file(snap.with_extension("dedup")); + + // First run: the client's writes are accepted once each. + let first = state_with_snapshot_path(snap.clone()); + assert!(!first.dedup_seen("client-a", 1)); + assert!(!first.dedup_seen("client-a", 2)); + assert!( + first.dedup_seen("client-a", 2), + "same id twice within a run" + ); + first.persist_dedup().await; + + // Restart: fresh process, same snapshot path. + let second = state_with_snapshot_path(snap.clone()); + assert!( + !second.dedup_seen("client-a", 3), + "a genuinely new id must still be accepted" + ); + + let third = state_with_snapshot_path(snap.clone()); + third.load_dedup().await; + assert!( + third.dedup_seen("client-a", 2), + "a replayed write must be recognised after a restart — this is the \ + caveat the sidecar exists to close" + ); + assert!(!third.dedup_seen("client-a", 99), "higher ids still apply"); + + let _ = std::fs::remove_file(snap.with_extension("dedup")); + } + + #[tokio::test] + async fn persist_dedup_is_a_no_op_when_nothing_advanced() { + // The flusher runs every second; it must not rewrite the file when no + // mark moved. + let snap = tmp_path("dedup_noop.rdb"); + let side = snap.with_extension("dedup"); + let _ = std::fs::remove_file(&side); + + let state = state_with_snapshot_path(snap.clone()); + state.dedup_seen("c", 1); + state.persist_dedup().await; + assert!(side.exists(), "first flush should write"); + + let before = std::fs::metadata(&side).unwrap().modified().unwrap(); + state.persist_dedup().await; // nothing changed since + let after = std::fs::metadata(&side).unwrap().modified().unwrap(); + assert_eq!(before, after, "unchanged marks must not rewrite the file"); + + let _ = std::fs::remove_file(&side); + } + + #[tokio::test] + async fn a_corrupt_dedup_sidecar_is_ignored_not_fatal() { + // Losing exactly-once bookkeeping is bad; refusing to boot is worse. + let snap = tmp_path("dedup_corrupt.rdb"); + let side = snap.with_extension("dedup"); + std::fs::write(&side, b"not messagepack").unwrap(); + + let state = state_with_snapshot_path(snap.clone()); + state.load_dedup().await; // must not panic + assert!(!state.dedup_seen("client-a", 1), "server still functions"); + + let _ = std::fs::remove_file(&side); + } + + #[tokio::test] + async fn a_missing_dedup_sidecar_is_a_clean_first_boot() { + let snap = tmp_path("dedup_absent.rdb"); + let _ = std::fs::remove_file(snap.with_extension("dedup")); + let state = state_with_snapshot_path(snap); + state.load_dedup().await; + assert!(!state.dedup_seen("fresh", 1)); + } + // ── Presence: connection-scoped keys ────────────────────────────────────── #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -4405,6 +4616,7 @@ mod tests { is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -4453,6 +4665,7 @@ mod tests { is_replica: AtomicBool::new(true), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); let rs = Arc::clone(&replica_store); let rst = Arc::clone(&replica_state); @@ -4526,6 +4739,7 @@ mod tests { is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -4644,6 +4858,7 @@ mod tests { is_replica: AtomicBool::new(true), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); let replica_store = Arc::new(KeyValueStore::new()); let rs = Arc::clone(&replica_store); @@ -4691,6 +4906,7 @@ mod tests { is_replica: AtomicBool::new(false), dedup: std::sync::Mutex::new(HashMap::new()), ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/wasm-edge/src/lib.rs b/wasm-edge/src/lib.rs index 2ac39a8..06f58a6 100644 --- a/wasm-edge/src/lib.rs +++ b/wasm-edge/src/lib.rs @@ -69,6 +69,21 @@ export function idbOutboxDelete(db, id) { req.onerror = (e) => reject(e.target.error); }); } +export function idbWalReplace(db, cmds) { + return new Promise((resolve, reject) => { + // One transaction: the clear and the rewrite commit together or not at + // all. Clearing in a separate transaction (as this used to) leaves the + // WAL empty if the tab closes before the snapshot is written, losing the + // entire persisted cache. + const tx = db.transaction('wal', 'readwrite'); + const store = tx.objectStore('wal'); + store.clear(); + for (let i = 0; i < cmds.length; i++) store.put(cmds[i], i); + tx.oncomplete = () => resolve(undefined); + tx.onerror = (e) => reject(e.target.error); + tx.onabort = (e) => reject(e.target.error); + }); +} export function idbWalClear(db) { return new Promise((resolve, reject) => { const tx = db.transaction('wal', 'readwrite'); @@ -118,6 +133,8 @@ extern "C" { fn idb_outbox_delete_js(db: &JsValue, id: f64) -> Promise; #[wasm_bindgen(js_name = "idbWalClear")] fn idb_wal_clear_js(db: &JsValue) -> Promise; + #[wasm_bindgen(js_name = "idbWalReplace")] + fn idb_wal_replace_js(db: &JsValue, cmds: &JsValue) -> Promise; #[wasm_bindgen(js_name = "idbMetaGet")] fn idb_meta_get(db: &JsValue, key: &str) -> Promise; #[wasm_bindgen(js_name = "idbMetaPut")] @@ -683,14 +700,17 @@ impl RecachedCache { let next_seq = if entry_count > WAL_COMPACT_THRESHOLD { // WAL only — the outbox holds writes still awaiting server // acknowledgment and must survive compaction. - JsFuture::from(idb_wal_clear_js(&db)).await?; + // Atomic: the old log is only dropped if the replacement + // commits with it. Previously this cleared first and wrote + // after, so an interruption in between left an empty WAL and + // the persisted cache was gone. let cmds = snapshot_to_resp_cmds(&store.snapshot()); - let mut seq: u64 = 0; + let arr = js_sys::Array::new(); for cmd_str in &cmds { - let _ = JsFuture::from(idb_append_js(&db, seq as f64, cmd_str)).await; - seq += 1; + arr.push(&JsValue::from_str(cmd_str)); } - seq + JsFuture::from(idb_wal_replace_js(&db, &arr)).await?; + cmds.len() as u64 } else if entry_count == 0 { 0 } else { @@ -1568,6 +1588,102 @@ mod browser_tests { .expect("deleting a missing row should resolve"); } + // ── Atomic WAL compaction ───────────────────────────────────────────────── + + #[wasm_bindgen_test] + async fn wal_replace_swaps_contents_in_one_transaction() { + let db = fresh_db().await; + for (seq, cmd) in [(0.0, "SET a 1"), (1.0, "SET b 2"), (2.0, "SET c 3")] { + JsFuture::from(idb_append_js(&db, seq, cmd)).await.unwrap(); + } + + let arr = js_sys::Array::new(); + arr.push(&JsValue::from_str("SET compacted 1")); + JsFuture::from(idb_wal_replace_js(&db, &arr)) + .await + .expect("replace"); + + let (keys, vals) = wal_rows(&db).await; + assert_eq!(keys, vec![0.0], "old entries must be gone"); + assert_eq!(vals, vec!["SET compacted 1".to_string()]); + } + + #[wasm_bindgen_test] + async fn wal_replace_with_an_empty_snapshot_empties_the_log() { + // An empty store compacts to zero commands; the WAL should end empty + // rather than retaining stale entries. + let db = fresh_db().await; + JsFuture::from(idb_append_js(&db, 0.0, "SET a 1")) + .await + .unwrap(); + + let empty = js_sys::Array::new(); + JsFuture::from(idb_wal_replace_js(&db, &empty)) + .await + .expect("replace"); + + assert!(wal_rows(&db).await.0.is_empty()); + } + + #[wasm_bindgen_test] + async fn wal_replace_leaves_the_outbox_untouched() { + // Compaction must never drop writes still awaiting acknowledgement. + let db = fresh_db().await; + JsFuture::from(idb_append_js(&db, 0.0, "SET a 1")) + .await + .unwrap(); + JsFuture::from(idb_outbox_put_js(&db, 5.0, "PENDING")) + .await + .unwrap(); + + let arr = js_sys::Array::new(); + arr.push(&JsValue::from_str("SET compacted 1")); + JsFuture::from(idb_wal_replace_js(&db, &arr)).await.unwrap(); + + let (ids, vals) = outbox_rows(&db).await; + assert_eq!(ids, vec![5.0]); + assert_eq!(vals[0], "PENDING"); + } + + #[wasm_bindgen_test] + async fn compacted_wal_replays_to_the_same_state() { + // End to end: snapshot a store, compact through the atomic path, read + // back, and rebuild — the persisted cache must be intact. + use core_engine::cmd::Command; + use core_engine::store::KeyValueStore; + + let db = fresh_db().await; + let source = KeyValueStore::new(); + source.execute(Command::Set( + "k".into(), + "v".into(), + core_engine::cmd::SetOptions::default(), + )); + source.execute(Command::RPush("l".into(), vec!["a".into(), "b".into()])); + + let cmds = snapshot_to_resp_cmds(&source.snapshot()); + let arr = js_sys::Array::new(); + for c in &cmds { + arr.push(&JsValue::from_str(c)); + } + JsFuture::from(idb_wal_replace_js(&db, &arr)).await.unwrap(); + + let (_, vals) = wal_rows(&db).await; + let restored = KeyValueStore::new(); + for frame in &vals { + let (value, _) = core_engine::resp::Value::parse(frame.as_bytes()).unwrap(); + restored.execute(Command::from_value(value).unwrap()); + } + assert_eq!( + restored.execute(Command::Get("k".into())), + core_engine::resp::Value::BulkString(Some(b"v".to_vec())) + ); + assert_eq!( + restored.execute(Command::LLen("l".into())), + core_engine::resp::Value::Integer(2) + ); + } + // ── Meta (client identity + epoch) ──────────────────────────────────────── #[wasm_bindgen_test] From cf08f1320a9e6df80108a57eccf085743ba23c2a Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 17:09:15 +0800 Subject: [PATCH 03/11] feat: surface outbox overflow, propagate FLUSHDB to live queries --- CHANGELOG.md | 16 ++++ docs/roadmap.md | 28 +++---- docs/server/commands.md | 5 +- docs/server/troubleshooting.md | 7 +- server-native/src/main.rs | 143 +++++++++++++++++++++++++++++++-- sync-client/src/lib.rs | 19 ++++- sync-client/src/tests.rs | 48 +++++++++++ wasm-edge/sdk.ts | 42 ++++++++++ wasm-edge/src/lib.rs | 30 +++++++ 9 files changed, 315 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6b503c..38196fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ All notable changes to Recached are documented here. offline. Replicas receive the write as a plain `SET` — they have no connection to scope a lifetime to, and the owning server broadcasts the deletion. +- **`onOutboxFull()` and `pendingWrites()` on the browser SDK.** The offline queue holds 10 000 + writes and evicts the oldest past that — previously in silence, with no error and no signal, so an + application could not tell that a user's write had been discarded. `onOutboxFull(cb)` reports the + dropped row id and the remaining depth; `pendingWrites()` exposes the depth for a "syncing…" + indicator or to apply back-pressure before the cap is reached. + - **Capacity and sync metrics.** Seven new series, sampled every 5 seconds because capacity is a level rather than an event: `recached_memory_bytes`, `recached_keys`, `recached_evictions_total`, `recached_replicas_connected`, `recached_live_queries`, `recached_watched_keys`, and @@ -52,6 +58,16 @@ All notable changes to Recached are documented here. versions. ::: +- **`FLUSHDB` now reaches live queries.** `primary_keys()` is empty for `FLUSHDB`, so the generic + notifier had nothing to announce and subscribers silently kept serving data the server had already + wiped. + + Announcing per deleted key would mean one frame per key in the keyspace for a single command, so + the server emits **one sentinel per registered pattern** instead — a `keychange` whose key is the + pattern and whose value is nil. Clients expand it locally to "every key matching this pattern is + gone", which is O(patterns) rather than O(keys). Explicitly `WATCH`ed keys are still notified + individually, since that set is bounded and callers expect per-key precision there. + - **Reconnect backoff is jittered.** Delays now land in `[nominal/2, nominal]` instead of an exact `500ms × 2^attempts`. Without jitter every client disconnected by the same event computes an identical schedule and reconnects in lockstep — a thundering herd that can keep a recovering server diff --git a/docs/roadmap.md b/docs/roadmap.md index 356d746..eb4c1de 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,14 +4,20 @@ Recached competes on **where the data can live** — the same engine on the serv ## Near-term -The next three things worth doing, in order: - -1. **[Byte-slice command arguments](#performance)** — the main remaining lever on unpipelined - latency, and the likeliest explanation for `HSET` trailing Redis. -2. **[Surface outbox overflow](#reliability)** — the client silently drops the oldest write past - 10 000; applications get no signal. -3. **[Live queries carry `FLUSHDB` diffs](#live-queries)** — subscribers currently miss a mass - deletion entirely. +**[Byte-slice command arguments](#performance)** is the one substantial piece left in the hardening +list, and the main remaining lever on unpipelined latency. It is a large mechanical refactor — 125 +parse arms, ~210 argument extractions — and its payoff needs a quiet machine to measure, so it wants +its own focused pass rather than being folded into a release alongside other work. + +After that, in order: + +1. **[Replication lag](#operability)** — replica *count* is exported, but not how far behind each one + is, which is the signal that matters before a failover. +2. **[Bound rate-limiter memory](#operability)** — one timestamp per attempt means ~800 KB for a + single `RLSET key 100000 3600` limiter. +3. **[Make the compiled-in limits configurable](#operability)** — outbox size, live queries per + connection, eviction sample size. + --- ## 6. Mobile SDKs — React Native, Flutter, Kotlin, Swift @@ -75,14 +81,8 @@ independent — pick them off in any order. ### Reliability -**Surface outbox overflow.** The client outbox holds 10 000 writes and silently evicts the oldest -past that. An `onOutboxFull` callback, or a `pendingWrites()` accessor, lets an application degrade -deliberately instead of losing writes without a signal. - ### Live queries -**`FLUSHDB` should emit per-key diffs.** Subscribers currently miss a mass deletion entirely. - ### Performance Both of these are diagnosed on the [benchmarks](/guide/benchmarks) page; the analysis is done, the diff --git a/docs/server/commands.md b/docs/server/commands.md index d32ec02..6fc6d0b 100644 --- a/docs/server/commands.md +++ b/docs/server/commands.md @@ -301,7 +301,10 @@ two clients receiving the same notification build identical local state. Each notification carries the **complete** current value, so the receiver replaces the key rather than merging — which is what allows a removed member to propagate. -Remaining limitation: `FLUSHDB` does not emit per-key diffs to live queries. +`FLUSHDB` is announced as a single sentinel per subscribed pattern — a `keychange` whose key is the +pattern and whose value is nil — meaning "every key matching this pattern is gone". Announcing each +deleted key would mean one frame per key in the keyspace for one command. The browser SDK expands the +sentinel locally; a hand-written client should do the same. ::: warning Changed in 0.2.2 Before 0.2.2 collections arrived as a bare type name (`"hash"`), and subscribers had to follow up with diff --git a/docs/server/troubleshooting.md b/docs/server/troubleshooting.md index 027e1df..4ce230b 100644 --- a/docs/server/troubleshooting.md +++ b/docs/server/troubleshooting.md @@ -122,6 +122,10 @@ connections — by design, since they would leak or destroy data outside the con The client outbox holds **10,000 pending writes**. Past that, each new write **evicts the oldest one** — silently, with no error surfaced to your code. +Register `cache.onOutboxFull((droppedId, pending) => …)` to be told when this happens — without it +the loss is invisible. `cache.pendingWrites()` reports the current depth, so an application can apply +back-pressure before the cap is reached. + If a client can be offline long enough to exceed 10,000 writes, do not rely on the outbox as the system of record for those mutations. Batch them, or persist them yourself and reconcile on reconnect. @@ -151,7 +155,8 @@ See [Offline & Reconnection](/browser/offline). A live query's initial state is capped at **10,000 keys**. Beyond that the snapshot is truncated. Narrow the pattern. -Also note a remaining limit: `FLUSHDB` does not emit per-key diffs. +`FLUSHDB` arrives as one sentinel per subscribed pattern rather than one frame per key — if your +client is hand-written, expand it locally. If collection values arrive as a bare type name (`"hash"`) rather than their contents, the server and SDK are on different versions — collection values ship complete from 0.2.2 onward, and the two are diff --git a/server-native/src/main.rs b/server-native/src/main.rs index f9cd89d..cb6eb88 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -1681,6 +1681,42 @@ async fn notify_watchers(registry: &WatchRegistry, cmd: &Command, store: &KeyVal } } +/// Announce a `FLUSHDB` to live queries. +/// +/// Emitting a keychange per deleted key would mean one frame per key in the +/// keyspace — potentially millions — for a single command. Instead each +/// registered pattern receives one sentinel, delivered as a keychange whose key +/// is the pattern and whose value is nil. Subscribers treat it as "every key +/// matching this pattern is gone", which is exactly what happened, at O(patterns) +/// instead of O(keys). +/// +/// Explicitly `WATCH`ed keys are notified individually — that set is bounded by +/// the connection limit and callers expect per-key precision there. +async fn notify_flushdb(registry: &WatchRegistry, watched_before: Vec) { + if registry.watched_keys.load(Ordering::Relaxed) > 0 && !watched_before.is_empty() { + let mut reg = registry.map.lock().await; + for key in &watched_before { + if let Some(subs) = reg.get_mut(key) { + subs.retain(|(_, tx)| tx.send((key.clone(), Value::BulkString(None))).is_ok()); + } + } + registry.sync_len(®); + } + if registry.watched_patterns.load(Ordering::Relaxed) > 0 { + let mut pats = registry.patterns.lock().await; + let mut emptied = false; + for (pattern, subs) in pats.iter_mut() { + let sentinel = pattern.clone(); + subs.retain(|(_, tx)| tx.send((sentinel.clone(), Value::BulkString(None))).is_ok()); + emptied |= subs.is_empty(); + } + if emptied { + pats.retain(|_, subs| !subs.is_empty()); + } + registry.sync_patterns_len(&pats); + } +} + /// Drop all of `conn_id`'s live-query subscriptions. Called on QUNSUB (all /// form) and on connection close. async fn unregister_all_qsubs( @@ -1729,7 +1765,17 @@ async fn apply_write_effects( state.on_write(&msg).await; } if has_watch { - notify_watchers(watch_registry, cmd, store).await; + if matches!(cmd, Command::FlushDb) { + // primary_keys() is empty for FLUSHDB, so the generic notifier has + // nothing to announce — subscribers would silently miss the wipe. + let watched: Vec = { + let reg = watch_registry.map.lock().await; + reg.keys().cloned().collect() + }; + notify_flushdb(watch_registry, watched).await; + } else { + notify_watchers(watch_registry, cmd, store).await; + } } if has_ws { let _ = tx.send(Arc::new(SyncPush { @@ -4362,6 +4408,70 @@ mod tests { assert!(!srv.state.is_replica()); } + // ── FLUSHDB reaches live queries ────────────────────────────────────────── + + /// Collect every frame arriving within `ms`, so a test can assert on the + /// keychange among the command-replay pushes that travel alongside it. + async fn drain_frames(c: &mut WsClient, ms: u64) -> Vec { + let mut out = Vec::new(); + while let Some(f) = c.recv_any(ms).await { + out.push(f); + } + out + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn flushdb_notifies_live_query_subscribers() { + // Previously FLUSHDB emitted nothing to live queries: primary_keys() is + // empty for it, so subscribers kept serving data the server had wiped. + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "cart:*"]).await; + + let mut writer = WsClient::connect(srv.tcp_addr).await; + writer.cmd(&["SET", "cart:item:1", "a"]).await; + drain_frames(&mut watcher, 400).await; + + writer.cmd(&["FLUSHDB"]).await; + + let frames = drain_frames(&mut watcher, 800).await; + assert!( + frames + .iter() + .any(|f| f.contains("keychange") && f.contains("cart:*")), + "expected a keychange sentinel naming the pattern, got: {frames:?}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn flushdb_sends_one_sentinel_per_pattern_not_per_key() { + // The reason for a sentinel: announcing per key would be one frame per + // key in the keyspace for a single command. + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "bulk:*"]).await; + + let mut writer = WsClient::connect(srv.tcp_addr).await; + for i in 0..25 { + writer.cmd(&["SET", &format!("bulk:{i}"), "v"]).await; + } + drain_frames(&mut watcher, 500).await; + + writer.cmd(&["FLUSHDB"]).await; + + let keychanges: Vec = drain_frames(&mut watcher, 800) + .await + .into_iter() + .filter(|f| f.contains("keychange")) + .collect(); + assert_eq!( + keychanges.len(), + 1, + "25 keys wiped must produce one sentinel, not 25 frames: {keychanges:?}" + ); + assert!(keychanges[0].contains("bulk:*")); + } + // ── Exactly-once across a restart ───────────────────────────────────────── /// Build a `ServerState` whose snapshot path (and therefore dedup sidecar) @@ -4506,11 +4616,13 @@ mod tests { let _ = watcher.recv_push(1000).await; } - let push = watcher - .recv_push(3000) - .await - .expect("expected a keychange for the departing peer"); - assert!(push.contains("presence:user:7"), "unexpected push: {push}"); + let frames = drain_frames(&mut watcher, 1500).await; + assert!( + frames + .iter() + .any(|f| f.contains("keychange") && f.contains("presence:user:7")), + "a live query must receive a keychange for the departing peer, got: {frames:?}" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -4964,6 +5076,25 @@ mod tests { self.next_reply().await } + /// Wait up to `ms` for the next frame of any kind — RESP3 Push + /// broadcasts *and* plain arrays. `keychange` notifications are encoded + /// as arrays, so `recv_push` skips them entirely. + async fn recv_any(&mut self, ms: u64) -> Option { + let fut = async { + loop { + match self.ws.next().await { + Some(Ok(Message::Text(t))) => return Some(t.to_string()), + Some(Ok(_)) => continue, + _ => return None, + } + } + }; + tokio::time::timeout(tokio::time::Duration::from_millis(ms), fut) + .await + .ok() + .flatten() + } + /// Wait up to `ms` for the next RESP3 Push broadcast frame, returning /// its raw text. `None` when nothing arrives in time. async fn recv_push(&mut self, ms: u64) -> Option { diff --git a/sync-client/src/lib.rs b/sync-client/src/lib.rs index 61d48e7..a68648a 100644 --- a/sync-client/src/lib.rs +++ b/sync-client/src/lib.rs @@ -440,7 +440,24 @@ impl SyncClient { )); } Value::BulkString(None) => { - self.store.execute(Command::Del(vec![key])); + // A nil value whose "key" is one of our registered live-query + // patterns is the FLUSHDB sentinel: every matching key is gone. + // Sending one frame per deleted key would mean millions of + // frames for a single command, so the server announces it once + // per pattern and the client expands it locally. + if self.live_queries.contains(&key) { + let matched: Vec = self + .store + .matching_key_values(&key, usize::MAX) + .into_iter() + .map(|(k, _)| k) + .collect(); + if !matched.is_empty() { + self.store.execute(Command::Del(matched)); + } + } else { + self.store.execute(Command::Del(vec![key])); + } } // Type-tagged collection: ["hash"|"list"|"set"|"zset"|"json", ...]. Value::Array(Some(items)) => self.apply_tagged(key, items), diff --git a/sync-client/src/tests.rs b/sync-client/src/tests.rs index 55c086a..15a9345 100644 --- a/sync-client/src/tests.rs +++ b/sync-client/src/tests.rs @@ -695,3 +695,51 @@ fn an_unknown_type_tag_is_ignored_rather_than_guessed() { ])); assert_eq!(get(&c, "k"), Value::BulkString(None)); } + +// ── FLUSHDB propagation ─────────────────────────────────────────────────────── + +#[test] +fn flushdb_sentinel_clears_every_key_matching_the_pattern() { + // The server announces a flush once per registered pattern rather than once + // per deleted key — a keyspace-sized frame storm for a single command. + let mut c = client(); + c.add_live_query("cart:*", false); + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$11\r\ncart:item:1\r\n$1\r\na\r\n"); + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$11\r\ncart:item:2\r\n$1\r\nb\r\n"); + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$7\r\nother:1\r\n$1\r\nc\r\n"); + assert_eq!(get(&c, "cart:item:1"), bulk("a")); + + // Sentinel: nil value whose "key" is the registered pattern. + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$6\r\ncart:*\r\n$-1\r\n"); + + assert_eq!(get(&c, "cart:item:1"), Value::BulkString(None)); + assert_eq!(get(&c, "cart:item:2"), Value::BulkString(None)); + assert_eq!( + get(&c, "other:1"), + bulk("c"), + "keys outside the pattern must be untouched" + ); +} + +#[test] +fn a_nil_for_an_unregistered_pattern_deletes_only_that_key() { + // Without this distinction, a literal key that happens to contain a glob + // character would wipe unrelated data. + let mut c = client(); + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$5\r\nkey:1\r\n$1\r\na\r\n"); + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$5\r\nkey:2\r\n$1\r\nb\r\n"); + + // Not a registered live query — treat it as an ordinary single-key delete. + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$5\r\nkey:1\r\n$-1\r\n"); + + assert_eq!(get(&c, "key:1"), Value::BulkString(None)); + assert_eq!(get(&c, "key:2"), bulk("b"), "unrelated key survives"); +} + +#[test] +fn flushdb_sentinel_is_harmless_when_nothing_matches() { + let mut c = client(); + c.add_live_query("cart:*", false); + c.handle_frame("*3\r\n$9\r\nkeychange\r\n$6\r\ncart:*\r\n$-1\r\n"); + assert_eq!(c.store().execute(Command::DbSize), Value::Integer(0)); +} diff --git a/wasm-edge/sdk.ts b/wasm-edge/sdk.ts index ffa5419..881f303 100644 --- a/wasm-edge/sdk.ts +++ b/wasm-edge/sdk.ts @@ -87,6 +87,8 @@ interface RawCache { live_unquery(pattern?: string): void; get_matching(pattern: string): Array<[string, string | null]>; set_mutation_callback(cb: () => void): void; + set_outbox_full_callback(cb: (droppedId: number, pending: number) => void): void; + pending_writes(): number; set_message_callback(cb: (channel: string, message: string) => void): void; free(): void; } @@ -130,12 +132,18 @@ export class Cache { private readonly _mutationListeners = new Set<() => void>(); private readonly _messageListeners = new Map void>>(); + private readonly _outboxFullListeners = new Set<(droppedId: number, pending: number) => void>(); /** @internal Arrow function so `this` is always bound when passed as a callback. */ private readonly _notifyMutation = (): void => { for (const cb of this._mutationListeners) cb(); }; + /** @internal */ + private readonly _notifyOutboxFull = (droppedId: number, pending: number): void => { + for (const cb of this._outboxFullListeners) cb(droppedId, pending); + }; + /** @internal */ private readonly _notifyMessage = (channel: string, message: string): void => { const listeners = this._messageListeners.get(channel); @@ -149,6 +157,7 @@ export class Cache { this.raw = raw; raw.set_mutation_callback(this._notifyMutation); raw.set_message_callback(this._notifyMessage); + raw.set_outbox_full_callback(this._notifyOutboxFull); } /** @@ -171,6 +180,39 @@ export class Cache { return () => this._mutationListeners.delete(cb); } + /** + * Called when the offline write queue overflows and the **oldest** queued + * write is discarded. + * + * The queue holds 10,000 writes. Past that, each new write evicts the oldest + * one — without this callback that is silent data loss: no error is thrown + * and the write is simply gone. If a client can plausibly be offline long + * enough to hit the cap, do not treat the outbox as the system of record; + * persist the mutation yourself and reconcile on reconnect. + * + * Returns an unsubscribe function. + * + * ```ts + * cache.onOutboxFull((droppedId, pending) => { + * console.error(`dropped queued write ${droppedId}; ${pending} still pending`); + * }); + * ``` + */ + onOutboxFull(cb: (droppedId: number, pending: number) => void): () => void { + this._outboxFullListeners.add(cb); + return () => this._outboxFullListeners.delete(cb); + } + + /** + * Writes queued locally and not yet acknowledged by the server. + * + * Useful for a "syncing…" indicator, or to apply back-pressure before the + * queue reaches its 10,000-write cap. + */ + pendingWrites(): number { + return this.raw.pending_writes(); + } + /** * Subscribe to pub/sub messages on `channel`. * diff --git a/wasm-edge/src/lib.rs b/wasm-edge/src/lib.rs index 06f58a6..f9ab998 100644 --- a/wasm-edge/src/lib.rs +++ b/wasm-edge/src/lib.rs @@ -311,6 +311,8 @@ struct WsShared { core: Rc>, on_mutation: Rc>>, on_message: Rc>>, + /// Invoked when the offline outbox overflows and a queued write is dropped. + on_outbox_full: Rc>>, ws: Rc>>, handlers: Rc>, /// Shared with `RecachedCache.idb` — the open IndexedDB handle, if any. @@ -395,6 +397,15 @@ fn queue_write(sh: &WsShared, encoded: &str, dedup: bool) { web_sys::console::warn_1(&JsValue::from_str( "recached: offline write queue full — dropped the oldest queued write", )); + // A silent drop is data loss the application cannot detect. Give it the + // dropped frame so it can persist or reconcile the write itself. + if let Some(cb) = sh.on_outbox_full.borrow().as_ref() { + let _ = cb.call2( + &JsValue::NULL, + &JsValue::from_f64(old as f64), + &JsValue::from_f64(sh.core.borrow().outbox_len() as f64), + ); + } } outbox_put(sh, e.id, &e.frame); if e.send_now @@ -535,6 +546,7 @@ impl RecachedCache { pub fn new() -> RecachedCache { let store = Arc::new(KeyValueStore::new()); let on_mutation = Rc::new(RefCell::new(None)); + let on_outbox_full: Rc>> = Rc::new(RefCell::new(None)); let on_message = Rc::new(RefCell::new(None)); let idb = Rc::new(RefCell::new(None)); let core = Rc::new(RefCell::new(SyncClient::new( @@ -546,6 +558,7 @@ impl RecachedCache { core, on_mutation: Rc::clone(&on_mutation), on_message: Rc::clone(&on_message), + on_outbox_full: Rc::clone(&on_outbox_full), ws: Rc::new(RefCell::new(None)), handlers: Rc::new(RefCell::new(WsHandlers::default())), idb: Rc::clone(&idb), @@ -584,6 +597,23 @@ impl RecachedCache { *self.on_mutation.borrow_mut() = Some(cb); } + /// Register a JS callback invoked when the offline outbox overflows and the + /// oldest queued write is dropped. Signature: + /// `cb(droppedId: number, pendingWrites: number)`. + /// + /// Without this an overflow is invisible to the application: the write is + /// discarded, no error is raised, and the data is simply gone. The SDK's + /// `onOutboxFull()` wires this up automatically. + pub fn set_outbox_full_callback(&mut self, cb: js_sys::Function) { + *self.shared.on_outbox_full.borrow_mut() = Some(cb); + } + + /// Writes queued locally and not yet acknowledged by the server. Poll this + /// to show sync state, or to back off before the queue overflows. + pub fn pending_writes(&self) -> usize { + self.shared.core.borrow().outbox_len() + } + /// Register a JS callback invoked when a pub/sub message arrives. /// Signature: `cb(channel: string, message: string)`. /// The SDK's `onMessage()` wires this up automatically. From 59e15fdf2f932a876d4719454670448d140d442d Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 18:13:14 +0800 Subject: [PATCH 04/11] perf: move command arguments instead of copying; bound rate-limiter memory; make limits configurable --- CHANGELOG.md | 41 +++++- core-engine/src/cmd.rs | 265 ++++++++++++++++++++++++++++------- core-engine/src/store.rs | 240 ++++++++++++++++++++++++++----- docs/roadmap.md | 43 ++---- docs/server/configuration.md | 5 + docs/server/operations.md | 20 +-- server-native/src/main.rs | 140 ++++++++++++++++-- sync-client/src/lib.rs | 18 ++- sync-client/src/tests.rs | 33 +++++ 9 files changed, 657 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38196fb..6451212 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,12 +28,51 @@ All notable changes to Recached are documented here. - **Capacity and sync metrics.** Seven new series, sampled every 5 seconds because capacity is a level rather than an event: `recached_memory_bytes`, `recached_keys`, `recached_evictions_total`, `recached_replicas_connected`, `recached_live_queries`, `recached_watched_keys`, and - `recached_dedup_clients_tracked`. Previously only traffic was exported, so an operator could not + `recached_dedup_clients_tracked`, and `recached_replication_queue_depth`. Previously only traffic + was exported, so an operator could not answer "am I near the cap?" or "is eviction thrashing?" from a dashboard. Replication *lag* and browser outbox depth remain unexported — see [Operations](docs/server/operations.md). ### Changed +- **Command arguments are moved rather than copied.** `extract_string` built every argument with + `from_utf8_lossy(..).into_owned()`, which allocates a fresh `String` and memcpys the payload even + when the bytes are already valid UTF-8 — which they nearly always are. Parsing now moves the byte + buffer the RESP parser already allocated, so a 1 MB `SET` value costs no copy at all. + + Applied to the commands that actually carry payloads: `SET`, `MSET`, `HSET`/`HMSET`, `APPEND`, + `GETSET`, `SETNX`, `JSET`, `JMERGE`, and all 22 bulk argument lists (`RPUSH`, `SADD`, `ZADD`, …). + The long tail of small-argument commands still copies; the remaining win there is negligible and + the risk of touching 125 parse arms is not. + + Argument slots are consumed, so an arm must read each index once — new tests cover the shapes + where that matters (`SET`'s option scan after moving key and value, `MSET`/`HSET` pair splitting, + a 1 MB round-trip, and invalid UTF-8 falling back to a lossy re-encode). + + **Not benchmarked.** The gain is a removed allocation and memcpy per argument, which is + structural, but the machine available could not produce trustworthy figures — see the + [benchmarks](docs/guide/benchmarks) caveat. Re-measure on a quiet host before quoting numbers. + +- **Rate-limiter memory is bounded.** A limiter stored one timestamp per attempt, so + `RLSET key 100000 3600` held 100 000 `u64`s — roughly 800 KB for a single key, and token-cost + limiting (roadmap #9) makes six-figure limits ordinary. Attempts are now counted into 64 buckets, + capping a limiter at about 1 KB whatever the limit. + + The trade-off is granularity: the window advances one bucket at a time, so a limiter is exact to + within `window / 64`. Attempts are never under-counted — a bucket leaves the window only once it + is entirely outside it — so the limiter errs toward rejecting slightly early rather than admitting + over the limit. `retry_after_ms` is clamped to the window. + + Limiter *attempt* state is no longer persisted in snapshots (configuration still is). It ages out + within a single window and a restart has already interrupted that window, so a restored limiter + enforces the same policy from a clean slate rather than a stale partial count. + +- **Per-connection limits are configurable** rather than compiled in, because the right value is + workload-dependent: `RECACHED_MAX_MULTI_QUEUE`, `RECACHED_MAX_WATCHES_PER_CONN`, + `RECACHED_MAX_LIVE_QUERIES`, `RECACHED_MAX_QSUB_INITIAL_KEYS`, and `RECACHED_EVICTION_SAMPLE` (the + knob Redis exposes as `maxmemory-samples`). Defaults are unchanged. The browser outbox cap is now + settable through `sync-client` rather than fixed at 10 000. + - **Live queries now carry collection values.** `qstate` and `keychange` previously delivered only a *type name* for hashes, lists, sets, sorted sets and JSON, so every subscriber had to follow up with `HGETALL`/`LRANGE`/`JGET` — a network round-trip in a system whose premise is that reads are local. diff --git a/core-engine/src/cmd.rs b/core-engine/src/cmd.rs index 621ee2a..85d0d52 100644 --- a/core-engine/src/cmd.rs +++ b/core-engine/src/cmd.rs @@ -198,7 +198,7 @@ pub enum Command { impl Command { pub fn from_value(value: Value) -> Result { match value { - Value::Array(Some(arr)) => { + Value::Array(Some(mut arr)) => { if arr.is_empty() { return Err("Empty command".to_string()); } @@ -237,8 +237,8 @@ impl Command { // ── Strings ─────────────────────────────────────────────── "SET" => { need!(3); - let key = extract_key(&arr[1])?; - let val = extract_string(&arr[2]).unwrap_or_default(); + let key = take_key(&mut arr[1])?; + let val = take_string(&mut arr[2]).unwrap_or_default(); let mut opts = SetOptions::default(); let mut i = 3usize; while i < arr.len() { @@ -329,10 +329,9 @@ impl Command { } "APPEND" => { need!(3); - Ok(Command::Append( - extract_key(&arr[1])?, - extract_string(&arr[2]).unwrap_or_default(), - )) + let key = take_key(&mut arr[1])?; + let val = take_string(&mut arr[2]).unwrap_or_default(); + Ok(Command::Append(key, val)) } "STRLEN" => { need!(2); @@ -340,10 +339,9 @@ impl Command { } "GETSET" => { need!(3); - Ok(Command::GetSet( - extract_key(&arr[1])?, - extract_string(&arr[2]).unwrap_or_default(), - )) + let key = take_key(&mut arr[1])?; + let val = take_string(&mut arr[2]).unwrap_or_default(); + Ok(Command::GetSet(key, val)) } "MGET" => { need!(2); @@ -356,11 +354,12 @@ impl Command { ); } let pairs = arr[1..] - .chunks(2) + .chunks_mut(2) .map(|c| { + let (k, v) = c.split_at_mut(1); Ok(( - extract_key(&c[0])?, - extract_string(&c[1]).unwrap_or_default(), + take_key(&mut k[0])?, + take_string(&mut v[0]).unwrap_or_default(), )) }) .collect::, String>>()?; @@ -368,10 +367,9 @@ impl Command { } "SETNX" => { need!(3); - Ok(Command::SetNx( - extract_key(&arr[1])?, - extract_string(&arr[2]).unwrap_or_default(), - )) + let key = take_key(&mut arr[1])?; + let val = take_string(&mut arr[2]).unwrap_or_default(); + Ok(Command::SetNx(key, val)) } "SETEX" => { need!(4); @@ -586,11 +584,10 @@ impl Command { // ── JSON ─────────────────────────────────────────────────── "JSET" => { need!(4); - Ok(Command::JSet( - extract_key(&arr[1])?, - extract_string(&arr[2]).unwrap_or_default(), - extract_string(&arr[3]).unwrap_or_default(), - )) + let key = take_key(&mut arr[1])?; + let path = take_string(&mut arr[2]).unwrap_or_default(); + let doc = take_string(&mut arr[3]).unwrap_or_default(); + Ok(Command::JSet(key, path, doc)) } "JGET" => { need!(2); @@ -603,10 +600,9 @@ impl Command { } "JMERGE" => { need!(3); - Ok(Command::JMerge( - extract_key(&arr[1])?, - extract_string(&arr[2]).unwrap_or_default(), - )) + let key = take_key(&mut arr[1])?; + let patch = take_string(&mut arr[2]).unwrap_or_default(); + Ok(Command::JMerge(key, patch)) } // ── Rate limiting ────────────────────────────────────────── @@ -640,11 +636,12 @@ impl Command { } let key = extract_key(&arr[1])?; let pairs = arr[2..] - .chunks(2) + .chunks_mut(2) .map(|c| { + let (f, v) = c.split_at_mut(1); ( - extract_string(&c[0]).unwrap_or_default(), - extract_string(&c[1]).unwrap_or_default(), + take_string(&mut f[0]).unwrap_or_default(), + take_string(&mut v[0]).unwrap_or_default(), ) }) .collect(); @@ -666,7 +663,7 @@ impl Command { "HDEL" => { need!(3); let key = extract_key(&arr[1])?; - let fields = arr[2..].iter().filter_map(extract_string).collect(); + let fields = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::HDel(key, fields)) } "HKEYS" => { @@ -716,7 +713,7 @@ impl Command { "HMGET" => { need!(3); let key = extract_key(&arr[1])?; - let fields = arr[2..].iter().filter_map(extract_string).collect(); + let fields = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::HMGet(key, fields)) } @@ -724,25 +721,25 @@ impl Command { "LPUSH" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter().filter_map(extract_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::LPush(key, vals)) } "RPUSH" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter().filter_map(extract_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::RPush(key, vals)) } "LPUSHX" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter().filter_map(extract_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::LPushX(key, vals)) } "RPUSHX" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter().filter_map(extract_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::RPushX(key, vals)) } "LPOP" => { @@ -812,8 +809,8 @@ impl Command { // ── Set ──────────────────────────────────────────────────── "SADD" => { need!(3); - let key = extract_key(&arr[1])?; - let members = arr[2..].iter().filter_map(extract_string).collect(); + let key = take_key(&mut arr[1])?; + let members = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::SAdd(key, members)) } "SMEMBERS" => { @@ -825,7 +822,7 @@ impl Command { "SREM" => { need!(3); let key = extract_key(&arr[1])?; - let members = arr[2..].iter().filter_map(extract_string).collect(); + let members = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::SRem(key, members)) } "SCARD" => { @@ -842,43 +839,43 @@ impl Command { "SMISMEMBER" => { need!(3); let key = extract_key(&arr[1])?; - let members = arr[2..].iter().filter_map(extract_string).collect(); + let members = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::SMIsMember(key, members)) } "SINTER" => { need!(2); Ok(Command::SInter( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )) } "SINTERSTORE" => { need!(3); let dst = extract_string(&arr[1]).unwrap_or_default(); - let keys = arr[2..].iter().filter_map(extract_string).collect(); + let keys = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::SInterStore(dst, keys)) } "SUNION" => { need!(2); Ok(Command::SUnion( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )) } "SUNIONSTORE" => { need!(3); let dst = extract_string(&arr[1]).unwrap_or_default(); - let keys = arr[2..].iter().filter_map(extract_string).collect(); + let keys = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::SUnionStore(dst, keys)) } "SDIFF" => { need!(2); Ok(Command::SDiff( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )) } "SDIFFSTORE" => { need!(3); let dst = extract_string(&arr[1]).unwrap_or_default(); - let keys = arr[2..].iter().filter_map(extract_string).collect(); + let keys = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::SDiffStore(dst, keys)) } "SPOP" => { @@ -1031,7 +1028,7 @@ impl Command { "ZMSCORE" => { need!(3); let key = extract_key(&arr[1])?; - let members = arr[2..].iter().filter_map(extract_string).collect(); + let members = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::ZMScore(key, members)) } "ZRANK" => { @@ -1051,7 +1048,7 @@ impl Command { "ZREM" => { need!(3); let key = extract_key(&arr[1])?; - let members = arr[2..].iter().filter_map(extract_string).collect(); + let members = arr[2..].iter_mut().filter_map(take_string).collect(); Ok(Command::ZRem(key, members)) } "ZCARD" => { @@ -1085,20 +1082,20 @@ impl Command { "SUBSCRIBE" => { need!(2); Ok(Command::Subscribe( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )) } "UNSUBSCRIBE" => Ok(Command::Unsubscribe( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )), "PSUBSCRIBE" => { need!(2); Ok(Command::PSubscribe( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )) } "PUNSUBSCRIBE" => Ok(Command::PUnsubscribe( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )), "PUBLISH" => { need!(3); @@ -1112,11 +1109,11 @@ impl Command { "WATCH" => { need!(2); Ok(Command::Watch( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )) } "UNWATCH" => Ok(Command::Unwatch( - arr[1..].iter().filter_map(extract_string).collect(), + arr[1..].iter_mut().filter_map(take_string).collect(), )), // ── Persistence ─────────────────────────────────────────── @@ -1186,6 +1183,35 @@ fn extract_keys(vals: &[Value]) -> Result, String> { vals.iter().map(extract_key).collect() } +/// Take a string argument by **moving** its bytes out of the parsed frame. +/// +/// `extract_string` borrows and therefore copies: `from_utf8_lossy(..).into_owned()` +/// allocates a fresh `String` and memcpys the payload even when the bytes are +/// already valid UTF-8, which they almost always are. Moving reuses the `Vec` +/// the parser already allocated, so a 1 MB `SET` value costs no copy at all. +/// +/// The slot is left as a nil array, so an arm must not read the same index +/// twice — call this last, once per argument. +fn take_string(val: &mut Value) -> Option { + match std::mem::replace(val, Value::Array(None)) { + // `from_utf8` reuses the buffer on success; only genuinely invalid + // UTF-8 pays for a lossy re-encode. + Value::BulkString(Some(data)) => Some( + String::from_utf8(data) + .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()), + ), + Value::SimpleString(s) => Some(s), + _ => None, + } +} + +/// Moving counterpart of `extract_key`, with the same validation. +fn take_key(val: &mut Value) -> Result { + let key = take_string(val).unwrap_or_default(); + validate_key(&key)?; + Ok(key) +} + fn extract_string(val: &Value) -> Option { match val { Value::BulkString(Some(data)) => Some(String::from_utf8_lossy(data).into_owned()), @@ -2604,3 +2630,134 @@ mod arity_and_error_tests { assert!(err.contains("key cannot be empty"), "got {err}"); } } + +#[cfg(test)] +mod zero_copy_parse_tests { + use super::*; + + fn bulk(s: &str) -> Value { + Value::BulkString(Some(s.as_bytes().to_vec())) + } + + fn parse(parts: &[&str]) -> Result { + Command::from_value(Value::Array(Some(parts.iter().map(|s| bulk(s)).collect()))) + } + + /// Arguments are now *moved* out of the parsed frame rather than copied, so + /// the failure mode is an arm reading the same index twice and getting an + /// empty string the second time. These assert full round-trips through the + /// converted commands. + + #[test] + fn set_preserves_key_value_and_options_together() { + // SET reads index 1 and 2 by move, then scans 3.. for options — the + // exact shape that breaks if a moved slot were re-read. + match parse(&["SET", "k", "v", "EX", "60", "NX"]).unwrap() { + Command::Set(key, val, opts) => { + assert_eq!(key, "k"); + assert_eq!(val, "v"); + assert_eq!(opts.expiry, Some(SetExpiry::Ex(60))); + assert_eq!(opts.condition, Some(SetCondition::Nx)); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_large_value_survives_the_move_intact() { + // The whole point of moving: a 1 MB payload is no longer memcpy'd. + let big = "x".repeat(1024 * 1024); + match parse(&["SET", "k", &big]).unwrap() { + Command::Set(_, val, _) => { + assert_eq!(val.len(), big.len()); + assert_eq!(val, big, "payload must be byte-identical after the move"); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn invalid_utf8_still_parses_losslessly_enough() { + // `from_utf8` reuses the buffer on success; invalid input falls back to + // a lossy re-encode rather than failing the command. + let frame = Value::Array(Some(vec![ + bulk("SET"), + bulk("k"), + Value::BulkString(Some(vec![0xff, 0xfe, b'o', b'k'])), + ])); + match Command::from_value(frame).unwrap() { + Command::Set(key, val, _) => { + assert_eq!(key, "k"); + assert!(val.ends_with("ok"), "got {val:?}"); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn paired_arguments_keep_their_pairing() { + // MSET and HSET split each chunk into two mutable halves; a mistake + // there would swap or blank alternate fields. + match parse(&["MSET", "a", "1", "b", "2"]).unwrap() { + Command::MSet(pairs) => assert_eq!( + pairs, + vec![ + ("a".to_string(), "1".to_string()), + ("b".to_string(), "2".to_string()) + ] + ), + other => panic!("{other:?}"), + } + match parse(&["HSET", "h", "f1", "v1", "f2", "v2"]).unwrap() { + Command::HSet(key, pairs) => { + assert_eq!(key, "h"); + assert_eq!( + pairs, + vec![ + ("f1".to_string(), "v1".to_string()), + ("f2".to_string(), "v2".to_string()) + ] + ); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn bulk_member_lists_keep_every_element_in_order() { + match parse(&["RPUSH", "l", "a", "b", "c"]).unwrap() { + Command::RPush(key, vals) => { + assert_eq!(key, "l"); + assert_eq!(vals, vec!["a", "b", "c"]); + } + other => panic!("{other:?}"), + } + match parse(&["SADD", "s", "m1", "m2"]).unwrap() { + Command::SAdd(key, members) => { + assert_eq!(key, "s"); + assert_eq!(members, vec!["m1", "m2"]); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn key_validation_still_applies_to_moved_keys() { + // take_key must validate exactly as extract_key did. + assert!(parse(&["SET", "", "v"]).is_err(), "empty key rejected"); + assert!(parse(&["APPEND", "", "v"]).is_err()); + assert!(parse(&["MSET", "", "1"]).is_err()); + } + + #[test] + fn json_commands_carry_path_and_document_separately() { + match parse(&["JSET", "doc", "$.a", "{\"x\":1}"]).unwrap() { + Command::JSet(key, path, val) => { + assert_eq!(key, "doc"); + assert_eq!(path, "$.a"); + assert_eq!(val, "{\"x\":1}"); + } + other => panic!("{other:?}"), + } + } +} diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 9964688..418e971 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -267,15 +267,29 @@ fn json_approx_size(v: &serde_json::Value) -> usize { // ── Sliding-window rate limiter (RLSET / RLCHECK) ───────────────────────────── +/// Buckets the sliding window is divided into. Memory per limiter is fixed at +/// this many `(start, count)` pairs regardless of the configured limit. +const RL_BUCKETS: u64 = 64; + #[derive(Clone)] struct RateLimiterInner { limit: u64, window_ms: u64, - /// Timestamps (ms) of recorded attempts, oldest first. Attempts arrive in - /// monotonically non-decreasing time, so the deque stays sorted and window - /// pruning is O(pruned) pops from the front — no sorted-set machinery - /// needed. Length is bounded by `limit` (denied attempts are not recorded). - events: VecDeque, + /// `(bucket_start_ms, attempts)` for buckets inside the window, oldest + /// first. + /// + /// Storing one timestamp per attempt was exact but unbounded in practice: + /// a `RLSET key 100000 3600` limiter held 100 000 `u64`s — roughly 800 KB + /// for a single key, and token-cost limiting (roadmap #9) makes six-figure + /// limits ordinary. Counting into a fixed number of buckets caps a limiter + /// at ~1 KB whatever the limit. + /// + /// The cost is granularity: the window advances one bucket at a time, so a + /// limiter is exact to within `window_ms / RL_BUCKETS`. Attempts are never + /// under-counted — a bucket only leaves the window once it is entirely + /// outside it — so the limiter errs toward rejecting slightly early rather + /// than admitting over the limit. + buckets: VecDeque<(u64, u64)>, } impl RateLimiterInner { @@ -283,28 +297,50 @@ impl RateLimiterInner { Self { limit, window_ms, - events: VecDeque::new(), + buckets: VecDeque::new(), } } + /// Width of one bucket, at least 1 ms. + fn bucket_ms(&self) -> u64 { + (self.window_ms / RL_BUCKETS).max(1) + } + /// Record an attempt at `now`, returning `(allowed, remaining, retry_after_ms)`. /// Denied attempts are not recorded — a client hammering a full limiter /// does not push its own recovery further away. fn check(&mut self, now: u64) -> (i64, u64, u64) { + let width = self.bucket_ms(); let cutoff = now.saturating_sub(self.window_ms); - while self.events.front().is_some_and(|&t| t <= cutoff) { - self.events.pop_front(); + // A bucket leaves the window only once its whole span is behind the + // cutoff, so attempts are never dropped early. + while self + .buckets + .front() + .is_some_and(|&(start, _)| start + width <= cutoff) + { + self.buckets.pop_front(); } - if (self.events.len() as u64) < self.limit { - self.events.push_back(now); - let remaining = self.limit - self.events.len() as u64; - (1, remaining, 0) + + let used: u64 = self.buckets.iter().map(|&(_, c)| c).sum(); + if used < self.limit { + let current = now - (now % width); + match self.buckets.back_mut() { + Some((start, count)) if *start == current => *count += 1, + _ => self.buckets.push_back((current, 1)), + } + (1, self.limit - used - 1, 0) } else { + // Recovery arrives when the oldest bucket falls out of the window. let retry_after = self - .events + .buckets .front() - .map(|&t| (t + self.window_ms).saturating_sub(now)) - .unwrap_or(0); + .map(|&(start, _)| (start + width + self.window_ms).saturating_sub(now)) + .unwrap_or(0) + // A bucket spans forward from its start, so the raw figure can + // land a fraction past the window. The wait never legitimately + // exceeds one window. + .min(self.window_ms); (0, 0, retry_after) } } @@ -510,6 +546,12 @@ pub struct KeyValueStore { max_memory_bytes: Option, eviction_policy: EvictionPolicy, dirty: Arc, + /// Keys sampled per eviction pass. Approximate-LRU quality rises with the + /// sample and so does the cost, so the right value is workload-dependent — + /// Redis exposes the same knob as `maxmemory-samples`. Configured rather + /// than read from the environment because this crate also runs in the + /// browser, where there is no environment to read. + eviction_sample: usize, /// Total keys evicted since start. Exported as a metric: without it an /// operator cannot tell a healthy cache from one thrashing at its cap. evicted: Arc, @@ -529,6 +571,7 @@ impl KeyValueStore { max_memory_bytes: None, eviction_policy: EvictionPolicy::NoEviction, dirty: Arc::new(AtomicU64::new(0)), + eviction_sample: 10, evicted: Arc::new(AtomicU64::new(0)), } } @@ -540,6 +583,7 @@ impl KeyValueStore { max_memory_bytes: None, eviction_policy: EvictionPolicy::NoEviction, dirty: Arc::new(AtomicU64::new(0)), + eviction_sample: 10, evicted: Arc::new(AtomicU64::new(0)), } } @@ -555,6 +599,7 @@ impl KeyValueStore { max_memory_bytes, eviction_policy, dirty: Arc::new(AtomicU64::new(0)), + eviction_sample: 10, evicted: Arc::new(AtomicU64::new(0)), } } @@ -720,6 +765,14 @@ impl KeyValueStore { /// Evict a single entry per the configured policy. Returns the number of /// bytes freed (`Some`), or `None` if nothing could be evicted. + /// Set how many keys each eviction pass samples (default 10, minimum 1). + /// + /// A larger sample approximates true LRU/TTL ordering more closely at the + /// cost of more work per eviction. + pub fn set_eviction_sample(&mut self, sample: usize) { + self.eviction_sample = sample.max(1); + } + /// Keys evicted since start. pub fn evicted_count(&self) -> u64 { self.evicted.load(Ordering::Relaxed) @@ -735,7 +788,7 @@ impl KeyValueStore { } fn evict_one(&self, now: u64) -> Option { - const SAMPLE: usize = 10; + let sample = self.eviction_sample; let mut rng = rand::rng(); let chosen: Option = match self.eviction_policy { EvictionPolicy::NoEviction => None, @@ -749,7 +802,7 @@ impl KeyValueStore { r.value().last_access_ms.load(Ordering::Relaxed), ) }) - .choose_multiple(&mut rng, SAMPLE); + .choose_multiple(&mut rng, sample); sample.into_iter().min_by_key(|(_, w)| *w).map(|(k, _)| k) } EvictionPolicy::AllKeysRandom => { @@ -766,7 +819,7 @@ impl KeyValueStore { r.value().last_access_ms.load(Ordering::Relaxed), ) }) - .choose_multiple(&mut rng, SAMPLE); + .choose_multiple(&mut rng, sample); sample.into_iter().min_by_key(|(_, w)| *w).map(|(k, _)| k) } EvictionPolicy::VolatileTtl => { @@ -781,7 +834,7 @@ impl KeyValueStore { Some((r.key().clone(), exp)) } }) - .choose_multiple(&mut rng, SAMPLE); + .choose_multiple(&mut rng, sample); sample .into_iter() .min_by_key(|(_, exp)| *exp) @@ -813,10 +866,14 @@ impl KeyValueStore { EntryValue::ZSet(z) => { SnapshotValue::ZSet(z.scores.iter().map(|(k, &v)| (k.clone(), v)).collect()) } + // Attempt counts are deliberately not persisted: they age + // out within a single window, and a restart has already + // interrupted that window. Only the configuration is + // restored. The field remains for snapshot compatibility. EntryValue::RateLimiter(rl) => SnapshotValue::RateLimiter { limit: rl.limit, window_ms: rl.window_ms, - events: rl.events.iter().copied().collect(), + events: Vec::new(), }, EntryValue::Json(doc) => { SnapshotValue::Json(serde_json::to_string(doc).unwrap_or_default()) @@ -851,11 +908,14 @@ impl KeyValueStore { limit, window_ms, events, - } => EntryValue::RateLimiter(RateLimiterInner { - limit, - window_ms, - events: events.into(), - }), + } => { + let _ = events; // older snapshots carried attempt timestamps + EntryValue::RateLimiter(RateLimiterInner { + limit, + window_ms, + buckets: VecDeque::new(), + }) + } SnapshotValue::Json(s) => { EntryValue::Json(serde_json::from_str(&s).unwrap_or(serde_json::Value::Null)) } @@ -2532,7 +2592,7 @@ fn entry_size(key: &str, e: &Entry) -> usize { EntryValue::List(l) => l.iter().map(|s| s.len()).sum(), EntryValue::Set(s) => s.iter().map(|m| m.len()).sum::(), EntryValue::ZSet(z) => z.scores.keys().map(|m| m.len() + 8).sum(), - EntryValue::RateLimiter(rl) => rl.events.len() * 8 + 16, + EntryValue::RateLimiter(rl) => rl.buckets.len() * 16 + 16, EntryValue::Json(doc) => json_approx_size(doc), }; key.len() + val_size + 64 @@ -4225,17 +4285,26 @@ mod tests { } #[test] - fn rl_snapshot_roundtrip_preserves_state() { + fn rl_snapshot_preserves_config_but_not_attempt_state() { + // Attempt counts are transient: they age out within one window, and a + // restart has already interrupted that window. Only the configuration + // is restored, so a limiter comes back enforcing the same policy with a + // clean slate rather than a stale partial count. let s = store(); - s.execute(Command::RlSet("api".into(), 2, 60)); - rl(s.execute(Command::RlCheck("api".into(), None))); - rl(s.execute(Command::RlCheck("api".into(), None))); + s.execute(Command::RlSet("api".into(), 3, 60)); + s.execute(Command::RlCheck("api".into(), None)); + s.execute(Command::RlCheck("api".into(), None)); - let s2 = store(); - s2.restore(s.snapshot()); - let (allowed, remaining, retry) = rl(s2.execute(Command::RlCheck("api".into(), None))); - assert_eq!((allowed, remaining), (0, 0)); - assert!(retry > 0); + let restored = store(); + restored.restore(s.snapshot()); + + // Config survived: still a 3-per-60s limiter. + let (allowed, remaining, _) = rl(restored.execute(Command::RlCheck("api".into(), None))); + assert_eq!(allowed, 1); + assert_eq!( + remaining, 2, + "attempts reset on restore — the limiter enforces the same policy afresh" + ); } // ── JSON (JSET / JGET / JMERGE) ─────────────────────────────────────────── @@ -5463,3 +5532,104 @@ mod metrics_tests { assert!(s.approximate_memory_bytes() >= empty + 4096); } } + +#[cfg(test)] +mod rate_limiter_memory_tests { + use super::*; + use crate::cmd::Command; + + fn rl(v: Value) -> (i64, u64, u64) { + match v { + Value::Array(Some(items)) => match (&items[0], &items[1], &items[2]) { + (Value::Integer(a), Value::Integer(r), Value::Integer(w)) => { + (*a, *r as u64, *w as u64) + } + _ => panic!("unexpected RLCHECK reply shape"), + }, + other => panic!("expected array, got {other:?}"), + } + } + + #[test] + fn memory_is_bounded_regardless_of_limit() { + // The reason for bucketing: one timestamp per attempt meant ~800 KB for + // a single `RLSET key 100000 3600` limiter. Buckets cap it at ~1 KB. + let s = KeyValueStore::new(); + s.execute(Command::RlSet("big".into(), 100_000, 3600)); + for _ in 0..5_000 { + s.execute(Command::RlCheck("big".into(), None)); + } + let bytes = s.approximate_memory_bytes(); + assert!( + bytes < 4_096, + "5000 attempts against a 100k limiter should stay small, got {bytes} bytes" + ); + } + + #[test] + fn a_high_limit_still_admits_every_attempt_under_it() { + // Bounding memory must not bound throughput. + let s = KeyValueStore::new(); + s.execute(Command::RlSet("api".into(), 10_000, 3600)); + for i in 0..2_000 { + let (allowed, _, _) = rl(s.execute(Command::RlCheck("api".into(), None))); + assert_eq!(allowed, 1, "attempt {i} should be allowed"); + } + } + + #[test] + fn the_limit_is_still_enforced_exactly_at_the_boundary() { + // Bucketing approximates *when* attempts age out, never *how many* are + // counted inside the window. + let s = KeyValueStore::new(); + s.execute(Command::RlSet("api".into(), 5, 60)); + for expected in [4, 3, 2, 1, 0] { + let (allowed, remaining, retry) = rl(s.execute(Command::RlCheck("api".into(), None))); + assert_eq!(allowed, 1); + assert_eq!(remaining, expected); + assert_eq!(retry, 0); + } + let (allowed, remaining, retry) = rl(s.execute(Command::RlCheck("api".into(), None))); + assert_eq!(allowed, 0, "the 6th attempt against a limit of 5 is denied"); + assert_eq!(remaining, 0); + assert!(retry > 0, "a denied attempt must say when to retry"); + } + + #[test] + fn retry_after_never_exceeds_the_window() { + // Retry-After is handed straight to HTTP clients; a value past the + // window would park them longer than the policy requires. + let s = KeyValueStore::new(); + s.execute(Command::RlSet("api".into(), 1, 60)); + s.execute(Command::RlCheck("api".into(), None)); + let (_, _, retry) = rl(s.execute(Command::RlCheck("api".into(), None))); + assert!( + retry > 0 && retry <= 60_000, + "retry_after_ms = {retry}, window is 60000" + ); + } + + #[test] + fn the_shortest_window_recovers_after_it_elapses() { + // RLSET takes the window in *seconds*, so one second is the floor. At + // that width each bucket is ~15 ms; the bucket-width floor of 1 ms + // exists so an even shorter window could never divide to zero. + let s = KeyValueStore::new(); + s.execute(Command::RlSet("fast".into(), 2, 1)); + assert_eq!(rl(s.execute(Command::RlCheck("fast".into(), None))).0, 1); + assert_eq!(rl(s.execute(Command::RlCheck("fast".into(), None))).0, 1); + assert_eq!( + rl(s.execute(Command::RlCheck("fast".into(), None))).0, + 0, + "third attempt against a limit of 2 is denied" + ); + + // Past the window the limiter admits traffic again. + std::thread::sleep(std::time::Duration::from_millis(1_100)); + assert_eq!( + rl(s.execute(Command::RlCheck("fast".into(), None))).0, + 1, + "buckets older than the window must age out" + ); + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index eb4c1de..620121c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,19 +4,11 @@ Recached competes on **where the data can live** — the same engine on the serv ## Near-term -**[Byte-slice command arguments](#performance)** is the one substantial piece left in the hardening -list, and the main remaining lever on unpipelined latency. It is a large mechanical refactor — 125 -parse arms, ~210 argument extractions — and its payoff needs a quiet machine to measure, so it wants -its own focused pass rather than being folded into a release alongside other work. - -After that, in order: - -1. **[Replication lag](#operability)** — replica *count* is exported, but not how far behind each one - is, which is the signal that matters before a failover. -2. **[Bound rate-limiter memory](#operability)** — one timestamp per attempt means ~800 KB for a - single `RLSET key 100000 3600` limiter. -3. **[Make the compiled-in limits configurable](#operability)** — outbox size, live queries per - connection, eviction sample size. +1. **[Replication offset acknowledgement](#operability)** — replicas do not report an applied + offset, so true lag cannot be computed. Queue depth is exported as a proxy today. +2. **[Binary-safe WebSocket values](#ongoing-drop-in-credibility)** — values over WS are UTF-8 and + therefore lossy for raw bytes; binary frames would make the two transports equivalent. +3. **[RESP3](#ongoing-drop-in-credibility)** — push protocol support on the TCP port. --- @@ -79,33 +71,16 @@ Under consideration behind these: a CRDT text type for collaborative editing (li Improvements to shipped functionality rather than new surface. Unnumbered because they are small and independent — pick them off in any order. -### Reliability - -### Live queries - ### Performance -Both of these are diagnosed on the [benchmarks](/guide/benchmarks) page; the analysis is done, the -work is not. - -**Byte-slice command arguments.** RESP parsing allocates a `String` per argument. This is the main -remaining lever on unpipelined latency and the likeliest explanation for `HSET` sitting at ~46 % of -Redis while *beating* it pipelined. - **Serialize `LRANGE` straight from the store**, instead of building the full reply `Value` first. +Diagnosed on the [benchmarks](/guide/benchmarks) page. ### Operability -**Replication lag.** The number of connected replicas is exported, but not how far behind each one -is — the signal that actually matters before a failover. - -**Make the compiled-in limits configurable.** The 10 000-write outbox, 64 live queries per -connection, and eviction's fixed 10-key sample are all constants. Redis exposes `maxmemory-samples` -for the same reason: the right value is workload-dependent. - -**Bound rate-limiter memory.** The limiter stores one timestamp per attempt, so `RLSET key 100000 -3600` holds 100 000 `u64`s — roughly 800 KB for a single key. The cost-weighted rework in -[#9](#_9-token-cost-rate-limiting) is the natural moment to move to bucketed counts. +**Replication offset acknowledgement.** Replicas do not report an applied offset, so true lag cannot +be computed — `recached_replication_queue_depth` is exported as a proxy. Closing this means a small +protocol addition: replicas periodically reporting how far they have applied. ### Security diff --git a/docs/server/configuration.md b/docs/server/configuration.md index ffccd42..cbfc1e6 100644 --- a/docs/server/configuration.md +++ b/docs/server/configuration.md @@ -19,6 +19,11 @@ Recached is configured entirely through environment variables. There is no confi | `RECACHED_AOF_PATH` | _(disabled)_ | Path to the append-only file. When set, every write command is appended to this file in addition to snapshot saves. On startup the snapshot is loaded first, then AOF commands are replayed for the delta. The AOF is truncated after each successful snapshot save. | | `RECACHED_AOF_SYNC` | `everysec` | AOF fsync policy. `always`: fsync after every write (safest, slowest). `everysec`: fsync once per second (default, good balance). `no`: let the OS decide (fastest, least safe). | | `RECACHED_MAX_CONNECTIONS` | `1024` | Maximum number of concurrent client connections (TCP + WebSocket combined). New connections are dropped when the limit is reached. | +| `RECACHED_EVICTION_SAMPLE` | `10` | Keys sampled per eviction pass. A larger sample approximates true LRU/TTL ordering more closely at the cost of more work per eviction — the knob Redis exposes as `maxmemory-samples`. | +| `RECACHED_MAX_MULTI_QUEUE` | `10000` | Commands that may be queued inside one `MULTI`. | +| `RECACHED_MAX_WATCHES_PER_CONN` | `1024` | Keys a single connection may `WATCH`. | +| `RECACHED_MAX_LIVE_QUERIES` | `64` | Live queries (`QSUB`) a single connection may hold. | +| `RECACHED_MAX_QSUB_INITIAL_KEYS` | `10000` | Keys returned in a live query's initial `qstate` reply. Beyond this the snapshot is truncated — narrow the pattern instead of raising it. | | `RECACHED_REPL_PORT` | `6381` | TCP port the primary listens on for incoming replica connections. Only active when `RECACHED_REPLICAOF` is not set (i.e. this server is a primary). | | `RECACHED_REPLICAOF` | _(none)_ | Set to `host:port` to run this server as a read-only replica. On startup it connects to the primary, receives a full snapshot, and then streams all subsequent writes. Reconnects automatically with exponential backoff on disconnect. | | `RECACHED_REPL_PASSWORD` | _(none)_ | Shared secret for the replication channel. When set, replicas must send this password during the handshake before receiving any data. Must match on both primary and replica. Strongly recommended for any network-exposed replication port. | diff --git a/docs/server/operations.md b/docs/server/operations.md index 6daf51c..4bbb673 100644 --- a/docs/server/operations.md +++ b/docs/server/operations.md @@ -48,11 +48,14 @@ Sampled every 5 seconds, because these are levels rather than events. | `recached_live_queries` | gauge | Registered `QSUB` patterns across all connections. | | `recached_watched_keys` | gauge | Keys under `WATCH`. | | `recached_dedup_clients_tracked` | gauge | Clients with exactly-once bookkeeping in memory. | +| `recached_replication_queue_depth` | gauge | Deepest replica send queue, in frames. Replicas never acknowledge an applied offset, so true lag is not observable — a backing-up queue is the available signal that one cannot keep up. | ### What is still not exported -- **Replication lag.** Replica connection count is exported, but not how far behind each one is. -- **Client outbox depth.** That state lives in the browser, not the server. +- **True replication offset lag.** Replicas do not report an applied offset, so the server cannot + compute how far behind one is. `recached_replication_queue_depth` is the available proxy. +- **Client outbox depth.** That state lives in the browser — read it there with + `cache.pendingWrites()`. ## Useful queries @@ -119,17 +122,18 @@ healthy — they are separate listeners. Probe the cache port. Hard limits compiled into the server. Exceeding them produces errors rather than degradation, so it is worth knowing where the walls are: -| Limit | Value | Configurable | +| Limit | Default | Configurable | |---|---|---| | Max connections | 1024 | `RECACHED_MAX_CONNECTIONS` | | Consecutive auth failures before disconnect | 5 | No | | Read buffer per TCP connection | 64 MB | No | -| Queued commands per `MULTI` | 10,000 | No | -| `WATCH`ed keys per connection | 1,024 | No | -| Live queries (`QSUB`) per connection | 64 | No | -| Keys returned in a live query's initial state | 10,000 | No | +| Queued commands per `MULTI` | 10,000 | `RECACHED_MAX_MULTI_QUEUE` | +| `WATCH`ed keys per connection | 1,024 | `RECACHED_MAX_WATCHES_PER_CONN` | +| Live queries (`QSUB`) per connection | 64 | `RECACHED_MAX_LIVE_QUERIES` | +| Keys returned in a live query's initial state | 10,000 | `RECACHED_MAX_QSUB_INITIAL_KEYS` | +| Keys sampled per eviction pass | 10 | `RECACHED_EVICTION_SAMPLE` | | Replication frame | 512 MB | No | -| Client outbox (browser, offline writes) | 10,000 writes | No | +| Client outbox (browser, offline writes) | 10,000 writes | via `sync-client` | The keyspace cap (`RECACHED_MAX_KEYS`) and memory cap (`RECACHED_MAX_MEMORY`) are configured rather than compiled — see [Configuration](/server/configuration#environment-variable-reference). diff --git a/server-native/src/main.rs b/server-native/src/main.rs index cb6eb88..034c824 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -338,12 +338,42 @@ fn load_tls_acceptor() -> Option { const TCP_READ_BUFFER_BYTES: usize = 16 * 1024; // 16 KB — matches Redis default const MAX_TCP_READ_BUFFER_BYTES: usize = 64 * 1024 * 1024; // 64 MB per connection -const MAX_MULTI_QUEUE_LEN: usize = 10_000; -const MAX_WATCHES_PER_CONN: usize = 1_024; -const MAX_QSUBS_PER_CONN: usize = 64; +/// Per-connection limits. Compiled-in defaults, overridable at startup because +/// the right value is workload-dependent — Redis exposes `maxmemory-samples` +/// for the same reason. Read once and cached; changing one needs a restart. +fn env_limit(var: &str, default: usize) -> usize { + std::env::var(var) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(default) +} + +/// Commands queued inside one `MULTI`. Override: `RECACHED_MAX_MULTI_QUEUE`. +fn max_multi_queue_len() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_MULTI_QUEUE", 10_000)) +} + +/// Keys one connection may `WATCH`. Override: `RECACHED_MAX_WATCHES_PER_CONN`. +fn max_watches_per_conn() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_WATCHES_PER_CONN", 1_024)) +} + +/// Live queries one connection may hold. Override: `RECACHED_MAX_LIVE_QUERIES`. +fn max_qsubs_per_conn() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_LIVE_QUERIES", 64)) +} /// Cap on the number of key/value pairs returned as QSUB initial state, so a /// pattern matching a huge keyspace cannot produce an unbounded reply frame. -const MAX_QSUB_INITIAL_KEYS: usize = 10_000; +/// Keys returned in a live query's initial state. +/// Override: `RECACHED_MAX_QSUB_INITIAL_KEYS`. +fn max_qsub_initial_keys() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_QSUB_INITIAL_KEYS", 10_000)) +} const BROADCAST_CHANNEL_CAPACITY: usize = 512; const DEFAULT_MAX_CONNECTIONS: usize = 1024; const MAX_AUTH_FAILURES: u32 = 5; @@ -566,6 +596,22 @@ struct ReplHub { } impl ReplHub { + /// Deepest send queue across connected replicas, in frames. + /// + /// Replication is fire-and-forget — replicas never acknowledge an applied + /// offset — so true offset lag is not observable without a protocol change. + /// Queue depth is the honest proxy available today: a replica that cannot + /// keep up backs its channel up, and a queue at capacity means frames are + /// about to be dropped. + async fn max_queue_depth(&self) -> usize { + let senders = self.senders.lock().await; + senders + .iter() + .map(|tx| tx.max_capacity().saturating_sub(tx.capacity())) + .max() + .unwrap_or(0) + } + fn new() -> ReplRegistry { Arc::new(ReplHub { senders: tokio::sync::Mutex::new(Vec::new()), @@ -2372,11 +2418,11 @@ async fn main() -> Result<(), Box> { ); } - let store = Arc::new(KeyValueStore::with_config( - max_keys, - max_memory_bytes, - eviction_policy, - )); + let mut store_inner = KeyValueStore::with_config(max_keys, max_memory_bytes, eviction_policy); + // Eviction sample size — the knob Redis exposes as `maxmemory-samples`. + // Configured before the store is shared, so no interior mutability is needed. + store_inner.set_eviction_sample(env_limit("RECACHED_EVICTION_SAMPLE", 10)); + let store = Arc::new(store_inner); // ── snapshot persistence ────────────────────────────────────────────── let save_path = PathBuf::from( @@ -2626,6 +2672,8 @@ async fn main() -> Result<(), Box> { counter!("recached_evictions_total").absolute(store_m.evicted_count()); gauge!("recached_replicas_connected") .set(state_m.replicas.count.load(Ordering::Relaxed) as f64); + gauge!("recached_replication_queue_depth") + .set(state_m.replicas.max_queue_depth().await as f64); gauge!("recached_live_queries") .set(registry_m.watched_patterns.load(Ordering::Relaxed) as f64); gauge!("recached_watched_keys") @@ -2967,7 +3015,7 @@ async fn handle_tcp( if writer.write_all(err).await.is_err() { break 'outer; } } _ => { - if queue.len() >= MAX_MULTI_QUEUE_LEN { + if queue.len() >= max_multi_queue_len() { let err = b"-ERR transaction queue limit reached\r\n"; if writer.write_all(err).await.is_err() { break 'outer; } } else { @@ -3041,7 +3089,7 @@ async fn handle_tcp( Command::Watch(keys) => { let new_count = keys.iter().filter(|k| !watched_keys.contains(*k)).count(); - if watched_keys.len() + new_count > MAX_WATCHES_PER_CONN { + if watched_keys.len() + new_count > max_watches_per_conn() { if writer.write_all(b"-ERR watch limit per connection reached\r\n").await.is_err() { break 'outer; } } else { { @@ -3411,7 +3459,7 @@ async fn handle_ws( ws_send!(b"-ERR Command not allowed inside a transaction\r\n"); } _ => { - if queue.len() >= MAX_MULTI_QUEUE_LEN { + if queue.len() >= max_multi_queue_len() { ws_send!(b"-ERR transaction queue limit reached\r\n"); } else { queue.push(cmd); @@ -3480,7 +3528,7 @@ async fn handle_ws( .iter() .filter(|k| !watched_keys.contains(*k)) .count(); - if watched_keys.len() + new_count > MAX_WATCHES_PER_CONN { + if watched_keys.len() + new_count > max_watches_per_conn() { ws_send!(b"-ERR watch limit per connection reached\r\n"); } else { { @@ -3542,7 +3590,7 @@ async fn handle_ws( } } if !qsub_patterns.contains(&pattern) - && qsub_patterns.len() >= MAX_QSUBS_PER_CONN + && qsub_patterns.len() >= max_qsubs_per_conn() { ws_send!(b"-ERR live query limit per connection reached\r\n"); continue 'outer; @@ -3558,7 +3606,7 @@ async fn handle_ws( .push((conn_id, q_tx.clone())); watch_registry.sync_patterns_len(&pats); } - let kvs = store.matching_key_values(&pattern, MAX_QSUB_INITIAL_KEYS); + let kvs = store.matching_key_values(&pattern, max_qsub_initial_keys()); // Tagged reply so clients can recognise it among // interleaved frames: ["qstate", pattern, k, v, ...] let mut items = Vec::with_capacity(kvs.len() * 2 + 2); @@ -6451,3 +6499,65 @@ mod tests { assert_eq!(key, "cart:42:item"); } } + +#[cfg(test)] +mod limit_config_tests { + use super::*; + + #[test] + fn env_limit_falls_back_to_the_default() { + // Unset, empty, non-numeric, and zero all mean "use the default" — + // a zero limit would disable the feature rather than tune it. + assert_eq!(env_limit("RECACHED_DEFINITELY_UNSET_VAR_XYZ", 64), 64); + for bad in ["", " ", "abc", "0", "-5", "1.5"] { + unsafe { std::env::set_var("RECACHED_TEST_LIMIT", bad) }; + assert_eq!(env_limit("RECACHED_TEST_LIMIT", 64), 64, "input {bad:?}"); + } + unsafe { std::env::remove_var("RECACHED_TEST_LIMIT") }; + } + + #[test] + fn env_limit_accepts_a_positive_override() { + unsafe { std::env::set_var("RECACHED_TEST_LIMIT_OK", " 256 ") }; + assert_eq!( + env_limit("RECACHED_TEST_LIMIT_OK", 64), + 256, + "whitespace tolerated" + ); + unsafe { std::env::remove_var("RECACHED_TEST_LIMIT_OK") }; + } + + #[test] + fn overrides_are_read_from_the_documented_variable_names() { + // A bulk rename once rewrote these string literals along with the + // function names, leaving variables like `RECACHED_max_watches_per_conn()` + // that no operator would ever set — the override silently did nothing. + // Assert the names the docs promise. + for (var, default) in [ + ("RECACHED_MAX_MULTI_QUEUE", 10_000usize), + ("RECACHED_MAX_WATCHES_PER_CONN", 1_024), + ("RECACHED_MAX_LIVE_QUERIES", 64), + ("RECACHED_MAX_QSUB_INITIAL_KEYS", 10_000), + ("RECACHED_EVICTION_SAMPLE", 10), + ] { + assert!( + var.chars() + .all(|c| c.is_ascii_uppercase() || c == '_' || c.is_ascii_digit()), + "{var} is not a plausible environment variable name" + ); + unsafe { std::env::set_var(var, "7") }; + assert_eq!(env_limit(var, default), 7, "{var} override ignored"); + unsafe { std::env::remove_var(var) }; + } + } + + #[test] + fn compiled_defaults_match_the_documented_values() { + // These appear in docs/server/operations.md; drift would mislead + // operators sizing a deployment. + assert_eq!(max_multi_queue_len(), 10_000); + assert_eq!(max_watches_per_conn(), 1_024); + assert_eq!(max_qsubs_per_conn(), 64); + assert_eq!(max_qsub_initial_keys(), 10_000); + } +} diff --git a/sync-client/src/lib.rs b/sync-client/src/lib.rs index a68648a..7eb9a56 100644 --- a/sync-client/src/lib.rs +++ b/sync-client/src/lib.rs @@ -93,6 +93,10 @@ pub struct SyncClient { sync_scopes_csv: Option, live_queries: Vec, attempts: u32, + /// Cap on queued-but-unacknowledged writes. Beyond it the oldest is + /// evicted. Configurable because the right depth depends on how long a + /// client is expected to be offline and how large its writes are. + max_pending: usize, /// Jitter source for reconnect backoff. Seeded from `client_id` rather than /// a system RNG so this crate stays dependency-free and I/O-free — and so /// the sequence is reproducible in tests. Different clients get different @@ -115,6 +119,7 @@ impl SyncClient { sync_scopes_csv: None, live_queries: Vec::new(), attempts: 0, + max_pending: MAX_PENDING_WRITES, jitter, } } @@ -127,6 +132,17 @@ impl SyncClient { &self.client_id } + /// Set the outbox cap (minimum 1). Writes past it evict the oldest, which + /// is reported through the adapter's overflow callback. + pub fn set_max_pending(&mut self, max: usize) { + self.max_pending = max.max(1); + } + + /// Current outbox cap. + pub fn max_pending(&self) -> usize { + self.max_pending + } + pub fn epoch(&self) -> u32 { self.epoch } @@ -290,7 +306,7 @@ impl SyncClient { } else { encoded.to_string() }; - let dropped = if self.outbox.len() >= MAX_PENDING_WRITES { + let dropped = if self.outbox.len() >= self.max_pending { self.outbox.pop_front().map(|(old, _)| old) } else { None diff --git a/sync-client/src/tests.rs b/sync-client/src/tests.rs index 15a9345..6bfb205 100644 --- a/sync-client/src/tests.rs +++ b/sync-client/src/tests.rs @@ -743,3 +743,36 @@ fn flushdb_sentinel_is_harmless_when_nothing_matches() { c.handle_frame("*3\r\n$9\r\nkeychange\r\n$6\r\ncart:*\r\n$-1\r\n"); assert_eq!(c.store().execute(Command::DbSize), Value::Integer(0)); } + +// ── Configurable outbox cap ─────────────────────────────────────────────────── + +#[test] +fn outbox_cap_defaults_to_the_documented_limit() { + assert_eq!(client().max_pending(), MAX_PENDING_WRITES); +} + +#[test] +fn a_lower_cap_evicts_sooner() { + // The right depth depends on how long a client may be offline and how large + // its writes are, so it is configurable rather than fixed. + let mut c = client(); + c.set_max_pending(3); + for i in 0..3 { + let e = c.enqueue_write(&to_resp(&["SET", &format!("k{i}"), "v"]), true, false); + assert!(e.dropped.is_none(), "within the cap, nothing is evicted"); + } + let e = c.enqueue_write(&to_resp(&["SET", "k3", "v"]), true, false); + assert!(e.dropped.is_some(), "past the cap the oldest is evicted"); + assert_eq!(c.outbox_len(), 3, "depth stays at the cap"); +} + +#[test] +fn a_cap_of_zero_is_clamped_to_one() { + // A zero cap would discard every write immediately; clamp rather than + // silently break the client. + let mut c = client(); + c.set_max_pending(0); + assert_eq!(c.max_pending(), 1); + c.enqueue_write(&to_resp(&["SET", "k", "v"]), true, false); + assert_eq!(c.outbox_len(), 1); +} From d07b717a744b1a0712b28f0cd7a8d3872afaa4cd Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 19:07:08 +0800 Subject: [PATCH 05/11] Add replication acks with lag metric, RESP3 HELLO negotiation, and binary WebSocket frames; fix unflushed and mis-typed pub/sub deliveries --- CHANGELOG.md | 51 ++- core-engine/src/cmd.rs | 11 + core-engine/src/resp.rs | 106 +++++ core-engine/src/store.rs | 3 + core-engine/tests/binary_values.rs | 45 ++ docs/roadmap.md | 27 +- docs/server/commands.md | 1 + docs/server/operations.md | 22 +- docs/server/protocol.md | 33 +- docs/server/troubleshooting.md | 24 + server-native/src/main.rs | 705 +++++++++++++++++++++++++---- 11 files changed, 928 insertions(+), 100 deletions(-) create mode 100644 core-engine/tests/binary_values.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6451212..b6b9f11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,32 @@ All notable changes to Recached are documented here. `recached_replicas_connected`, `recached_live_queries`, `recached_watched_keys`, and `recached_dedup_clients_tracked`, and `recached_replication_queue_depth`. Previously only traffic was exported, so an operator could not - answer "am I near the cap?" or "is eviction thrashing?" from a dashboard. Replication *lag* and - browser outbox depth remain unexported — see [Operations](docs/server/operations.md). + answer "am I near the cap?" or "is eviction thrashing?" from a dashboard. Replication lag landed + separately in this release; browser outbox depth remains unexported because it lives in the client + — see [Operations](docs/server/operations.md). + +- **`HELLO` and RESP3 negotiation on the TCP port.** A connection starts in RESP2 and `HELLO 3` + switches it to RESP3; `HELLO 2` switches back, a bare `HELLO` reports without changing, and an + unsupported version returns `-NOPROTO` while leaving the connection on what it had. The reply is a + RESP3 map on a RESP3 connection and a flat array on a RESP2 one, matching Redis. `HELLO` requires + authentication — the pre-auth reply carries no server details, so it cannot be used to fingerprint + a deployment. + + This also adds a `Map` type (`%N`) to the RESP codec, with the header counting *pairs* rather than + elements. + +- **Replication offset acknowledgement, and a true lag metric.** Replicas now acknowledge each frame + they apply on the existing replication socket, and the primary exports + `recached_replication_lag_frames` — frames sent to the furthest-behind replica but not yet + acknowledged. + + `recached_replication_queue_depth` only ever showed work stuck in the primary's send queue, so the + case that matters most read as healthy: a replica that has received everything and is not applying + it shows an empty queue and unbounded lag. Acknowledgements are monotonic, so a reordered or + replayed ack cannot walk the high-water mark backwards. + + A replica older than 0.2.2 never acknowledges, so its lag climbs while replication works normally — + upgrade both ends together. ### Changed @@ -115,6 +139,29 @@ All notable changes to Recached are documented here. ### Fixed +- **Pub/sub deliveries never reached a TCP subscriber that only listened.** Deliveries were written + into a 32 KB buffered writer and flushed only at the end of a client-command batch, so a + connection that subscribed and then waited received nothing until it happened to send another + command or 32 KB of messages accumulated. A subscriber that also polled looked fine, which is how + this survived. Deliveries are now flushed on write. + +- **Pub/sub frames were RESP3 Push on RESP2 connections.** Every delivery was a `>` frame regardless + of protocol. RESP2 has no push type, so a standard Redis client that subscribed without sending + `HELLO 3` could not parse what it was sent. Frame type now follows the negotiated version. The + WebSocket transport is unchanged — it is RESP3 by definition, and `HELLO 2` on it is refused + rather than silently ignored. + +- **WebSocket command frames must now be accepted in binary as well as text.** The handler matched + text frames only, so a binary frame was dropped without a reply — and the WebSocket spec requires + text frames to be well-formed UTF-8, which left no way to send bytes at all. Replies are sent as + text when the RESP bytes are valid UTF-8 and binary otherwise, so existing clients see no change. + + This makes the *transport* byte-clean. It does **not** make values byte-transparent: the engine + stores values as `String`, so invalid UTF-8 is still replaced with U+FFFD before storage, on every + transport including TCP. That was true before this release and remains true; it is pinned by + `core-engine/tests/binary_values.rs` and is a separate breaking change. Do not store raw binary in + Recached today. + - **Exactly-once delivery now survives a server restart.** Dedup high-water marks were held only in memory, so a restart inside the acknowledgement window let a client's replayed write apply twice — the last standing caveat on the guarantee. Marks are now persisted to a `.dedup` sidecar beside the diff --git a/core-engine/src/cmd.rs b/core-engine/src/cmd.rs index 85d0d52..9befc8b 100644 --- a/core-engine/src/cmd.rs +++ b/core-engine/src/cmd.rs @@ -47,6 +47,9 @@ pub struct ZAddOptions { pub enum Command { Ping(Option), Auth(String), + /// `HELLO [protover]` — protocol negotiation. Connection-level like AUTH: + /// the store never sees it, because the answer depends on the connection. + Hello(Option), // ── Strings ────────────────────────────────────────────────────────────── Set(String, String, SetOptions), Get(String), @@ -233,6 +236,14 @@ impl Command { need!(2); Ok(Command::Auth(extract_string(&arr[1]).unwrap_or_default())) } + // Bare HELLO reports the current version without changing it. + // Trailing AUTH/SETNAME arguments are not supported and are + // rejected by the connection layer rather than ignored. + "HELLO" => Ok(Command::Hello(if arr.len() > 1 { + Some(extract_string(&arr[1]).unwrap_or_default()) + } else { + None + })), // ── Strings ─────────────────────────────────────────────── "SET" => { diff --git a/core-engine/src/resp.rs b/core-engine/src/resp.rs index 027c162..09af0b6 100644 --- a/core-engine/src/resp.rs +++ b/core-engine/src/resp.rs @@ -13,6 +13,12 @@ pub enum Value { /// RESP3 Push frame (`>N\r\n...`). Used for server-initiated out-of-band messages /// (mutation fan-out, pub/sub) on the WebSocket channel. Never sent as a command response. Push(Vec), + /// RESP3 Map (`%N\r\n` followed by `N` key/value pairs). + /// + /// Only `HELLO 3` replies with one today. A RESP2 connection must never be + /// sent a map — the type does not exist in RESP2 and the client will fail + /// to parse it — so the caller picks the shape from the negotiated version. + Map(Vec<(Value, Value)>), } impl Value { @@ -65,6 +71,15 @@ impl Value { v.serialize_into(out); } } + Value::Map(pairs) => { + // The header counts *pairs*, not elements, so a 3-entry map is + // `%3` followed by six values. + let _ = write!(out, "%{}\r\n", pairs.len()); + for (k, v) in pairs { + k.serialize_into(out); + v.serialize_into(out); + } + } } } @@ -84,6 +99,7 @@ impl Value { b'$' => Self::parse_bulk_string(buffer), b'*' => Self::parse_array(buffer, depth), b'>' => Self::parse_push(buffer, depth), + b'%' => Self::parse_map(buffer, depth), _ => Err("Invalid RESP type".to_string()), } } @@ -195,6 +211,39 @@ impl Value { } } + fn parse_map(buffer: &[u8], depth: usize) -> Result<(Value, usize), String> { + if depth >= MAX_ARRAY_DEPTH { + return Err("ERR max nesting depth exceeded".to_string()); + } + match Self::read_until_crlf(buffer) { + Some((data, mut offset)) => { + let s = String::from_utf8_lossy(data); + let count: u64 = s.parse().map_err(|_| "Invalid map length".to_string())?; + // Each pair is two values, so the element budget is halved. + if count as usize > MAX_ARRAY_ELEMENTS / 2 { + return Err(format!( + "ERR map too large ({} > {} pairs)", + count, + MAX_ARRAY_ELEMENTS / 2 + )); + } + let mut pairs = Vec::with_capacity(count as usize); + for _ in 0..count { + let (k, klen) = Self::parse_inner(&buffer[offset..], depth + 1)?; + offset += klen; + let (v, vlen) = Self::parse_inner(&buffer[offset..], depth + 1)?; + offset += vlen; + pairs.push((k, v)); + if offset > MAX_TOTAL_MESSAGE_BYTES { + return Err("ERR message too large".to_string()); + } + } + Ok((Value::Map(pairs), offset)) + } + None => Err("Incomplete".to_string()), + } + } + fn parse_array(buffer: &[u8], depth: usize) -> Result<(Value, usize), String> { if depth >= MAX_ARRAY_DEPTH { return Err("ERR max nesting depth exceeded".to_string()); @@ -341,6 +390,21 @@ mod tests { #[test] fn push_round_trip() { + round_trip(&Value::Map(vec![])); + round_trip(&Value::Map(vec![( + Value::BulkString(Some(b"proto".to_vec())), + Value::Integer(3), + )])); + round_trip(&Value::Map(vec![ + ( + Value::BulkString(Some(b"server".to_vec())), + Value::BulkString(Some(b"recached".to_vec())), + ), + ( + Value::BulkString(Some(b"modules".to_vec())), + Value::Array(Some(vec![])), + ), + ])); round_trip(&Value::Push(vec![])); round_trip(&Value::Push(vec![ Value::BulkString(Some(b"SET".to_vec())), @@ -354,6 +418,48 @@ mod tests { ])); } + #[test] + fn map_header_counts_pairs_not_elements() { + // `%N` means N key/value *pairs* — 2N values follow. Emitting the + // element count instead would desynchronise every downstream parser. + let m = Value::Map(vec![ + (Value::BulkString(Some(b"a".to_vec())), Value::Integer(1)), + (Value::BulkString(Some(b"b".to_vec())), Value::Integer(2)), + ]); + let bytes = m.serialize(); + assert!( + bytes.starts_with(b"%2\r\n"), + "got {:?}", + String::from_utf8_lossy(&bytes) + ); + let (parsed, n) = Value::parse(&bytes).unwrap(); + assert_eq!(parsed, m); + assert_eq!(n, bytes.len(), "must consume exactly the frame"); + } + + #[test] + fn map_rejects_a_malformed_length() { + assert!(Value::parse(b"%x\r\n").is_err()); + assert!(Value::parse(b"%-1\r\n").is_err()); + } + + #[test] + fn truncated_map_is_incomplete_not_an_error() { + // A partial frame must be retried once more bytes arrive, not rejected. + let full = Value::Map(vec![( + Value::BulkString(Some(b"k".to_vec())), + Value::Integer(7), + )]) + .serialize(); + for cut in 1..full.len() { + assert_eq!( + Value::parse(&full[..cut]), + Err("Incomplete".to_string()), + "prefix of length {cut} should be incomplete" + ); + } + } + #[test] fn push_prefix_distinct_from_array() { let push = Value::Push(vec![Value::BulkString(Some(b"x".to_vec()))]).serialize(); diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 418e971..47f4619 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -948,6 +948,9 @@ impl KeyValueStore { Command::Auth(_) => Value::Error( "ERR AUTH is handled by the connection layer, not the store".to_string(), ), + Command::Hello(_) => Value::Error( + "ERR HELLO is handled by the connection layer, not the store".to_string(), + ), // ── Strings ─────────────────────────────────────────────────────── Command::Set(key, val, opts) => { diff --git a/core-engine/tests/binary_values.rs b/core-engine/tests/binary_values.rs new file mode 100644 index 0000000..09f4a22 --- /dev/null +++ b/core-engine/tests/binary_values.rs @@ -0,0 +1,45 @@ +//! Tripwire for the engine's handling of values that are not valid UTF-8. +//! +//! `EntryValue::Str` is a `String`, so a value is forced through a lossy UTF-8 +//! conversion at command-parse time. This is a property of the *engine*, not of +//! any transport: the round-trip below is lossy over TCP exactly as it is over +//! WebSocket, so the two are already equivalent — just equivalently lossy. +//! +//! This test asserts the current (lossy) behaviour deliberately. When byte-safe +//! values land, it will fail — flip it to assert the bytes survive. + +use core_engine::{cmd::Command, resp::Value, store::KeyValueStore}; + +#[test] +fn non_utf8_values_are_lossily_replaced() { + // 0xFF 0xFE is not valid UTF-8 in any position. + let raw: &[u8] = b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$3\r\n\xff\xfe\x41\r\n"; + let (v, _) = Value::parse(raw).unwrap(); + let store = KeyValueStore::new(); + store.execute(Command::from_value(v).unwrap()); + + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing after SET"); + }; + + // Each invalid byte becomes U+FFFD (EF BF BD), so 3 bytes in, 7 bytes out. + assert_eq!( + got, + b"\xef\xbf\xbd\xef\xbf\xbd\x41".to_vec(), + "engine is now byte-safe — update this test to assert the round-trip" + ); +} + +#[test] +fn utf8_values_survive_unchanged() { + let store = KeyValueStore::new(); + store.execute(Command::Set( + "k".into(), + "héllo ✓".into(), + Default::default(), + )); + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing after SET"); + }; + assert_eq!(String::from_utf8(got).unwrap(), "héllo ✓"); +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 620121c..5c2a0bf 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,11 +4,8 @@ Recached competes on **where the data can live** — the same engine on the serv ## Near-term -1. **[Replication offset acknowledgement](#operability)** — replicas do not report an applied - offset, so true lag cannot be computed. Queue depth is exported as a proxy today. -2. **[Binary-safe WebSocket values](#ongoing-drop-in-credibility)** — values over WS are UTF-8 and - therefore lossy for raw bytes; binary frames would make the two transports equivalent. -3. **[RESP3](#ongoing-drop-in-credibility)** — push protocol support on the TCP port. +1. **[Byte-transparent values](#byte-transparent-values)** — values are stored as UTF-8 strings, so + raw binary is corrupted on every transport. The largest remaining drop-in gap. --- @@ -76,11 +73,21 @@ independent — pick them off in any order. **Serialize `LRANGE` straight from the store**, instead of building the full reply `Value` first. Diagnosed on the [benchmarks](/guide/benchmarks) page. -### Operability +### Byte-transparent values -**Replication offset acknowledgement.** Replicas do not report an applied offset, so true lag cannot -be computed — `recached_replication_queue_depth` is exported as a proxy. Closing this means a small -protocol addition: replicas periodically reporting how far they have applied. +**Store values as bytes rather than `String`.** `EntryValue::Str` is a `String`, so a value is forced +through a lossy UTF-8 conversion when the command is parsed: `SET k <0xFF 0xFE>` stores two U+FFFD +replacement characters instead. This is a property of the engine, so it applies to **every** +transport — TCP included, not just WebSocket as previously recorded here. + +The practical effect is that compressed blobs, protobuf, images, and anything else Redis users +routinely cache cannot be stored without base64-encoding first. It is the largest remaining gap in +"any Redis client works today". + +Closing it means `Vec`/`Bytes` values through `cmd.rs`, `store.rs`, and the collection types; a +snapshot format change; and an API decision for the browser SDK, whose `set(key, value)` takes a +string. That is a breaking change and wants its own release rather than being folded into a patch. +Current behaviour is pinned by `core-engine/tests/binary_values.rs`, which fails when it changes. ### Security @@ -99,8 +106,6 @@ both are procurement checkboxes worth building when someone actually asks. Not features, but continuous work that keeps "any Redis client works today" honest: -- **Binary-safe WebSocket values** — values over WS are currently UTF-8 (lossy for raw bytes); binary frames would make the two transports equivalent. -- **RESP3** — push protocol support on the TCP port. - **Command coverage** — closing gaps in the supported command set as real workloads surface them (see [Commands](/server/commands)). --- diff --git a/docs/server/commands.md b/docs/server/commands.md index 6fc6d0b..3a05fd0 100644 --- a/docs/server/commands.md +++ b/docs/server/commands.md @@ -8,6 +8,7 @@ Recached implements the subset of RESP commands that most applications use. Comm |---|---| | `PING [message]` | Returns `PONG`, or echoes `message` if provided. Used to test connectivity and measure latency. | | `AUTH password` | Authenticates the connection. Required on the first command if `RECACHED_PASSWORD` is set. 5 consecutive failures close the connection. | +| `HELLO [protover]` | Reports server info and negotiates the protocol version. `3` switches the connection to RESP3, `2` back to RESP2, no argument reports without changing. Unsupported versions return `-NOPROTO` and leave the connection unchanged. Requires authentication. See [Wire Protocol](/server/protocol#protocol-version-tcp). | --- diff --git a/docs/server/operations.md b/docs/server/operations.md index 4bbb673..5b24de6 100644 --- a/docs/server/operations.md +++ b/docs/server/operations.md @@ -48,12 +48,27 @@ Sampled every 5 seconds, because these are levels rather than events. | `recached_live_queries` | gauge | Registered `QSUB` patterns across all connections. | | `recached_watched_keys` | gauge | Keys under `WATCH`. | | `recached_dedup_clients_tracked` | gauge | Clients with exactly-once bookkeeping in memory. | -| `recached_replication_queue_depth` | gauge | Deepest replica send queue, in frames. Replicas never acknowledge an applied offset, so true lag is not observable — a backing-up queue is the available signal that one cannot keep up. | +| `recached_replication_queue_depth` | gauge | Deepest replica send queue, in frames — work the primary has not yet put on the wire. | +| `recached_replication_lag_frames` | gauge | Frames the furthest-behind replica has been sent but has not acknowledged applying. Zero means every replica is caught up. | + +### Reading the two replication gauges + +They fail differently, which is why both exist: + +- **Queue depth high, lag high** — the primary cannot hand frames off fast enough. The replica's + channel is backing up, usually a slow or saturated network link. A replica whose queue fills is + disconnected outright so it resyncs from a snapshot rather than falling further behind. +- **Queue depth zero, lag high** — everything was written to the socket and the replica is not + acknowledging it. The frames are in flight, or the replica is applying them slowly, or it is + wedged. This is the case queue depth alone cannot see, and it is the one worth alerting on. + +Lag is measured in frames, not bytes or seconds: one frame is one replicated write command. + +A replica running a build older than 0.2.2 never acknowledges, so its lag climbs without bound while +replication works normally. Upgrade both ends together. ### What is still not exported -- **True replication offset lag.** Replicas do not report an applied offset, so the server cannot - compute how far behind one is. `recached_replication_queue_depth` is the available proxy. - **Client outbox depth.** That state lives in the browser — read it there with `cache.pendingWrites()`. @@ -91,6 +106,7 @@ Thresholds are starting points — tune to your traffic. | Memory pressure | `recached_memory_bytes` > 80% of `RECACHED_MAX_MEMORY` | Eviction is about to start, or already has. | | Eviction churn | `rate(recached_evictions_total[5m])` climbing | The working set no longer fits; results will start missing. | | Replica lost | `recached_replicas_connected` drops | Failover risk — the standby is no longer following. | +| Replica falling behind | `recached_replication_lag_frames` > 1000 for 5m | The standby is not keeping up; a failover now would lose those writes. | ## Health checking diff --git a/docs/server/protocol.md b/docs/server/protocol.md index 95ea837..d0603f5 100644 --- a/docs/server/protocol.md +++ b/docs/server/protocol.md @@ -6,10 +6,35 @@ This page is **normative**: client SDKs (browser `recached-edge`, the planned mo | Port | Transport | Framing | Audience | |---|---|---|---| -| 6379 | TCP | RESP2, pipelined | Trusted backends (any Redis client) | -| 6380 | WebSocket | One RESP value per **text** frame | Untrusted browsers / apps | +| 6379 | TCP | RESP2 by default, RESP3 after `HELLO 3`, pipelined | Trusted backends (any Redis client) | +| 6380 | WebSocket | One RESP value per frame — **text**, or **binary** for bytes that are not valid UTF-8 | Untrusted browsers / apps | -WebSocket text frames imply UTF-8: raw binary values are only fully round-trippable over TCP. +### Protocol version (TCP) + +A TCP connection starts in **RESP2**. `HELLO 3` switches it to RESP3; `HELLO 2` switches back; a +bare `HELLO` reports without changing anything. An unsupported version is refused with `-NOPROTO` +and leaves the connection on the protocol it already had, so a client can probe and fall back. + +The version changes exactly one thing on the wire today: **pub/sub deliveries are RESP3 Push (`>`) +frames on a RESP3 connection and plain arrays (`*`) on a RESP2 one.** RESP2 has no push type, so +sending `>` to a RESP2 client is unparseable — before 0.2.2 the server did exactly that, which broke +standard Redis clients that subscribed without negotiating. + +`HELLO` requires authentication when a password is set; the pre-auth reply is `-NOAUTH` and carries +no server details. + +### Binary frames (WebSocket) + +The WebSocket spec requires text frames to be well-formed UTF-8. A command or reply carrying bytes +that are not valid UTF-8 therefore travels in a **binary** frame instead; everything else stays in +text frames, so existing clients are unaffected. A client must accept both. + +::: warning Values are not yet byte-transparent +Binary frames make the *transport* byte-clean, but the engine stores values as UTF-8 strings, so a +value containing invalid UTF-8 is still replaced with U+FFFD before it is stored — on **every** +transport, TCP included. Byte-transparent values are a separate, breaking change. Do not store raw +binary (compressed blobs, protobuf, images) in Recached today; base64-encode it or keep it elsewhere. +::: ## Frame taxonomy (WebSocket) @@ -19,7 +44,7 @@ Every frame a client receives is exactly one of: |---|---|---| | **Reply** | any RESP value not matching the rows below | Response to one command this connection sent | | **Mutation push** | RESP3 Push `>N` whose elements form a replayable command (`SET`, `HSET`, `JSET`, `JMERGE`, …) | Another client/backend mutated a key in scope — apply to the local store | -| **Pub/sub push** | RESP3 Push `>3` = `["message", channel, payload]` | Pub/sub delivery | +| **Pub/sub push** | RESP3 Push `>3` = `["message", channel, payload]` | Pub/sub delivery. The WebSocket transport is always RESP3 — `HELLO 2` on it is refused, because the frame taxonomy below depends on the push type existing | | **Keychange push** | Array `["keychange", key, value]` | A watched / live-queried key changed. `value`: full string, nil (deleted), or a type-name marker (`hash`, `list`, `set`, `zset`, `json`, `ratelimit`) whose content travels via mutation pushes instead | | **Query state** | Array `["qstate", pattern, k1, v1, …]` | **Both** the reply to a `QSUB` **and** initial state to apply (same value encoding as keychange) | diff --git a/docs/server/troubleshooting.md b/docs/server/troubleshooting.md index 4ce230b..12db9c8 100644 --- a/docs/server/troubleshooting.md +++ b/docs/server/troubleshooting.md @@ -78,12 +78,36 @@ traffic you believed was encrypted was not. If you are on an older version, veri assume: a `rediss://` client should connect and a plaintext client should be refused. See [Security → Transport encryption](/server/security#transport-encryption). +### A replica is connected but falling behind + +`recached_replication_lag_frames` counts frames sent but not acknowledged by the furthest-behind +replica. Unlike `recached_replication_queue_depth`, it stays high when the primary has written +everything to the socket and the replica is not keeping up — see +[Operations → Reading the two replication gauges](/server/operations#reading-the-two-replication-gauges). + +Lag that climbs without bound while replication otherwise works usually means the replica predates +0.2.2 and never acknowledges. Upgrade both ends together. + ### Memory keeps growing There is no built-in memory metric — monitor process RSS. Set `RECACHED_MAX_MEMORY` and `RECACHED_EVICTION` so the cache bounds itself, and `RECACHED_MAX_KEYS` if key count rather than value size is the driver. `DBSIZE` reports the current key count on demand. +### A subscriber receives nothing, or cannot parse what it receives + +Two separate faults, both fixed in **0.2.2**: + +- **Nothing arrives until the subscriber sends another command.** Deliveries were written into a + buffered writer that only flushed when handling a client command, so a connection that purely + listened saw nothing. A subscriber that also polls appeared to work, which is why this went + unnoticed. +- **Frames arrive but the client errors on them.** Pub/sub was delivered as RESP3 Push (`>`) frames + on every connection. RESP2 has no push type, so a standard Redis client that subscribed without + sending `HELLO 3` could not parse the frame. Deliveries now follow the negotiated version. + +On an older server, neither has a client-side workaround — upgrade. + ### `ERR key too large` A key exceeded the maximum key length. Keys are identifiers, not payloads — put the data in the diff --git a/server-native/src/main.rs b/server-native/src/main.rs index 034c824..b31abcc 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -58,6 +58,7 @@ fn command_name(cmd: &Command) -> &'static str { match cmd { Command::Ping(_) => "ping", Command::Auth(_) => "auth", + Command::Hello(_) => "hello", Command::Get(_) => "get", Command::ESet(_, _) => "eset", Command::Set(_, _, _) => "set", @@ -587,11 +588,26 @@ async fn replay_aof(store: &KeyValueStore, path: &std::path::Path) -> usize { type ReplSender = mpsc::Sender>; +/// A connected replica: its write channel plus the counters that make lag +/// observable. +/// +/// Replication was previously one-way, so the primary could only report how +/// many replicas were attached — never how far behind one had fallen. The +/// replica now acknowledges each applied frame, and the difference between +/// what was queued and what was acknowledged is the lag. +struct ReplicaHandle { + tx: ReplSender, + /// Frames handed to this replica's channel. + sent: Arc, + /// Frames the replica reports as applied. + acked: Arc, +} + /// Connected-replica registry. `count` mirrors `senders.len()` (updated by /// every writer while holding the lock) so the per-write hot path can skip /// the mutex entirely when no replica is connected. struct ReplHub { - senders: tokio::sync::Mutex>, + senders: tokio::sync::Mutex>, count: AtomicUsize, } @@ -607,7 +623,46 @@ impl ReplHub { let senders = self.senders.lock().await; senders .iter() - .map(|tx| tx.max_capacity().saturating_sub(tx.capacity())) + .map(|r| r.tx.max_capacity().saturating_sub(r.tx.capacity())) + .max() + .unwrap_or(0) + } + + /// Send one frame to every attached replica, dropping any that cannot keep + /// up. Increments each surviving replica's sent counter, which is one half + /// of the lag calculation. + async fn fan_out(&self, bytes: Vec) { + let mut reg = self.senders.lock().await; + reg.retain(|r| match r.tx.try_send(bytes.clone()) { + Ok(()) => { + r.sent.fetch_add(1, Ordering::Relaxed); + true + } + Err(mpsc::error::TrySendError::Full(_)) => { + warn!( + "Replica fell too far behind (channel full) — disconnecting so it can resync" + ); + false + } + Err(mpsc::error::TrySendError::Closed(_)) => false, + }); + self.count.store(reg.len(), Ordering::Relaxed); + } + + /// Frames the furthest-behind replica has yet to acknowledge. + /// + /// This is true lag: how much of what the primary sent has actually been + /// applied downstream. Queue depth only shows what is stuck locally, and + /// reads zero for a replica that has received frames but cannot apply them. + async fn max_lag_frames(&self) -> u64 { + let senders = self.senders.lock().await; + senders + .iter() + .map(|r| { + r.sent + .load(Ordering::Relaxed) + .saturating_sub(r.acked.load(Ordering::Relaxed)) + }) .max() .unwrap_or(0) } @@ -750,19 +805,7 @@ impl ServerState { if self.replicas.is_empty() { return; } - let bytes = resp.as_bytes().to_vec(); - let mut reg = self.replicas.senders.lock().await; - reg.retain(|tx| match tx.try_send(bytes.clone()) { - Ok(()) => true, - Err(mpsc::error::TrySendError::Full(_)) => { - warn!( - "Replica fell too far behind (channel full) — disconnecting so it can resync" - ); - false - } - Err(mpsc::error::TrySendError::Closed(_)) => false, - }); - self.replicas.count.store(reg.len(), Ordering::Relaxed); + self.replicas.fan_out(resp.as_bytes().to_vec()).await; } /// Path of the dedup sidecar, alongside the snapshot. @@ -990,9 +1033,15 @@ async fn handle_replica( // 1. Register channel first so subsequent writes are buffered let (tx, mut rx) = mpsc::channel::>(repl_channel_capacity); + let sent = Arc::new(AtomicU64::new(0)); + let acked = Arc::new(AtomicU64::new(0)); { let mut reg = replicas.senders.lock().await; - reg.push(tx); + reg.push(ReplicaHandle { + tx, + sent: Arc::clone(&sent), + acked: Arc::clone(&acked), + }); replicas.count.store(reg.len(), Ordering::Relaxed); } @@ -1004,12 +1053,37 @@ async fn handle_replica( socket.write_all(&snap_bytes).await?; socket.flush().await?; - // 3. Stream buffered + ongoing writes - while let Some(bytes) = rx.recv().await { - let len = bytes.len() as u32; - socket.write_all(&len.to_le_bytes()).await?; - socket.write_all(&bytes).await?; - socket.flush().await?; + // 3. Stream buffered + ongoing writes, and read acknowledgements + // + // The socket is bidirectional but used to carry frames one way only, which + // left the primary unable to say how far behind a replica was. The replica + // now writes back a cumulative count of applied frames; `sent - acked` is + // the lag. Reading and writing are selected over so a replica that stops + // acknowledging cannot stall the write side, and vice versa. + let (mut rd, mut wr) = socket.split(); + let mut ack_buf = [0u8; 8]; + loop { + tokio::select! { + frame = rx.recv() => { + let Some(bytes) = frame else { break }; + let len = bytes.len() as u32; + wr.write_all(&len.to_le_bytes()).await?; + wr.write_all(&bytes).await?; + wr.flush().await?; + } + res = rd.read_exact(&mut ack_buf) => { + // A replica that closes its read side, or one running a build + // that predates acks, simply stops updating the gauge — it is + // not an error, so the stream continues either way. + if res.is_err() { + break; + } + let applied = u64::from_le_bytes(ack_buf); + // Monotonic: a reordered or replayed ack must never walk the + // high-water mark backwards and report negative lag. + acked.fetch_max(applied, Ordering::Relaxed); + } + } } Ok(()) } @@ -1123,7 +1197,12 @@ async fn sync_from_primary( } } - // 2. Stream write commands from primary + // 2. Stream write commands from primary, acknowledging what we apply + // + // Every frame is counted, including one that fails to parse: the primary + // counts frames it sent, so skipping a bad frame here would desynchronise + // the two offsets and understate lag forever after. + let mut applied: u64 = 0; loop { let mut len_buf = [0u8; 4]; socket.read_exact(&mut len_buf).await?; @@ -1136,6 +1215,7 @@ async fn sync_from_primary( } let mut cmd_bytes = vec![0u8; cmd_len]; socket.read_exact(&mut cmd_bytes).await?; + applied += 1; match Value::parse(&cmd_bytes) { Ok((value, _)) => { @@ -1162,6 +1242,13 @@ async fn sync_from_primary( } Err(e) => warn!("Replica: bad command from primary: {}", e), } + + // Acknowledge on the same socket. TcpStream is unbuffered, so this is a + // single 8-byte write with no flush; a failure means the primary is + // gone, which the next read will surface with a better error. + if socket.write_all(&applied.to_le_bytes()).await.is_err() { + warn!("Replica: failed to send replication acknowledgement"); + } } } @@ -1268,6 +1355,7 @@ fn command_scope(cmd: &Command) -> CommandScope { match cmd { Command::Ping(_) | Command::Auth(_) + | Command::Hello(_) | Command::Multi | Command::Exec | Command::Discard @@ -1857,9 +1945,89 @@ async fn unregister_all_watches( // ── helpers ────────────────────────────────────────────────────────────────── -fn encode_pubsub_msg(msg: PubSubMsg) -> Vec { +/// Encode a pub/sub delivery for a connection speaking protocol `protover`. +/// +/// RESP2 has no push type, so a subscribed RESP2 client expects a plain array +/// and cannot parse a `>` frame at all. RESP3 clients want the push type so +/// deliveries are distinguishable from command replies on a multiplexed +/// connection. The WebSocket transport is RESP3 by definition — the sync +/// protocol is specified in terms of push frames — and passes 3. +/// Handle `HELLO [protover]`, updating `protover` in place on success. +/// +/// Returns the serialized reply. An unsupported version leaves the connection's +/// current protocol untouched and replies `-NOPROTO`, which is what lets a +/// client probe for RESP3 and fall back cleanly rather than being disconnected. +fn process_hello( + requested: Option<&str>, + protover: &mut u8, + is_authenticated: bool, + is_replica: bool, +) -> Vec { + if let Some(raw) = requested { + match raw.parse::() { + Ok(v @ (2 | 3)) => *protover = v, + _ => { + return Value::Error("NOPROTO unsupported protocol version".to_string()) + .serialize(); + } + } + } + + // Pre-auth HELLO reports the protocol but nothing about the server, so an + // unauthenticated client cannot use it to fingerprint the deployment. + if !is_authenticated { + return Value::Error("NOAUTH HELLO must be called with authentication".to_string()) + .serialize(); + } + + let fields = vec![ + ("server", Value::BulkString(Some(b"recached".to_vec()))), + ( + "version", + Value::BulkString(Some(env!("CARGO_PKG_VERSION").as_bytes().to_vec())), + ), + ("proto", Value::Integer(*protover as i64)), + ("mode", Value::BulkString(Some(b"standalone".to_vec()))), + ( + "role", + Value::BulkString(Some(if is_replica { + b"replica".to_vec() + } else { + b"master".to_vec() + })), + ), + ("modules", Value::Array(Some(vec![]))), + ]; + + if *protover >= 3 { + Value::Map( + fields + .into_iter() + .map(|(k, v)| (Value::BulkString(Some(k.as_bytes().to_vec())), v)) + .collect(), + ) + .serialize() + } else { + // RESP2 has no map type; Redis flattens to alternating key/value. + let mut flat = Vec::with_capacity(fields.len() * 2); + for (k, v) in fields { + flat.push(Value::BulkString(Some(k.as_bytes().to_vec()))); + flat.push(v); + } + Value::Array(Some(flat)).serialize() + } +} + +fn encode_pubsub_msg(msg: PubSubMsg, protover: u8) -> Vec { + let frame = |parts: Vec| { + if protover >= 3 { + Value::Push(parts) + } else { + Value::Array(Some(parts)) + } + }; match msg { - PubSubMsg::Message { channel, message } => Value::Push(vec![ + PubSubMsg::Message { channel, message } => frame(vec![ Value::BulkString(Some(b"message".to_vec())), Value::BulkString(Some(channel.into_bytes())), Value::BulkString(Some(message.into_bytes())), @@ -1869,7 +2037,7 @@ fn encode_pubsub_msg(msg: PubSubMsg) -> Vec { pattern, channel, message, - } => Value::Push(vec![ + } => frame(vec![ Value::BulkString(Some(b"pmessage".to_vec())), Value::BulkString(Some(pattern.into_bytes())), Value::BulkString(Some(channel.into_bytes())), @@ -2674,6 +2842,8 @@ async fn main() -> Result<(), Box> { .set(state_m.replicas.count.load(Ordering::Relaxed) as f64); gauge!("recached_replication_queue_depth") .set(state_m.replicas.max_queue_depth().await as f64); + gauge!("recached_replication_lag_frames") + .set(state_m.replicas.max_lag_frames().await as f64); gauge!("recached_live_queries") .set(registry_m.watched_patterns.load(Ordering::Relaxed) as f64); gauge!("recached_watched_keys") @@ -2878,6 +3048,10 @@ async fn handle_tcp( let mut resp_buf = Vec::::with_capacity(4 * 1024); let mut is_authenticated = password.is_none(); let mut auth_failures: u32 = 0; + // RESP2 until the client negotiates otherwise. Defaulting to 2 keeps every + // existing client working: they never send HELLO and must not start + // receiving RESP3-only types. + let mut protover: u8 = 2; let mut multi_queue: Option> = None; let mut subscribed_channels: HashSet = HashSet::new(); let mut subscribed_patterns: HashSet = HashSet::new(); @@ -2928,6 +3102,17 @@ async fn handle_tcp( continue 'parse; } + if let Command::Hello(ref requested) = cmd { + let resp = process_hello( + requested.as_deref(), + &mut protover, + is_authenticated, + state.is_replica(), + ); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + if !is_authenticated { if writer.write_all(b"-NOAUTH Authentication required.\r\n").await.is_err() { break 'outer; @@ -3213,7 +3398,15 @@ async fn handle_tcp( msg = ps_rx.recv(), if is_subscribed => { match msg { Some(m) => { - if writer.write_all(&encode_pubsub_msg(m)).await.is_err() { + if writer.write_all(&encode_pubsub_msg(m, protover)).await.is_err() { + break; + } + // `writer` is a BufWriter, and a delivery is not a + // response to anything this connection sent — nothing + // else is going to flush it. Without this a subscriber + // that only listens receives nothing until it happens + // to send a command or 32 KB of pushes accumulate. + if writer.flush().await.is_err() { break; } } @@ -3285,14 +3478,19 @@ async fn handle_ws( let mut qsub_patterns: HashSet = HashSet::new(); let (q_tx, mut q_rx) = mpsc::unbounded_channel::(); - // NOTE: the WebSocket transport uses *text* frames, so values must be valid - // UTF-8. Non-UTF-8 bytes are replaced (lossy) on the way out. This is safe - // for the SDK, whose `set(key, value)` API only accepts `&str` values; raw - // binary values are only fully round-trippable over the TCP (RESP) port. + // Replies go out as *text* frames whenever the RESP bytes are valid UTF-8, + // which is the overwhelming majority and is what every existing client + // expects. A reply carrying a value that is not valid UTF-8 goes out as a + // *binary* frame instead of being mangled by a lossy conversion, which is + // what made raw binary values round-trip only over the TCP port. macro_rules! ws_send { ($bytes:expr) => {{ - let text = String::from_utf8_lossy($bytes).into_owned(); - if ws_sender.send(Message::Text(text.into())).await.is_err() { + let bytes: &[u8] = $bytes; + let msg = match std::str::from_utf8(bytes) { + Ok(text) => Message::Text(text.into()), + Err(_) => Message::Binary(bytes.to_vec().into()), + }; + if ws_sender.send(msg).await.is_err() { break; } }}; @@ -3304,8 +3502,17 @@ async fn handle_ws( tokio::select! { msg = ws_receiver.next() => { match msg { - Some(Ok(Message::Text(text))) => { - let (value, _) = match Value::parse(text.as_bytes()) { + // Binary frames carry the same RESP bytes as text frames. + // They exist so a client can write a value that is not + // valid UTF-8 — impossible over a text frame, which the + // WebSocket spec requires to be well-formed UTF-8. + Some(Ok(frame @ (Message::Text(_) | Message::Binary(_)))) => { + let raw: Vec = match &frame { + Message::Text(t) => t.as_bytes().to_vec(), + Message::Binary(b) => b.to_vec(), + _ => unreachable!("pattern restricts to text and binary"), + }; + let (value, _) = match Value::parse(&raw) { Ok(v) => v, Err(e) => { let err = Value::Error(format!("ERR Protocol error: {}", e)).serialize(); @@ -3333,6 +3540,28 @@ async fn handle_ws( continue; } + // The WebSocket sync protocol is specified in terms of + // RESP3 push frames, so this transport is always RESP3 + // and HELLO cannot downgrade it — a client asking for 2 + // is refused rather than silently left on 3. + if let Command::Hello(ref requested) = cmd { + let mut ws_protover: u8 = 3; + let resp = match requested.as_deref() { + Some("2") => Value::Error( + "NOPROTO the WebSocket transport requires RESP3".to_string(), + ) + .serialize(), + other => process_hello( + other, + &mut ws_protover, + is_authenticated, + state.is_replica(), + ), + }; + ws_send!(&resp); + continue; + } + if !is_authenticated { let resp = Value::Error("NOAUTH Authentication required.".to_string()).serialize(); ws_send!(&resp); @@ -3754,11 +3983,8 @@ async fn handle_ws( msg = ps_rx.recv(), if is_subscribed => { match msg { Some(m) => { - let bytes = encode_pubsub_msg(m); - let text = String::from_utf8_lossy(&bytes).into_owned(); - if ws_sender.send(Message::Text(text.into())).await.is_err() { - break; - } + let bytes = encode_pubsub_msg(m, 3); + ws_send!(&bytes); } None => break, } @@ -3771,10 +3997,7 @@ async fn handle_ws( // client for the observable-keys feature. watch_dirty = true; let bytes = encode_keychange(&key, &value); - let text = String::from_utf8_lossy(&bytes).into_owned(); - if ws_sender.send(Message::Text(text.into())).await.is_err() { - break; - } + ws_send!(&bytes); } } @@ -3783,10 +4006,7 @@ async fn handle_ws( notif = q_rx.recv(), if !qsub_patterns.is_empty() => { if let Some((key, value)) = notif { let bytes = encode_keychange(&key, &value); - let text = String::from_utf8_lossy(&bytes).into_owned(); - if ws_sender.send(Message::Text(text.into())).await.is_err() { - break; - } + ws_send!(&bytes); } } } @@ -3940,12 +4160,16 @@ mod tests { } } + /// Send `args` and read one value. An empty `args` sends nothing and + /// just reads the next frame — used to await an out-of-band push. async fn cmd(&mut self, args: &[&str]) -> Value { - let mut req = format!("*{}\r\n", args.len()); - for a in args { - req.push_str(&format!("${}\r\n{}\r\n", a.len(), a)); + if !args.is_empty() { + let mut req = format!("*{}\r\n", args.len()); + for a in args { + req.push_str(&format!("${}\r\n{}\r\n", a.len(), a)); + } + self.stream.write_all(req.as_bytes()).await.unwrap(); } - self.stream.write_all(req.as_bytes()).await.unwrap(); loop { match Value::parse(&self.buf[..self.filled]) { Ok((val, n)) => { @@ -4130,6 +4354,72 @@ mod tests { assert_eq!(c.cmd(&["DEL", "k"]).await, int(0)); // already gone } + #[tokio::test] + async fn integration_hello_negotiates_the_protocol() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + // Default is RESP2: the reply is a flat array, not a map. + let v = c.cmd(&["HELLO"]).await; + let Value::Array(Some(items)) = v else { + panic!("RESP2 HELLO must reply with an array, got {v:?}") + }; + assert!(items.contains(&bulk("recached"))); + assert!(items.contains(&Value::Integer(2))); + + // Upgrading yields a map keyed the same way. + let v = c.cmd(&["HELLO", "3"]).await; + let Value::Map(pairs) = v else { + panic!("RESP3 HELLO must reply with a map, got {v:?}") + }; + let proto = pairs + .iter() + .find(|(k, _)| *k == bulk("proto")) + .map(|(_, v)| v.clone()); + assert_eq!(proto, Some(Value::Integer(3))); + + // An unsupported version is refused and the connection stays usable. + let v = c.cmd(&["HELLO", "9"]).await; + assert!( + matches!(&v, Value::Error(e) if e.starts_with("NOPROTO")), + "expected NOPROTO, got {v:?}" + ); + assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); + } + + #[tokio::test] + async fn integration_pubsub_frame_type_follows_the_negotiated_protocol() { + // The bug this pins: pub/sub deliveries were RESP3 push frames on every + // connection, including RESP2 ones that cannot parse `>` at all. + for (protover, want_push) in [(None, false), (Some("3"), true)] { + let srv = spawn_server().await; + let mut sub = RespClient::connect(srv.tcp_addr).await; + if let Some(v) = protover { + sub.cmd(&["HELLO", v]).await; + } + assert!(matches!( + sub.cmd(&["SUBSCRIBE", "news"]).await, + Value::Array(_) | Value::Push(_) + )); + + let mut pubr = RespClient::connect(srv.tcp_addr).await; + // Wait for the subscription to register before publishing. + for _ in 0..50 { + if pubr.cmd(&["PUBLISH", "news", "hi"]).await == int(1) { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + + let delivery = sub.cmd(&[]).await; + match (&delivery, want_push) { + (Value::Push(_), true) => {} + (Value::Array(Some(_)), false) => {} + _ => panic!("protover {protover:?}: expected push={want_push}, got {delivery:?}"), + } + } + } + #[tokio::test] async fn integration_incr_and_expiry() { let srv = spawn_server().await; @@ -4849,6 +5139,148 @@ mod tests { replica_store.execute(Command::Get("replkey".into())), Value::BulkString(Some(b"replval".to_vec())) ); + + // The replica acknowledges what it applies, so once it has caught up the + // primary must observe zero lag. Before acknowledgements existed the + // primary had no way to distinguish this from a replica that had + // received the frame and silently failed to apply it. + let mut lag = u64::MAX; + for _ in 0..50 { + lag = repl_registry.max_lag_frames().await; + if lag == 0 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + assert_eq!(lag, 0, "caught-up replica must report zero lag"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn integration_ws_accepts_binary_command_frames() { + // A WebSocket text frame must be well-formed UTF-8, so a command + // carrying raw bytes can only travel in a binary frame. The server + // previously handled text frames only and dropped binary ones. + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n$3\r\n\xff\xfe\x41\r\n".to_vec(); + assert_eq!(c.cmd_binary(raw).await, ok()); + + // The command was accepted and executed — the key exists. + assert_eq!(c.cmd(&["EXISTS", "bin"]).await, int(1)); + + // NOTE: the *value* is still lossy, because the engine stores values as + // `String` (see core-engine/tests/binary_values.rs). Binary frames make + // the transport byte-clean; byte-clean storage is a separate change. + let Value::BulkString(Some(got)) = c.cmd(&["GET", "bin"]).await else { + panic!("expected a value") + }; + assert_eq!(got, b"\xef\xbf\xbd\xef\xbf\xbd\x41".to_vec()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn integration_ws_hello_reports_resp3_and_refuses_downgrade() { + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + // The sync protocol is defined in terms of RESP3 push frames. + let v = c.cmd(&["HELLO"]).await; + let Value::Map(pairs) = v else { + panic!("WS HELLO must reply with a RESP3 map, got {v:?}") + }; + assert!( + pairs + .iter() + .any(|(k, v)| *k == bulk("proto") && *v == Value::Integer(3)) + ); + + // Downgrading would silently break push delivery, so it is refused + // rather than accepted-and-ignored. + let v = c.cmd(&["HELLO", "2"]).await; + assert!( + matches!(&v, Value::Error(e) if e.starts_with("NOPROTO")), + "expected NOPROTO, got {v:?}" + ); + } + + #[tokio::test] + async fn replication_lag_counts_unacknowledged_frames() { + // A replica that receives frames but never acknowledges them is exactly + // the case queue depth cannot see: the frames left the primary's + // channel, so the queue reads empty while the replica is arbitrarily + // far behind. Lag must report them. + let store = Arc::new(KeyValueStore::new()); + let registry: ReplRegistry = ReplHub::new(); + let snap_cfg = Arc::new(SnapshotConfig { + path: tmp_path("lag_snap.rdb"), + last_save: AtomicI64::new(0), + }); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + { + let (s, sc, r) = ( + Arc::clone(&store), + Arc::clone(&snap_cfg), + Arc::clone(®istry), + ); + tokio::spawn(async move { + if let Ok((socket, _)) = listener.accept().await { + let _ = + handle_replica(socket, s, sc, r, None, DEFAULT_REPL_CHANNEL_CAPACITY).await; + } + }); + } + + // A replica that reads the snapshot and then goes silent. + let mut sock = TcpStream::connect(addr).await.unwrap(); + let mut len_buf = [0u8; 4]; + sock.read_exact(&mut len_buf).await.unwrap(); + let mut snap = vec![0u8; u32::from_le_bytes(len_buf) as usize]; + sock.read_exact(&mut snap).await.unwrap(); + + // Wait for registration, then fan out three writes. + for _ in 0..50 { + if registry.count.load(Ordering::Relaxed) == 1 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + for i in 0..3 { + registry + .fan_out(format!("*1\r\n$4\r\nPING{i}\r\n").into_bytes()) + .await; + } + + let mut lag = 0; + for _ in 0..50 { + lag = registry.max_lag_frames().await; + if lag == 3 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + assert_eq!(lag, 3, "three unacknowledged frames must show as lag 3"); + + // Acknowledging two of them retires exactly two frames of lag. + sock.write_all(&2u64.to_le_bytes()).await.unwrap(); + for _ in 0..50 { + lag = registry.max_lag_frames().await; + if lag == 1 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + assert_eq!(lag, 1, "after acking 2 of 3, one frame remains outstanding"); + + // A stale ack must not walk the high-water mark backwards. + sock.write_all(&1u64.to_le_bytes()).await.unwrap(); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + assert_eq!( + registry.max_lag_frames().await, + 1, + "a replayed lower ack must not increase reported lag" + ); } // ── Integration: 3e load (ignored in normal CI) ─────────────────────────── @@ -5124,6 +5556,14 @@ mod tests { self.next_reply().await } + /// Send pre-encoded RESP bytes in a *binary* frame. Text frames must be + /// well-formed UTF-8 per the WebSocket spec, so this is the only way to + /// put arbitrary bytes on the wire. + async fn cmd_binary(&mut self, raw: Vec) -> Value { + self.ws.send(Message::Binary(raw.into())).await.unwrap(); + self.next_reply().await + } + /// Wait up to `ms` for the next frame of any kind — RESP3 Push /// broadcasts *and* plain arrays. `keychange` notifications are encoded /// as arrays, so `recv_push` skips them entirely. @@ -5206,24 +5646,24 @@ mod tests { /// (RESP3 Push broadcasts and `keychange` observable-key pushes). async fn next_reply(&mut self) -> Value { loop { - match self.ws.next().await { - Some(Ok(Message::Text(t))) => { - let Ok((v, _)) = Value::parse(t.as_bytes()) else { - continue; - }; - if matches!(v, Value::Push(_)) { - continue; - } - if let Value::Array(Some(items)) = &v - && matches!(items.first(), Some(Value::BulkString(Some(k))) if k == b"keychange") - { - continue; - } - return v; - } + let raw: Vec = match self.ws.next().await { + Some(Ok(Message::Text(t))) => t.as_bytes().to_vec(), + Some(Ok(Message::Binary(b))) => b.to_vec(), Some(Ok(_)) => continue, _ => panic!("ws closed unexpectedly"), + }; + let Ok((v, _)) = Value::parse(&raw) else { + continue; + }; + if matches!(v, Value::Push(_)) { + continue; + } + if let Value::Array(Some(items)) = &v + && matches!(items.first(), Some(Value::BulkString(Some(k))) if k == b"keychange") + { + continue; } + return v; } } } @@ -6026,11 +6466,14 @@ mod tests { // ── Wire encoding ───────────────────────────────────────────────────────── #[test] - fn pubsub_message_encodes_as_a_resp_push_frame() { - let bytes = encode_pubsub_msg(PubSubMsg::Message { - channel: "news".into(), - message: "hello".into(), - }); + fn pubsub_message_encodes_as_a_resp3_push_frame() { + let bytes = encode_pubsub_msg( + PubSubMsg::Message { + channel: "news".into(), + message: "hello".into(), + }, + 3, + ); let text = String::from_utf8_lossy(&bytes); assert!( text.starts_with('>'), @@ -6041,19 +6484,121 @@ mod tests { assert!(text.contains("hello")); } + #[test] + fn pubsub_message_encodes_as_an_array_for_resp2() { + // RESP2 has no push type. Sending `>` to a RESP2 client — which is + // every client that has not sent HELLO 3 — is unparseable, so a + // subscribed connection would break outright. + let bytes = encode_pubsub_msg( + PubSubMsg::Message { + channel: "news".into(), + message: "hello".into(), + }, + 2, + ); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.starts_with("*3\r\n"), + "RESP2 delivery must be a 3-element array: {text:?}" + ); + assert!(!text.contains('>'), "no push frame on RESP2: {text:?}"); + } + #[test] fn pattern_message_carries_the_matching_pattern() { // A pmessage must name the pattern that matched, or a client // subscribed to several patterns cannot tell them apart. - let bytes = encode_pubsub_msg(PubSubMsg::PMessage { - pattern: "news.*".into(), - channel: "news.tech".into(), - message: "hi".into(), - }); + for protover in [2u8, 3u8] { + let bytes = encode_pubsub_msg( + PubSubMsg::PMessage { + pattern: "news.*".into(), + channel: "news.tech".into(), + message: "hi".into(), + }, + protover, + ); + let text = String::from_utf8_lossy(&bytes); + assert!(text.contains("pmessage"), "protover {protover}"); + assert!(text.contains("news.*"), "protover {protover}"); + assert!(text.contains("news.tech"), "protover {protover}"); + } + } + + // ── HELLO / protocol negotiation ───────────────────────────────────────── + + #[test] + fn hello_defaults_to_the_connections_current_version() { + // Bare HELLO reports, it does not change. A client using it purely to + // read server info must not be silently switched to another protocol. + let mut protover = 2u8; + let bytes = process_hello(None, &mut protover, true, false); + assert_eq!(protover, 2, "bare HELLO must not change the version"); let text = String::from_utf8_lossy(&bytes); - assert!(text.contains("pmessage")); - assert!(text.contains("news.*")); - assert!(text.contains("news.tech")); + assert!( + text.starts_with('*'), + "RESP2 reply must be an array: {text:?}" + ); + assert!(text.contains("recached")); + assert!(text.contains(":2\r\n"), "proto must report 2: {text:?}"); + } + + #[test] + fn hello_3_upgrades_and_replies_with_a_map() { + let mut protover = 2u8; + let bytes = process_hello(Some("3"), &mut protover, true, false); + assert_eq!(protover, 3); + let text = String::from_utf8_lossy(&bytes); + assert!(text.starts_with("%6\r\n"), "must be a 6-pair map: {text:?}"); + assert!(text.contains(":3\r\n"), "proto must report 3: {text:?}"); + } + + #[test] + fn hello_3_then_2_downgrades_again() { + let mut protover = 2u8; + process_hello(Some("3"), &mut protover, true, false); + assert_eq!(protover, 3); + let bytes = process_hello(Some("2"), &mut protover, true, false); + assert_eq!(protover, 2, "HELLO 2 must downgrade"); + assert!(String::from_utf8_lossy(&bytes).starts_with('*')); + } + + #[test] + fn hello_rejects_unsupported_versions_without_changing_protocol() { + // A client probing for a version the server does not speak must get a + // clean NOPROTO and stay on what it had — not be left in a half-state. + for bad in ["4", "1", "0", "abc", "", "255", "-1", "3.0"] { + let mut protover = 2u8; + let bytes = process_hello(Some(bad), &mut protover, true, false); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.starts_with("-NOPROTO"), + "HELLO {bad:?} must be refused: {text:?}" + ); + assert_eq!(protover, 2, "HELLO {bad:?} must not change the version"); + } + } + + #[test] + fn hello_does_not_leak_server_details_before_auth() { + let mut protover = 2u8; + let bytes = process_hello(Some("3"), &mut protover, false, false); + let text = String::from_utf8_lossy(&bytes); + assert!(text.starts_with("-NOAUTH"), "{text:?}"); + assert!( + !text.contains("recached") && !text.contains(env!("CARGO_PKG_VERSION")), + "unauthenticated HELLO must not fingerprint the server: {text:?}" + ); + } + + #[test] + fn hello_reports_replica_role() { + let mut protover = 3u8; + let primary = + String::from_utf8_lossy(&process_hello(None, &mut protover, true, false)).into_owned(); + let replica = + String::from_utf8_lossy(&process_hello(None, &mut protover, true, true)).into_owned(); + assert!(primary.contains("master"), "{primary:?}"); + assert!(replica.contains("replica"), "{replica:?}"); } #[test] From deee0acb174f028c776b5caa9ab3ebb08cbda9eb Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 19:17:22 +0800 Subject: [PATCH 06/11] Reject non-UTF-8 values instead of silently corrupting them; add replication acks with lag metric, RESP3 HELLO negotiation, and binary WebSocket frames --- CHANGELOG.md | 32 ++++++++-- README.md | 2 +- core-engine/src/cmd.rs | 43 ++++++++++--- core-engine/tests/binary_values.rs | 96 ++++++++++++++++++++---------- docs/guide/introduction.md | 38 +++++++++++- docs/guide/use-cases.md | 20 +++++-- docs/index.md | 4 +- docs/roadmap.md | 18 +++--- docs/server/protocol.md | 17 ++++-- docs/server/troubleshooting.md | 22 +++++++ server-native/src/main.rs | 49 +++++++++++---- 11 files changed, 263 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b9f11..cbc84b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,32 @@ All notable changes to Recached are documented here. ### Fixed +- **Non-UTF-8 values were silently corrupted; they are now rejected.** Values are stored as `String`, + so a value containing invalid UTF-8 was converted to U+FFFD replacement characters on the way in. + `SET` returned `OK`, `GET` returned bytes that differed from what was written, and nothing anywhere + reported a problem — the original bytes were destroyed at parse time, before storage, so there was + nothing to recover. + + This affected **every transport, TCP included** — not only WebSocket, as the roadmap previously + recorded. `SET k <0xFF 0xFE 0x41>` over plain RESP came back as `EF BF BD EF BF BD 41`. + + Such a command is now refused before anything is stored: + + ``` + ERR argument 2 is not valid UTF-8. Recached stores values as text; + base64-encode binary payloads before caching them + ``` + + Keys are checked the same way, and the connection stays usable after a rejection. Nothing that + worked before can break: the affected values were already being destroyed, so no application can + have depended on the old behaviour. + + **Action required if you cache binary payloads** — compressed responses, protobuf, pickled objects, + images — base64-encode them, or keep them elsewhere. Check whether your client compresses + transparently; several do by default. Byte-transparent values are on the + [roadmap](docs/roadmap.md) as a breaking change; this release makes the limitation visible rather + than silent. + - **Pub/sub deliveries never reached a TCP subscriber that only listened.** Deliveries were written into a 32 KB buffered writer and flushed only at the end of a client-command batch, so a connection that subscribed and then waited received nothing until it happened to send another @@ -156,11 +182,7 @@ All notable changes to Recached are documented here. text frames to be well-formed UTF-8, which left no way to send bytes at all. Replies are sent as text when the RESP bytes are valid UTF-8 and binary otherwise, so existing clients see no change. - This makes the *transport* byte-clean. It does **not** make values byte-transparent: the engine - stores values as `String`, so invalid UTF-8 is still replaced with U+FFFD before storage, on every - transport including TCP. That was true before this release and remains true; it is pinned by - `core-engine/tests/binary_values.rs` and is a separate breaking change. Do not store raw binary in - Recached today. + This makes the *transport* byte-clean. Values are a separate matter — see below. - **Exactly-once delivery now survives a server restart.** Dedup high-water marks were held only in memory, so a restart inside the acknowledgement window let a client's replayed write apply twice — diff --git a/README.md b/README.md index a3e2ee7..65e88e4 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Every caching solution forces a choice: server-side caches like Redis mean every frontend read is a network round-trip; client-side state like Zustand or SWR means two caches — one on the server and one in every client, with manual staleness code gluing them together. **Recached removes the choice.** -The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. +The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes; values are text, so binary payloads need base64) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. > [!NOTE] > Recached is not a full Redis replacement. It covers the subset most applications actually need: strings, expiry, counters, all collection types, transactions, pub/sub, and observable keys. Best fit: reactive UIs, session caches, browser-side API response caching, and rate limiting. diff --git a/core-engine/src/cmd.rs b/core-engine/src/cmd.rs index 9befc8b..5037da7 100644 --- a/core-engine/src/cmd.rs +++ b/core-engine/src/cmd.rs @@ -205,6 +205,22 @@ impl Command { if arr.is_empty() { return Err("Empty command".to_string()); } + // Reject the whole command if any argument is not valid UTF-8. + // + // Values are stored as `String`, so a non-UTF-8 byte would + // otherwise be silently replaced with U+FFFD: SET returns OK, + // GET returns different bytes than were written, and nothing + // anywhere reports a problem. Failing loudly here — before + // anything is stored — is the only honest option until values + // are byte-transparent. See docs/roadmap.md. + if let Some(pos) = arr.iter().position( + |v| matches!(v, Value::BulkString(Some(b)) if std::str::from_utf8(b).is_err()), + ) { + return Err(format!( + "ERR argument {pos} is not valid UTF-8. Recached stores values as text; \ + base64-encode binary payloads before caching them" + )); + } let cmd_name = match &arr[0] { Value::BulkString(Some(data)) => String::from_utf8_lossy(data).to_uppercase(), Value::SimpleString(s) => s.to_uppercase(), @@ -1205,8 +1221,10 @@ fn extract_keys(vals: &[Value]) -> Result, String> { /// twice — call this last, once per argument. fn take_string(val: &mut Value) -> Option { match std::mem::replace(val, Value::Array(None)) { - // `from_utf8` reuses the buffer on success; only genuinely invalid - // UTF-8 pays for a lossy re-encode. + // `from_utf8` reuses the buffer, so this is a move rather than a copy. + // `from_value` has already rejected non-UTF-8 arguments, so the lossy + // branch is unreachable in practice — it is kept only so a future + // caller that bypasses that check degrades rather than panics. Value::BulkString(Some(data)) => Some( String::from_utf8(data) .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()), @@ -2688,18 +2706,27 @@ mod zero_copy_parse_tests { } #[test] - fn invalid_utf8_still_parses_losslessly_enough() { - // `from_utf8` reuses the buffer on success; invalid input falls back to - // a lossy re-encode rather than failing the command. + fn invalid_utf8_fails_the_command_instead_of_being_re_encoded() { + // The move-based parse must not resurrect the lossy fallback: a value + // that cannot be stored faithfully has to fail loudly, not arrive + // half-corrupted behind a successful reply. let frame = Value::Array(Some(vec![ bulk("SET"), bulk("k"), Value::BulkString(Some(vec![0xff, 0xfe, b'o', b'k'])), ])); - match Command::from_value(frame).unwrap() { + let err = Command::from_value(frame).expect_err("must be refused"); + assert!(err.contains("not valid UTF-8"), "got {err:?}"); + } + + #[test] + fn utf8_multibyte_arguments_are_not_mistaken_for_binary() { + // The validation runs over every argument, so an over-eager check would + // break ordinary non-ASCII payloads. + match parse(&["SET", "ключ", "日本語 ✓"]).unwrap() { Command::Set(key, val, _) => { - assert_eq!(key, "k"); - assert!(val.ends_with("ok"), "got {val:?}"); + assert_eq!(key, "ключ"); + assert_eq!(val, "日本語 ✓"); } other => panic!("{other:?}"), } diff --git a/core-engine/tests/binary_values.rs b/core-engine/tests/binary_values.rs index 09f4a22..a65e455 100644 --- a/core-engine/tests/binary_values.rs +++ b/core-engine/tests/binary_values.rs @@ -1,45 +1,79 @@ -//! Tripwire for the engine's handling of values that are not valid UTF-8. +//! Values that are not valid UTF-8 are rejected, not silently corrupted. //! -//! `EntryValue::Str` is a `String`, so a value is forced through a lossy UTF-8 -//! conversion at command-parse time. This is a property of the *engine*, not of -//! any transport: the round-trip below is lossy over TCP exactly as it is over -//! WebSocket, so the two are already equivalent — just equivalently lossy. -//! -//! This test asserts the current (lossy) behaviour deliberately. When byte-safe -//! values land, it will fail — flip it to assert the bytes survive. +//! `EntryValue::Str` is a `String`, so a non-UTF-8 value cannot be stored +//! faithfully. Until values are byte-transparent (see docs/roadmap.md), the +//! command is refused at parse time — before anything is written — rather than +//! being lossily converted to U+FFFD behind a successful `+OK`. use core_engine::{cmd::Command, resp::Value, store::KeyValueStore}; -#[test] -fn non_utf8_values_are_lossily_replaced() { - // 0xFF 0xFE is not valid UTF-8 in any position. - let raw: &[u8] = b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$3\r\n\xff\xfe\x41\r\n"; - let (v, _) = Value::parse(raw).unwrap(); - let store = KeyValueStore::new(); - store.execute(Command::from_value(v).unwrap()); +/// `SET k <0xFF 0xFE 'A'>` — invalid UTF-8 in the value position. +const SET_BINARY_VALUE: &[u8] = b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$3\r\n\xff\xfe\x41\r\n"; - let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { - panic!("key missing after SET"); - }; +#[test] +fn non_utf8_values_are_rejected() { + let (v, _) = Value::parse(SET_BINARY_VALUE).unwrap(); + let err = Command::from_value(v).expect_err("binary value must be refused"); + assert!(err.starts_with("ERR "), "must be a RESP error: {err:?}"); + assert!( + err.contains("base64"), + "the error must say what to do instead: {err:?}" + ); +} - // Each invalid byte becomes U+FFFD (EF BF BD), so 3 bytes in, 7 bytes out. +#[test] +fn a_rejected_command_stores_nothing() { + // The dangerous failure was a silent partial success: OK returned, wrong + // bytes stored. Nothing may reach the store. + let store = KeyValueStore::new(); + let (v, _) = Value::parse(SET_BINARY_VALUE).unwrap(); + assert!(Command::from_value(v).is_err()); assert_eq!( - got, - b"\xef\xbf\xbd\xef\xbf\xbd\x41".to_vec(), - "engine is now byte-safe — update this test to assert the round-trip" + store.execute(Command::Get("k".into())), + Value::BulkString(None) ); } +#[test] +fn non_utf8_keys_are_rejected_too() { + // A key is as lossy as a value, and a corrupted key is unreachable + // afterwards — the write would be silently unretrievable. + let raw: &[u8] = b"*3\r\n$3\r\nSET\r\n$2\r\n\xff\xfe\r\n$1\r\nv\r\n"; + let (v, _) = Value::parse(raw).unwrap(); + assert!(Command::from_value(v).is_err()); +} + +#[test] +fn the_error_names_which_argument_was_bad() { + // MSET k1 v1 k2 — the bad argument is index 4, not the first. + let raw: &[u8] = b"*5\r\n$4\r\nMSET\r\n$2\r\nk1\r\n$2\r\nv1\r\n$2\r\nk2\r\n$2\r\n\xff\xfe\r\n"; + let (v, _) = Value::parse(raw).unwrap(); + let err = Command::from_value(v).expect_err("must be refused"); + assert!(err.contains("argument 4"), "got {err:?}"); +} + #[test] fn utf8_values_survive_unchanged() { + // Multi-byte UTF-8 is valid and must not be caught by the new check. + let store = KeyValueStore::new(); + for value in ["héllo ✓", "日本語", "\u{1F600}", "", "plain"] { + store.execute(Command::Set("k".into(), value.into(), Default::default())); + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing after SET of {value:?}"); + }; + assert_eq!(String::from_utf8(got).unwrap(), value); + } +} + +#[test] +fn utf8_values_still_parse_from_the_wire() { + // Guards against the validation being over-eager and rejecting valid input. + let raw = "*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$9\r\n日本語\r\n".as_bytes(); + let (v, _) = Value::parse(raw).unwrap(); let store = KeyValueStore::new(); - store.execute(Command::Set( - "k".into(), - "héllo ✓".into(), - Default::default(), - )); - let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { - panic!("key missing after SET"); - }; - assert_eq!(String::from_utf8(got).unwrap(), "héllo ✓"); + store.execute(Command::from_value(v).expect("valid UTF-8 must be accepted")); + assert_eq!( + store.execute(Command::Get("k".into())), + Value::BulkString(Some("日本語".as_bytes().to_vec())) + ); } diff --git a/docs/guide/introduction.md b/docs/guide/introduction.md index 5951e30..f771f8b 100644 --- a/docs/guide/introduction.md +++ b/docs/guide/introduction.md @@ -4,6 +4,11 @@ Recached is an in-memory cache server written in Rust. It speaks RESP (the Redis Serialization Protocol) on port 6379, so any Redis client — `ioredis`, `node-redis`, `redis-py`, `Jedis` — works against it today with no code changes. +One caveat worth knowing before you start: **values are text, not arbitrary bytes.** A value that is +not valid UTF-8 is rejected with an error rather than stored, so binary payloads — compressed blobs, +protobuf, pickled objects, images — must be base64-encoded first. See +[Binary values](#binary-values) below. + That is where the similarity with Redis ends. The distinguishing feature is the `core-engine` crate: a pure Rust state machine with no network dependencies, no file I/O, and no OS-specific code. It compiles to native x86-64/ARM64 for the server **and** to `wasm32-unknown-unknown` for the browser. Both targets run the same cache logic from the same source. The WebSocket sync layer (port 6380) keeps the two sides consistent in real time. @@ -42,14 +47,43 @@ Recached is a good fit when: - **You want live UI without polling.** The WebSocket sync replaces a polling loop without requiring you to build a separate SSE or WebSocket server. - **You want a frontend-only cache with TTL.** The WASM module works entirely without a server. Call `createCache()` without `connect()` and you get a local in-memory cache with built-in TTL — no Recached server, no Redis, no backend changes required. - **You need cross-tab sync.** BroadcastChannel support means all open tabs in the same browser share mutations automatically. -- **You want a drop-in Redis replacement** for the subset of commands most applications actually use (strings, expiry, counters, collections, transactions, pub/sub). +- **You want a drop-in Redis replacement** for the subset of commands most applications actually use (strings, expiry, counters, collections, transactions, pub/sub) — and your values are text or JSON rather than binary blobs. ## When Recached is not the right fit - **You need very high-durability persistence.** Recached supports snapshots (RDB-style) and AOF, but it is still primarily an in-memory cache. If you cannot tolerate any data loss between fsync intervals, a purpose-built database is the right tool. - **You need multi-replica consensus failover.** Recached supports leader–follower replication with automatic single-replica failover (`RECACHED_FAILOVER_TIMEOUT`). If the primary is unreachable for the configured duration, the designated replica promotes itself. What it does not include is multi-replica quorum election: in a setup with several replicas, split-brain prevention requires you to designate one replica for auto-failover and keep the others as passive standbys. -- **You depend on uncommon Redis commands.** Recached implements the commands most applications use, not all 250+. Server introspection (`INFO`, `SLOWLOG`, `COMMAND`), Lua scripting, RESP3, and cluster mode are out of scope. +- **You depend on uncommon Redis commands.** Recached implements the commands most applications use, not all 250+. Server introspection (`INFO`, `SLOWLOG`, `COMMAND`), Lua scripting, and cluster mode are out of scope. RESP3 is supported for protocol negotiation and pub/sub delivery (`HELLO 3`), not for the full RESP3 type surface. - **You need very large datasets.** Recached is an in-memory cache — it is not a database. If your working set does not fit in RAM, Redis with RDB persistence or a proper database is the right tool. +- **You cache binary payloads.** Values are text. See below. + +## Binary values + +**Values must be valid UTF-8.** A command carrying a value that is not — a gzip blob, protobuf, +a pickled Python object, an image — is rejected with an error: + +``` +ERR argument 2 is not valid UTF-8. Recached stores values as text; + base64-encode binary payloads before caching them +``` + +Nothing is stored when this happens, and the connection stays usable. Keys are checked the same way. + +This is a real difference from Redis, where values are binary-safe. It matters most if you are +migrating a cache that stores compressed responses or serialized objects, since many client +libraries compress transparently — check your client's configuration before assuming your values are +text. + +The workaround is base64, at roughly 33% extra memory and a small CPU cost on both ends. JSON, +strings, numbers, and counters are unaffected, which covers most of what Recached is used for. + +::: tip Why an error rather than best-effort storage +The engine stores values as UTF-8 strings, so a binary value cannot be kept faithfully. Before 0.2.2 +it was silently converted — invalid bytes became U+FFFD, `SET` still returned `OK`, and `GET` +returned data that differed from what was written, with no signal anywhere that anything had +happened. Failing loudly is worse ergonomics and much better behaviour. Byte-transparent values are +on the [roadmap](/roadmap#byte-transparent-values) as a breaking change. +::: ## Maturity diff --git a/docs/guide/use-cases.md b/docs/guide/use-cases.md index 7449c92..50f3b5f 100644 --- a/docs/guide/use-cases.md +++ b/docs/guide/use-cases.md @@ -88,8 +88,13 @@ It is not durable storage. See the persistence caveats in **Working set larger than RAM → a database, or Redis with eviction tuned.** There is no disk-backed tier. -**You need Lua scripting, cluster mode, RESP3, or `INFO`/`SLOWLOG` introspection → Redis.** These are -explicitly out of scope; see [Commands](/server/commands). +**You need Lua scripting, cluster mode, or `INFO`/`SLOWLOG` introspection → Redis.** These are +explicitly out of scope; see [Commands](/server/commands). RESP3 exists only for protocol +negotiation and pub/sub framing (`HELLO 3`), not the full type surface. + +**You cache binary payloads → Redis.** Redis values are binary-safe; Recached values must be valid +UTF-8 and a binary value is rejected outright. Compressed responses, protobuf, and serialized +objects need base64 first — see [Binary values](/guide/introduction#binary-values). ## Compared directly @@ -139,9 +144,14 @@ Before switching a workload over, check: 1. **Command coverage.** Diff your actual command usage against [Commands](/server/commands). `MONITOR` on your existing Redis for a representative window is the fastest way to get that list. Lua scripts, cluster commands, streams, and `INFO`-based tooling will not carry over. -2. **Persistence expectations.** Confirm snapshot + AOF semantics match what you assume today. -3. **Replication topology.** Single-replica auto-failover only; no Sentinel or quorum election. -4. **Eviction.** Review the key cap and TTL behaviour in +2. **Value encoding.** Recached values must be valid UTF-8 — binary is rejected, not stored. Check + whether your client compresses transparently (many do), and whether anything caches serialized + objects rather than JSON. This is the migration surprise most likely to bite, because the client + works perfectly right up until the value is not text. See + [Binary values](/guide/introduction#binary-values). +3. **Persistence expectations.** Confirm snapshot + AOF semantics match what you assume today. +4. **Replication topology.** Single-replica auto-failover only; no Sentinel or quorum election. +5. **Eviction.** Review the key cap and TTL behaviour in [Configuration](/server/configuration) against your memory budget. There is no data migration path from an RDB file — treat it as a cold cache and let it fill. diff --git a/docs/index.md b/docs/index.md index 77f6682..75943d9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,7 +33,7 @@ features: details: Any mutation on the server is pushed to all connected browser instances instantly. Any write from the browser is pushed to the server and fanned out to other tabs. - icon: 🦀 title: Redis-compatible server - details: Speaks RESP on port 6379. Drop it in front of any Redis client — ioredis, node-redis, redis-py — with no code changes. + details: Speaks RESP on port 6379. Drop it in front of any Redis client — ioredis, node-redis, redis-py — with no code changes. Values are text; base64-encode binary payloads. - icon: 🌐 title: Offline-first browser cache details: IndexedDB persistence means the cache survives page refreshes. Users see their data immediately, before any network request completes. @@ -49,7 +49,7 @@ features: Every caching solution forces a choice: server-side caches like Redis mean every frontend read is a network round-trip; client-side state like Zustand or SWR means two caches — one on the server and one in every client, with manual staleness code gluing them together. **Recached removes the choice.** -The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. +The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes; values are text, so binary payloads need base64) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. ```typescript import { createCache } from 'recached-edge' diff --git a/docs/roadmap.md b/docs/roadmap.md index 5c2a0bf..7e944a0 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -5,7 +5,7 @@ Recached competes on **where the data can live** — the same engine on the serv ## Near-term 1. **[Byte-transparent values](#byte-transparent-values)** — values are stored as UTF-8 strings, so - raw binary is corrupted on every transport. The largest remaining drop-in gap. + binary payloads are rejected. The largest remaining drop-in gap. --- @@ -75,19 +75,21 @@ Diagnosed on the [benchmarks](/guide/benchmarks) page. ### Byte-transparent values -**Store values as bytes rather than `String`.** `EntryValue::Str` is a `String`, so a value is forced -through a lossy UTF-8 conversion when the command is parsed: `SET k <0xFF 0xFE>` stores two U+FFFD -replacement characters instead. This is a property of the engine, so it applies to **every** -transport — TCP included, not just WebSocket as previously recorded here. +**Store values as bytes rather than `String`.** `EntryValue::Str` is a `String`, so a non-UTF-8 value +cannot be stored faithfully. As of 0.2.2 such a command is **rejected** with an error; before that it +was silently converted to U+FFFD behind a successful `+OK`. This is a property of the engine, so it +applies to **every** transport — TCP included, not just WebSocket as previously recorded here. The practical effect is that compressed blobs, protobuf, images, and anything else Redis users -routinely cache cannot be stored without base64-encoding first. It is the largest remaining gap in -"any Redis client works today". +routinely cache must be base64-encoded first, at ~33% memory overhead. It is the largest remaining +gap in "any Redis client works today" — the rejection makes it honest and visible, not fixed. Closing it means `Vec`/`Bytes` values through `cmd.rs`, `store.rs`, and the collection types; a snapshot format change; and an API decision for the browser SDK, whose `set(key, value)` takes a string. That is a breaking change and wants its own release rather than being folded into a patch. -Current behaviour is pinned by `core-engine/tests/binary_values.rs`, which fails when it changes. +Current behaviour is pinned by `core-engine/tests/binary_values.rs`. When values become +byte-transparent the rejection goes away, which is a behaviour change in the *permissive* direction — +no application that works today can break on it. ### Security diff --git a/docs/server/protocol.md b/docs/server/protocol.md index d0603f5..f5f1b7d 100644 --- a/docs/server/protocol.md +++ b/docs/server/protocol.md @@ -29,11 +29,18 @@ The WebSocket spec requires text frames to be well-formed UTF-8. A command or re that are not valid UTF-8 therefore travels in a **binary** frame instead; everything else stays in text frames, so existing clients are unaffected. A client must accept both. -::: warning Values are not yet byte-transparent -Binary frames make the *transport* byte-clean, but the engine stores values as UTF-8 strings, so a -value containing invalid UTF-8 is still replaced with U+FFFD before it is stored — on **every** -transport, TCP included. Byte-transparent values are a separate, breaking change. Do not store raw -binary (compressed blobs, protobuf, images) in Recached today; base64-encode it or keep it elsewhere. +::: warning Values must be valid UTF-8 +Binary frames make the *transport* byte-clean, but the engine stores values as UTF-8 strings. A +command carrying an argument that is not valid UTF-8 is **rejected** — on every transport, TCP +included — with: + +``` +ERR argument is not valid UTF-8. Recached stores values as text; + base64-encode binary payloads before caching them +``` + +Nothing is stored, and the connection stays usable. Base64-encode binary payloads. Byte-transparent +values are a separate, breaking change — see the [roadmap](/roadmap#byte-transparent-values). ::: ## Frame taxonomy (WebSocket) diff --git a/docs/server/troubleshooting.md b/docs/server/troubleshooting.md index 12db9c8..937dadd 100644 --- a/docs/server/troubleshooting.md +++ b/docs/server/troubleshooting.md @@ -108,6 +108,28 @@ Two separate faults, both fixed in **0.2.2**: On an older server, neither has a client-side workaround — upgrade. +### `ERR argument N is not valid UTF-8` + +The command carried a value (or key) containing bytes that are not valid UTF-8. Recached stores +values as text, so it refuses rather than storing something different from what you sent. Nothing +was written and the connection is still usable. + +Base64-encode the payload: + +```js +await cache.set('blob:1', Buffer.from(binary).toString('base64')); +const binary = Buffer.from(await cache.get('blob:1'), 'base64'); +``` + +The usual cause during a Redis migration is a client that compresses transparently, or a cache of +serialized objects (protobuf, pickle, Java serialization) rather than JSON — the client works +normally right up until the value is not text. See +[Binary values](/guide/introduction#binary-values). + +Before 0.2.2 these values were silently converted to U+FFFD and stored corrupted, with `SET` +returning `OK`. If you are on an older server and reading back mangled binary, that is why — and the +stored data cannot be recovered, since the bytes were destroyed on the way in. + ### `ERR key too large` A key exceeded the maximum key length. Keys are identifiers, not payloads — put the data in the diff --git a/server-native/src/main.rs b/server-native/src/main.rs index b31abcc..db58845 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -4354,6 +4354,28 @@ mod tests { assert_eq!(c.cmd(&["DEL", "k"]).await, int(0)); // already gone } + #[tokio::test] + async fn integration_binary_value_is_refused_over_resp() { + // The drop-in claim runs through this port, so the refusal has to be a + // clean RESP error that leaves the connection usable — not a + // disconnect, and emphatically not a silent lossy store. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n$3\r\n\xff\xfe\x41\r\n"; + c.stream.write_all(raw).await.unwrap(); + let reply = c.cmd(&[]).await; + let Value::Error(e) = &reply else { + panic!("binary value must be refused, got {reply:?}") + }; + assert!(e.contains("base64"), "error must be actionable: {e:?}"); + + // Nothing was stored, and the connection still works. + assert_eq!(c.cmd(&["EXISTS", "bin"]).await, int(0)); + assert_eq!(c.cmd(&["SET", "bin", "ok"]).await, ok()); + assert_eq!(c.cmd(&["GET", "bin"]).await, bulk("ok")); + } + #[tokio::test] async fn integration_hello_negotiates_the_protocol() { let srv = spawn_server().await; @@ -5163,19 +5185,24 @@ mod tests { let srv = spawn_ws_server().await; let mut c = WsClient::connect(srv.tcp_addr).await; - let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n$3\r\n\xff\xfe\x41\r\n".to_vec(); + // A binary frame whose contents are valid UTF-8 is a normal command. + let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n$5\r\nhello\r\n".to_vec(); assert_eq!(c.cmd_binary(raw).await, ok()); - - // The command was accepted and executed — the key exists. - assert_eq!(c.cmd(&["EXISTS", "bin"]).await, int(1)); - - // NOTE: the *value* is still lossy, because the engine stores values as - // `String` (see core-engine/tests/binary_values.rs). Binary frames make - // the transport byte-clean; byte-clean storage is a separate change. - let Value::BulkString(Some(got)) = c.cmd(&["GET", "bin"]).await else { - panic!("expected a value") + assert_eq!(c.cmd(&["GET", "bin"]).await, bulk("hello")); + + // A binary *value* is refused rather than silently corrupted, and + // nothing is stored — the transport accepted the frame, the engine + // declined the payload. + let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nraw\r\n$3\r\n\xff\xfe\x41\r\n".to_vec(); + let reply = c.cmd_binary(raw).await; + let Value::Error(e) = &reply else { + panic!("binary value must be refused, got {reply:?}") }; - assert_eq!(got, b"\xef\xbf\xbd\xef\xbf\xbd\x41".to_vec()); + assert!(e.contains("base64"), "error must be actionable: {e:?}"); + assert_eq!(c.cmd(&["EXISTS", "raw"]).await, int(0)); + + // The connection stays usable after a rejection. + assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From eb588353ded3a3d0d1e74e8c7766b775bd90a3bf Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 20:12:31 +0800 Subject: [PATCH 07/11] Make values byte-transparent across store, replication, AOF, pub/sub and browser WAL; add replication acks with lag metric, RESP3 HELLO negotiation, and binary WebSocket frames --- CHANGELOG.md | 48 ++-- Cargo.lock | 1 + README.md | 2 +- core-engine/Cargo.toml | 5 + core-engine/src/cmd.rs | 203 ++++++++++----- core-engine/src/store.rs | 230 +++++++++++++---- core-engine/tests/binary_values.rs | 153 ++++++++---- core-engine/tests/snapshot_compat.rs | 118 +++++++++ docs/guide/introduction.md | 51 ++-- docs/guide/use-cases.md | 12 +- docs/index.md | 4 +- docs/roadmap.md | 27 +- docs/server/protocol.md | 21 +- docs/server/troubleshooting.md | 23 +- server-native/src/main.rs | 355 ++++++++++++++++++--------- sync-client/src/lib.rs | 54 ++-- wasm-edge/sdk.ts | 34 ++- wasm-edge/src/lib.rs | 258 +++++++++++++------ 18 files changed, 1131 insertions(+), 468 deletions(-) create mode 100644 core-engine/tests/snapshot_compat.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc84b6..5da7ad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,31 +139,44 @@ All notable changes to Recached are documented here. ### Fixed -- **Non-UTF-8 values were silently corrupted; they are now rejected.** Values are stored as `String`, - so a value containing invalid UTF-8 was converted to U+FFFD replacement characters on the way in. - `SET` returned `OK`, `GET` returned bytes that differed from what was written, and nothing anywhere - reported a problem — the original bytes were destroyed at parse time, before storage, so there was - nothing to recover. +- **Values were silently corrupted unless they were valid UTF-8; they are now byte-transparent.** + Values were stored as `String`, so a value containing invalid UTF-8 was converted to U+FFFD + replacement characters on the way in. `SET` returned `OK`, `GET` returned bytes that differed from + what was written, and nothing anywhere reported a problem — the original bytes were destroyed at + parse time, before storage, so there was nothing to recover. This affected **every transport, TCP included** — not only WebSocket, as the roadmap previously recorded. `SET k <0xFF 0xFE 0x41>` over plain RESP came back as `EF BF BD EF BF BD 41`. - Such a command is now refused before anything is stored: + Values are now stored and returned as the exact bytes sent, matching Redis. The change runs the + full depth of the stack: the store's string, list and hash types; command parsing; the RESP + encoder used for replication, AOF and browser sync; pub/sub payloads; and the browser's IndexedDB + write-ahead log. + + **Identifiers stay text.** Keys, hash fields, set and sorted-set members, glob patterns and channel + names must be valid UTF-8, and a command carrying a binary one is refused before anything is + written: ``` - ERR argument 2 is not valid UTF-8. Recached stores values as text; - base64-encode binary payloads before caching them + ERR argument 1 is not valid UTF-8. Keys, fields, members and patterns must be text; + only values may be binary ``` - Keys are checked the same way, and the connection stays usable after a rejection. Nothing that - worked before can break: the affected values were already being destroyed, so no application can - have depended on the old behaviour. + Redis permits binary there too, but those positions are looked up, glob-matched and checked against + sync scopes as text, so a binary identifier would be unreachable through its own access paths. It + is recorded on the [roadmap](docs/roadmap.md) rather than scheduled. + + **Snapshots remain compatible.** A snapshot written by 0.2.1 or earlier still loads: values were + msgpack strings then and are msgpack binary now, and the decoder accepts either. Binary values + encode as msgpack `bin` rather than an array of integers, so snapshot size is unchanged for text + and roughly halved versus the naive encoding for binary. + + **Browser SDK: binary is read-only.** The browser cache stores and syncs binary faithfully and + `cache.getBytes(key)` returns a `Uint8Array`, but `cache.set()` still takes a string — binary + values originate from a backend writing over the RESP port. `cache.get()` now **throws** on a + binary value instead of returning mangled text, and `getJSON()` treats one as a miss. - **Action required if you cache binary payloads** — compressed responses, protobuf, pickled objects, - images — base64-encode them, or keep them elsewhere. Check whether your client compresses - transparently; several do by default. Byte-transparent values are on the - [roadmap](docs/roadmap.md) as a breaking change; this release makes the limitation visible rather - than silent. + Data corrupted by an earlier version cannot be recovered and must be re-populated. - **Pub/sub deliveries never reached a TCP subscriber that only listened.** Deliveries were written into a 32 KB buffered writer and flushed only at the end of a client-command batch, so a @@ -182,7 +195,8 @@ All notable changes to Recached are documented here. text frames to be well-formed UTF-8, which left no way to send bytes at all. Replies are sent as text when the RESP bytes are valid UTF-8 and binary otherwise, so existing clients see no change. - This makes the *transport* byte-clean. Values are a separate matter — see below. + This makes the transport byte-clean; the values travelling over it became byte-transparent in the + same release — see below. - **Exactly-once delivery now survives a server restart.** Dedup high-water marks were held only in memory, so a restart inside the acknowledgement window let a client's replayed write apply twice — diff --git a/Cargo.lock b/Cargo.lock index 7be72c6..f18282f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,6 +130,7 @@ dependencies = [ "indexmap", "js-sys", "rand", + "rmp-serde", "serde", "serde_json", ] diff --git a/README.md b/README.md index 65e88e4..a3e2ee7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Every caching solution forces a choice: server-side caches like Redis mean every frontend read is a network round-trip; client-side state like Zustand or SWR means two caches — one on the server and one in every client, with manual staleness code gluing them together. **Recached removes the choice.** -The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes; values are text, so binary payloads need base64) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. +The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. > [!NOTE] > Recached is not a full Redis replacement. It covers the subset most applications actually need: strings, expiry, counters, all collection types, transactions, pub/sub, and observable keys. Best fit: reactive UIs, session caches, browser-side API response caching, and rate limiting. diff --git a/core-engine/Cargo.toml b/core-engine/Cargo.toml index 990112c..2fbc2d8 100644 --- a/core-engine/Cargo.toml +++ b/core-engine/Cargo.toml @@ -15,3 +15,8 @@ serde_json.workspace = true # crate is compiled into the browser SDK, so it needs the platform clock there. [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys.workspace = true + +[dev-dependencies] +# Snapshot back-compatibility tests need to write a pre-0.2.2 snapshot, where +# values were msgpack strings rather than binary. +rmp-serde.workspace = true diff --git a/core-engine/src/cmd.rs b/core-engine/src/cmd.rs index 5037da7..73871ae 100644 --- a/core-engine/src/cmd.rs +++ b/core-engine/src/cmd.rs @@ -51,22 +51,22 @@ pub enum Command { /// the store never sees it, because the answer depends on the connection. Hello(Option), // ── Strings ────────────────────────────────────────────────────────────── - Set(String, String, SetOptions), + Set(String, Vec, SetOptions), Get(String), /// ESET — a SET whose key is owned by the connection that wrote it. /// The engine stores it like any string; the *server* deletes it when that /// connection closes. Presence, cursors, "who is online". - ESet(String, String), + ESet(String, Vec), Del(Vec), Unlink(Vec), - Append(String, String), + Append(String, Vec), Strlen(String), - GetSet(String, String), + GetSet(String, Vec), MGet(Vec), - MSet(Vec<(String, String)>), - SetNx(String, String), - SetEx(String, u64, String), - PSetEx(String, u64, String), + MSet(Vec<(String, Vec)>), + SetNx(String, Vec), + SetEx(String, u64, Vec), + PSetEx(String, u64, Vec), Incr(String), Decr(String), IncrBy(String, i64), @@ -88,7 +88,7 @@ pub enum Command { Rename(String, String), Type(String), // ── Hash ───────────────────────────────────────────────────────────────── - HSet(String, Vec<(String, String)>), + HSet(String, Vec<(String, Vec)>), HGet(String, String), HGetAll(String), HDel(String, Vec), @@ -98,20 +98,20 @@ pub enum Command { HIncrBy(String, String, i64), HIncrByFloat(String, String, f64), HExists(String, String), - HSetNx(String, String, String), + HSetNx(String, String, Vec), HMGet(String, Vec), // ── List ───────────────────────────────────────────────────────────────── - LPush(String, Vec), - RPush(String, Vec), - LPushX(String, Vec), - RPushX(String, Vec), + LPush(String, Vec>), + RPush(String, Vec>), + LPushX(String, Vec>), + RPushX(String, Vec>), LPop(String, Option), RPop(String, Option), LRange(String, i64, i64), LLen(String), LIndex(String, i64), - LSet(String, i64, String), - LRem(String, i64, String), + LSet(String, i64, Vec), + LRem(String, i64, Vec), LTrim(String, i64, i64), // ── Set ────────────────────────────────────────────────────────────────── SAdd(String, Vec), @@ -167,7 +167,7 @@ pub enum Command { Unsubscribe(Vec), PSubscribe(Vec), PUnsubscribe(Vec), - Publish(String, String), + Publish(String, Vec), // ── Observable keys ─────────────────────────────────────────────────────── Watch(Vec), Unwatch(Vec), @@ -205,28 +205,32 @@ impl Command { if arr.is_empty() { return Err("Empty command".to_string()); } - // Reject the whole command if any argument is not valid UTF-8. - // - // Values are stored as `String`, so a non-UTF-8 byte would - // otherwise be silently replaced with U+FFFD: SET returns OK, - // GET returns different bytes than were written, and nothing - // anywhere reports a problem. Failing loudly here — before - // anything is stored — is the only honest option until values - // are byte-transparent. See docs/roadmap.md. - if let Some(pos) = arr.iter().position( - |v| matches!(v, Value::BulkString(Some(b)) if std::str::from_utf8(b).is_err()), - ) { - return Err(format!( - "ERR argument {pos} is not valid UTF-8. Recached stores values as text; \ - base64-encode binary payloads before caching them" - )); - } let cmd_name = match &arr[0] { Value::BulkString(Some(data)) => String::from_utf8_lossy(data).to_uppercase(), Value::SimpleString(s) => s.to_uppercase(), _ => return Err("Invalid command name type".to_string()), }; + // Payload arguments may be arbitrary bytes; identifier + // arguments may not. + // + // Keys, hash fields, set and sorted-set members, glob patterns + // and channel names are looked up, matched and routed as text, + // so a non-UTF-8 one cannot be handled faithfully. Rather than + // lossily converting it — which returns OK and stores something + // different from what was sent — the command is refused here, + // before anything is written. Values are exempt: they are + // stored verbatim. See `is_payload_index`. + if let Some(pos) = (1..arr.len()).find(|&i| { + !is_payload_index(&cmd_name, i) + && matches!(&arr[i], Value::BulkString(Some(b)) if std::str::from_utf8(b).is_err()) + }) { + return Err(format!( + "ERR argument {pos} is not valid UTF-8. Keys, fields, members and \ + patterns must be text; only values may be binary" + )); + } + macro_rules! need { ($n:expr) => { if arr.len() < $n { @@ -265,7 +269,7 @@ impl Command { "SET" => { need!(3); let key = take_key(&mut arr[1])?; - let val = take_string(&mut arr[2]).unwrap_or_default(); + let val = take_bytes(&mut arr[2]).unwrap_or_default(); let mut opts = SetOptions::default(); let mut i = 3usize; while i < arr.len() { @@ -343,7 +347,7 @@ impl Command { need!(3); Ok(Command::ESet( extract_key(&arr[1])?, - extract_string(&arr[2]).unwrap_or_default(), + extract_bytes(&arr[2]).unwrap_or_default(), )) } "DEL" => { @@ -357,7 +361,7 @@ impl Command { "APPEND" => { need!(3); let key = take_key(&mut arr[1])?; - let val = take_string(&mut arr[2]).unwrap_or_default(); + let val = take_bytes(&mut arr[2]).unwrap_or_default(); Ok(Command::Append(key, val)) } "STRLEN" => { @@ -367,7 +371,7 @@ impl Command { "GETSET" => { need!(3); let key = take_key(&mut arr[1])?; - let val = take_string(&mut arr[2]).unwrap_or_default(); + let val = take_bytes(&mut arr[2]).unwrap_or_default(); Ok(Command::GetSet(key, val)) } "MGET" => { @@ -386,7 +390,7 @@ impl Command { let (k, v) = c.split_at_mut(1); Ok(( take_key(&mut k[0])?, - take_string(&mut v[0]).unwrap_or_default(), + take_bytes(&mut v[0]).unwrap_or_default(), )) }) .collect::, String>>()?; @@ -395,7 +399,7 @@ impl Command { "SETNX" => { need!(3); let key = take_key(&mut arr[1])?; - let val = take_string(&mut arr[2]).unwrap_or_default(); + let val = take_bytes(&mut arr[2]).unwrap_or_default(); Ok(Command::SetNx(key, val)) } "SETEX" => { @@ -407,7 +411,7 @@ impl Command { Ok(Command::SetEx( extract_key(&arr[1])?, secs as u64, - extract_string(&arr[3]).unwrap_or_default(), + extract_bytes(&arr[3]).unwrap_or_default(), )) } "PSETEX" => { @@ -419,7 +423,7 @@ impl Command { Ok(Command::PSetEx( extract_key(&arr[1])?, ms as u64, - extract_string(&arr[3]).unwrap_or_default(), + extract_bytes(&arr[3]).unwrap_or_default(), )) } "INCR" => { @@ -668,7 +672,7 @@ impl Command { let (f, v) = c.split_at_mut(1); ( take_string(&mut f[0]).unwrap_or_default(), - take_string(&mut v[0]).unwrap_or_default(), + take_bytes(&mut v[0]).unwrap_or_default(), ) }) .collect(); @@ -734,7 +738,7 @@ impl Command { Ok(Command::HSetNx( extract_string(&arr[1]).unwrap_or_default(), extract_string(&arr[2]).unwrap_or_default(), - extract_string(&arr[3]).unwrap_or_default(), + extract_bytes(&arr[3]).unwrap_or_default(), )) } "HMGET" => { @@ -748,25 +752,25 @@ impl Command { "LPUSH" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter_mut().filter_map(take_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_bytes).collect(); Ok(Command::LPush(key, vals)) } "RPUSH" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter_mut().filter_map(take_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_bytes).collect(); Ok(Command::RPush(key, vals)) } "LPUSHX" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter_mut().filter_map(take_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_bytes).collect(); Ok(Command::LPushX(key, vals)) } "RPUSHX" => { need!(3); let key = extract_key(&arr[1])?; - let vals = arr[2..].iter_mut().filter_map(take_string).collect(); + let vals = arr[2..].iter_mut().filter_map(take_bytes).collect(); Ok(Command::RPushX(key, vals)) } "LPOP" => { @@ -813,7 +817,7 @@ impl Command { Ok(Command::LSet( extract_string(&arr[1]).unwrap_or_default(), extract_int(&arr[2])?, - extract_string(&arr[3]).unwrap_or_default(), + extract_bytes(&arr[3]).unwrap_or_default(), )) } "LREM" => { @@ -821,7 +825,7 @@ impl Command { Ok(Command::LRem( extract_string(&arr[1]).unwrap_or_default(), extract_int(&arr[2])?, - extract_string(&arr[3]).unwrap_or_default(), + extract_bytes(&arr[3]).unwrap_or_default(), )) } "LTRIM" => { @@ -1128,7 +1132,7 @@ impl Command { need!(3); Ok(Command::Publish( extract_string(&arr[1]).unwrap_or_default(), - extract_string(&arr[2]).unwrap_or_default(), + extract_bytes(&arr[2]).unwrap_or_default(), )) } @@ -1219,6 +1223,43 @@ fn extract_keys(vals: &[Value]) -> Result, String> { /// /// The slot is left as a nil array, so an arm must not read the same index /// twice — call this last, once per argument. +/// Is argument `i` of `cmd_name` a value payload rather than an identifier? +/// +/// Payloads are stored verbatim and may be any bytes. Everything else is used +/// as a lookup key, a glob pattern or a routing name, and must be text. Index 0 +/// is the command name and is never a payload. +fn is_payload_index(cmd_name: &str, i: usize) -> bool { + match cmd_name { + // key value + "SET" | "ESET" | "APPEND" | "GETSET" | "SETNX" => i == 2, + // key ttl value + "SETEX" | "PSETEX" => i == 3, + // key field value + "HSETNX" => i == 3, + // key index value / key count value + "LSET" | "LREM" => i == 3, + // channel message + "PUBLISH" => i == 2, + // key v1 v2 … + "LPUSH" | "RPUSH" | "LPUSHX" | "RPUSHX" => i >= 2, + // key f1 v1 f2 v2 … — values are the odd positions from 3 + "HSET" | "HMSET" => i >= 3 && !i.is_multiple_of(2), + // k1 v1 k2 v2 … — values are the even positions from 2 + "MSET" => i >= 2 && i.is_multiple_of(2), + _ => false, + } +} + +/// Moving counterpart of `take_string` for payload arguments: takes the bytes +/// as they arrived, with no UTF-8 conversion of any kind. +fn take_bytes(val: &mut Value) -> Option> { + match std::mem::replace(val, Value::Array(None)) { + Value::BulkString(Some(data)) => Some(data), + Value::SimpleString(s) => Some(s.into_bytes()), + _ => None, + } +} + fn take_string(val: &mut Value) -> Option { match std::mem::replace(val, Value::Array(None)) { // `from_utf8` reuses the buffer, so this is a move rather than a copy. @@ -1241,6 +1282,16 @@ fn take_key(val: &mut Value) -> Result { Ok(key) } +/// Borrowing counterpart of `take_bytes`, for payload arguments read without +/// consuming the frame. +fn extract_bytes(val: &Value) -> Option> { + match val { + Value::BulkString(Some(data)) => Some(data.clone()), + Value::SimpleString(s) => Some(s.as_bytes().to_vec()), + _ => None, + } +} + fn extract_string(val: &Value) -> Option { match val { Value::BulkString(Some(data)) => Some(String::from_utf8_lossy(data).into_owned()), @@ -2684,7 +2735,7 @@ mod zero_copy_parse_tests { match parse(&["SET", "k", "v", "EX", "60", "NX"]).unwrap() { Command::Set(key, val, opts) => { assert_eq!(key, "k"); - assert_eq!(val, "v"); + assert_eq!(val, b"v"); assert_eq!(opts.expiry, Some(SetExpiry::Ex(60))); assert_eq!(opts.condition, Some(SetCondition::Nx)); } @@ -2699,24 +2750,46 @@ mod zero_copy_parse_tests { match parse(&["SET", "k", &big]).unwrap() { Command::Set(_, val, _) => { assert_eq!(val.len(), big.len()); - assert_eq!(val, big, "payload must be byte-identical after the move"); + assert_eq!( + val, + big.as_bytes(), + "payload must be byte-identical after the move" + ); } other => panic!("{other:?}"), } } #[test] - fn invalid_utf8_fails_the_command_instead_of_being_re_encoded() { - // The move-based parse must not resurrect the lossy fallback: a value - // that cannot be stored faithfully has to fail loudly, not arrive - // half-corrupted behind a successful reply. + fn a_binary_value_reaches_the_command_untouched() { + // The move-based parse must not reintroduce a UTF-8 conversion on the + // value path: the bytes have to arrive exactly as they were sent. + let raw = vec![0xff, 0xfe, b'o', b'k']; let frame = Value::Array(Some(vec![ bulk("SET"), bulk("k"), - Value::BulkString(Some(vec![0xff, 0xfe, b'o', b'k'])), + Value::BulkString(Some(raw.clone())), + ])); + match Command::from_value(frame).expect("a binary value is valid") { + Command::Set(key, val, _) => { + assert_eq!(key, "k"); + assert_eq!(val, raw); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_binary_key_is_still_refused() { + // Keys are matched and routed as text, so this position stays strict + // even though the value beside it does not. + let frame = Value::Array(Some(vec![ + bulk("SET"), + Value::BulkString(Some(vec![0xff, 0xfe])), + bulk("v"), ])); - let err = Command::from_value(frame).expect_err("must be refused"); - assert!(err.contains("not valid UTF-8"), "got {err:?}"); + let err = Command::from_value(frame).expect_err("binary key must be refused"); + assert!(err.contains("must be text"), "got {err:?}"); } #[test] @@ -2726,7 +2799,7 @@ mod zero_copy_parse_tests { match parse(&["SET", "ключ", "日本語 ✓"]).unwrap() { Command::Set(key, val, _) => { assert_eq!(key, "ключ"); - assert_eq!(val, "日本語 ✓"); + assert_eq!(val, "日本語 ✓".as_bytes()); } other => panic!("{other:?}"), } @@ -2740,8 +2813,8 @@ mod zero_copy_parse_tests { Command::MSet(pairs) => assert_eq!( pairs, vec![ - ("a".to_string(), "1".to_string()), - ("b".to_string(), "2".to_string()) + ("a".to_string(), b"1".to_vec()), + ("b".to_string(), b"2".to_vec()) ] ), other => panic!("{other:?}"), @@ -2752,8 +2825,8 @@ mod zero_copy_parse_tests { assert_eq!( pairs, vec![ - ("f1".to_string(), "v1".to_string()), - ("f2".to_string(), "v2".to_string()) + ("f1".to_string(), b"v1".to_vec()), + ("f2".to_string(), b"v2".to_vec()) ] ); } @@ -2766,7 +2839,7 @@ mod zero_copy_parse_tests { match parse(&["RPUSH", "l", "a", "b", "c"]).unwrap() { Command::RPush(key, vals) => { assert_eq!(key, "l"); - assert_eq!(vals, vec!["a", "b", "c"]); + assert_eq!(vals, vec![b"a".to_vec(), b"b".to_vec(), b"c".to_vec()]); } other => panic!("{other:?}"), } diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 47f4619..5f8a873 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -103,13 +103,136 @@ fn in_score_range(score: f64, min: &ScoreBound, max: &ScoreBound) -> bool { above && below } +// ── Byte payloads ───────────────────────────────────────────────────────────── + +/// A stored value's bytes. +/// +/// Values are payloads and may be arbitrary bytes — compressed blobs, protobuf, +/// images. *Identifiers* (keys, hash fields, set and sorted-set members) stay +/// `String`: they are looked up, pattern-matched and scope-checked as text, and +/// making them bytes would spread through the glob matcher, sync scopes and +/// pub/sub routing for no practical gain. +/// +/// The serde impls are hand-written for two reasons. `Vec` serializes as an +/// array of integers under rmp-serde, which would roughly double snapshot size; +/// `serialize_bytes` emits a compact msgpack `bin`. And deserialization accepts +/// *either* a string or bytes, so snapshots written by 0.2.1 and earlier — where +/// values were `String` — still load. +#[derive(Clone, PartialEq, Eq, Hash, Default)] +pub struct Blob(pub Vec); + +impl Blob { + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub fn into_vec(self) -> Vec { + self.0 + } + /// Alias of `into_vec`, mirroring `String::into_bytes` at the many call + /// sites that build a RESP `BulkString` straight from a stored value. + pub fn into_bytes(self) -> Vec { + self.0 + } + /// Append bytes, for `APPEND`. + pub fn extend(&mut self, other: &[u8]) { + self.0.extend_from_slice(other); + } + /// Parse the payload as text, for the commands that require a number. + /// Non-UTF-8 fails the same way non-numeric text does. + pub fn parse_as(&self) -> Option { + self.as_str()?.parse().ok() + } + /// Interpret the payload as text. Commands that need a number or a JSON + /// document (`INCR`, `JSET`, …) go through this and error when it fails, + /// rather than the storage layer refusing the write in the first place. + pub fn as_str(&self) -> Option<&str> { + std::str::from_utf8(&self.0).ok() + } +} + +impl From> for Blob { + fn from(v: Vec) -> Self { + Blob(v) + } +} +impl From<&[u8]> for Blob { + fn from(v: &[u8]) -> Self { + Blob(v.to_vec()) + } +} +impl From for Blob { + fn from(v: String) -> Self { + Blob(v.into_bytes()) + } +} +impl From<&str> for Blob { + fn from(v: &str) -> Self { + Blob(v.as_bytes().to_vec()) + } +} + +impl std::fmt::Debug for Blob { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Text payloads dominate, so print them readably and fall back to hex. + match self.as_str() { + Some(t) => write!(f, "{t:?}"), + None => write!(f, "<{} bytes>", self.0.len()), + } + } +} + +impl Serialize for Blob { + fn serialize(&self, ser: S) -> Result { + ser.serialize_bytes(&self.0) + } +} + +impl<'de> Deserialize<'de> for Blob { + fn deserialize>(de: D) -> Result { + struct V; + impl<'de> serde::de::Visitor<'de> for V { + type Value = Blob; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("bytes or a string") + } + // Pre-0.2.2 snapshots stored values as msgpack strings. + fn visit_str(self, v: &str) -> Result { + Ok(Blob(v.as_bytes().to_vec())) + } + fn visit_string(self, v: String) -> Result { + Ok(Blob(v.into_bytes())) + } + fn visit_bytes(self, v: &[u8]) -> Result { + Ok(Blob(v.to_vec())) + } + fn visit_byte_buf(self, v: Vec) -> Result { + Ok(Blob(v)) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); + while let Some(b) = seq.next_element::()? { + out.push(b); + } + Ok(Blob(out)) + } + } + de.deserialize_any(V) + } +} + // ── Entry value type ────────────────────────────────────────────────────────── #[derive(Clone)] enum EntryValue { - Str(String), - Hash(HashMap), - List(VecDeque), + Str(Blob), + Hash(HashMap), + List(VecDeque), // IndexSet rather than HashSet: SPOP / SRANDMEMBER need O(1) access to a // random member by index, which a hash table cannot provide. Set(IndexSet), @@ -368,17 +491,17 @@ impl Clone for Entry { } impl Entry { - fn new_str(value: String) -> Self { + fn new_str(value: impl Into) -> Self { Self { - value: EntryValue::Str(value), + value: EntryValue::Str(value.into()), expires_at_ms: None, last_access_ms: AtomicU64::new(now_ms()), } } - fn new_str_ex(value: String, expires_at_ms: u64) -> Self { + fn new_str_ex(value: impl Into, expires_at_ms: u64) -> Self { Self { - value: EntryValue::Str(value), + value: EntryValue::Str(value.into()), expires_at_ms: Some(expires_at_ms), last_access_ms: AtomicU64::new(now_ms()), } @@ -516,9 +639,9 @@ pub enum EvictionPolicy { #[derive(Serialize, Deserialize)] pub enum SnapshotValue { - Str(String), - Hash(HashMap), - List(Vec), + Str(Blob), + Hash(HashMap), + List(Vec), Set(Vec), ZSet(Vec<(String, f64)>), // Appended after the original variants: rmp-serde encodes variants by @@ -694,6 +817,9 @@ impl KeyValueStore { fn bulk(s: &str) -> Value { Value::BulkString(Some(s.as_bytes().to_vec())) } + fn blob(b: &Blob) -> Value { + Value::BulkString(Some(b.as_slice().to_vec())) + } let now = now_ms(); match self.data.get(key) { @@ -702,15 +828,15 @@ impl KeyValueStore { Some(e) => match &e.value { EntryValue::Str(s) => Value::BulkString(Some(s.clone().into_bytes())), EntryValue::Hash(m) => { - let mut fields: Vec<(&String, &String)> = m.iter().collect(); + let mut fields: Vec<(&String, &Blob)> = m.iter().collect(); fields.sort_by(|a, b| a.0.cmp(b.0)); let items = fields .into_iter() - .flat_map(|(f, v)| [bulk(f), bulk(v)]) + .flat_map(|(f, v)| [bulk(f), blob(v)]) .collect(); tagged("hash", items) } - EntryValue::List(l) => tagged("list", l.iter().map(|v| bulk(v)).collect()), + EntryValue::List(l) => tagged("list", l.iter().map(blob).collect()), EntryValue::Set(st) => tagged("set", st.iter().map(|m| bulk(m)).collect()), EntryValue::ZSet(z) => { let mut pairs: Vec<(&String, &f64)> = z.scores.iter().collect(); @@ -1012,7 +1138,7 @@ impl KeyValueStore { self.data.insert( key, Entry { - value: EntryValue::Str(val), + value: EntryValue::Str(val.into()), expires_at_ms, last_access_ms: AtomicU64::new(now), }, @@ -1058,12 +1184,12 @@ impl KeyValueStore { .entry(key) .or_insert_with(|| Entry::new_str(String::new())); if was_expired { - entry.value = EntryValue::Str(String::new()); + entry.value = EntryValue::Str(Blob::default()); entry.expires_at_ms = None; } match &mut entry.value { EntryValue::Str(s) => { - s.push_str(&suffix); + s.extend(&suffix); Value::Integer(s.len() as i64) } _ => unreachable!(), @@ -1361,7 +1487,7 @@ impl KeyValueStore { .filter(|(f, _)| !h.contains_key(f.as_str())) .count(); for (field, val) in pairs { - h.insert(field, val); + h.insert(field, val.into()); } Value::Integer(new_count as i64) } @@ -1391,15 +1517,15 @@ impl KeyValueStore { Some(e) => match &e.value { EntryValue::Hash(h) => { e.touch(now); - let mut pairs: Vec<(&str, &str)> = - h.iter().map(|(f, v)| (f.as_str(), v.as_str())).collect(); + let mut pairs: Vec<(&str, &Blob)> = + h.iter().map(|(f, v)| (f.as_str(), v)).collect(); pairs.sort_unstable_by_key(|(f, _)| *f); let out = pairs .into_iter() .flat_map(|(f, v)| { [ Value::BulkString(Some(f.as_bytes().to_vec())), - Value::BulkString(Some(v.as_bytes().to_vec())), + Value::BulkString(Some(v.as_slice().to_vec())), ] }) .collect(); @@ -1453,13 +1579,13 @@ impl KeyValueStore { Some(e) if e.is_expired(now) => Value::Array(Some(vec![])), Some(e) => match &e.value { EntryValue::Hash(h) => { - let mut pairs: Vec<(&str, &str)> = - h.iter().map(|(f, v)| (f.as_str(), v.as_str())).collect(); + let mut pairs: Vec<(&str, &Blob)> = + h.iter().map(|(f, v)| (f.as_str(), v)).collect(); pairs.sort_unstable_by_key(|(f, _)| *f); Value::Array(Some( pairs .into_iter() - .map(|(_, v)| Value::BulkString(Some(v.as_bytes().to_vec()))) + .map(|(_, v)| Value::BulkString(Some(v.as_slice().to_vec()))) .collect(), )) } @@ -1517,7 +1643,7 @@ impl KeyValueStore { _ => unreachable!(), }; if let std::collections::hash_map::Entry::Vacant(e) = h.entry(field) { - e.insert(val); + e.insert(val.into()); Value::Integer(1) } else { Value::Integer(0) @@ -1567,7 +1693,7 @@ impl KeyValueStore { _ => unreachable!(), }; for v in vals { - list.push_front(v); + list.push_front(v.into()); } Value::Integer(list.len() as i64) } @@ -1589,7 +1715,7 @@ impl KeyValueStore { _ => unreachable!(), }; for v in vals { - list.push_back(v); + list.push_back(v.into()); } Value::Integer(list.len() as i64) } @@ -1602,7 +1728,7 @@ impl KeyValueStore { Some(mut e) => match &mut e.value { EntryValue::List(list) => { for v in vals { - list.push_front(v); + list.push_front(v.into()); } Value::Integer(list.len() as i64) } @@ -1619,7 +1745,7 @@ impl KeyValueStore { Some(mut e) => match &mut e.value { EntryValue::List(list) => { for v in vals { - list.push_back(v); + list.push_back(v.into()); } Value::Integer(list.len() as i64) } @@ -1688,13 +1814,13 @@ impl KeyValueStore { Some(e) => match &e.value { EntryValue::List(list) => { e.touch(now); - let slice: Vec<&String> = list.iter().collect(); + let slice: Vec<&Blob> = list.iter().collect(); match resolve_range(start, stop, slice.len()) { None => Value::Array(Some(vec![])), Some((s, e)) => Value::Array(Some( slice[s..=e] .iter() - .map(|v| Value::BulkString(Some(v.as_bytes().to_vec()))) + .map(|v| Value::BulkString(Some(v.as_slice().to_vec()))) .collect(), )), } @@ -1723,9 +1849,9 @@ impl KeyValueStore { Some(e) if e.is_expired(now) => Value::BulkString(None), Some(e) => match &e.value { EntryValue::List(list) => { - let slice: Vec<&String> = list.iter().collect(); + let slice: Vec<&Blob> = list.iter().collect(); resolve_idx(idx, slice.len()) - .map(|i| Value::BulkString(Some(slice[i].as_bytes().to_vec()))) + .map(|i| Value::BulkString(Some(slice[i].as_slice().to_vec()))) .unwrap_or(Value::BulkString(None)) } _ => Value::Error(WRONGTYPE.to_string()), @@ -1744,7 +1870,7 @@ impl KeyValueStore { match resolve_idx(idx, len) { None => Value::Error("ERR index out of range".to_string()), Some(i) => { - list[i] = val; + list[i] = val.into(); Value::SimpleString("OK".to_string()) } } @@ -1766,7 +1892,7 @@ impl KeyValueStore { if count >= 0 { let mut i = 0; while i < list.len() && (count == 0 || removed < abs as i64) { - if list[i] == element { + if list[i].as_slice() == element.as_slice() { list.remove(i); removed += 1; } else { @@ -1777,7 +1903,7 @@ impl KeyValueStore { let mut i = list.len(); while i > 0 && removed < abs as i64 { i -= 1; - if list[i] == element { + if list[i].as_slice() == element.as_slice() { list.remove(i); removed += 1; } @@ -1801,7 +1927,7 @@ impl KeyValueStore { match resolve_range(start, stop, len) { None => list.clear(), Some((s, e)) => { - let trimmed: VecDeque = list.drain(s..=e).collect(); + let trimmed: VecDeque = list.drain(s..=e).collect(); *list = trimmed; } } @@ -2615,16 +2741,18 @@ fn incr_by(data: &DashMap, key: String, delta: i64) -> Value { .entry(key) .or_insert_with(|| Entry::new_str("0".to_string())); if was_expired { - entry.value = EntryValue::Str("0".to_string()); + entry.value = EntryValue::Str("0".into()); entry.expires_at_ms = None; } match &mut entry.value { - EntryValue::Str(s) => match s.parse::() { - Err(_) => Value::Error("ERR value is not an integer or out of range".to_string()), - Ok(n) => match n.checked_add(delta) { + // A non-UTF-8 value fails here exactly as non-numeric text does: the + // bytes are stored faithfully, they are simply not a number. + EntryValue::Str(s) => match s.parse_as::() { + None => Value::Error("ERR value is not an integer or out of range".to_string()), + Some(n) => match n.checked_add(delta) { None => Value::Error("ERR increment or decrement would overflow".to_string()), Some(new) => { - *s = new.to_string(); + *s = new.to_string().into(); Value::Integer(new) } }, @@ -2813,11 +2941,11 @@ fn hash_incr_int(data: &DashMap, key: String, field: String, delt EntryValue::Hash(h) => h, _ => unreachable!(), }; - let cur: i64 = h.get(&field).and_then(|s| s.parse().ok()).unwrap_or(0); + let cur: i64 = h.get(&field).and_then(|s| s.parse_as()).unwrap_or(0); match cur.checked_add(delta) { None => Value::Error("ERR increment or decrement would overflow".to_string()), Some(new) => { - h.insert(field, new.to_string()); + h.insert(field, new.to_string().into()); Value::Integer(new) } } @@ -2846,13 +2974,13 @@ fn hash_incr_float(data: &DashMap, key: String, field: String, de EntryValue::Hash(h) => h, _ => unreachable!(), }; - let cur: f64 = h.get(&field).and_then(|s| s.parse().ok()).unwrap_or(0.0); + let cur: f64 = h.get(&field).and_then(|s| s.parse_as()).unwrap_or(0.0); let new = cur + delta; if new.is_nan() || new.is_infinite() { return Value::Error("ERR increment would produce NaN or Infinity".to_string()); } let new_str = format_score(new); - h.insert(field, new_str.clone()); + h.insert(field, new_str.clone().into()); Value::BulkString(Some(new_str.into_bytes())) } @@ -4696,9 +4824,9 @@ mod capacity_tests { let base = s.approximate_memory_bytes(); s.execute(Command::HSet( "h".into(), - vec![("f".into(), "v".repeat(500))], + vec![("f".into(), "v".repeat(500).into())], )); - s.execute(Command::LPush("l".into(), vec!["v".repeat(500)])); + s.execute(Command::LPush("l".into(), vec!["v".repeat(500).into()])); s.execute(Command::SAdd("st".into(), vec!["v".repeat(500)])); s.execute(Command::ZAdd( "z".into(), @@ -4826,7 +4954,7 @@ mod capacity_tests { for (k, ttl) in [("keep", 600_000u64), ("drop", 1_000)] { s.execute(Command::Set( k.into(), - "x".repeat(400), + "x".repeat(400).into(), SetOptions { expiry: Some(crate::cmd::SetExpiry::Px(ttl)), ..Default::default() @@ -5279,7 +5407,7 @@ mod critical_path_tests { let s = KeyValueStore::new(); s.execute(Command::Set( "n".into(), - i64::MAX.to_string(), + i64::MAX.to_string().into(), SetOptions::default(), )); let r = s.execute(Command::Incr("n".into())); @@ -5299,7 +5427,7 @@ mod critical_path_tests { let s = KeyValueStore::new(); s.execute(Command::Set( "n".into(), - i64::MIN.to_string(), + i64::MIN.to_string().into(), SetOptions::default(), )); assert!(matches!( @@ -5529,7 +5657,7 @@ mod metrics_tests { let empty = s.approximate_memory_bytes(); s.execute(Command::Set( "k".into(), - "x".repeat(4096), + "x".repeat(4096).into(), SetOptions::default(), )); assert!(s.approximate_memory_bytes() >= empty + 4096); diff --git a/core-engine/tests/binary_values.rs b/core-engine/tests/binary_values.rs index a65e455..4c4cc39 100644 --- a/core-engine/tests/binary_values.rs +++ b/core-engine/tests/binary_values.rs @@ -1,60 +1,134 @@ -//! Values that are not valid UTF-8 are rejected, not silently corrupted. +//! Values are byte-transparent; identifiers are text. //! -//! `EntryValue::Str` is a `String`, so a non-UTF-8 value cannot be stored -//! faithfully. Until values are byte-transparent (see docs/roadmap.md), the -//! command is refused at parse time — before anything is written — rather than -//! being lossily converted to U+FFFD behind a successful `+OK`. +//! A stored value may be arbitrary bytes — compressed blobs, protobuf, images — +//! and must come back exactly as it went in. Keys, hash fields, set and +//! sorted-set members and glob patterns are looked up and matched as text, so a +//! non-UTF-8 one is refused rather than lossily converted. use core_engine::{cmd::Command, resp::Value, store::KeyValueStore}; -/// `SET k <0xFF 0xFE 'A'>` — invalid UTF-8 in the value position. -const SET_BINARY_VALUE: &[u8] = b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$3\r\n\xff\xfe\x41\r\n"; +/// Invalid UTF-8 in any position: a lone continuation byte and a truncated +/// sequence, plus an embedded NUL and a byte that is legal only inside one. +const BINARY: &[u8] = &[0xff, 0xfe, 0x00, 0x41, 0x80, 0xc3]; + +fn parse(raw: &[u8]) -> Result { + let (v, _) = Value::parse(raw).unwrap(); + Command::from_value(v) +} + +/// Build a RESP array frame from raw byte arguments. +fn frame(args: &[&[u8]]) -> Vec { + let mut out = format!("*{}\r\n", args.len()).into_bytes(); + for a in args { + out.extend_from_slice(format!("${}\r\n", a.len()).as_bytes()); + out.extend_from_slice(a); + out.extend_from_slice(b"\r\n"); + } + out +} + +#[test] +fn a_binary_value_round_trips_byte_for_byte() { + let store = KeyValueStore::new(); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); + + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing after SET"); + }; + assert_eq!(got, BINARY, "value must survive unchanged"); +} #[test] -fn non_utf8_values_are_rejected() { - let (v, _) = Value::parse(SET_BINARY_VALUE).unwrap(); - let err = Command::from_value(v).expect_err("binary value must be refused"); - assert!(err.starts_with("ERR "), "must be a RESP error: {err:?}"); - assert!( - err.contains("base64"), - "the error must say what to do instead: {err:?}" +fn binary_values_work_in_lists_and_hashes() { + let store = KeyValueStore::new(); + + store.execute(parse(&frame(&[b"RPUSH", b"l", BINARY, b"plain"])).unwrap()); + let Value::Array(Some(items)) = store.execute(Command::LRange("l".into(), 0, -1)) else { + panic!("list missing"); + }; + assert_eq!(items[0], Value::BulkString(Some(BINARY.to_vec()))); + + store.execute(parse(&frame(&[b"HSET", b"h", b"f", BINARY])).unwrap()); + assert_eq!( + store.execute(Command::HGet("h".into(), "f".into())), + Value::BulkString(Some(BINARY.to_vec())) ); } #[test] -fn a_rejected_command_stores_nothing() { - // The dangerous failure was a silent partial success: OK returned, wrong - // bytes stored. Nothing may reach the store. +fn append_concatenates_bytes_rather_than_text() { let store = KeyValueStore::new(); - let (v, _) = Value::parse(SET_BINARY_VALUE).unwrap(); - assert!(Command::from_value(v).is_err()); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); + store.execute(parse(&frame(&[b"APPEND", b"k", BINARY])).unwrap()); + + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing"); + }; + assert_eq!(got.len(), BINARY.len() * 2); + assert_eq!(&got[..BINARY.len()], BINARY); + assert_eq!(&got[BINARY.len()..], BINARY); +} + +#[test] +fn strlen_counts_bytes_not_characters() { + let store = KeyValueStore::new(); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); assert_eq!( - store.execute(Command::Get("k".into())), - Value::BulkString(None) + store.execute(Command::Strlen("k".into())), + Value::Integer(BINARY.len() as i64) ); } #[test] -fn non_utf8_keys_are_rejected_too() { - // A key is as lossy as a value, and a corrupted key is unreachable - // afterwards — the write would be silently unretrievable. - let raw: &[u8] = b"*3\r\n$3\r\nSET\r\n$2\r\n\xff\xfe\r\n$1\r\nv\r\n"; - let (v, _) = Value::parse(raw).unwrap(); - assert!(Command::from_value(v).is_err()); +fn incr_on_a_binary_value_errors_like_any_non_numeric_value() { + // The bytes are stored faithfully; they are simply not a number. This must + // read as a type error, not as corruption. + let store = KeyValueStore::new(); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); + let Value::Error(e) = store.execute(Command::Incr("k".into())) else { + panic!("INCR on binary must error"); + }; + assert!(e.contains("not an integer"), "got {e:?}"); +} + +#[test] +fn a_binary_key_is_rejected() { + // Keys are matched by glob and checked against sync scopes as text, so a + // corrupted key would be silently unretrievable. + let err = parse(&frame(&[b"SET", BINARY, b"v"])).expect_err("binary key must be refused"); + assert!(err.starts_with("ERR "), "{err:?}"); + assert!(err.contains("must be text"), "{err:?}"); +} + +#[test] +fn binary_fields_and_members_are_rejected() { + for args in [ + vec![b"HSET".as_slice(), b"h", BINARY, b"v"], // hash field + vec![b"SADD".as_slice(), b"s", BINARY], // set member + vec![b"ZADD".as_slice(), b"z", b"1", BINARY], // zset member + vec![b"KEYS".as_slice(), BINARY], // glob pattern + ] { + let err = parse(&frame(&args)).expect_err("identifier must be refused"); + assert!(err.contains("must be text"), "{:?} -> {err:?}", args[0]); + } } #[test] fn the_error_names_which_argument_was_bad() { - // MSET k1 v1 k2 — the bad argument is index 4, not the first. - let raw: &[u8] = b"*5\r\n$4\r\nMSET\r\n$2\r\nk1\r\n$2\r\nv1\r\n$2\r\nk2\r\n$2\r\n\xff\xfe\r\n"; - let (v, _) = Value::parse(raw).unwrap(); - let err = Command::from_value(v).expect_err("must be refused"); - assert!(err.contains("argument 4"), "got {err:?}"); + // MSET k1 v1 v2 — index 3 is a key, so it is refused. + let err = parse(&frame(&[b"MSET", b"k1", b"v1", BINARY, b"v2"])).expect_err("must be refused"); + assert!(err.contains("argument 3"), "got {err:?}"); +} + +#[test] +fn a_rejected_command_stores_nothing() { + let store = KeyValueStore::new(); + assert!(parse(&frame(&[b"SET", BINARY, b"v"])).is_err()); + assert_eq!(store.execute(Command::DbSize), Value::Integer(0)); } #[test] fn utf8_values_survive_unchanged() { - // Multi-byte UTF-8 is valid and must not be caught by the new check. let store = KeyValueStore::new(); for value in ["héllo ✓", "日本語", "\u{1F600}", "", "plain"] { store.execute(Command::Set("k".into(), value.into(), Default::default())); @@ -64,16 +138,3 @@ fn utf8_values_survive_unchanged() { assert_eq!(String::from_utf8(got).unwrap(), value); } } - -#[test] -fn utf8_values_still_parse_from_the_wire() { - // Guards against the validation being over-eager and rejecting valid input. - let raw = "*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$9\r\n日本語\r\n".as_bytes(); - let (v, _) = Value::parse(raw).unwrap(); - let store = KeyValueStore::new(); - store.execute(Command::from_value(v).expect("valid UTF-8 must be accepted")); - assert_eq!( - store.execute(Command::Get("k".into())), - Value::BulkString(Some("日本語".as_bytes().to_vec())) - ); -} diff --git a/core-engine/tests/snapshot_compat.rs b/core-engine/tests/snapshot_compat.rs new file mode 100644 index 0000000..59f814e --- /dev/null +++ b/core-engine/tests/snapshot_compat.rs @@ -0,0 +1,118 @@ +//! A snapshot written before values became bytes must still load. +//! +//! `SnapshotValue` held `String` up to 0.2.1 and holds `Blob` from 0.2.2. +//! rmp-serde encodes those differently — msgpack `str` versus `bin` — so +//! `Blob`'s deserializer accepts either. Without that, upgrading a server would +//! silently start from an empty cache, or fail to boot. + +use core_engine::store::{KeyValueStore, SnapshotEntry, SnapshotValue}; +use serde::Serialize; +use std::collections::HashMap; + +/// Mirror of the pre-0.2.2 `SnapshotValue`, used to produce a genuine old-format +/// payload rather than a hand-rolled byte string. Variant order matters: +/// rmp-serde encodes variants by index. +#[derive(Serialize)] +#[allow(dead_code)] // variants exist to fix the discriminant order, not to be built +enum LegacySnapshotValue { + Str(String), + Hash(HashMap), + List(Vec), + Set(Vec), + ZSet(Vec<(String, f64)>), + RateLimiter { + limit: u64, + window_ms: u64, + events: Vec, + }, + Json(String), +} + +#[derive(Serialize)] +struct LegacyEntry { + key: String, + value: LegacySnapshotValue, + expires_at_ms: Option, +} + +#[test] +fn a_pre_0_2_2_snapshot_still_restores() { + let legacy = vec![ + LegacyEntry { + key: "s".into(), + value: LegacySnapshotValue::Str("hello".into()), + expires_at_ms: None, + }, + LegacyEntry { + key: "l".into(), + value: LegacySnapshotValue::List(vec!["a".into(), "b".into()]), + expires_at_ms: None, + }, + LegacyEntry { + key: "h".into(), + value: LegacySnapshotValue::Hash(HashMap::from([("f".to_string(), "v".to_string())])), + expires_at_ms: None, + }, + ]; + let bytes = rmp_serde::to_vec(&legacy).expect("legacy snapshot must encode"); + + // Decode with the *current* types — this is what a restarted server does. + let entries: Vec = + rmp_serde::from_slice(&bytes).expect("a pre-0.2.2 snapshot must still decode"); + + let store = KeyValueStore::new(); + store.restore(entries); + + use core_engine::{cmd::Command, resp::Value}; + assert_eq!( + store.execute(Command::Get("s".into())), + Value::BulkString(Some(b"hello".to_vec())) + ); + assert_eq!( + store.execute(Command::HGet("h".into(), "f".into())), + Value::BulkString(Some(b"v".to_vec())) + ); + assert_eq!(store.execute(Command::LLen("l".into())), Value::Integer(2)); +} + +#[test] +fn binary_values_survive_a_snapshot_round_trip() { + let binary = vec![0xff, 0xfe, 0x00, 0x41, 0x80]; + let store = KeyValueStore::new(); + store.restore(vec![SnapshotEntry { + key: "b".into(), + value: SnapshotValue::Str(binary.clone().into()), + expires_at_ms: None, + }]); + + let bytes = rmp_serde::to_vec(&store.snapshot()).unwrap(); + let entries: Vec = rmp_serde::from_slice(&bytes).unwrap(); + + let restored = KeyValueStore::new(); + restored.restore(entries); + + use core_engine::{cmd::Command, resp::Value}; + assert_eq!( + restored.execute(Command::Get("b".into())), + Value::BulkString(Some(binary)), + "binary must survive snapshot and restore" + ); +} + +#[test] +fn a_binary_value_encodes_as_msgpack_bin_not_an_int_array() { + // Vec serializes as an array of integers by default, which would roughly + // double snapshot size for binary payloads. Blob emits a compact `bin`. + let store = KeyValueStore::new(); + store.restore(vec![SnapshotEntry { + key: "b".into(), + value: SnapshotValue::Str(vec![0xffu8; 1000].into()), + expires_at_ms: None, + }]); + let bytes = rmp_serde::to_vec(&store.snapshot()).unwrap(); + assert!( + bytes.len() < 1200, + "1000 bytes encoded to {} — likely an int array, not msgpack bin", + bytes.len() + ); +} diff --git a/docs/guide/introduction.md b/docs/guide/introduction.md index f771f8b..64c2c58 100644 --- a/docs/guide/introduction.md +++ b/docs/guide/introduction.md @@ -4,10 +4,8 @@ Recached is an in-memory cache server written in Rust. It speaks RESP (the Redis Serialization Protocol) on port 6379, so any Redis client — `ioredis`, `node-redis`, `redis-py`, `Jedis` — works against it today with no code changes. -One caveat worth knowing before you start: **values are text, not arbitrary bytes.** A value that is -not valid UTF-8 is rejected with an error rather than stored, so binary payloads — compressed blobs, -protobuf, pickled objects, images — must be base64-encoded first. See -[Binary values](#binary-values) below. +Values are binary-safe, as they are in Redis: a value is stored and returned as the exact bytes you +sent. *Keys* and other identifiers must be text — see [Binary values](#binary-values). That is where the similarity with Redis ends. @@ -47,7 +45,7 @@ Recached is a good fit when: - **You want live UI without polling.** The WebSocket sync replaces a polling loop without requiring you to build a separate SSE or WebSocket server. - **You want a frontend-only cache with TTL.** The WASM module works entirely without a server. Call `createCache()` without `connect()` and you get a local in-memory cache with built-in TTL — no Recached server, no Redis, no backend changes required. - **You need cross-tab sync.** BroadcastChannel support means all open tabs in the same browser share mutations automatically. -- **You want a drop-in Redis replacement** for the subset of commands most applications actually use (strings, expiry, counters, collections, transactions, pub/sub) — and your values are text or JSON rather than binary blobs. +- **You want a drop-in Redis replacement** for the subset of commands most applications actually use (strings, expiry, counters, collections, transactions, pub/sub). ## When Recached is not the right fit @@ -55,36 +53,41 @@ Recached is a good fit when: - **You need multi-replica consensus failover.** Recached supports leader–follower replication with automatic single-replica failover (`RECACHED_FAILOVER_TIMEOUT`). If the primary is unreachable for the configured duration, the designated replica promotes itself. What it does not include is multi-replica quorum election: in a setup with several replicas, split-brain prevention requires you to designate one replica for auto-failover and keep the others as passive standbys. - **You depend on uncommon Redis commands.** Recached implements the commands most applications use, not all 250+. Server introspection (`INFO`, `SLOWLOG`, `COMMAND`), Lua scripting, and cluster mode are out of scope. RESP3 is supported for protocol negotiation and pub/sub delivery (`HELLO 3`), not for the full RESP3 type surface. - **You need very large datasets.** Recached is an in-memory cache — it is not a database. If your working set does not fit in RAM, Redis with RDB persistence or a proper database is the right tool. -- **You cache binary payloads.** Values are text. See below. ## Binary values -**Values must be valid UTF-8.** A command carrying a value that is not — a gzip blob, protobuf, -a pickled Python object, an image — is rejected with an error: +**Values are binary-safe.** A value is stored and returned as the exact bytes you sent — compressed +payloads, protobuf, images, serialized objects — with no encoding step and no size penalty. + +**Identifiers must be text.** Keys, hash fields, set and sorted-set members, glob patterns and +pub/sub channel names must be valid UTF-8, and a command carrying a binary one is rejected: ``` -ERR argument 2 is not valid UTF-8. Recached stores values as text; - base64-encode binary payloads before caching them +ERR argument 1 is not valid UTF-8. Keys, fields, members and patterns must be text; + only values may be binary ``` -Nothing is stored when this happens, and the connection stays usable. Keys are checked the same way. - -This is a real difference from Redis, where values are binary-safe. It matters most if you are -migrating a cache that stores compressed responses or serialized objects, since many client -libraries compress transparently — check your client's configuration before assuming your values are -text. +Nothing is stored when this happens and the connection stays usable. This is narrower than Redis, +where keys are binary-safe too — but keys are looked up, glob-matched and checked against sync scopes +as text, and a binary key would be unreachable through those paths. Keys are identifiers in practice, +so this is rarely felt. -The workaround is base64, at roughly 33% extra memory and a small CPU cost on both ends. JSON, -strings, numbers, and counters are unaffected, which covers most of what Recached is used for. +Commands that interpret a value still require the right shape: `INCR` on a binary value returns +`ERR value is not an integer`, and JSON documents must be UTF-8 because JSON is defined that way. +Those are type errors, not encoding losses — the stored bytes are unchanged either way. -::: tip Why an error rather than best-effort storage -The engine stores values as UTF-8 strings, so a binary value cannot be kept faithfully. Before 0.2.2 -it was silently converted — invalid bytes became U+FFFD, `SET` still returned `OK`, and `GET` -returned data that differed from what was written, with no signal anywhere that anything had -happened. Failing loudly is worse ergonomics and much better behaviour. Byte-transparent values are -on the [roadmap](/roadmap#byte-transparent-values) as a breaking change. +::: warning Browser SDK: binary is read-only +The browser cache stores and syncs binary values faithfully, and `cache.getBytes(key)` returns them +as a `Uint8Array`. But `cache.set()` takes a string, so binary values can only *originate* from a +backend writing over the RESP port. `cache.get()` throws on a binary value rather than returning +mangled text. ::: +Before 0.2.2 values were stored as UTF-8 strings and binary was silently replaced with U+FFFD: `SET` +returned `OK` and `GET` returned different bytes than were written, on every transport. If you are +upgrading from an earlier version, data already corrupted that way cannot be recovered — the bytes +were destroyed on the way in. + ## Maturity Honest status, per layer: diff --git a/docs/guide/use-cases.md b/docs/guide/use-cases.md index 50f3b5f..dae0856 100644 --- a/docs/guide/use-cases.md +++ b/docs/guide/use-cases.md @@ -92,10 +92,6 @@ tier. explicitly out of scope; see [Commands](/server/commands). RESP3 exists only for protocol negotiation and pub/sub framing (`HELLO 3`), not the full type surface. -**You cache binary payloads → Redis.** Redis values are binary-safe; Recached values must be valid -UTF-8 and a binary value is rejected outright. Compressed responses, protobuf, and serialized -objects need base64 first — see [Binary values](/guide/introduction#binary-values). - ## Compared directly | | Recached | Redis | Memcached | @@ -144,11 +140,9 @@ Before switching a workload over, check: 1. **Command coverage.** Diff your actual command usage against [Commands](/server/commands). `MONITOR` on your existing Redis for a representative window is the fastest way to get that list. Lua scripts, cluster commands, streams, and `INFO`-based tooling will not carry over. -2. **Value encoding.** Recached values must be valid UTF-8 — binary is rejected, not stored. Check - whether your client compresses transparently (many do), and whether anything caches serialized - objects rather than JSON. This is the migration surprise most likely to bite, because the client - works perfectly right up until the value is not text. See - [Binary values](/guide/introduction#binary-values). +2. **Key encoding.** Values are binary-safe, but keys, hash fields, set members and glob patterns + must be valid UTF-8 — Redis allows binary there. Applications that use binary keys are rare; if + yours does, that is a blocker. See [Binary values](/guide/introduction#binary-values). 3. **Persistence expectations.** Confirm snapshot + AOF semantics match what you assume today. 4. **Replication topology.** Single-replica auto-failover only; no Sentinel or quorum election. 5. **Eviction.** Review the key cap and TTL behaviour in diff --git a/docs/index.md b/docs/index.md index 75943d9..77f6682 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,7 +33,7 @@ features: details: Any mutation on the server is pushed to all connected browser instances instantly. Any write from the browser is pushed to the server and fanned out to other tabs. - icon: 🦀 title: Redis-compatible server - details: Speaks RESP on port 6379. Drop it in front of any Redis client — ioredis, node-redis, redis-py — with no code changes. Values are text; base64-encode binary payloads. + details: Speaks RESP on port 6379. Drop it in front of any Redis client — ioredis, node-redis, redis-py — with no code changes. - icon: 🌐 title: Offline-first browser cache details: IndexedDB persistence means the cache survives page refreshes. Users see their data immediately, before any network request completes. @@ -49,7 +49,7 @@ features: Every caching solution forces a choice: server-side caches like Redis mean every frontend read is a network round-trip; client-side state like Zustand or SWR means two caches — one on the server and one in every client, with manual staleness code gluing them together. **Recached removes the choice.** -The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes; values are text, so binary payloads need base64) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. +The same Rust cache engine runs natively on your server (RESP on port 6379 — any Redis client works today, zero code changes) and as WebAssembly inside the browser. Reads always come from local WASM memory. The WebSocket is only a sync path, not a read path. ```typescript import { createCache } from 'recached-edge' diff --git a/docs/roadmap.md b/docs/roadmap.md index 7e944a0..cbfa76a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,10 +2,6 @@ Recached competes on **where the data can live** — the same engine on the server and in the browser, with sync in between. The [benchmarks](/guide/benchmarks) show this costs nothing in raw speed. -## Near-term - -1. **[Byte-transparent values](#byte-transparent-values)** — values are stored as UTF-8 strings, so - binary payloads are rejected. The largest remaining drop-in gap. --- @@ -73,23 +69,16 @@ independent — pick them off in any order. **Serialize `LRANGE` straight from the store**, instead of building the full reply `Value` first. Diagnosed on the [benchmarks](/guide/benchmarks) page. -### Byte-transparent values - -**Store values as bytes rather than `String`.** `EntryValue::Str` is a `String`, so a non-UTF-8 value -cannot be stored faithfully. As of 0.2.2 such a command is **rejected** with an error; before that it -was silently converted to U+FFFD behind a successful `+OK`. This is a property of the engine, so it -applies to **every** transport — TCP included, not just WebSocket as previously recorded here. +### Binary keys -The practical effect is that compressed blobs, protobuf, images, and anything else Redis users -routinely cache must be base64-encoded first, at ~33% memory overhead. It is the largest remaining -gap in "any Redis client works today" — the rejection makes it honest and visible, not fixed. +**Keys are text; values are not.** Values became byte-transparent in 0.2.2, but keys, hash fields, +set and sorted-set members, glob patterns and channel names must still be valid UTF-8. Redis allows +binary in all of those positions. -Closing it means `Vec`/`Bytes` values through `cmd.rs`, `store.rs`, and the collection types; a -snapshot format change; and an API decision for the browser SDK, whose `set(key, value)` takes a -string. That is a breaking change and wants its own release rather than being folded into a patch. -Current behaviour is pinned by `core-engine/tests/binary_values.rs`. When values become -byte-transparent the rejection goes away, which is a behaviour change in the *permissive* direction — -no application that works today can break on it. +Closing the gap means carrying bytes through the store's key map, the glob matcher, sync-scope prefix +matching, live-query patterns and pub/sub routing — several of which are security-relevant and were +hardened separately. The practical demand is low, since keys are identifiers in every workload seen +so far, so this is recorded rather than scheduled. ### Security diff --git a/docs/server/protocol.md b/docs/server/protocol.md index f5f1b7d..7472145 100644 --- a/docs/server/protocol.md +++ b/docs/server/protocol.md @@ -29,18 +29,15 @@ The WebSocket spec requires text frames to be well-formed UTF-8. A command or re that are not valid UTF-8 therefore travels in a **binary** frame instead; everything else stays in text frames, so existing clients are unaffected. A client must accept both. -::: warning Values must be valid UTF-8 -Binary frames make the *transport* byte-clean, but the engine stores values as UTF-8 strings. A -command carrying an argument that is not valid UTF-8 is **rejected** — on every transport, TCP -included — with: - -``` -ERR argument is not valid UTF-8. Recached stores values as text; - base64-encode binary payloads before caching them -``` - -Nothing is stored, and the connection stays usable. Base64-encode binary payloads. Byte-transparent -values are a separate, breaking change — see the [roadmap](/roadmap#byte-transparent-values). +::: tip Values are binary-safe; identifiers are not +A value is stored and returned as the exact bytes sent. Keys, hash fields, set and sorted-set +members, glob patterns and channel names must be valid UTF-8 — they are looked up, matched and routed +as text — and a command carrying a binary one is rejected with +`ERR argument is not valid UTF-8. Keys, fields, members and patterns must be text; only values +may be binary`. Nothing is stored, and the connection stays usable. + +Before 0.2.2 values were stored as UTF-8 strings and binary was silently replaced with U+FFFD on +every transport. ::: ## Frame taxonomy (WebSocket) diff --git a/docs/server/troubleshooting.md b/docs/server/troubleshooting.md index 937dadd..044a9ec 100644 --- a/docs/server/troubleshooting.md +++ b/docs/server/troubleshooting.md @@ -110,27 +110,22 @@ On an older server, neither has a client-side workaround — upgrade. ### `ERR argument N is not valid UTF-8` -The command carried a value (or key) containing bytes that are not valid UTF-8. Recached stores -values as text, so it refuses rather than storing something different from what you sent. Nothing +A key, hash field, set or sorted-set member, or glob pattern contained bytes that are not valid +UTF-8. **Values are binary-safe** — this error only ever refers to an identifier position. Nothing was written and the connection is still usable. -Base64-encode the payload: +Identifiers are looked up, glob-matched and checked against sync scopes as text, so a binary one +would be unreachable through those paths. Hex- or base64-encode the identifier: ```js -await cache.set('blob:1', Buffer.from(binary).toString('base64')); -const binary = Buffer.from(await cache.get('blob:1'), 'base64'); +await cache.set(`blob:${id.toString('hex')}`, binaryPayload); // value stays raw ``` -The usual cause during a Redis migration is a client that compresses transparently, or a cache of -serialized objects (protobuf, pickle, Java serialization) rather than JSON — the client works -normally right up until the value is not text. See -[Binary values](/guide/introduction#binary-values). +Before 0.2.2 *values* were also required to be text and binary was silently corrupted. If you are +reading back mangled binary written by an older server, that data cannot be recovered — the bytes +were destroyed on the way in. Re-populate the affected keys. -Before 0.2.2 these values were silently converted to U+FFFD and stored corrupted, with `SET` -returning `OK`. If you are on an older server and reading back mangled binary, that is why — and the -stored data cannot be recovered, since the bytes were destroyed on the way in. - -### `ERR key too large` +### `ERR key too large`### `ERR key too large` A key exceeded the maximum key length. Keys are identifiers, not payloads — put the data in the value. diff --git a/server-native/src/main.rs b/server-native/src/main.rs index db58845..6c8c3c4 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -522,9 +522,9 @@ impl AofWriter { }) } - async fn append(&self, resp: &str) { + async fn append(&self, resp: &[u8]) { let mut f = self.file.lock().await; - if f.write_all(resp.as_bytes()).await.is_err() { + if f.write_all(resp).await.is_err() { warn!("AOF write failed"); return; } @@ -798,14 +798,14 @@ impl ServerState { } /// Called after every successful write: appends to AOF and fans out to replicas. - async fn on_write(&self, resp: &str) { + async fn on_write(&self, resp: &[u8]) { if let Some(aof) = &self.aof { aof.append(resp).await; } if self.replicas.is_empty() { return; } - self.replicas.fan_out(resp.as_bytes().to_vec()).await; + self.replicas.fan_out(resp.to_vec()).await; } /// Path of the dedup sidecar, alongside the snapshot. @@ -1231,13 +1231,12 @@ async fn sync_from_primary( // Relay the applied write so this replica's own WebSocket // clients see it, and any sub-replicas / AOF get it too // (enables multi-tier replication and replica WS push). - let frame = String::from_utf8_lossy(&cmd_bytes).into_owned(); let _ = tx.send(Arc::new(SyncPush { origin: 0, keys, - resp: frame.clone(), + resp: cmd_bytes.clone(), })); - state.on_write(&frame).await; + state.on_write(&cmd_bytes).await; } } Err(e) => warn!("Replica: bad command from primary: {}", e), @@ -1274,7 +1273,7 @@ fn ct_eq_bytes(a: &[u8], b: &[u8]) -> bool { struct SyncPush { origin: u64, keys: Vec, - resp: String, + resp: Vec, } type SyncMsg = Arc; @@ -1549,12 +1548,12 @@ fn next_conn_id() -> u64 { enum PubSubMsg { Message { channel: String, - message: String, + message: Vec, }, PMessage { pattern: String, channel: String, - message: String, + message: Vec, }, } @@ -1607,7 +1606,7 @@ impl PubSubHub { } /// Deliver to all matching subscribers; returns the count delivered. - fn publish(&mut self, channel: &str, message: &str) -> i64 { + fn publish(&mut self, channel: &str, message: &[u8]) -> i64 { let mut count = 0i64; if let std::collections::hash_map::Entry::Occupied(mut e) = @@ -1618,7 +1617,7 @@ impl PubSubHub { let ok = tx .send(PubSubMsg::Message { channel: channel.to_string(), - message: message.to_string(), + message: message.to_vec(), }) .is_ok(); if ok { @@ -1642,7 +1641,7 @@ impl PubSubHub { .send(PubSubMsg::PMessage { pattern, channel: channel.to_string(), - message: message.to_string(), + message: message.to_vec(), }) .is_ok() { @@ -2030,7 +2029,7 @@ fn encode_pubsub_msg(msg: PubSubMsg, protover: u8) -> Vec { PubSubMsg::Message { channel, message } => frame(vec![ Value::BulkString(Some(b"message".to_vec())), Value::BulkString(Some(channel.into_bytes())), - Value::BulkString(Some(message.into_bytes())), + Value::BulkString(Some(message)), ]) .serialize(), PubSubMsg::PMessage { @@ -2041,7 +2040,7 @@ fn encode_pubsub_msg(msg: PubSubMsg, protover: u8) -> Vec { Value::BulkString(Some(b"pmessage".to_vec())), Value::BulkString(Some(pattern.into_bytes())), Value::BulkString(Some(channel.into_bytes())), - Value::BulkString(Some(message.into_bytes())), + Value::BulkString(Some(message)), ]) .serialize(), } @@ -2058,21 +2057,29 @@ fn resp_subscribe_ack(kind: &str, channel: &str, count: usize) -> Vec { /// Encodes a list of string parts as a RESP3 Push frame for WebSocket fan-out. /// Uses `>` prefix so clients can distinguish server-initiated pushes from command responses. -fn resp_push(parts: &[&str]) -> String { - let mut s = format!(">{}\r\n", parts.len()); +/// Build a RESP3 Push frame from raw byte arguments. +/// +/// Bytes rather than `&str` because these frames carry stored values, which may +/// be arbitrary binary. Building them as a `String` would have required a lossy +/// conversion — silently corrupting the replicated, AOF-logged and +/// browser-synced copy of a value the store itself holds faithfully. +fn resp_push(parts: &[&[u8]]) -> Vec { + let mut out = format!(">{}\r\n", parts.len()).into_bytes(); for part in parts { - s.push_str(&format!("${}\r\n{}\r\n", part.len(), part)); + out.extend_from_slice(format!("${}\r\n", part.len()).as_bytes()); + out.extend_from_slice(part); + out.extend_from_slice(b"\r\n"); } - s + out } /// Returns the RESP-encoded mutation to broadcast to WebSocket peers, or `None` /// if the command mutated nothing (read-only or conditional-and-failed). -fn broadcast_for(cmd: &Command, response: &Value) -> Option { +fn broadcast_for(cmd: &Command, response: &Value) -> Option> { match cmd { // Replays as SET: a replica has no connection to scope the lifetime to, // and the owning server broadcasts the DEL when the connection closes. - Command::ESet(k, v) => Some(resp_push(&["SET", k, v])), + Command::ESet(k, v) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), Command::Set(k, v, opts) => { // Without GET: nil response means NX/XX condition failed — don't broadcast. // With GET: nil means key didn't exist before, but SET still happened. @@ -2081,125 +2088,163 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { return None; } match &opts.expiry { - None => Some(resp_push(&["SET", k, v])), + None => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), Some(SetExpiry::Ex(s)) => { let px = s.saturating_mul(1000).to_string(); - Some(resp_push(&["SET", k, v, "PX", &px])) + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PX", + px.as_bytes(), + ])) } Some(SetExpiry::Px(ms)) => { let ms_s = ms.to_string(); - Some(resp_push(&["SET", k, v, "PX", &ms_s])) + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PX", + ms_s.as_bytes(), + ])) } Some(SetExpiry::Exat(ts)) => { let pxat = ts.saturating_mul(1000).to_string(); - Some(resp_push(&["SET", k, v, "PXAT", &pxat])) + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + pxat.as_bytes(), + ])) } Some(SetExpiry::Pxat(ts)) => { let ts_s = ts.to_string(); - Some(resp_push(&["SET", k, v, "PXAT", &ts_s])) + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + ts_s.as_bytes(), + ])) + } + Some(SetExpiry::KeepTtl) => { + Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice(), b"KEEPTTL"])) } - Some(SetExpiry::KeepTtl) => Some(resp_push(&["SET", k, v, "KEEPTTL"])), } } Command::Del(keys) | Command::Unlink(keys) => { - let mut parts: Vec<&str> = vec!["DEL"]; - let key_refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"DEL"]; + let key_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&key_refs); Some(resp_push(&parts)) } Command::MSet(pairs) => { - let mut parts: Vec<&str> = vec!["MSET"]; - let flat: Vec = pairs + let mut parts: Vec<&[u8]> = vec![b"MSET"]; + let flat: Vec> = pairs .iter() - .flat_map(|(k, v)| [k.clone(), v.clone()]) + .flat_map(|(k, v)| [k.as_bytes().to_vec(), v.clone()]) .collect(); - let flat_refs: Vec<&str> = flat.iter().map(|s| s.as_str()).collect(); + let flat_refs: Vec<&[u8]> = flat.iter().map(|s| s.as_slice()).collect(); parts.extend_from_slice(&flat_refs); Some(resp_push(&parts)) } Command::SetNx(k, v) => match response { - Value::Integer(1) => Some(resp_push(&["SET", k, v])), + Value::Integer(1) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), _ => None, }, Command::SetEx(k, secs, v) => { let px = secs.saturating_mul(1000).to_string(); - Some(resp_push(&["SET", k, v, "PX", &px])) + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PX", + px.as_bytes(), + ])) } Command::PSetEx(k, ms, v) => { let ms_s = ms.to_string(); - Some(resp_push(&["SET", k, v, "PX", &ms_s])) + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PX", + ms_s.as_bytes(), + ])) } Command::Append(k, v) => match response { - Value::Integer(_) => Some(resp_push(&["APPEND", k, v])), + Value::Integer(_) => Some(resp_push(&[b"APPEND", k.as_bytes(), v.as_slice()])), _ => None, }, - Command::GetSet(k, v) => Some(resp_push(&["SET", k, v])), + Command::GetSet(k, v) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), Command::Incr(k) | Command::Decr(k) => match response { Value::Integer(n) => { let s = n.to_string(); - Some(resp_push(&["SET", k, &s])) + Some(resp_push(&[b"SET", k.as_bytes(), s.as_bytes()])) } _ => None, }, Command::IncrBy(k, _) | Command::DecrBy(k, _) => match response { Value::Integer(n) => { let s = n.to_string(); - Some(resp_push(&["SET", k, &s])) + Some(resp_push(&[b"SET", k.as_bytes(), s.as_bytes()])) } _ => None, }, Command::Expire(k, secs) => match response { Value::Integer(1) => { let ms = secs.saturating_mul(1000).to_string(); - Some(resp_push(&["PEXPIRE", k, &ms])) + Some(resp_push(&[b"PEXPIRE", k.as_bytes(), ms.as_bytes()])) } _ => None, }, Command::PExpire(k, ms) => match response { Value::Integer(1) => { let ms_s = ms.to_string(); - Some(resp_push(&["PEXPIRE", k, &ms_s])) + Some(resp_push(&[b"PEXPIRE", k.as_bytes(), ms_s.as_bytes()])) } _ => None, }, Command::ExpireAt(k, ts) => match response { Value::Integer(1) => { let ts_ms = ts.saturating_mul(1000).to_string(); - Some(resp_push(&["PEXPIREAT", k, &ts_ms])) + Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts_ms.as_bytes()])) } _ => None, }, Command::PExpireAt(k, ts) => match response { Value::Integer(1) => { let ts_s = ts.to_string(); - Some(resp_push(&["PEXPIREAT", k, &ts_s])) + Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts_s.as_bytes()])) } _ => None, }, Command::Persist(k) => match response { - Value::Integer(1) => Some(resp_push(&["PERSIST", k])), + Value::Integer(1) => Some(resp_push(&[b"PERSIST", k.as_bytes()])), _ => None, }, - Command::FlushDb => Some(resp_push(&["FLUSHDB"])), + Command::FlushDb => Some(resp_push(&[b"FLUSHDB"])), Command::Rename(src, dst) => match response { Value::Error(_) => None, - _ => Some(resp_push(&["RENAME", src, dst])), + _ => Some(resp_push(&[b"RENAME", src.as_bytes(), dst.as_bytes()])), }, // ── Hash ───────────────────────────────────────────────────────────── Command::HSet(k, pairs) => { - let mut parts: Vec = vec!["HSET".into(), k.clone()]; + let mut parts: Vec> = vec![b"HSET".to_vec(), k.as_bytes().to_vec()]; for (f, v) in pairs { - parts.push(f.clone()); + parts.push(f.as_bytes().to_vec()); parts.push(v.clone()); } - let refs: Vec<&str> = parts.iter().map(|s| s.as_str()).collect(); + let refs: Vec<&[u8]> = parts.iter().map(|s| s.as_slice()).collect(); Some(resp_push(&refs)) } Command::HDel(k, fields) => match response { Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&str> = vec!["HDEL", k]; - let field_refs: Vec<&str> = fields.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"HDEL", k.as_bytes()]; + let field_refs: Vec<&[u8]> = fields.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&field_refs); Some(resp_push(&parts)) } @@ -2208,19 +2253,34 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { Command::HIncrBy(k, f, _) => match response { Value::Integer(n) => { let s = n.to_string(); - Some(resp_push(&["HSET", k, f, &s])) + Some(resp_push(&[ + b"HSET", + k.as_bytes(), + f.as_bytes(), + s.as_bytes(), + ])) } _ => None, }, Command::HIncrByFloat(k, f, _) => match response { Value::BulkString(Some(data)) => { let s = String::from_utf8_lossy(data); - Some(resp_push(&["HSET", k, f, &s])) + Some(resp_push(&[ + b"HSET", + k.as_bytes(), + f.as_bytes(), + s.as_bytes(), + ])) } _ => None, }, Command::HSetNx(k, f, v) => match response { - Value::Integer(1) => Some(resp_push(&["HSET", k, f, v])), + Value::Integer(1) => Some(resp_push(&[ + b"HSET", + k.as_bytes(), + f.as_bytes(), + v.as_slice(), + ])), _ => None, }, @@ -2231,8 +2291,8 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { } else { "RPUSH" }; - let mut parts: Vec<&str> = vec![cmd_name, k]; - let val_refs: Vec<&str> = vals.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![cmd_name.as_bytes(), k.as_bytes()]; + let val_refs: Vec<&[u8]> = vals.iter().map(|v| v.as_slice()).collect(); parts.extend_from_slice(&val_refs); Some(resp_push(&parts)) } @@ -2243,8 +2303,8 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { } else { "RPUSH" }; - let mut parts: Vec<&str> = vec![cmd_name, k]; - let val_refs: Vec<&str> = vals.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![cmd_name.as_bytes(), k.as_bytes()]; + let val_refs: Vec<&[u8]> = vals.iter().map(|v| v.as_slice()).collect(); parts.extend_from_slice(&val_refs); Some(resp_push(&parts)) } @@ -2256,8 +2316,8 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { _ => { let n = count.map(|c| c.to_string()); match &n { - Some(ns) => Some(resp_push(&["LPOP", k, ns])), - None => Some(resp_push(&["LPOP", k])), + Some(ns) => Some(resp_push(&[b"LPOP", k.as_bytes(), ns.as_bytes()])), + None => Some(resp_push(&[b"LPOP", k.as_bytes()])), } } }, @@ -2267,36 +2327,51 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { _ => { let n = count.map(|c| c.to_string()); match &n { - Some(ns) => Some(resp_push(&["RPOP", k, ns])), - None => Some(resp_push(&["RPOP", k])), + Some(ns) => Some(resp_push(&[b"RPOP", k.as_bytes(), ns.as_bytes()])), + None => Some(resp_push(&[b"RPOP", k.as_bytes()])), } } }, Command::LSet(k, idx, v) => match response { Value::SimpleString(_) => { let idx_s = idx.to_string(); - Some(resp_push(&["LSET", k, &idx_s, v])) + Some(resp_push(&[ + b"LSET", + k.as_bytes(), + idx_s.as_bytes(), + v.as_slice(), + ])) } _ => None, }, Command::LRem(k, count, elem) => match response { Value::Integer(n) if *n > 0 => { let count_s = count.to_string(); - Some(resp_push(&["LREM", k, &count_s, elem])) + Some(resp_push(&[ + b"LREM", + k.as_bytes(), + count_s.as_bytes(), + elem.as_slice(), + ])) } _ => None, }, Command::LTrim(k, start, stop) => { let start_s = start.to_string(); let stop_s = stop.to_string(); - Some(resp_push(&["LTRIM", k, &start_s, &stop_s])) + Some(resp_push(&[ + b"LTRIM", + k.as_bytes(), + start_s.as_bytes(), + stop_s.as_bytes(), + ])) } // ── Set ─────────────────────────────────────────────────────────────── Command::SAdd(k, members) => match response { Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&str> = vec!["SADD", k]; - let m_refs: Vec<&str> = members.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"SADD", k.as_bytes()]; + let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&m_refs); Some(resp_push(&parts)) } @@ -2304,8 +2379,8 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { }, Command::SRem(k, members) => match response { Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&str> = vec!["SREM", k]; - let m_refs: Vec<&str> = members.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"SREM", k.as_bytes()]; + let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&m_refs); Some(resp_push(&parts)) } @@ -2332,31 +2407,36 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { let _ = count; None } else { - let mut parts: Vec<&str> = vec!["SREM", k]; - let m_refs: Vec<&str> = popped.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"SREM", k.as_bytes()]; + let m_refs: Vec<&[u8]> = popped.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&m_refs); Some(resp_push(&parts)) } } Command::SMove(src, dst, member) => match response { - Value::Integer(1) => Some(resp_push(&["SMOVE", src, dst, member])), + Value::Integer(1) => Some(resp_push(&[ + b"SMOVE", + src.as_bytes(), + dst.as_bytes(), + member.as_bytes(), + ])), _ => None, }, Command::SInterStore(dst, keys) => { - let mut parts: Vec<&str> = vec!["SINTERSTORE", dst]; - let k_refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"SINTERSTORE", dst.as_bytes()]; + let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&k_refs); Some(resp_push(&parts)) } Command::SUnionStore(dst, keys) => { - let mut parts: Vec<&str> = vec!["SUNIONSTORE", dst]; - let k_refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"SUNIONSTORE", dst.as_bytes()]; + let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&k_refs); Some(resp_push(&parts)) } Command::SDiffStore(dst, keys) => { - let mut parts: Vec<&str> = vec!["SDIFFSTORE", dst]; - let k_refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"SDIFFSTORE", dst.as_bytes()]; + let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&k_refs); Some(resp_push(&parts)) } @@ -2380,13 +2460,13 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { parts.push(format_f64_score(*score)); parts.push(member.clone()); } - let refs: Vec<&str> = parts.iter().map(|s| s.as_str()).collect(); + let refs: Vec<&[u8]> = parts.iter().map(|s| s.as_bytes()).collect(); Some(resp_push(&refs)) } Command::ZRem(k, members) => match response { Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&str> = vec!["ZREM", k]; - let m_refs: Vec<&str> = members.iter().map(|s| s.as_str()).collect(); + let mut parts: Vec<&[u8]> = vec![b"ZREM", k.as_bytes()]; + let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); parts.extend_from_slice(&m_refs); Some(resp_push(&parts)) } @@ -2394,18 +2474,28 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { }, Command::ZIncrBy(k, delta, member) => { let delta_s = format_f64_score(*delta); - Some(resp_push(&["ZINCRBY", k, &delta_s, member])) + Some(resp_push(&[ + b"ZINCRBY", + k.as_bytes(), + delta_s.as_bytes(), + member.as_bytes(), + ])) } // ── JSON ───────────────────────────────────────────────────────────── // Replayable as-is on replicas, AOF, and browser stores. Only // successful writes replicate (errors reply -ERR, not +OK). Command::JSet(k, path, value) => match response { - Value::SimpleString(_) => Some(resp_push(&["JSET", k, path, value])), + Value::SimpleString(_) => Some(resp_push(&[ + b"JSET", + k.as_bytes(), + path.as_bytes(), + value.as_bytes(), + ])), _ => None, }, Command::JMerge(k, patch) => match response { - Value::SimpleString(_) => Some(resp_push(&["JMERGE", k, patch])), + Value::SimpleString(_) => Some(resp_push(&[b"JMERGE", k.as_bytes(), patch.as_bytes()])), _ => None, }, @@ -2417,7 +2507,12 @@ fn broadcast_for(cmd: &Command, response: &Value) -> Option { Command::RlSet(k, limit, window_secs) => { let limit_s = limit.to_string(); let window_s = window_secs.to_string(); - Some(resp_push(&["RLSET", k, &limit_s, &window_s])) + Some(resp_push(&[ + b"RLSET", + k.as_bytes(), + limit_s.as_bytes(), + window_s.as_bytes(), + ])) } // Pub/Sub and transactions carry no store state — no broadcast needed. @@ -3965,10 +4060,8 @@ async fn handle_ws( Some(scopes) => scopes_match(scopes, &push.keys), None => !strict, }; - if visible - && ws_sender.send(Message::Text(push.resp.clone().into())).await.is_err() - { - break; + if visible { + ws_send!(&push.resp); } } Ok(_) => {} @@ -4328,7 +4421,7 @@ mod tests { async fn aof_writer_append_and_truncate() { let path = tmp_path("aof_writer.aof"); let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); - aof.append("*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$1\r\nv\r\n") + aof.append(b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$1\r\nv\r\n") .await; aof.flush().await; let len_before = tokio::fs::metadata(&path).await.unwrap().len(); @@ -4355,25 +4448,52 @@ mod tests { } #[tokio::test] - async fn integration_binary_value_is_refused_over_resp() { - // The drop-in claim runs through this port, so the refusal has to be a - // clean RESP error that leaves the connection usable — not a - // disconnect, and emphatically not a silent lossy store. + async fn integration_binary_value_round_trips_over_resp() { + // The drop-in claim runs through this port: a value that is not valid + // UTF-8 must come back byte-for-byte, exactly as Redis would. let srv = spawn_server().await; let mut c = RespClient::connect(srv.tcp_addr).await; - let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n$3\r\n\xff\xfe\x41\r\n"; - c.stream.write_all(raw).await.unwrap(); + let binary: &[u8] = &[0xff, 0xfe, 0x00, 0x41, 0x80]; + let mut req = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n".to_vec(); + req.extend_from_slice(format!("${}\r\n", binary.len()).as_bytes()); + req.extend_from_slice(binary); + req.extend_from_slice(b"\r\n"); + c.stream.write_all(&req).await.unwrap(); + assert_eq!(c.cmd(&[]).await, ok()); + + assert_eq!( + c.cmd(&["GET", "bin"]).await, + Value::BulkString(Some(binary.to_vec())), + "binary value must survive the round trip" + ); + assert_eq!( + c.cmd(&["STRLEN", "bin"]).await, + int(binary.len() as i64), + "length is counted in bytes" + ); + } + + #[tokio::test] + async fn integration_binary_key_is_refused_over_resp() { + // Keys stay text: they are glob-matched and scope-checked, so a + // corrupted one would be silently unreachable. The refusal must be a + // clean RESP error that leaves the connection usable. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.stream + .write_all(b"*3\r\n$3\r\nSET\r\n$2\r\n\xff\xfe\r\n$1\r\nv\r\n") + .await + .unwrap(); let reply = c.cmd(&[]).await; let Value::Error(e) = &reply else { - panic!("binary value must be refused, got {reply:?}") + panic!("binary key must be refused, got {reply:?}") }; - assert!(e.contains("base64"), "error must be actionable: {e:?}"); + assert!(e.contains("must be text"), "error must explain: {e:?}"); - // Nothing was stored, and the connection still works. - assert_eq!(c.cmd(&["EXISTS", "bin"]).await, int(0)); - assert_eq!(c.cmd(&["SET", "bin", "ok"]).await, ok()); - assert_eq!(c.cmd(&["GET", "bin"]).await, bulk("ok")); + assert_eq!(c.cmd(&["DBSIZE"]).await, int(0), "nothing may be stored"); + assert_eq!(c.cmd(&["SET", "ok", "v"]).await, ok()); } #[tokio::test] @@ -4698,10 +4818,10 @@ mod tests { // Simulate writes captured by AOF state - .on_write("*3\r\n$3\r\nSET\r\n$5\r\nhello\r\n$5\r\nworld\r\n") + .on_write(b"*3\r\n$3\r\nSET\r\n$5\r\nhello\r\n$5\r\nworld\r\n") .await; state - .on_write("*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n") + .on_write(b"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n") .await; if let Some(ref a) = state.aof { a.flush().await; @@ -5190,18 +5310,20 @@ mod tests { assert_eq!(c.cmd_binary(raw).await, ok()); assert_eq!(c.cmd(&["GET", "bin"]).await, bulk("hello")); - // A binary *value* is refused rather than silently corrupted, and - // nothing is stored — the transport accepted the frame, the engine - // declined the payload. - let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nraw\r\n$3\r\n\xff\xfe\x41\r\n".to_vec(); - let reply = c.cmd_binary(raw).await; - let Value::Error(e) = &reply else { - panic!("binary value must be refused, got {reply:?}") - }; - assert!(e.contains("base64"), "error must be actionable: {e:?}"); - assert_eq!(c.cmd(&["EXISTS", "raw"]).await, int(0)); + // A binary value round-trips byte-for-byte, and the reply comes back in + // a binary frame because it is not valid UTF-8. + let binary: &[u8] = &[0xff, 0xfe, 0x00, 0x41]; + let mut req = b"*3\r\n$3\r\nSET\r\n$3\r\nraw\r\n".to_vec(); + req.extend_from_slice(format!("${}\r\n", binary.len()).as_bytes()); + req.extend_from_slice(binary); + req.extend_from_slice(b"\r\n"); + assert_eq!(c.cmd_binary(req).await, ok()); + assert_eq!( + c.cmd(&["GET", "raw"]).await, + Value::BulkString(Some(binary.to_vec())) + ); - // The connection stays usable after a rejection. + // The connection stays usable. assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); } @@ -6369,6 +6491,7 @@ mod tests { &Value::SimpleString("OK".into()), ) .expect("ESET must broadcast"); + let frame = String::from_utf8_lossy(&frame).into_owned(); assert!(frame.contains("SET"), "{frame}"); assert!(frame.contains("presence:1")); assert!( diff --git a/sync-client/src/lib.rs b/sync-client/src/lib.rs index 7eb9a56..c83ce96 100644 --- a/sync-client/src/lib.rs +++ b/sync-client/src/lib.rs @@ -449,11 +449,8 @@ impl SyncClient { let key = String::from_utf8_lossy(key).into_owned(); match &items[2] { Value::BulkString(Some(v)) => { - self.store.execute(Command::Set( - key, - String::from_utf8_lossy(v).into_owned(), - Default::default(), - )); + self.store + .execute(Command::Set(key, v.clone(), Default::default())); } Value::BulkString(None) => { // A nil value whose "key" is one of our registered live-query @@ -497,37 +494,54 @@ impl SyncClient { _ => None, } } + fn bytes(v: &Value) -> Option> { + match v { + Value::BulkString(Some(b)) => Some(b.clone()), + Value::SimpleString(s) => Some(s.as_bytes().to_vec()), + _ => None, + } + } let Some(tag) = items.first().and_then(text) else { return; }; - let rest: Vec = items[1..].iter().filter_map(text).collect(); + // Values may be arbitrary bytes; hash fields, set and sorted-set + // members are identifiers and stay text. `raw` keeps the bytes, `as_text` + // reinterprets a slot when the position calls for an identifier. + let raw: Vec> = items[1..].iter().filter_map(bytes).collect(); + let as_text = |b: &Vec| String::from_utf8_lossy(b).into_owned(); // Clear first so removed members do not linger. self.store.execute(Command::Del(vec![key.clone()])); match tag.as_str() { "hash" => { - let pairs: Vec<(String, String)> = rest + let pairs: Vec<(String, Vec)> = raw .chunks_exact(2) - .map(|c| (c[0].clone(), c[1].clone())) + .map(|c| (as_text(&c[0]), c[1].clone())) .collect(); if !pairs.is_empty() { self.store.execute(Command::HSet(key, pairs)); } } "list" => { - if !rest.is_empty() { - self.store.execute(Command::RPush(key, rest)); + if !raw.is_empty() { + self.store.execute(Command::RPush(key, raw)); } } "set" => { - if !rest.is_empty() { - self.store.execute(Command::SAdd(key, rest)); + if !raw.is_empty() { + self.store + .execute(Command::SAdd(key, raw.iter().map(as_text).collect())); } } "zset" => { - let members: Vec<(f64, String)> = rest + let members: Vec<(f64, String)> = raw .chunks_exact(2) - .filter_map(|c| c[1].parse::().ok().map(|sc| (sc, c[0].clone()))) + .filter_map(|c| { + as_text(&c[1]) + .parse::() + .ok() + .map(|sc| (sc, as_text(&c[0]))) + }) .collect(); if !members.is_empty() { self.store @@ -535,9 +549,10 @@ impl SyncClient { } } "json" => { - if let Some(doc) = rest.first() { + // JSON is UTF-8 by definition, so this position is text. + if let Some(doc) = raw.first() { self.store - .execute(Command::JSet(key, "$".to_string(), doc.clone())); + .execute(Command::JSet(key, "$".to_string(), as_text(doc))); } } _ => {} @@ -552,11 +567,8 @@ impl SyncClient { let key = String::from_utf8_lossy(k).into_owned(); match &pair[1] { Value::BulkString(Some(v)) => { - self.store.execute(Command::Set( - key, - String::from_utf8_lossy(v).into_owned(), - Default::default(), - )); + self.store + .execute(Command::Set(key, v.clone(), Default::default())); } // Collections arrive type-tagged, so the initial state of a // live query is complete — no follow-up typed read needed. diff --git a/wasm-edge/sdk.ts b/wasm-edge/sdk.ts index 881f303..3625187 100644 --- a/wasm-edge/sdk.ts +++ b/wasm-edge/sdk.ts @@ -69,6 +69,7 @@ interface RawCache { set(key: string, value: string): string; set_ex(key: string, value: string, seconds: number): string; get(key: string): string | undefined; + getBytes(key: string): Uint8Array | undefined; del(key: string): number; incr_by(key: string, delta: number): number; disconnect(): void; @@ -246,11 +247,35 @@ export class Cache { * Return the value for `key`, or `null` if the key does not exist or has expired. * * Always served from local WASM memory — zero network latency. + * + * @throws if the stored value is not valid UTF-8. Values are byte-transparent, + * so a backend can write binary that syncs into this cache; returning it as a + * mangled string would be worse than failing. Use {@link getBytes} for those. */ get(key: string): string | null { return this.raw.get(key) ?? null; } + /** + * Return the value for `key` as raw bytes, or `null` if it does not exist. + * + * Use this for values a backend wrote as binary — compressed payloads, + * protobuf, images — which cannot be represented as a JS string. Text values + * work here too, as their UTF-8 bytes. + * + * ```ts + * const bytes = cache.getBytes('thumb:42'); + * if (bytes) img.src = URL.createObjectURL(new Blob([bytes])); + * ``` + * + * Note: the browser SDK can *read* binary values but cannot yet *write* them + * — `set()` takes a string. Binary values originate from a backend writing + * over the RESP port. + */ + getBytes(key: string): Uint8Array | null { + return this.raw.getBytes(key) ?? null; + } + /** * Return a JSON-parsed value stored under `key`, or `null` if the key is * missing, expired, or not valid JSON. @@ -261,7 +286,14 @@ export class Cache { * ``` */ getJSON(key: string): T | null { - const raw = this.get(key); + let raw: string | null; + try { + raw = this.get(key); + } catch { + // Binary value: not JSON by definition, so this is a miss rather than an + // error — getJSON is documented to return null for anything unparseable. + return null; + } if (raw === null) return null; try { return JSON.parse(raw) as T; diff --git a/wasm-edge/src/lib.rs b/wasm-edge/src/lib.rs index f9ab998..e7898ea 100644 --- a/wasm-edge/src/lib.rs +++ b/wasm-edge/src/lib.rs @@ -126,7 +126,7 @@ extern "C" { #[wasm_bindgen(js_name = "idbOutboxReadAll")] fn idb_outbox_read_all(db: &JsValue) -> Promise; #[wasm_bindgen(js_name = "idbAppend")] - fn idb_append_js(db: &JsValue, seq: f64, cmd: &str) -> Promise; + fn idb_append_js(db: &JsValue, seq: f64, cmd: &[u8]) -> Promise; #[wasm_bindgen(js_name = "idbOutboxPut")] fn idb_outbox_put_js(db: &JsValue, id: f64, cmd: &str) -> Promise; #[wasm_bindgen(js_name = "idbOutboxDelete")] @@ -145,9 +145,19 @@ extern "C" { // ── RESP helper ─────────────────────────────────────────────────────────────── -fn to_resp_owned(parts: &[String]) -> String { - let refs: Vec<&str> = parts.iter().map(|s| s.as_str()).collect(); - to_resp(&refs) +/// Build a RESP array frame from owned byte arguments. +/// +/// Bytes rather than `String`: WAL compaction re-encodes stored values, and a +/// value pushed from the server may be arbitrary binary. Going through a +/// `String` here would corrupt exactly the values the store holds correctly. +fn to_resp_owned(parts: &[Vec]) -> Vec { + let mut out = format!("*{}\r\n", parts.len()).into_bytes(); + for part in parts { + out.extend_from_slice(format!("${}\r\n", part.len()).as_bytes()); + out.extend_from_slice(part); + out.extend_from_slice(b"\r\n"); + } + out } // ── WAL compaction ──────────────────────────────────────────────────────────── @@ -174,7 +184,7 @@ fn wall_clock_ms() -> u64 { /// Convert snapshot entries into minimal RESP command strings suitable for /// storing in the WAL. Each entry produces one command; entries with a TTL on /// collection types produce an extra PEXPIREAT command. -fn snapshot_to_resp_cmds(entries: &[SnapshotEntry]) -> Vec { +fn snapshot_to_resp_cmds(entries: &[SnapshotEntry]) -> Vec> { // `SystemTime::now()` is unsupported on wasm32-unknown-unknown and panics // outright. This runs inside WAL compaction, immediately after the WAL has // been cleared — a panic here therefore destroys the entire persisted cache @@ -187,29 +197,32 @@ fn snapshot_to_resp_cmds(entries: &[SnapshotEntry]) -> Vec { if e.expires_at_ms.is_some_and(|exp| now_ms >= exp) { continue; } - let data_parts: Vec = match &e.value { + // Identifiers are text and values are bytes, so every part is built as + // bytes and the text ones are converted on the way in. + let t = |x: &str| x.as_bytes().to_vec(); + let data_parts: Vec> = match &e.value { SnapshotValue::Str(s) => { if let Some(exp) = e.expires_at_ms { let rem_ms = exp.saturating_sub(now_ms); vec![ - "SET".into(), - e.key.clone(), - s.clone(), - "PX".into(), - rem_ms.to_string(), + t("SET"), + t(&e.key), + s.as_slice().to_vec(), + t("PX"), + t(&rem_ms.to_string()), ] } else { - vec!["SET".into(), e.key.clone(), s.clone()] + vec![t("SET"), t(&e.key), s.as_slice().to_vec()] } } SnapshotValue::Hash(map) => { if map.is_empty() { continue; } - let mut parts = vec!["HSET".to_string(), e.key.clone()]; + let mut parts = vec![t("HSET"), t(&e.key)]; for (f, v) in map { - parts.push(f.clone()); - parts.push(v.clone()); + parts.push(t(f)); + parts.push(v.as_slice().to_vec()); } parts } @@ -217,26 +230,26 @@ fn snapshot_to_resp_cmds(entries: &[SnapshotEntry]) -> Vec { if list.is_empty() { continue; } - let mut parts = vec!["RPUSH".to_string(), e.key.clone()]; - parts.extend(list.iter().cloned()); + let mut parts = vec![t("RPUSH"), t(&e.key)]; + parts.extend(list.iter().map(|v| v.as_slice().to_vec())); parts } SnapshotValue::Set(set) => { if set.is_empty() { continue; } - let mut parts = vec!["SADD".to_string(), e.key.clone()]; - parts.extend(set.iter().cloned()); + let mut parts = vec![t("SADD"), t(&e.key)]; + parts.extend(set.iter().map(|m| t(m))); parts } SnapshotValue::ZSet(pairs) => { if pairs.is_empty() { continue; } - let mut parts = vec!["ZADD".to_string(), e.key.clone()]; + let mut parts = vec![t("ZADD"), t(&e.key)]; for (member, score) in pairs { - parts.push(format_score(*score)); - parts.push(member.clone()); + parts.push(t(&format_score(*score))); + parts.push(t(member)); } parts } @@ -244,7 +257,7 @@ fn snapshot_to_resp_cmds(entries: &[SnapshotEntry]) -> Vec { // not persisted in the browser WAL. SnapshotValue::RateLimiter { .. } => continue, SnapshotValue::Json(doc) => { - vec!["JSET".into(), e.key.clone(), "$".into(), doc.clone()] + vec![t("JSET"), t(&e.key), t("$"), t(doc)] } }; out.push(to_resp_owned(&data_parts)); @@ -253,9 +266,9 @@ fn snapshot_to_resp_cmds(entries: &[SnapshotEntry]) -> Vec { && let Some(exp) = e.expires_at_ms { out.push(to_resp_owned(&[ - "PEXPIREAT".to_string(), - e.key.clone(), - exp.to_string(), + t("PEXPIREAT"), + t(&e.key), + t(&exp.to_string()), ])); } } @@ -520,12 +533,12 @@ impl Default for RecachedCache { // ── persistence helper ──────────────────────────────────────────────────────── -fn persist_cmd(idb: &Rc>>, seq: &Rc>, encoded: &str) { +fn persist_cmd(idb: &Rc>>, seq: &Rc>, encoded: &[u8]) { let maybe_db = idb.borrow().as_ref().cloned(); if let Some(db) = maybe_db { let s = seq.get(); seq.set(s + 1); - let cmd = encoded.to_string(); + let cmd = encoded.to_vec(); spawn_local(async move { let _ = JsFuture::from(idb_append_js(&db, s as f64, &cmd)).await; }); @@ -710,8 +723,13 @@ impl RecachedCache { let mut max_seq: u64 = 0; for i in 0..entry_count { let s = keys.get(i).as_f64().unwrap_or(0.0) as u64; - let cmd_str = vals.get(i).as_string().unwrap_or_default(); - if let Ok((value, _)) = Value::parse(cmd_str.as_bytes()) + let raw = vals.get(i); + let cmd_bytes = match raw.as_string() { + // Written by 0.2.1 and earlier, when frames were strings. + Some(t) => t.into_bytes(), + None => js_sys::Uint8Array::new(&raw).to_vec(), + }; + if let Ok((value, _)) = Value::parse(&cmd_bytes) && let Ok(cmd) = Command::from_value(value) { store.execute(cmd); @@ -736,8 +754,8 @@ impl RecachedCache { // the persisted cache was gone. let cmds = snapshot_to_resp_cmds(&store.snapshot()); let arr = js_sys::Array::new(); - for cmd_str in &cmds { - arr.push(&JsValue::from_str(cmd_str)); + for cmd in &cmds { + arr.push(&js_sys::Uint8Array::from(&cmd[..])); } JsFuture::from(idb_wal_replace_js(&db, &arr)).await?; cmds.len() as u64 @@ -936,7 +954,7 @@ impl RecachedCache { if let Some(bc) = &self.bc { let _ = bc.post_message(&JsValue::from_str(&encoded)); } - persist_cmd(&self.idb, &self.seq, &encoded); + persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); notify_mutation(&self.on_mutation); Ok(n) } @@ -958,7 +976,7 @@ impl RecachedCache { if let Some(bc) = &self.bc { let _ = bc.post_message(&JsValue::from_str(&encoded)); } - persist_cmd(&self.idb, &self.seq, &encoded); + persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); notify_mutation(&self.on_mutation); "OK".to_string() } @@ -990,7 +1008,7 @@ impl RecachedCache { if let Some(bc) = &self.bc { let _ = bc.post_message(&JsValue::from_str(&encoded)); } - persist_cmd(&self.idb, &self.seq, &encoded); + persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); notify_mutation(&self.on_mutation); "OK".to_string() } @@ -1024,7 +1042,7 @@ impl RecachedCache { pub fn set(&self, key: &str, value: &str) -> String { let resp = self.store.execute(Command::Set( key.to_string(), - value.to_string(), + value.as_bytes().to_vec(), SetOptions::default(), )); @@ -1033,7 +1051,7 @@ impl RecachedCache { if let Some(bc) = &self.bc { let _ = bc.post_message(&JsValue::from_str(&encoded)); } - persist_cmd(&self.idb, &self.seq, &encoded); + persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); notify_mutation(&self.on_mutation); match resp { @@ -1049,16 +1067,18 @@ impl RecachedCache { expiry: Some(core_engine::cmd::SetExpiry::Ex(seconds as u64)), ..Default::default() }; - let resp = self - .store - .execute(Command::Set(key.to_string(), value.to_string(), opts)); + let resp = self.store.execute(Command::Set( + key.to_string(), + value.as_bytes().to_vec(), + opts, + )); let encoded = to_resp(&["SET", key, value, "EX", &seconds.to_string()]); self.ws_enqueue(&encoded); if let Some(bc) = &self.bc { let _ = bc.post_message(&JsValue::from_str(&encoded)); } - persist_cmd(&self.idb, &self.seq, &encoded); + persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); notify_mutation(&self.on_mutation); match resp { @@ -1069,9 +1089,31 @@ impl RecachedCache { } /// Get a value from the local store (zero latency). - pub fn get(&self, key: &str) -> Option { + pub fn get(&self, key: &str) -> Result, JsValue> { + match self.store.execute(Command::Get(key.to_string())) { + Value::BulkString(Some(data)) => match String::from_utf8(data) { + Ok(text) => Ok(Some(text)), + // A backend can write binary that syncs into this cache. + // Returning it lossily would hand the application text that is + // not what was stored, so this throws instead. + Err(_) => Err(JsValue::from_str( + "value is not valid UTF-8 — use getBytes() to read binary values", + )), + }, + _ => Ok(None), + } + } + + /// Read a key as raw bytes, whatever it contains. + /// + /// `get()` is the right call for text; this exists for values a backend + /// wrote as binary — compressed payloads, protobuf, images — which cannot + /// be represented as a JS string. Returns `undefined` when the key is + /// absent. + #[wasm_bindgen(js_name = "getBytes")] + pub fn get_bytes(&self, key: &str) -> Option { match self.store.execute(Command::Get(key.to_string())) { - Value::BulkString(Some(data)) => Some(String::from_utf8_lossy(&data).into_owned()), + Value::BulkString(Some(data)) => Some(js_sys::Uint8Array::from(&data[..])), _ => None, } } @@ -1085,7 +1127,7 @@ impl RecachedCache { if let Some(bc) = &self.bc { let _ = bc.post_message(&JsValue::from_str(&encoded)); } - persist_cmd(&self.idb, &self.seq, &encoded); + persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); notify_mutation(&self.on_mutation); match resp { @@ -1158,15 +1200,30 @@ mod tests { } /// Join the emitted RESP frames for substring assertions. + /// + /// Frames are bytes now that values can be binary; these tests all use text + /// payloads, so rendering them lossily for assertions is safe here and + /// keeps the assertions readable. fn joined(entries: &[SnapshotEntry]) -> String { - snapshot_to_resp_cmds(entries).join("") + as_text(&snapshot_to_resp_cmds(entries)).join("") + } + + fn as_text(frames: &[Vec]) -> Vec { + frames + .iter() + .map(|f| String::from_utf8_lossy(f).into_owned()) + .collect() } // ── WAL encoding: every value type must survive a page reload ───────────── #[test] fn strings_persist_as_set() { - let out = snapshot_to_resp_cmds(&[entry("k", SnapshotValue::Str("v".into()), None)]); + let out = as_text(&snapshot_to_resp_cmds(&[entry( + "k", + SnapshotValue::Str("v".into()), + None, + )])); assert_eq!(out.len(), 1); assert!(out[0].contains("SET") && out[0].contains("k") && out[0].contains("v")); } @@ -1174,8 +1231,8 @@ mod tests { #[test] fn hashes_persist_every_field() { let mut m = HashMap::new(); - m.insert("f1".to_string(), "v1".to_string()); - m.insert("f2".to_string(), "v2".to_string()); + m.insert("f1".to_string(), "v1".into()); + m.insert("f2".to_string(), "v2".into()); let s = joined(&[entry("h", SnapshotValue::Hash(m), None)]); assert!(s.contains("HSET")); for part in ["f1", "v1", "f2", "v2"] { @@ -1238,10 +1295,10 @@ mod tests { #[test] fn already_expired_entries_are_dropped_not_resurrected() { // Replaying an expired key would bring deleted data back on reload. - let out = snapshot_to_resp_cmds(&[ + let out = as_text(&snapshot_to_resp_cmds(&[ entry("dead", SnapshotValue::Str("v".into()), Some(1)), entry("live", SnapshotValue::Str("v".into()), None), - ]); + ])); let s = out.join(""); assert!(!s.contains("dead"), "expired key resurrected: {s}"); assert!(s.contains("live")); @@ -1265,8 +1322,11 @@ mod tests { // Collection commands cannot carry an inline TTL, so the expiry rides // in a second command — as an absolute timestamp this time. let exp = now_ms() + 60_000; - let out = - snapshot_to_resp_cmds(&[entry("l", SnapshotValue::List(vec!["a".into()]), Some(exp))]); + let out = as_text(&snapshot_to_resp_cmds(&[entry( + "l", + SnapshotValue::List(vec!["a".into()]), + Some(exp), + )])); assert_eq!(out.len(), 2, "expected RPUSH + PEXPIREAT: {out:?}"); assert!(out[1].contains("PEXPIREAT") && out[1].contains(&exp.to_string())); } @@ -1274,7 +1334,11 @@ mod tests { #[test] fn a_string_with_a_ttl_emits_only_one_command() { let exp = now_ms() + 60_000; - let out = snapshot_to_resp_cmds(&[entry("k", SnapshotValue::Str("v".into()), Some(exp))]); + let out = as_text(&snapshot_to_resp_cmds(&[entry( + "k", + SnapshotValue::Str("v".into()), + Some(exp), + )])); assert_eq!(out.len(), 1, "SET carries PX inline: {out:?}"); } @@ -1284,12 +1348,12 @@ mod tests { fn empty_collections_emit_nothing() { // `RPUSH key` with no members is a syntax error on replay, which would // abort WAL hydration part-way through. - let out = snapshot_to_resp_cmds(&[ + let out = as_text(&snapshot_to_resp_cmds(&[ entry("h", SnapshotValue::Hash(HashMap::new()), None), entry("l", SnapshotValue::List(vec![]), None), entry("s", SnapshotValue::Set(vec![]), None), entry("z", SnapshotValue::ZSet(vec![]), None), - ]); + ])); assert!(out.is_empty(), "empty collections must be skipped: {out:?}"); } @@ -1297,7 +1361,7 @@ mod tests { fn rate_limiter_state_is_not_persisted_in_the_browser() { // Attempt state is server-side and transient; replaying it locally // would let a client fabricate its own rate-limit history. - let out = snapshot_to_resp_cmds(&[entry( + let out = as_text(&snapshot_to_resp_cmds(&[entry( "rl", SnapshotValue::RateLimiter { limit: 10, @@ -1305,7 +1369,7 @@ mod tests { events: vec![1, 2, 3], }, None, - )]); + )])); assert!( out.is_empty(), "rate-limiter state leaked into the WAL: {out:?}" @@ -1320,13 +1384,13 @@ mod tests { #[test] fn every_emitted_command_is_a_well_formed_resp_array() { let mut m = HashMap::new(); - m.insert("f".to_string(), "v".to_string()); - let out = snapshot_to_resp_cmds(&[ + m.insert("f".to_string(), "v".into()); + let out = as_text(&snapshot_to_resp_cmds(&[ entry("s", SnapshotValue::Str("v".into()), None), entry("h", SnapshotValue::Hash(m), None), entry("l", SnapshotValue::List(vec!["a".into()]), None), entry("z", SnapshotValue::ZSet(vec![("m".into(), 1.0)]), None), - ]); + ])); assert_eq!(out.len(), 4); for frame in &out { assert!(frame.starts_with('*'), "not a RESP array: {frame}"); @@ -1347,7 +1411,7 @@ mod tests { use core_engine::store::KeyValueStore; let mut m = HashMap::new(); - m.insert("f".to_string(), "v".to_string()); + m.insert("f".to_string(), "v".into()); let cmds = snapshot_to_resp_cmds(&[ entry("str", SnapshotValue::Str("hello".into()), None), entry("h", SnapshotValue::Hash(m), None), @@ -1358,7 +1422,7 @@ mod tests { let store = KeyValueStore::new(); for frame in &cmds { - let (value, _) = core_engine::resp::Value::parse(frame.as_bytes()).unwrap(); + let (value, _) = core_engine::resp::Value::parse(frame).unwrap(); store.execute(Command::from_value(value).unwrap()); } @@ -1428,7 +1492,14 @@ mod browser_tests { ( keys.iter().map(|k| k.as_f64().unwrap_or(-1.0)).collect(), vals.iter() - .map(|v| v.as_string().unwrap_or_default()) + .map(|v| match v.as_string() { + Some(t) => t, + // WAL frames are stored as Uint8Array so binary values + // persist; these tests all use text payloads. + None => { + String::from_utf8_lossy(&js_sys::Uint8Array::new(&v).to_vec()).into_owned() + } + }) .collect(), ) } @@ -1511,7 +1582,7 @@ mod browser_tests { // order reconstructs the wrong state on reload. let db = fresh_db().await; for (seq, cmd) in [(1.0, "SET a 1"), (2.0, "SET b 2"), (3.0, "SET a 3")] { - JsFuture::from(idb_append_js(&db, seq, cmd)) + JsFuture::from(idb_append_js(&db, seq, cmd.as_bytes())) .await .expect("append"); } @@ -1525,7 +1596,7 @@ mod browser_tests { // Compaction clears the WAL only. Wiping the outbox here would discard // writes the server has not acknowledged yet — silent data loss. let db = fresh_db().await; - JsFuture::from(idb_append_js(&db, 1.0, "SET a 1")) + JsFuture::from(idb_append_js(&db, 1.0, b"SET a 1")) .await .expect("append"); JsFuture::from(idb_outbox_put_js(&db, 10.0, "PENDING WRITE")) @@ -1544,7 +1615,7 @@ mod browser_tests { #[wasm_bindgen_test] async fn full_clear_empties_every_store() { let db = open_db().await; - JsFuture::from(idb_append_js(&db, 1.0, "SET a 1")) + JsFuture::from(idb_append_js(&db, 1.0, b"SET a 1")) .await .unwrap(); JsFuture::from(idb_outbox_put_js(&db, 1.0, "SET b 2")) @@ -1624,7 +1695,9 @@ mod browser_tests { async fn wal_replace_swaps_contents_in_one_transaction() { let db = fresh_db().await; for (seq, cmd) in [(0.0, "SET a 1"), (1.0, "SET b 2"), (2.0, "SET c 3")] { - JsFuture::from(idb_append_js(&db, seq, cmd)).await.unwrap(); + JsFuture::from(idb_append_js(&db, seq, cmd.as_bytes())) + .await + .unwrap(); } let arr = js_sys::Array::new(); @@ -1643,7 +1716,7 @@ mod browser_tests { // An empty store compacts to zero commands; the WAL should end empty // rather than retaining stale entries. let db = fresh_db().await; - JsFuture::from(idb_append_js(&db, 0.0, "SET a 1")) + JsFuture::from(idb_append_js(&db, 0.0, b"SET a 1")) .await .unwrap(); @@ -1659,7 +1732,7 @@ mod browser_tests { async fn wal_replace_leaves_the_outbox_untouched() { // Compaction must never drop writes still awaiting acknowledgement. let db = fresh_db().await; - JsFuture::from(idb_append_js(&db, 0.0, "SET a 1")) + JsFuture::from(idb_append_js(&db, 0.0, b"SET a 1")) .await .unwrap(); JsFuture::from(idb_outbox_put_js(&db, 5.0, "PENDING")) @@ -1675,6 +1748,51 @@ mod browser_tests { assert_eq!(vals[0], "PENDING"); } + #[wasm_bindgen_test] + async fn binary_values_survive_the_wal() { + // A backend can write a binary value that reaches this browser through + // a live query. The WAL persists frames in IndexedDB, so it has to + // carry those bytes intact — a UTF-8 round trip here would corrupt on + // reload exactly the values the store holds correctly in memory. + let db = fresh_db().await; + let binary = vec![0xffu8, 0xfe, 0x00, 0x41, 0x80]; + + let source = KeyValueStore::new(); + source.execute(Command::Set( + "b".into(), + binary.clone(), + core_engine::cmd::SetOptions::default(), + )); + + let cmds = snapshot_to_resp_cmds(&source.snapshot()); + let arr = js_sys::Array::new(); + for c in &cmds { + arr.push(&js_sys::Uint8Array::from(&c[..])); + } + JsFuture::from(idb_wal_replace_js(&db, &arr)).await.unwrap(); + + // Read the raw rows back and replay them, as a page load would. + let res = JsFuture::from(idb_read_all(&db)).await.expect("wal read"); + let pair = js_sys::Array::from(&res); + let vals = js_sys::Array::from(&pair.get(1)); + let restored = KeyValueStore::new(); + for i in 0..vals.length() { + let raw = vals.get(i); + let bytes = match raw.as_string() { + Some(t) => t.into_bytes(), + None => js_sys::Uint8Array::new(&raw).to_vec(), + }; + let (value, _) = core_engine::resp::Value::parse(&bytes).unwrap(); + restored.execute(Command::from_value(value).unwrap()); + } + + assert_eq!( + restored.execute(Command::Get("b".into())), + Value::BulkString(Some(binary)), + "binary value must survive the WAL round trip" + ); + } + #[wasm_bindgen_test] async fn compacted_wal_replays_to_the_same_state() { // End to end: snapshot a store, compact through the atomic path, read @@ -1694,7 +1812,7 @@ mod browser_tests { let cmds = snapshot_to_resp_cmds(&source.snapshot()); let arr = js_sys::Array::new(); for c in &cmds { - arr.push(&JsValue::from_str(c)); + arr.push(&js_sys::Uint8Array::from(&c[..])); } JsFuture::from(idb_wal_replace_js(&db, &arr)).await.unwrap(); From cf063b92e58b9faa45c0c6fdb38227a1528562f1 Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 20:14:02 +0800 Subject: [PATCH 08/11] Make values byte-transparent across store, replication, AOF, pub/sub and browser WAL; add replication acks with lag metric, RESP3 HELLO negotiation, and binary WebSocket frames --- docs/roadmap.md | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index cbfa76a..c271bd6 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -59,46 +59,4 @@ Under consideration behind these: a CRDT text type for collaborative editing (li --- -## Hardening & enhancements - -Improvements to shipped functionality rather than new surface. Unnumbered because they are small and -independent — pick them off in any order. - -### Performance - -**Serialize `LRANGE` straight from the store**, instead of building the full reply `Value` first. -Diagnosed on the [benchmarks](/guide/benchmarks) page. - -### Binary keys - -**Keys are text; values are not.** Values became byte-transparent in 0.2.2, but keys, hash fields, -set and sorted-set members, glob patterns and channel names must still be valid UTF-8. Redis allows -binary in all of those positions. - -Closing the gap means carrying bytes through the store's key map, the glob matcher, sync-scope prefix -matching, live-query patterns and pub/sub routing — several of which are security-relevant and were -hardened separately. The practical demand is low, since keys are identifiers in every workload seen -so far, so this is recorded rather than scheduled. - -### Security - -**Warn on sync tokens minted without an expiry.** Expiry is optional in the token payload, so a token -issued without one is valid forever. Refusing to mint it — or at minimum logging loudly — would match -how half-configured TLS is now handled. - -**Per-command ACLs and an audit log.** Both are named as gaps in the -[threat model](/server/security#threat-model-stated-plainly): authentication on the RESP port is -all-or-nothing, and there is no record of who read or wrote what. Neither is interesting engineering; -both are procurement checkboxes worth building when someone actually asks. - ---- - -## Ongoing: drop-in credibility - -Not features, but continuous work that keeps "any Redis client works today" honest: - -- **Command coverage** — closing gaps in the supported command set as real workloads surface them (see [Commands](/server/commands)). - ---- - Feedback on priorities is welcome — [open an issue](https://github.com/thinkgrid-labs/recached/issues) or write to [dennis@thinkgrid.dev](mailto:dennis@thinkgrid.dev). From d959b297a1bc604dea9a78185e02b44abc00731e Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 21:21:20 +0800 Subject: [PATCH 09/11] =?UTF-8?q?Make=20values=20byte-transparent=20end=20?= =?UTF-8?q?to=20end=20=E2=80=94=20store,=20replication,=20AOF,=20pub/sub,?= =?UTF-8?q?=20browser=20outbox=20and=20WAL=20=E2=80=94=20plus=20setBytes/g?= =?UTF-8?q?etBytes/publishBytes;=20add=20replication=20acks=20with=20lag?= =?UTF-8?q?=20metric=20and=20RESP3=20HELLO=20negotiation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 13 +- docs/browser/api-reference.md | 46 ++++++- docs/guide/introduction.md | 13 +- docs/roadmap.md | 14 +-- sync-client/src/lib.rs | 89 ++++++++----- sync-client/src/tests.rs | 205 +++++++++++++++++++++--------- wasm-edge/Cargo.toml | 3 + wasm-edge/sdk.ts | 42 ++++++- wasm-edge/src/lib.rs | 229 ++++++++++++++++++++++++++-------- 9 files changed, 490 insertions(+), 164 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5da7ad7..1f7c448 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -171,10 +171,15 @@ All notable changes to Recached are documented here. encode as msgpack `bin` rather than an array of integers, so snapshot size is unchanged for text and roughly halved versus the naive encoding for binary. - **Browser SDK: binary is read-only.** The browser cache stores and syncs binary faithfully and - `cache.getBytes(key)` returns a `Uint8Array`, but `cache.set()` still takes a string — binary - values originate from a backend writing over the RESP port. `cache.get()` now **throws** on a - binary value instead of returning mangled text, and `getJSON()` treats one as a miss. + **The browser SDK handles binary end to end.** `setBytes()` / `getBytes()` and `publishBytes()` + are new, and binary survives the offline outbox, the exactly-once `DEDUP` envelope, cross-tab + `BroadcastChannel` sync and IndexedDB persistence unchanged. Frames that carry binary now travel + in WebSocket *binary* frames in both directions — the socket's `binaryType` is `arraybuffer`, so + an inbound binary frame is no longer silently dropped by the message handler. + + `cache.get()` now **throws** on a binary value instead of returning mangled text, `getJSON()` + treats one as a miss, and an `onMessage` listener receives a `Uint8Array` for a binary pub/sub + payload — so the listener signature widened to `string | Uint8Array`. Data corrupted by an earlier version cannot be recovered and must be re-populated. diff --git a/docs/browser/api-reference.md b/docs/browser/api-reference.md index 4a9267d..4895968 100644 --- a/docs/browser/api-reference.md +++ b/docs/browser/api-reference.md @@ -109,6 +109,26 @@ cache.get('name') // 'Alice' cache.get('missing') // null ``` +::: warning Throws on binary values +Values are byte-transparent, so a backend can write bytes that are not valid UTF-8 and they will sync +into this cache. `get()` throws rather than returning a mangled string — use +[`getBytes()`](#getbytes-key) when a value may not be text. +::: + +#### `getBytes(key)` + +Returns the value for a key as raw bytes, or `null` if it does not exist or has expired. Works for +any value; text values come back as their UTF-8 bytes. + +```typescript +getBytes(key: string): Uint8Array | null +``` + +```typescript +const bytes = cache.getBytes('thumb:42') +if (bytes) img.src = URL.createObjectURL(new Blob([bytes])) +``` + #### `getJSON(key)` Returns a JSON-parsed value, or `null` if the key is missing, expired, or not valid JSON. @@ -152,6 +172,19 @@ Sets a key to a string value. Overwrites any existing value and removes any exis set(key: string, value: string): void ``` +#### `setBytes(key, value)` + +Sets a key to raw bytes. Values are byte-transparent: the exact bytes are stored, synced to the +server, replicated, and persisted — through the offline outbox and IndexedDB unchanged. + +```typescript +setBytes(key: string, value: Uint8Array): void +``` + +```typescript +cache.setBytes('thumb:42', new Uint8Array(await blob.arrayBuffer())) +``` + #### `setEx(key, value, seconds)` Sets a key with a TTL in seconds. The key is deleted automatically when the TTL elapses. @@ -349,7 +382,7 @@ handler only** — it does not leave the channel; call `unsubscribe(channel)` fo handlers can be registered on the same channel. ```typescript -onMessage(channel: string, cb: (msg: string) => void): () => void +onMessage(channel: string, cb: (msg: string | Uint8Array) => void): () => void ``` ```typescript @@ -380,6 +413,15 @@ Publish a message to a pub/sub channel. All server-side and browser-side subscri publish(channel: string, message: string): void ``` +#### `publishBytes(channel, message)` + +Publish raw bytes to a pub/sub channel. Subscribers receive a `Uint8Array` rather than a string when +the payload is not valid UTF-8. + +```typescript +publishBytes(channel: string, message: Uint8Array): void +``` + --- ### Persistence @@ -413,6 +455,6 @@ Direct access to the underlying WASM instance (`RecachedCache` from wasm-bindgen get raw(): RawCache ``` -Available methods on `raw`: `set()`, `set_ex()`, `get()`, `del()`, `ttl()`, `exists()`, `subscribe()`, `unsubscribe()`, `publish()`, `connect()`, `auth()`, `broadcast()`, `enable_persistence()`, `clear_persistence()`, `set_mutation_callback()`, `free()`. +Available methods on `raw`: `set()`, `setBytes()`, `set_ex()`, `get()`, `getBytes()`, `del()`, `ttl()`, `exists()`, `subscribe()`, `unsubscribe()`, `publish()`, `publishBytes()`, `connect()`, `auth()`, `broadcast()`, `enable_persistence()`, `clear_persistence()`, `set_mutation_callback()`, `free()`. > Writes through `cache.raw` bypass the `onMutation` notification bus. Use the typed `Cache` methods when possible. diff --git a/docs/guide/introduction.md b/docs/guide/introduction.md index 64c2c58..700c107 100644 --- a/docs/guide/introduction.md +++ b/docs/guide/introduction.md @@ -76,12 +76,13 @@ Commands that interpret a value still require the right shape: `INCR` on a binar `ERR value is not an integer`, and JSON documents must be UTF-8 because JSON is defined that way. Those are type errors, not encoding losses — the stored bytes are unchanged either way. -::: warning Browser SDK: binary is read-only -The browser cache stores and syncs binary values faithfully, and `cache.getBytes(key)` returns them -as a `Uint8Array`. But `cache.set()` takes a string, so binary values can only *originate* from a -backend writing over the RESP port. `cache.get()` throws on a binary value rather than returning -mangled text. -::: +**The browser SDK handles binary too.** `cache.setBytes(key, uint8array)` writes it, +`cache.getBytes(key)` reads it back, and `cache.publishBytes(channel, uint8array)` publishes it. +Binary values survive the offline outbox, cross-tab sync and IndexedDB persistence unchanged. + +`cache.get()` **throws** on a binary value rather than returning mangled text, and `getJSON()` +treats one as a miss — reach for `getBytes()` when a value may not be text. A binary pub/sub payload +arrives at an `onMessage` listener as a `Uint8Array` instead of a string. Before 0.2.2 values were stored as UTF-8 strings and binary was silently replaced with U+FFFD: `SET` returned `OK` and `GET` returned different bytes than were written, on every transport. If you are diff --git a/docs/roadmap.md b/docs/roadmap.md index c271bd6..11b4672 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -5,19 +5,19 @@ Recached competes on **where the data can live** — the same engine on the serv --- -## 6. Mobile SDKs — React Native, Flutter, Kotlin, Swift +## Mobile SDKs — React Native, Flutter, Kotlin, Swift - **Kotlin + Swift first**, via a single `uniffi`-annotated Rust crate that generates bindings for both. The platform WebSocket (OkHttp / URLSession) feeds frames into `sync-client` — no embedded async runtime. Persistence: a file/SQLite adapter over the same outbox/meta effects the browser maps to IndexedDB. Reactivity: Kotlin `Flow` / Swift `Observation` over keychange pushes. - **Flutter** via `flutter_rust_bridge`: synchronous local reads into Rust memory, `watchKey()` → `Stream` for rebuilds. - **React Native** last (Hermes has no WASM): `uniffi-bindgen-react-native` reuses the same binding layer, and the existing React hooks API carries over — same `useKey` in React DOM and React Native. -## 7. WASM server-side scripting +## WASM server-side scripting Run `.wasm` stored procedures in place of Lua scripts. The scripting VM would be sandboxed (no network, no file I/O, bounded execution time), accept any WASM module that exports a specific entry function, and execute it against the cache store. Supports any language that compiles to WASM: Rust, Go (TinyGo), AssemblyScript, Python. -## 8. WASI target +## WASI target A `wasm32-wasip1` build of `wasm-edge` for Cloudflare Workers and Deno Deploy, running Recached as a cache layer at the edge with the same API as the browser client. @@ -28,7 +28,7 @@ A `wasm32-wasip1` build of `wasm-edge` for Cloudflare Workers and Deno Deploy, r Recached's unfair advantage is *where the data lives* — so the winning AI features put the intelligence layer **next to the user** instead of behind another network hop. Ordered by intended sequence. -### 9. Token-cost rate limiting +### Token-cost rate limiting AI providers meter tokens, not requests. One optional argument extends the existing limiter to weighted budgets: @@ -37,7 +37,7 @@ RLCHECK user:42 100000 3600 COST 1850 # consume 1,850 tokens of a 100k/hour bu ``` -### 10. Semantic caching (`SEMSET` / `SEMGET`) +### Semantic caching (`SEMSET` / `SEMGET`) LLM calls are expensive and repeats are *paraphrases*, so exact-key caching misses them. A semantic cache returns a hit when a query's embedding is close enough to a cached one: @@ -46,12 +46,12 @@ SEMSET prompts "" EX 3600 SEMGET prompts 0.92 # → cached response or nil ``` -### 11. Streaming values — "watch the agent think" +### Streaming values — "watch the agent think" An agent streams tokens into a key with `APPEND`; every subscribed browser renders it live. Live queries already deliver the subscription — the missing piece is an append *delta* frame (keychange currently re-sends the whole value) plus catch-up-then-follow on reconnect. `useKey('agent:run:42:output')` becomes a live-typing agent visible to any number of viewers. Redis Streams end at the backend; this reaches the UI. -### 12. Computed keys — the reactive cache +### Computed keys — the reactive cache Declare a key as a function of other keys; the server recomputes on change and the diff flows through live queries — cache becomes spreadsheet. `cart:42:total` recomputes when any `cart:42:item:*` changes, and every subscribed UI updates. Uses WASM scripting (#7) as the function runtime. Biggest lift, biggest ceiling. diff --git a/sync-client/src/lib.rs b/sync-client/src/lib.rs index c83ce96..e9e909a 100644 --- a/sync-client/src/lib.rs +++ b/sync-client/src/lib.rs @@ -46,7 +46,9 @@ pub enum Incoming { /// local store — notify UI subscribers. Applied, /// A pub/sub message — deliver to channel subscribers. - PubSub { channel: String, message: String }, + /// A pub/sub message. The payload is bytes because a publisher may send + /// binary; the adapter decides how to surface it. + PubSub { channel: String, message: Vec }, /// A command reply. When `retired` is set, that outbox row is now /// acknowledged — delete it from durable storage. Reply { retired: Option }, @@ -62,9 +64,13 @@ pub enum Incoming { pub struct Enqueued { /// Outbox row id — persist the row under this key. pub id: u64, - /// The wire frame (dedup-wrapped when requested). This exact string is + /// The wire frame (dedup-wrapped when requested). These exact bytes are /// stored in the outbox and replayed on reconnect. - pub frame: String, + /// + /// Bytes rather than `String`: a frame carries a stored value, which may be + /// arbitrary binary. Holding it as text would corrupt the queued write and, + /// on reconnect, replay the corrupted version to the server. + pub frame: Vec, /// Send `frame` now (the caller reported the socket open, and the write /// was recorded as inflight). pub send_now: bool, @@ -78,7 +84,7 @@ pub struct Enqueued { pub struct SyncClient { store: Arc, /// Writes not yet acknowledged by the server, in send order. - outbox: VecDeque<(u64, String)>, + outbox: VecDeque<(u64, Vec)>, outbox_seq: u64, /// One entry per frame sent on the current socket: `Some(outbox id)` for /// data writes, `None` for session commands. Replies arrive in send @@ -175,19 +181,19 @@ impl SyncClient { // inflight — session replies must occupy a reply slot or they would // falsely acknowledge a data write). - pub fn set_password(&mut self, password: &str, connected: bool) -> Option { + pub fn set_password(&mut self, password: &str, connected: bool) -> Option> { self.password = Some(password.to_string()); self.session_frame(to_resp(&["AUTH", password]), connected) } - pub fn set_sync_token(&mut self, token: &str, connected: bool) -> Option { + pub fn set_sync_token(&mut self, token: &str, connected: bool) -> Option> { self.sync_token = Some(token.to_string()); self.session_frame(to_resp(&["SYNC", "TOKEN", token]), connected) } /// Comma-separated glob patterns. Returns `None` (and records nothing) /// when the list is empty. - pub fn set_sync_scopes(&mut self, patterns_csv: &str, connected: bool) -> Option { + pub fn set_sync_scopes(&mut self, patterns_csv: &str, connected: bool) -> Option> { let frame = sync_scopes_frame(patterns_csv)?; self.sync_scopes_csv = Some(patterns_csv.to_string()); self.session_frame(frame, connected) @@ -195,14 +201,14 @@ impl SyncClient { /// Register a live query (idempotent). The returned frame re-hydrates /// matching keys via the `qstate` reply. - pub fn add_live_query(&mut self, pattern: &str, connected: bool) -> Option { + pub fn add_live_query(&mut self, pattern: &str, connected: bool) -> Option> { if !self.live_queries.iter().any(|p| p == pattern) { self.live_queries.push(pattern.to_string()); } self.session_frame(to_resp(&["QSUB", pattern]), connected) } - pub fn remove_live_query(&mut self, pattern: Option<&str>, connected: bool) -> Option { + pub fn remove_live_query(&mut self, pattern: Option<&str>, connected: bool) -> Option> { let frame = match pattern { Some(p) => { self.live_queries.retain(|q| q != p); @@ -216,7 +222,7 @@ impl SyncClient { self.session_frame(frame, connected) } - fn session_frame(&mut self, frame: String, connected: bool) -> Option { + fn session_frame(&mut self, frame: Vec, connected: bool) -> Option> { if connected { self.inflight.push_back(None); Some(frame) @@ -231,7 +237,7 @@ impl SyncClient { /// state first (AUTH → SYNC → live queries), then the full outbox replay. /// Live-query re-subscription re-hydrates local state; outbox entries /// stay queued until their replies acknowledge them. - pub fn on_open(&mut self) -> Vec { + pub fn on_open(&mut self) -> Vec> { self.attempts = 0; self.inflight.clear(); let mut frames = Vec::new(); @@ -298,13 +304,13 @@ impl SyncClient { /// envelope — store writes want this; connection-scoped commands /// (pub/sub) must not be wrapped, or a legitimately replayed SUBSCRIBE /// would be skipped as a duplicate. - pub fn enqueue_write(&mut self, encoded: &str, dedup: bool, connected: bool) -> Enqueued { + pub fn enqueue_write(&mut self, encoded: &[u8], dedup: bool, connected: bool) -> Enqueued { let id = self.outbox_seq; self.outbox_seq += 1; let frame = if dedup { self.wrap_dedup(encoded, id) } else { - encoded.to_string() + encoded.to_vec() }; let dropped = if self.outbox.len() >= self.max_pending { self.outbox.pop_front().map(|(old, _)| old) @@ -325,25 +331,35 @@ impl SyncClient { /// Splice a `DEDUP ` envelope into an already-encoded /// RESP array: bump the element count and prepend three elements. - fn wrap_dedup(&self, plain: &str, outbox_id: u64) -> String { - let Some((head, rest)) = plain.split_once("\r\n") else { - return plain.to_string(); + fn wrap_dedup(&self, plain: &[u8], outbox_id: u64) -> Vec { + // Only the header is text; `rest` is copied through untouched so a + // binary value inside the frame is never reinterpreted. + let Some(split) = plain.windows(2).position(|w| w == b"\r\n") else { + return plain.to_vec(); }; - let n: usize = head.trim_start_matches('*').parse().unwrap_or(0); + let head = &plain[..split]; + let rest = &plain[split + 2..]; + let n: usize = std::str::from_utf8(head) + .ok() + .map(|h| h.trim_start_matches('*')) + .and_then(|h| h.parse().ok()) + .unwrap_or(0); if n == 0 { - return plain.to_string(); + return plain.to_vec(); } let wire_id = ((self.epoch as u64) << 32) | (outbox_id & 0xFFFF_FFFF); let id_s = wire_id.to_string(); - format!( - "*{}\r\n$5\r\nDEDUP\r\n${}\r\n{}\r\n${}\r\n{}\r\n{}", + let mut out = format!( + "*{}\r\n$5\r\nDEDUP\r\n${}\r\n{}\r\n${}\r\n{}\r\n", n + 3, self.client_id.len(), self.client_id, id_s.len(), id_s, - rest ) + .into_bytes(); + out.extend_from_slice(rest); + out } /// Restore outbox rows persisted by a previous session (ordered by their @@ -352,7 +368,7 @@ impl SyncClient { /// verbatim, so their embedded dedup ids still match what the server may /// have already applied. Returns `(old_id, new_id, frame)` for the /// adapter to rewrite durable storage. - pub fn restore_outbox(&mut self, mut rows: Vec<(u64, String)>) -> Vec<(u64, u64, String)> { + pub fn restore_outbox(&mut self, mut rows: Vec<(u64, Vec)>) -> Vec<(u64, u64, Vec)> { rows.sort_by_key(|(id, _)| *id); let mut rewrites = Vec::with_capacity(rows.len()); for (old_id, frame) in rows.into_iter().rev() { @@ -382,8 +398,8 @@ impl SyncClient { /// or surfaced; everything else is a command reply that acknowledges the /// oldest inflight command — `qstate` arrays are both the reply to a /// QSUB *and* state to apply. See `docs/server/protocol.md`. - pub fn handle_frame(&mut self, text: &str) -> Incoming { - match Value::parse(text.as_bytes()) { + pub fn handle_frame(&mut self, raw: &[u8]) -> Incoming { + match Value::parse(raw) { Ok((Value::Push(arr), _)) => { if arr.len() == 3 && let ( @@ -395,7 +411,7 @@ impl SyncClient { { return Incoming::PubSub { channel: String::from_utf8_lossy(channel).into_owned(), - message: String::from_utf8_lossy(payload).into_owned(), + message: payload.clone(), }; } if let Ok(cmd) = Command::from_value(Value::Array(Some(arr))) @@ -579,7 +595,7 @@ impl SyncClient { } } -fn sync_scopes_frame(patterns_csv: &str) -> Option { +fn sync_scopes_frame(patterns_csv: &str) -> Option> { let pats: Vec<&str> = patterns_csv .split(',') .map(str::trim) @@ -638,12 +654,25 @@ pub fn is_replayable_mutation(cmd: &Command) -> bool { } /// RESP array encoding, shared by adapters. -pub fn to_resp(parts: &[&str]) -> String { - let mut s = format!("*{}\r\n", parts.len()); +/// Encode a RESP array frame from text arguments. +/// +/// Returns bytes, not a `String`: frames are concatenated with, and replayed +/// alongside, frames carrying binary values, so the whole pipeline is byte-based. +pub fn to_resp(parts: &[&str]) -> Vec { + let byte_parts: Vec<&[u8]> = parts.iter().map(|p| p.as_bytes()).collect(); + to_resp_bytes(&byte_parts) +} + +/// Encode a RESP array frame from raw byte arguments, for commands carrying a +/// binary value. +pub fn to_resp_bytes(parts: &[&[u8]]) -> Vec { + let mut out = format!("*{}\r\n", parts.len()).into_bytes(); for part in parts { - s.push_str(&format!("${}\r\n{}\r\n", part.len(), part)); + out.extend_from_slice(format!("${}\r\n", part.len()).as_bytes()); + out.extend_from_slice(part); + out.extend_from_slice(b"\r\n"); } - s + out } #[cfg(test)] diff --git a/sync-client/src/tests.rs b/sync-client/src/tests.rs index 6bfb205..89694a5 100644 --- a/sync-client/src/tests.rs +++ b/sync-client/src/tests.rs @@ -15,6 +15,12 @@ fn bulk(s: &str) -> Value { // ── dedup envelopes ─────────────────────────────────────────────────────────── +/// Render a wire frame as text for assertions. Frames are bytes because values +/// may be binary; every frame in these tests has a text payload. +fn t(frame: &[u8]) -> String { + String::from_utf8_lossy(frame).into_owned() +} + #[test] fn enqueue_wraps_writes_in_dedup_envelope() { let mut c = client(); @@ -22,13 +28,13 @@ fn enqueue_wraps_writes_in_dedup_envelope() { assert_eq!(e.id, 0); assert!(!e.send_now); assert_eq!( - e.frame, + t(&e.frame), "*6\r\n$5\r\nDEDUP\r\n$8\r\nclient-a\r\n$1\r\n0\r\n$6\r\nINCRBY\r\n$1\r\nn\r\n$1\r\n2\r\n" ); // Ids are monotonic. let e2 = c.enqueue_write(&to_resp(&["SET", "k", "v"]), true, false); assert_eq!(e2.id, 1); - assert!(e2.frame.contains("$1\r\n1\r\n")); + assert!(t(&e2.frame).contains("$1\r\n1\r\n")); } #[test] @@ -37,7 +43,7 @@ fn epoch_forms_upper_bits_of_wire_id() { c.set_epoch(2); let e = c.enqueue_write(&to_resp(&["SET", "k", "v"]), true, false); let wire_id = (2u64 << 32).to_string(); - assert!(e.frame.contains(&wire_id), "frame: {}", e.frame); + assert!(t(&e.frame).contains(&wire_id), "frame: {}", t(&e.frame)); } #[test] @@ -48,6 +54,72 @@ fn nodedup_writes_pass_through_unwrapped() { assert_eq!(e.frame, plain); } +// ── binary values on the write path ────────────────────────────────────────── + +#[test] +fn dedup_envelope_preserves_a_binary_payload() { + // wrap_dedup splices a header onto an already-encoded frame. Parsing that + // frame as text to do so would corrupt a binary value on the way out — and + // again on every reconnect replay, since the outbox stores the wrapped form. + let mut c = client(); + let binary: &[u8] = &[0xff, 0xfe, 0x00, 0x41, 0x80]; + let plain = to_resp_bytes(&[b"SET", b"k", binary]); + + let e = c.enqueue_write(&plain, true, false); + + // The envelope is spliced in front and the original payload is intact. + assert!(e.frame.starts_with(b"*6\r\n$5\r\nDEDUP\r\n")); + assert!( + e.frame.windows(binary.len()).any(|w| w == binary), + "binary payload must survive dedup wrapping" + ); + // Byte count: the wrapped frame is the plain frame plus the envelope, so + // nothing was re-encoded or replaced along the way. + // Both headers are one digit wide (*3 -> *6), so the wrapped frame is + // exactly the plain frame plus the three spliced elements. + assert_eq!( + e.frame.len(), + plain.len() + b"$5\r\nDEDUP\r\n$8\r\nclient-a\r\n$1\r\n0\r\n".len(), + "frame: {:?}", + e.frame + ); +} + +#[test] +fn a_binary_write_replays_unchanged_after_reconnect() { + let mut c = client(); + let binary: &[u8] = &[0x00, 0xff, 0x1b, 0x80]; + let e = c.enqueue_write(&to_resp_bytes(&[b"SET", b"k", binary]), true, false); + + let frames = c.on_open(); + assert_eq!(frames.len(), 1); + assert_eq!( + frames[0], e.frame, + "replayed frame must be byte-identical to the queued one" + ); +} + +#[test] +fn a_binary_pubsub_payload_is_delivered_as_bytes() { + // Rendering the payload as text here would hand the application a mangled + // message with no indication anything was lost. + let mut c = client(); + let binary = vec![0xffu8, 0xfe, 0x41]; + let mut frame = b"*3\r\n$7\r\nmessage\r\n$4\r\nnews\r\n$3\r\n".to_vec(); + frame.extend_from_slice(&binary); + frame.extend_from_slice(b"\r\n"); + // Re-tag as a Push frame, which is how pub/sub arrives. + frame[0] = b'>'; + + match c.handle_frame(&frame) { + Incoming::PubSub { channel, message } => { + assert_eq!(channel, "news"); + assert_eq!(message, binary); + } + other => panic!("expected a pub/sub delivery, got {other:?}"), + } +} + // ── connection lifecycle & replay order ─────────────────────────────────────── #[test] @@ -61,9 +133,9 @@ fn on_open_replays_session_then_outbox_in_order() { let frames = c.on_open(); assert_eq!(frames.len(), 5); - assert!(frames[0].contains("AUTH")); - assert!(frames[1].contains("TOKEN")); - assert!(frames[2].contains("QSUB")); + assert!(t(&frames[0]).contains("AUTH")); + assert!(t(&frames[1]).contains("TOKEN")); + assert!(t(&frames[2]).contains("QSUB")); assert_eq!(frames[3], w1.frame); assert_eq!(frames[4], w2.frame); } @@ -80,11 +152,11 @@ fn session_commands_sent_while_open_occupy_reply_slots() { let w = c.enqueue_write(&to_resp(&["SET", "a", "1"]), true, true); assert!(w.send_now); assert_eq!( - c.handle_frame("*1\r\n$6\r\ncart:*\r\n"), + c.handle_frame(b"*1\r\n$6\r\ncart:*\r\n"), Incoming::Reply { retired: None } ); assert_eq!( - c.handle_frame("+OK\r\n"), + c.handle_frame(b"+OK\r\n"), Incoming::Reply { retired: Some(w.id) } @@ -175,7 +247,7 @@ fn replies_retire_outbox_rows_in_order() { assert_eq!(frames.len(), 2); assert_eq!( - c.handle_frame("+OK\r\n"), + c.handle_frame(b"+OK\r\n"), Incoming::Reply { retired: Some(w1.id) } @@ -183,7 +255,7 @@ fn replies_retire_outbox_rows_in_order() { assert_eq!(c.outbox_len(), 1); // +DUP is a reply like any other — the row still retires. assert_eq!( - c.handle_frame("+DUP\r\n"), + c.handle_frame(b"+DUP\r\n"), Incoming::Reply { retired: Some(w2.id) } @@ -211,15 +283,15 @@ fn pushes_are_not_replies() { // A mutation push and a keychange arrive before our reply — neither may // consume the reply slot. assert_eq!( - c.handle_frame(">3\r\n$3\r\nSET\r\n$1\r\nx\r\n$1\r\ny\r\n"), + c.handle_frame(b">3\r\n$3\r\nSET\r\n$1\r\nx\r\n$1\r\ny\r\n"), Incoming::Applied ); assert_eq!( - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$1\r\nx\r\n$1\r\nz\r\n"), + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$1\r\nx\r\n$1\r\nz\r\n"), Incoming::Applied ); assert_eq!( - c.handle_frame("+OK\r\n"), + c.handle_frame(b"+OK\r\n"), Incoming::Reply { retired: Some(w.id) } @@ -247,7 +319,7 @@ fn qstate_applies_state_and_counts_as_reply() { c.on_open(); let qstate = "*4\r\n$6\r\nqstate\r\n$6\r\ncart:*\r\n$6\r\ncart:1\r\n$5\r\napple\r\n"; assert_eq!( - c.handle_frame(qstate), + c.handle_frame(qstate.as_bytes()), Incoming::AppliedReply { retired: None } ); assert_eq!(get(&c, "cart:1"), bulk("apple")); @@ -256,16 +328,16 @@ fn qstate_applies_state_and_counts_as_reply() { #[test] fn keychange_sets_and_deletes() { let mut c = client(); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$1\r\nk\r\n$1\r\nv\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$1\r\nk\r\n$1\r\nv\r\n"); assert_eq!(get(&c, "k"), bulk("v")); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$1\r\nk\r\n$-1\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$1\r\nk\r\n$-1\r\n"); assert_eq!(get(&c, "k"), Value::BulkString(None)); } #[test] fn pubsub_messages_surface_without_touching_store() { let mut c = client(); - let msg = c.handle_frame(">3\r\n$7\r\nmessage\r\n$4\r\nnews\r\n$5\r\nhello\r\n"); + let msg = c.handle_frame(b">3\r\n$7\r\nmessage\r\n$4\r\nnews\r\n$5\r\nhello\r\n"); assert_eq!( msg, Incoming::PubSub { @@ -304,8 +376,8 @@ fn restore_renumbers_but_preserves_frames_and_order() { // Replay order: restored rows first (oldest first), then this session's. let frames = c.on_open(); assert_eq!(frames.len(), 3); - assert_eq!(frames[0], old_frame_a); - assert_eq!(frames[1], old_frame_b); + assert_eq!(t(&frames[0]), old_frame_a); + assert_eq!(t(&frames[1]), old_frame_b); assert_eq!(frames[2], session_write.frame); } @@ -315,9 +387,9 @@ fn restore_renumbers_but_preserves_frames_and_order() { fn sync_scopes_builds_a_sync_frame_from_csv() { let mut c = client(); let frame = c.set_sync_scopes("cart:*,user:1:*", true).unwrap(); - assert!(frame.contains("SYNC"), "{frame}"); - assert!(frame.contains("cart:*")); - assert!(frame.contains("user:1:*")); + assert!(t(&frame).contains("SYNC"), "{}", t(&frame)); + assert!(t(&frame).contains("cart:*")); + assert!(t(&frame).contains("user:1:*")); } #[test] @@ -325,8 +397,12 @@ fn sync_scopes_ignore_blank_entries_and_whitespace() { let mut c = client(); let frame = c.set_sync_scopes(" cart:* , , user:1:* ,", true).unwrap(); // Three commas but only two real patterns → SYNC + 2 args. - assert!(frame.starts_with("*3\r\n"), "expected 3 parts, got {frame}"); - assert!(frame.contains("cart:*") && frame.contains("user:1:*")); + assert!( + t(&frame).starts_with("*3\r\n"), + "expected 3 parts, got {}", + t(&frame) + ); + assert!(t(&frame).contains("cart:*") && t(&frame).contains("user:1:*")); } #[test] @@ -344,7 +420,7 @@ fn scopes_are_replayed_on_reconnect() { c.set_sync_scopes("cart:*", false); let frames = c.on_open(); assert!( - frames.iter().any(|f| f.contains("cart:*")), + frames.iter().any(|f| t(f).contains("cart:*")), "scopes must be re-established after reconnect: {frames:?}" ); } @@ -357,9 +433,9 @@ fn a_sync_token_takes_precedence_over_raw_scope_patterns() { c.set_sync_scopes("cart:*", false); c.set_sync_token("tok-123", false); let frames = c.on_open(); - assert!(frames.iter().any(|f| f.contains("tok-123"))); + assert!(frames.iter().any(|f| t(f).contains("tok-123"))); assert!( - !frames.iter().any(|f| f.contains("cart:*")), + !frames.iter().any(|f| t(f).contains("cart:*")), "raw scopes must not be sent alongside a token: {frames:?}" ); } @@ -374,7 +450,7 @@ fn live_queries_register_once_and_replay_on_open() { c.add_live_query("user:*", false); let frames = c.on_open(); - let qsubs = frames.iter().filter(|f| f.contains("QSUB")).count(); + let qsubs = frames.iter().filter(|f| t(f).contains("QSUB")).count(); assert_eq!( qsubs, 2, "duplicate patterns must not double-subscribe: {frames:?}" @@ -389,17 +465,18 @@ fn removing_one_live_query_leaves_the_others() { let frame = c.remove_live_query(Some("cart:*"), true).unwrap(); assert!( - frame.contains("QUNSUB") && frame.contains("cart:*"), - "{frame}" + t(&frame).contains("QUNSUB") && t(&frame).contains("cart:*"), + "{}", + t(&frame) ); let frames = c.on_open(); assert!( - frames.iter().any(|f| f.contains("user:*")), + frames.iter().any(|f| t(f).contains("user:*")), "survivor replays" ); assert!( - !frames.iter().any(|f| f.contains("cart:*")), + !frames.iter().any(|f| t(f).contains("cart:*")), "removed query must not replay: {frames:?}" ); } @@ -411,15 +488,16 @@ fn removing_all_live_queries_sends_a_bare_qunsub() { c.add_live_query("user:*", false); let frame = c.remove_live_query(None, true).unwrap(); - assert!(frame.contains("QUNSUB"), "{frame}"); + assert!(t(&frame).contains("QUNSUB"), "{}", t(&frame)); assert!( - !frame.contains("cart:*"), - "bare QUNSUB carries no pattern: {frame}" + !t(&frame).contains("cart:*"), + "bare QUNSUB carries no pattern: {}", + t(&frame) ); let frames = c.on_open(); assert!( - !frames.iter().any(|f| f.contains("QSUB")), + !frames.iter().any(|f| t(f).contains("QSUB")), "nothing should replay after clearing: {frames:?}" ); } @@ -436,7 +514,7 @@ fn on_open_sends_auth_before_scopes_before_queries() { c.add_live_query("cart:*", false); let frames = c.on_open(); - let pos = |needle: &str| frames.iter().position(|f| f.contains(needle)); + let pos = |needle: &str| frames.iter().position(|f| t(f).contains(needle)); let (auth, sync, qsub) = ( pos("AUTH").unwrap(), pos("TOKEN").unwrap(), @@ -485,7 +563,7 @@ fn clear_outbox_drops_queued_and_inflight_writes() { assert_eq!(c.outbox_len(), 0); // A reply arriving after a clear must not retire a row that no longer // exists or panic on an empty inflight queue. - c.handle_frame("+OK\r\n"); + c.handle_frame(b"+OK\r\n"); assert_eq!(c.outbox_len(), 0); } @@ -538,7 +616,7 @@ fn keychange_rebuilds_a_hash_without_a_re_read() { )); let mut c = client(); - c.handle_frame(&push(keychange_frame(&source, "cart:42"))); + c.handle_frame(push(keychange_frame(&source, "cart:42")).as_bytes()); assert_eq!( c.store() @@ -561,7 +639,7 @@ fn keychange_rebuilds_lists_in_order() { )); let mut c = client(); - c.handle_frame(&push(keychange_frame(&source, "queue"))); + c.handle_frame(push(keychange_frame(&source, "queue")).as_bytes()); assert_eq!( c.store().execute(Command::LRange("queue".into(), 0, -1)), @@ -584,8 +662,8 @@ fn keychange_rebuilds_sets_and_zsets() { )); let mut c = client(); - c.handle_frame(&push(keychange_frame(&source, "tags"))); - c.handle_frame(&push(keychange_frame(&source, "board"))); + c.handle_frame(push(keychange_frame(&source, "tags")).as_bytes()); + c.handle_frame(push(keychange_frame(&source, "board")).as_bytes()); assert_eq!( c.store().execute(Command::SCard("tags".into())), @@ -608,7 +686,7 @@ fn keychange_rebuilds_json_documents() { )); let mut c = client(); - c.handle_frame(&push(keychange_frame(&source, "doc"))); + c.handle_frame(push(keychange_frame(&source, "doc")).as_bytes()); assert_eq!( c.store().execute(Command::JGet("doc".into(), None)), @@ -627,14 +705,14 @@ fn a_removed_member_disappears_from_the_local_copy() { )); let mut c = client(); - c.handle_frame(&push(keychange_frame(&source, "tags"))); + c.handle_frame(push(keychange_frame(&source, "tags")).as_bytes()); assert_eq!( c.store().execute(Command::SCard("tags".into())), Value::Integer(2) ); source.execute(Command::SRem("tags".into(), vec!["red".into()])); - c.handle_frame(&push(keychange_frame(&source, "tags"))); + c.handle_frame(push(keychange_frame(&source, "tags")).as_bytes()); assert_eq!( c.store() @@ -667,7 +745,7 @@ fn qstate_delivers_complete_collections_on_subscribe() { Value::BulkString(Some(b"cart:2".to_vec())), source.get_current("cart:2"), ]); - c.handle_frame(&frame); + c.handle_frame(frame.as_bytes()); assert_eq!( c.store() @@ -685,14 +763,17 @@ fn an_unknown_type_tag_is_ignored_rather_than_guessed() { // Forward compatibility: a newer server sending a type this client does not // know must not corrupt the local copy. let mut c = client(); - c.handle_frame(&push(vec![ - Value::BulkString(Some(b"keychange".to_vec())), - Value::BulkString(Some(b"k".to_vec())), - Value::Array(Some(vec![ - Value::BulkString(Some(b"futuretype".to_vec())), - Value::BulkString(Some(b"payload".to_vec())), - ])), - ])); + c.handle_frame( + push(vec![ + Value::BulkString(Some(b"keychange".to_vec())), + Value::BulkString(Some(b"k".to_vec())), + Value::Array(Some(vec![ + Value::BulkString(Some(b"futuretype".to_vec())), + Value::BulkString(Some(b"payload".to_vec())), + ])), + ]) + .as_bytes(), + ); assert_eq!(get(&c, "k"), Value::BulkString(None)); } @@ -704,13 +785,13 @@ fn flushdb_sentinel_clears_every_key_matching_the_pattern() { // per deleted key — a keyspace-sized frame storm for a single command. let mut c = client(); c.add_live_query("cart:*", false); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$11\r\ncart:item:1\r\n$1\r\na\r\n"); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$11\r\ncart:item:2\r\n$1\r\nb\r\n"); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$7\r\nother:1\r\n$1\r\nc\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$11\r\ncart:item:1\r\n$1\r\na\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$11\r\ncart:item:2\r\n$1\r\nb\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$7\r\nother:1\r\n$1\r\nc\r\n"); assert_eq!(get(&c, "cart:item:1"), bulk("a")); // Sentinel: nil value whose "key" is the registered pattern. - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$6\r\ncart:*\r\n$-1\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$6\r\ncart:*\r\n$-1\r\n"); assert_eq!(get(&c, "cart:item:1"), Value::BulkString(None)); assert_eq!(get(&c, "cart:item:2"), Value::BulkString(None)); @@ -726,11 +807,11 @@ fn a_nil_for_an_unregistered_pattern_deletes_only_that_key() { // Without this distinction, a literal key that happens to contain a glob // character would wipe unrelated data. let mut c = client(); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$5\r\nkey:1\r\n$1\r\na\r\n"); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$5\r\nkey:2\r\n$1\r\nb\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$5\r\nkey:1\r\n$1\r\na\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$5\r\nkey:2\r\n$1\r\nb\r\n"); // Not a registered live query — treat it as an ordinary single-key delete. - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$5\r\nkey:1\r\n$-1\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$5\r\nkey:1\r\n$-1\r\n"); assert_eq!(get(&c, "key:1"), Value::BulkString(None)); assert_eq!(get(&c, "key:2"), bulk("b"), "unrelated key survives"); @@ -740,7 +821,7 @@ fn a_nil_for_an_unregistered_pattern_deletes_only_that_key() { fn flushdb_sentinel_is_harmless_when_nothing_matches() { let mut c = client(); c.add_live_query("cart:*", false); - c.handle_frame("*3\r\n$9\r\nkeychange\r\n$6\r\ncart:*\r\n$-1\r\n"); + c.handle_frame(b"*3\r\n$9\r\nkeychange\r\n$6\r\ncart:*\r\n$-1\r\n"); assert_eq!(c.store().execute(Command::DbSize), Value::Integer(0)); } diff --git a/wasm-edge/Cargo.toml b/wasm-edge/Cargo.toml index f8e9586..5145b1d 100644 --- a/wasm-edge/Cargo.toml +++ b/wasm-edge/Cargo.toml @@ -18,6 +18,9 @@ js-sys.workspace = true workspace = true features = [ "WebSocket", + # WebSocket.binaryType, so binary frames arrive as ArrayBuffer rather than + # a Blob that would need an async read. + "BinaryType", "BroadcastChannel", "MessageEvent", "ErrorEvent", diff --git a/wasm-edge/sdk.ts b/wasm-edge/sdk.ts index 3625187..fc0489f 100644 --- a/wasm-edge/sdk.ts +++ b/wasm-edge/sdk.ts @@ -67,6 +67,7 @@ interface RawCache { connect(url: string): void; auth(password: string): string; set(key: string, value: string): string; + setBytes(key: string, value: Uint8Array): string; set_ex(key: string, value: string, seconds: number): string; get(key: string): string | undefined; getBytes(key: string): Uint8Array | undefined; @@ -77,6 +78,7 @@ interface RawCache { ttl(key: string): number; exists(key: string): boolean; publish(channel: string, message: string): void; + publishBytes(channel: string, message: Uint8Array): void; subscribe(channel: string): void; unsubscribe(channel: string): void; jset(key: string, path: string, value: string): string; @@ -132,7 +134,10 @@ export class Cache { readonly raw: RawCache; private readonly _mutationListeners = new Set<() => void>(); - private readonly _messageListeners = new Map void>>(); + private readonly _messageListeners = new Map< + string, + Set<(msg: string | Uint8Array) => void> + >(); private readonly _outboxFullListeners = new Set<(droppedId: number, pending: number) => void>(); /** @internal Arrow function so `this` is always bound when passed as a callback. */ @@ -146,7 +151,10 @@ export class Cache { }; /** @internal */ - private readonly _notifyMessage = (channel: string, message: string): void => { + private readonly _notifyMessage = ( + channel: string, + message: string | Uint8Array, + ): void => { const listeners = this._messageListeners.get(channel); if (listeners) { for (const cb of listeners) cb(message); @@ -228,7 +236,10 @@ export class Cache { * cache.unsubscribe('notifications'); * ``` */ - onMessage(channel: string, cb: (msg: string) => void): () => void { + onMessage( + channel: string, + cb: (msg: string | Uint8Array) => void, + ): () => void { let listeners = this._messageListeners.get(channel); if (!listeners) { listeners = new Set(); @@ -327,6 +338,21 @@ export class Cache { this.raw.set(key, value); } + /** + * Store raw bytes. Syncs to the server and other tabs when connected. + * + * Values are byte-transparent: the exact bytes given are stored, replicated, + * and persisted. Use this for compressed payloads, protobuf, or images — + * anything a JS string cannot hold. Read them back with {@link getBytes}. + * + * ```ts + * cache.setBytes('thumb:42', new Uint8Array(await blob.arrayBuffer())); + * ``` + */ + setBytes(key: string, value: Uint8Array): void { + this.raw.setBytes(key, value); + } + /** * Store a string value with a TTL (seconds). The key is deleted automatically * once the TTL elapses. @@ -465,6 +491,16 @@ export class Cache { this.raw.publish(channel, message); } + /** + * Publish raw bytes to a server pub/sub channel. + * + * Subscribers receive a `Uint8Array` rather than a string when the payload is + * not valid UTF-8 — see {@link onMessage}. + */ + publishBytes(channel: string, message: Uint8Array): void { + this.raw.publishBytes(channel, message); + } + // ── Sync scoping & live queries ─────────────────────────────────────────── /** diff --git a/wasm-edge/src/lib.rs b/wasm-edge/src/lib.rs index e7898ea..765b8dc 100644 --- a/wasm-edge/src/lib.rs +++ b/wasm-edge/src/lib.rs @@ -5,7 +5,7 @@ use js_sys::Promise; use std::cell::{Cell, RefCell}; use std::rc::Rc; use std::sync::Arc; -use sync_client::{Incoming, SyncClient, is_replayable_mutation, to_resp}; +use sync_client::{Incoming, SyncClient, is_replayable_mutation, to_resp, to_resp_bytes}; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::{JsFuture, spawn_local}; use web_sys::{BroadcastChannel, Event, MessageEvent, WebSocket}; @@ -128,7 +128,7 @@ extern "C" { #[wasm_bindgen(js_name = "idbAppend")] fn idb_append_js(db: &JsValue, seq: f64, cmd: &[u8]) -> Promise; #[wasm_bindgen(js_name = "idbOutboxPut")] - fn idb_outbox_put_js(db: &JsValue, id: f64, cmd: &str) -> Promise; + fn idb_outbox_put_js(db: &JsValue, id: f64, cmd: &[u8]) -> Promise; #[wasm_bindgen(js_name = "idbOutboxDelete")] fn idb_outbox_delete_js(db: &JsValue, id: f64) -> Promise; #[wasm_bindgen(js_name = "idbWalClear")] @@ -353,9 +353,9 @@ fn outbox_delete(sh: &WsShared, id: u64) { } /// Persist an outbox row to durable storage. -fn outbox_put(sh: &WsShared, id: u64, frame: &str) { +fn outbox_put(sh: &WsShared, id: u64, frame: &[u8]) { if let Some(db) = sh.idb.borrow().clone() { - let cmd = frame.to_string(); + let cmd = frame.to_vec(); spawn_local(async move { let _ = JsFuture::from(idb_outbox_put_js(&db, id as f64, &cmd)).await; }); @@ -368,11 +368,13 @@ fn dispatch_incoming(sh: &WsShared, incoming: Incoming) { Incoming::Applied => notify_mutation(&sh.on_mutation), Incoming::PubSub { channel, message } => { if let Some(f) = sh.on_message.borrow().as_ref() { - let _ = f.call2( - &JsValue::NULL, - &JsValue::from_str(&channel), - &JsValue::from_str(&message), - ); + let payload = match std::str::from_utf8(&message) { + Ok(text) => JsValue::from_str(text), + // A binary publish cannot be a JS string; the listener gets + // the bytes rather than a lossy rendering of them. + Err(_) => js_sys::Uint8Array::from(&message[..]).into(), + }; + let _ = f.call2(&JsValue::NULL, &JsValue::from_str(&channel), &payload); } } Incoming::Reply { retired } => { @@ -392,7 +394,7 @@ fn dispatch_incoming(sh: &WsShared, incoming: Incoming) { /// Queue a write through the core and mirror its effects: durable outbox row, /// possible overflow eviction, and an immediate send when connected. -fn queue_write(sh: &WsShared, encoded: &str, dedup: bool) { +fn queue_write(sh: &WsShared, encoded: &[u8], dedup: bool) { // Local-only mode (no connect() and no persistence): nothing to sync to. if sh.url.borrow().is_none() && sh.idb.borrow().is_none() { return; @@ -424,7 +426,50 @@ fn queue_write(sh: &WsShared, encoded: &str, dedup: bool) { if e.send_now && let Some(ws) = sh.ws.borrow().as_ref() { - let _ = ws.send_with_str(&e.frame); + ws_send_frame(ws, &e.frame); + } +} + +/// Extract the RESP bytes from a message event, whether the peer sent a text +/// frame or a binary one. +fn event_frame_bytes(e: &MessageEvent) -> Option> { + let data = e.data(); + if let Some(text) = data.as_string() { + return Some(text.into_bytes()); + } + if let Ok(buf) = data.clone().dyn_into::() { + return Some(js_sys::Uint8Array::new(&buf).to_vec()); + } + if let Ok(arr) = data.dyn_into::() { + return Some(arr.to_vec()); + } + None +} + +/// Post a RESP frame to other tabs, as a string when the bytes are text and a +/// `Uint8Array` when they are not. +fn bc_post(bc: &BroadcastChannel, frame: &[u8]) { + let msg = match std::str::from_utf8(frame) { + Ok(text) => JsValue::from_str(text), + Err(_) => js_sys::Uint8Array::from(frame).into(), + }; + let _ = bc.post_message(&msg); +} + +/// Send one RESP frame over the socket. +/// +/// Text frames must be well-formed UTF-8 per the WebSocket spec, so a frame +/// carrying a binary value goes out as a binary frame — which the server +/// accepts from 0.2.2. Everything else stays text, so nothing changes for the +/// overwhelming majority of traffic. +fn ws_send_frame(ws: &WebSocket, frame: &[u8]) { + match std::str::from_utf8(frame) { + Ok(text) => { + let _ = ws.send_with_str(text); + } + Err(_) => { + let _ = ws.send_with_u8_array(frame); + } } } @@ -446,12 +491,19 @@ fn open_socket(sh: &WsShared) { }; // ── onmessage: classify + apply via the core, execute its effects ──── + // + // ArrayBuffer rather than the default Blob: the server sends a *binary* + // frame whenever a reply carries a value that is not valid UTF-8, and a + // Blob would have to be read asynchronously — the handler below would drop + // it on the floor. + ws.set_binary_type(web_sys::BinaryType::Arraybuffer); let sh_msg = sh.clone(); let onmessage = Closure::wrap(Box::new(move |e: MessageEvent| { - if let Ok(text) = e.data().dyn_into::() { - let incoming = sh_msg.core.borrow_mut().handle_frame(&String::from(text)); - dispatch_incoming(&sh_msg, incoming); - } + let Some(raw) = event_frame_bytes(&e) else { + return; + }; + let incoming = sh_msg.core.borrow_mut().handle_frame(&raw); + dispatch_incoming(&sh_msg, incoming); }) as Box); ws.set_onmessage(Some(onmessage.as_ref().unchecked_ref())); @@ -460,7 +512,7 @@ fn open_socket(sh: &WsShared) { let ws_for_open = ws.clone(); let onopen = Closure::wrap(Box::new(move |_e: Event| { for frame in sh_open.core.borrow_mut().on_open() { - let _ = ws_for_open.send_with_str(&frame); + ws_send_frame(&ws_for_open, &frame); } }) as Box); ws.set_onopen(Some(onopen.as_ref().unchecked_ref())); @@ -592,14 +644,14 @@ impl RecachedCache { } /// Queue a store write (dedup-wrapped for exactly-once delivery). - fn ws_enqueue(&self, encoded: &str) { + fn ws_enqueue(&self, encoded: &[u8]) { queue_write(&self.shared, encoded, true); } /// Queue a connection-scoped command (pub/sub) — replayed on reconnect /// but never dedup-wrapped: skipping a re-sent SUBSCRIBE would silently /// drop the subscription. - fn ws_enqueue_nodedup(&self, encoded: &str) { + fn ws_enqueue_nodedup(&self, encoded: &[u8]) { queue_write(&self.shared, encoded, false); } @@ -694,10 +746,16 @@ impl RecachedCache { let pair = js_sys::Array::from(&result); let keys = js_sys::Array::from(&pair.get(0)); let vals = js_sys::Array::from(&pair.get(1)); - let mut restored: Vec<(u64, String)> = Vec::with_capacity(keys.length() as usize); + let mut restored: Vec<(u64, Vec)> = Vec::with_capacity(keys.length() as usize); for i in 0..keys.length() { let id = keys.get(i).as_f64().unwrap_or(0.0) as u64; - let cmd = vals.get(i).as_string().unwrap_or_default(); + let raw = vals.get(i); + let cmd = match raw.as_string() { + // Rows written by 0.2.1 and earlier, when frames were + // strings rather than Uint8Array. + Some(t) => t.into_bytes(), + None => js_sys::Uint8Array::new(&raw).to_vec(), + }; if !cmd.is_empty() { restored.push((id, cmd)); } @@ -802,15 +860,13 @@ impl RecachedCache { let on_mut = Rc::clone(&self.on_mutation); let onbc = Closure::wrap(Box::new(move |e: MessageEvent| { - if let Ok(text) = e.data().dyn_into::() { - let s = String::from(text); - if let Ok((value, _)) = Value::parse(s.as_bytes()) - && let Ok(cmd) = Command::from_value(value) - && is_replayable_mutation(&cmd) - { - store_clone.execute(cmd); - notify_mutation(&on_mut); - } + if let Some(raw) = event_frame_bytes(&e) + && let Ok((value, _)) = Value::parse(&raw) + && let Ok(cmd) = Command::from_value(value) + && is_replayable_mutation(&cmd) + { + store_clone.execute(cmd); + notify_mutation(&on_mut); } }) as Box); @@ -866,11 +922,11 @@ impl RecachedCache { /// Send a session frame the core produced (it has already recorded the /// inflight reply slot). - fn send_session_frame(&self, frame: Option) { + fn send_session_frame(&self, frame: Option>) { if let Some(frame) = frame && let Some(ws) = self.shared.ws.borrow().as_ref() { - let _ = ws.send_with_str(&frame); + ws_send_frame(ws, &frame); } } @@ -952,9 +1008,9 @@ impl RecachedCache { let encoded = to_resp(&["INCRBY", key, &delta_s]); self.ws_enqueue(&encoded); if let Some(bc) = &self.bc { - let _ = bc.post_message(&JsValue::from_str(&encoded)); + bc_post(bc, &encoded); } - persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); + persist_cmd(&self.idb, &self.seq, &encoded); notify_mutation(&self.on_mutation); Ok(n) } @@ -974,9 +1030,9 @@ impl RecachedCache { let encoded = to_resp(&["JSET", key, path, value]); self.ws_enqueue(&encoded); if let Some(bc) = &self.bc { - let _ = bc.post_message(&JsValue::from_str(&encoded)); + bc_post(bc, &encoded); } - persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); + persist_cmd(&self.idb, &self.seq, &encoded); notify_mutation(&self.on_mutation); "OK".to_string() } @@ -1006,9 +1062,9 @@ impl RecachedCache { let encoded = to_resp(&["JMERGE", key, patch]); self.ws_enqueue(&encoded); if let Some(bc) = &self.bc { - let _ = bc.post_message(&JsValue::from_str(&encoded)); + bc_post(bc, &encoded); } - persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); + persist_cmd(&self.idb, &self.seq, &encoded); notify_mutation(&self.on_mutation); "OK".to_string() } @@ -1049,9 +1105,37 @@ impl RecachedCache { let encoded = to_resp(&["SET", key, value]); self.ws_enqueue(&encoded); if let Some(bc) = &self.bc { - let _ = bc.post_message(&JsValue::from_str(&encoded)); + bc_post(bc, &encoded); + } + persist_cmd(&self.idb, &self.seq, &encoded); + notify_mutation(&self.on_mutation); + + match resp { + Value::SimpleString(s) => s, + Value::Error(e) => e, + _ => "ERR".to_string(), + } + } + + /// Set a key to raw bytes, synced to the server and other tabs. + /// + /// Values are byte-transparent, so this stores and replicates exactly the + /// bytes given — compressed payloads, protobuf, images. `set()` is the + /// right call for text; this exists for everything a JS string cannot hold. + #[wasm_bindgen(js_name = "setBytes")] + pub fn set_bytes(&self, key: &str, value: &[u8]) -> String { + let resp = self.store.execute(Command::Set( + key.to_string(), + value.to_vec(), + SetOptions::default(), + )); + + let encoded = to_resp_bytes(&[b"SET", key.as_bytes(), value]); + self.ws_enqueue(&encoded); + if let Some(bc) = &self.bc { + bc_post(bc, &encoded); } - persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); + persist_cmd(&self.idb, &self.seq, &encoded); notify_mutation(&self.on_mutation); match resp { @@ -1076,9 +1160,9 @@ impl RecachedCache { let encoded = to_resp(&["SET", key, value, "EX", &seconds.to_string()]); self.ws_enqueue(&encoded); if let Some(bc) = &self.bc { - let _ = bc.post_message(&JsValue::from_str(&encoded)); + bc_post(bc, &encoded); } - persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); + persist_cmd(&self.idb, &self.seq, &encoded); notify_mutation(&self.on_mutation); match resp { @@ -1125,9 +1209,9 @@ impl RecachedCache { let encoded = to_resp(&["DEL", key]); self.ws_enqueue(&encoded); if let Some(bc) = &self.bc { - let _ = bc.post_message(&JsValue::from_str(&encoded)); + bc_post(bc, &encoded); } - persist_cmd(&self.idb, &self.seq, encoded.as_bytes()); + persist_cmd(&self.idb, &self.seq, &encoded); notify_mutation(&self.on_mutation); match resp { @@ -1157,6 +1241,15 @@ impl RecachedCache { self.ws_enqueue_nodedup(&to_resp(&["PUBLISH", channel, message])); } + /// Publish raw bytes to a channel on the server. + /// + /// Subscribers receive a `Uint8Array` rather than a string when the payload + /// is not valid UTF-8. + #[wasm_bindgen(js_name = "publishBytes")] + pub fn publish_bytes(&self, channel: &str, message: &[u8]) { + self.ws_enqueue_nodedup(&to_resp_bytes(&[b"PUBLISH", channel.as_bytes(), message])); + } + /// Subscribe to a channel on the server. Push messages arrive via the `onmessage` callback. pub fn subscribe(&self, channel: &str) { self.ws_enqueue_nodedup(&to_resp(&["SUBSCRIBE", channel])); @@ -1599,7 +1692,7 @@ mod browser_tests { JsFuture::from(idb_append_js(&db, 1.0, b"SET a 1")) .await .expect("append"); - JsFuture::from(idb_outbox_put_js(&db, 10.0, "PENDING WRITE")) + JsFuture::from(idb_outbox_put_js(&db, 10.0, b"PENDING WRITE")) .await .expect("outbox put"); @@ -1618,7 +1711,7 @@ mod browser_tests { JsFuture::from(idb_append_js(&db, 1.0, b"SET a 1")) .await .unwrap(); - JsFuture::from(idb_outbox_put_js(&db, 1.0, "SET b 2")) + JsFuture::from(idb_outbox_put_js(&db, 1.0, b"SET b 2")) .await .unwrap(); @@ -1635,7 +1728,7 @@ mod browser_tests { // The durability guarantee: a write made offline must still be there // after the page is reloaded and the database reopened. let db = fresh_db().await; - JsFuture::from(idb_outbox_put_js(&db, 7.0, "SET offline 1")) + JsFuture::from(idb_outbox_put_js(&db, 7.0, b"SET offline 1")) .await .expect("put"); drop(db); @@ -1650,7 +1743,7 @@ mod browser_tests { async fn deleting_an_acknowledged_row_leaves_the_others() { let db = fresh_db().await; for (id, cmd) in [(1.0, "SET a 1"), (2.0, "SET b 2"), (3.0, "SET c 3")] { - JsFuture::from(idb_outbox_put_js(&db, id, cmd)) + JsFuture::from(idb_outbox_put_js(&db, id, cmd.as_bytes())) .await .unwrap(); } @@ -1667,10 +1760,10 @@ mod browser_tests { // Renumbering on restore re-puts rows; duplicates would replay a write // twice. let db = fresh_db().await; - JsFuture::from(idb_outbox_put_js(&db, 5.0, "FIRST")) + JsFuture::from(idb_outbox_put_js(&db, 5.0, b"FIRST")) .await .unwrap(); - JsFuture::from(idb_outbox_put_js(&db, 5.0, "SECOND")) + JsFuture::from(idb_outbox_put_js(&db, 5.0, b"SECOND")) .await .unwrap(); @@ -1735,7 +1828,7 @@ mod browser_tests { JsFuture::from(idb_append_js(&db, 0.0, b"SET a 1")) .await .unwrap(); - JsFuture::from(idb_outbox_put_js(&db, 5.0, "PENDING")) + JsFuture::from(idb_outbox_put_js(&db, 5.0, b"PENDING")) .await .unwrap(); @@ -1748,6 +1841,42 @@ mod browser_tests { assert_eq!(vals[0], "PENDING"); } + #[wasm_bindgen_test] + async fn binary_writes_survive_the_outbox() { + // An offline write is stored in IndexedDB and replayed on reconnect. A + // binary value has to come back out of that queue byte-identical, or + // the replayed write differs from the one the application made. + let db = fresh_db().await; + let binary = vec![0xffu8, 0xfe, 0x00, 0x41, 0x80]; + let frame = to_resp_bytes(&[b"SET", b"k", &binary]); + + JsFuture::from(idb_outbox_put_js(&db, 0.0, &frame)) + .await + .expect("outbox put"); + + let res = JsFuture::from(idb_outbox_read_all(&db)) + .await + .expect("outbox read"); + let pair = js_sys::Array::from(&res); + let vals = js_sys::Array::from(&pair.get(1)); + let raw = vals.get(0); + let stored = match raw.as_string() { + Some(t) => t.into_bytes(), + None => js_sys::Uint8Array::new(&raw).to_vec(), + }; + + assert_eq!(stored, frame, "outbox row must round-trip unchanged"); + + // And it still parses into the write the application made. + let (value, _) = core_engine::resp::Value::parse(&stored).unwrap(); + let store = KeyValueStore::new(); + store.execute(Command::from_value(value).unwrap()); + assert_eq!( + store.execute(Command::Get("k".into())), + Value::BulkString(Some(binary)) + ); + } + #[wasm_bindgen_test] async fn binary_values_survive_the_wal() { // A backend can write a binary value that reaches this browser through From 9f7bcde11bcb2e9e5d517b6c1f08754fd7ac59d4 Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Sun, 19 Jul 2026 21:53:14 +0800 Subject: [PATCH 10/11] =?UTF-8?q?Make=20values=20byte-transparent=20end=20?= =?UTF-8?q?to=20end=20=E2=80=94=20store,=20replication,=20AOF,=20pub/sub,?= =?UTF-8?q?=20browser=20outbox=20and=20WAL=20=E2=80=94=20plus=20setBytes/g?= =?UTF-8?q?etBytes/publishBytes;=20add=20replication=20acks=20with=20lag?= =?UTF-8?q?=20metric=20and=20RESP3=20HELLO=20negotiation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 3 + .github/workflows/docker.yml | 51 ---- .github/workflows/npm.yml | 128 ---------- .github/workflows/release.yml | 204 +++++++++++++++- CHANGELOG.md | 15 +- README.md | 1 + core-engine/src/store.rs | 277 ++++++++++++++++++++++ core-engine/tests/binary_values.rs | 140 ----------- core-engine/tests/snapshot_compat.rs | 118 --------- sdks/recached-react/package-lock.json | 6 +- sdks/recached-react/src/index.ts | 2 +- sdks/recached-react/src/useKey.ts | 42 +++- sdks/recached-react/src/useKeys.ts | 5 +- sdks/recached-react/src/usePubSub.ts | 9 +- sdks/recached-vue/package-lock.json | 6 +- sdks/recached-vue/src/index.ts | 2 +- sdks/recached-vue/src/useKey.ts | 32 ++- sdks/recached-vue/src/useKeys.ts | 5 +- sdks/recached-vue/src/usePubSub.ts | 9 +- wasm-edge/node_modules/.package-lock.json | 2 +- wasm-edge/package-lock.json | 6 +- wasm-edge/sdk.d.ts | 85 ++++++- wasm-edge/sdk.d.ts.map | 2 +- wasm-edge/sdk.js | 99 +++++++- wasm-edge/sdk.ts | 9 +- wasm-edge/src/lib.rs | 8 +- 26 files changed, 787 insertions(+), 479 deletions(-) delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/npm.yml delete mode 100644 core-engine/tests/binary_values.rs delete mode 100644 core-engine/tests/snapshot_compat.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f90dfc..22ffa6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: ["main", "dev"] pull_request: branches: ["main", "dev"] + # Callable so the release workflow gates on the *same* checks rather than a + # re-declared subset that can drift out of step with this file. + workflow_call: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index d839f58..0000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Publish Docker Image - -on: - push: - tags: - - "v*" - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-and-push-image: - name: Build and push multi-arch image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - # Required for multi-platform builds (linux/amd64 + linux/arm64) - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml deleted file mode 100644 index 8fa0237..0000000 --- a/.github/workflows/npm.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Publish Packages to NPM - -on: - push: - tags: - - "v*" - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - -jobs: - publish-wasm: - name: Build and publish recached-edge to NPM - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-unknown-unknown - - - name: Rust cache - uses: Swatinem/rust-cache@v2 - with: - shared-key: "recached-wasm-cache" - - - name: Install wasm-pack - uses: jetli/wasm-pack-action@v0.4.0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "20.x" - registry-url: "https://registry.npmjs.org" - - - name: Build Wasm package - run: wasm-pack build wasm-edge --target web --release --out-name recached-edge - - - name: Copy LICENSE + NOTICE into package - run: cp LICENSE.md NOTICE wasm-edge/pkg/ - - - name: Set package name and version - # wasm-pack derives the npm name and version from the crate — patch both so the - # published package always matches the git tag (e.g. v0.1.5 → 0.1.5). - run: | - node -e " - const fs = require('fs'); - const path = 'wasm-edge/pkg/package.json'; - const pkg = JSON.parse(fs.readFileSync(path, 'utf8')); - pkg.name = 'recached-edge'; - pkg.version = process.env.TAG_VERSION.replace(/^v/, ''); - fs.writeFileSync(path, JSON.stringify(pkg, null, 2) + '\n'); - console.log('Package:', pkg.name, pkg.version); - " - env: - TAG_VERSION: ${{ github.ref_name }} - - - name: Publish to NPM - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: cd wasm-edge/pkg && npm publish --access public - - publish-react: - name: Build and publish @recached/react to NPM - runs-on: ubuntu-latest - needs: publish-wasm - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "20.x" - registry-url: "https://registry.npmjs.org" - - - name: Install dependencies - run: cd sdks/recached-react && npm install --legacy-peer-deps - - - name: Typecheck - run: cd sdks/recached-react && npm run typecheck - - - name: Build - run: cd sdks/recached-react && npm run build - - - name: Copy LICENSE + NOTICE into package - run: cp LICENSE.md NOTICE sdks/recached-react/ - - - name: Publish to NPM - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: cd sdks/recached-react && npm publish --access public - - publish-vue: - name: Build and publish @recached/vue to NPM - runs-on: ubuntu-latest - needs: publish-wasm - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "20.x" - registry-url: "https://registry.npmjs.org" - - - name: Install dependencies - run: cd sdks/recached-vue && npm install --legacy-peer-deps - - - name: Typecheck - run: cd sdks/recached-vue && npm run typecheck - - - name: Build - run: cd sdks/recached-vue && npm run build - - - name: Copy LICENSE + NOTICE into package - run: cp LICENSE.md NOTICE sdks/recached-vue/ - - - name: Publish to NPM - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: cd sdks/recached-vue && npm publish --access public diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33c1343..eb7a690 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,13 @@ -name: Release Binaries +name: Release + +# One workflow for everything a tag ships: GitHub Release binaries, the +# multi-arch container image, and the three npm packages. +# +# These were three separate workflows on the same `v*` trigger, which produced +# three entries per tag in the Actions list and, more importantly, three +# independent runs — npm could publish while the binaries failed, leaving a +# half-released version. An npm publish cannot be undone after 72 hours, so +# nothing publishes until the full CI suite and the tag checks have passed. on: push: @@ -7,10 +16,62 @@ on: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# Two tags pushed close together must not publish over each other. +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false jobs: - build-and-upload: + # ── Gates ─────────────────────────────────────────────────────────────────── + # + # The full CI suite: fmt, clippy, tests, both coverage floors, browser tests, + # and the React/Vue typechecks. Reused rather than restated so the release + # gate cannot drift from what CI actually enforces on the branch. + ci: + name: CI + uses: ./.github/workflows/ci.yml + + tag-checks: + name: Verify tag + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # The npm job rewrites each package version from the tag, so a tag that + # disagrees with Cargo.toml would publish artifacts labelled with a + # version the source never claimed. Catch it before anything ships. + - name: Check tag matches workspace version + run: | + TAG_VERSION="${GITHUB_REF_NAME#v}" + CARGO_VERSION="$(grep -m1 '^version' Cargo.toml | cut -d'"' -f2)" + echo "tag=$TAG_VERSION cargo=$CARGO_VERSION" + if [ "$TAG_VERSION" != "$CARGO_VERSION" ]; then + echo "::error::Tag $GITHUB_REF_NAME does not match Cargo.toml version $CARGO_VERSION" + exit 1 + fi + + # A release whose notes were never written is a release nobody can read. + - name: Check CHANGELOG has an entry for this version + run: | + TAG_VERSION="${GITHUB_REF_NAME#v}" + if ! grep -qE "^## \[?${TAG_VERSION}\]?" CHANGELOG.md; then + echo "::error::CHANGELOG.md has no '## [$TAG_VERSION]' section" + exit 1 + fi + if grep -qE "^## \[?${TAG_VERSION}\]?.*Unreleased" CHANGELOG.md; then + echo "::error::CHANGELOG.md still marks $TAG_VERSION as Unreleased" + exit 1 + fi + + # ── Binaries ──────────────────────────────────────────────────────────────── + binaries: name: Build (${{ matrix.target }}) + needs: [ci, tag-checks] runs-on: ${{ matrix.runner }} permissions: contents: write @@ -65,3 +126,142 @@ jobs: asset_name: ${{ matrix.artifact }} tag: ${{ github.ref }} overwrite: true + + # ── Container image ───────────────────────────────────────────────────────── + docker: + name: Build and push multi-arch image + needs: [ci, tag-checks] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Required for multi-platform builds (linux/amd64 + linux/arm64) + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ── npm packages ──────────────────────────────────────────────────────────── + # + # The framework SDKs depend on recached-edge, so it publishes first. + npm-wasm: + name: Publish recached-edge to NPM + needs: [ci, tag-checks] + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "recached-wasm-cache" + + - name: Install wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + + - name: Build Wasm package + run: wasm-pack build wasm-edge --target web --release --out-name recached-edge + + - name: Copy LICENSE + NOTICE into package + run: cp LICENSE.md NOTICE wasm-edge/pkg/ + + - name: Set package name and version + # wasm-pack derives the npm name and version from the crate — patch both so the + # published package always matches the git tag (e.g. v0.1.5 → 0.1.5). + run: | + node -e " + const fs = require('fs'); + const path = 'wasm-edge/pkg/package.json'; + const pkg = JSON.parse(fs.readFileSync(path, 'utf8')); + pkg.name = 'recached-edge'; + pkg.version = process.env.TAG_VERSION.replace(/^v/, ''); + fs.writeFileSync(path, JSON.stringify(pkg, null, 2) + '\n'); + console.log('Package:', pkg.name, pkg.version); + " + env: + TAG_VERSION: ${{ github.ref_name }} + + - name: Publish to NPM + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: cd wasm-edge/pkg && npm publish --access public + + npm-sdks: + name: Publish @recached/${{ matrix.package }} to NPM + needs: npm-wasm + runs-on: ubuntu-latest + + strategy: + # One SDK failing must not stop the other from publishing: they are + # independent packages, and finishing a partial release is easier than + # unpicking one. + fail-fast: false + matrix: + package: [react, vue] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + + - name: Install dependencies + run: cd sdks/recached-${{ matrix.package }} && npm install --legacy-peer-deps + + - name: Typecheck + run: cd sdks/recached-${{ matrix.package }} && npm run typecheck + + - name: Build + run: cd sdks/recached-${{ matrix.package }} && npm run build + + - name: Copy LICENSE + NOTICE into package + run: cp LICENSE.md NOTICE sdks/recached-${{ matrix.package }}/ + + - name: Publish to NPM + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: cd sdks/recached-${{ matrix.package }} && npm publish --access public diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f7c448..7224bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Recached are documented here. --- -## [0.2.2] — Unreleased +## [0.2.2] — 2026-07-20 ### Added @@ -179,7 +179,18 @@ All notable changes to Recached are documented here. `cache.get()` now **throws** on a binary value instead of returning mangled text, `getJSON()` treats one as a miss, and an `onMessage` listener receives a `Uint8Array` for a binary pub/sub - payload — so the listener signature widened to `string | Uint8Array`. + payload — so the listener signature widened to `string | Uint8Array`. `getMatching()` — the read + behind every live query — returns a `Uint8Array` for a binary value rather than a lossy string, + which was the last silent conversion left on the browser read path. + + **`@recached/react` and `@recached/vue`** follow: `useKeyBytes()` is new, `usePubSub` handlers and + the `KeyValuePair` value type widened to `string | Uint8Array`, and `useKey` returns `null` for a + binary value rather than letting `get()` throw out of a React `getSnapshot` or a Vue reactive + update — which would have taken down the render tree over a value the hook cannot represent. + + These are **compile-time breaking changes for TypeScript users** who annotated a `usePubSub` + handler or an `onMessage` listener as `(msg: string) => void`, or who destructured a + `KeyValuePair` value as `string | null`. Widen the annotation, or narrow with `typeof v === 'string'`. Data corrupted by an earlier version cannot be recovered and must be re-populated. diff --git a/README.md b/README.md index a3e2ee7..b3d40a2 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ See [recached.dev/roadmap](https://recached.dev/roadmap) for what's planned. Reach out: [dennis@thinkgrid.dev](mailto:dennis@thinkgrid.dev) + ## Support Recached Recached is free and open-source, maintained by one person. If it saves you infrastructure cost or development time, [sponsoring on GitHub](https://github.com/sponsors/thinkgrid-labs) directly funds continued development: more Redis commands, RESP3, cluster support, and performance work. diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 5f8a873..20a4067 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -5764,3 +5764,280 @@ mod rate_limiter_memory_tests { ); } } + +// ── Byte transparency ───────────────────────────────────────────────────────── +// +// Values are byte-transparent; identifiers are text. A stored value may be +// arbitrary bytes — compressed blobs, protobuf, images — and must come back +// exactly as it went in. Keys, hash fields, set and sorted-set members and glob +// patterns are looked up and matched as text, so a non-UTF-8 one is refused +// rather than lossily converted. +// +// These live in-crate rather than in `tests/`: a separate integration binary +// links its own copy of every function into the coverage map, and the copies it +// does not exercise drag the measured figure down without changing what is +// actually tested. +#[cfg(test)] +mod byte_transparency_tests { + use super::*; + use crate::cmd::Command; + use crate::resp::Value; + + /// Invalid UTF-8 in any position: a lone continuation byte and a truncated + /// sequence, plus an embedded NUL and a byte that is legal only inside one. + const BINARY: &[u8] = &[0xff, 0xfe, 0x00, 0x41, 0x80, 0xc3]; + + fn parse(raw: &[u8]) -> Result { + let (v, _) = Value::parse(raw).unwrap(); + Command::from_value(v) + } + + /// Build a RESP array frame from raw byte arguments. + fn frame(args: &[&[u8]]) -> Vec { + let mut out = format!("*{}\r\n", args.len()).into_bytes(); + for a in args { + out.extend_from_slice(format!("${}\r\n", a.len()).as_bytes()); + out.extend_from_slice(a); + out.extend_from_slice(b"\r\n"); + } + out + } + + #[test] + fn a_binary_value_round_trips_byte_for_byte() { + let store = KeyValueStore::new(); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); + + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing after SET"); + }; + assert_eq!(got, BINARY, "value must survive unchanged"); + } + + #[test] + fn binary_values_work_in_lists_and_hashes() { + let store = KeyValueStore::new(); + + store.execute(parse(&frame(&[b"RPUSH", b"l", BINARY, b"plain"])).unwrap()); + let Value::Array(Some(items)) = store.execute(Command::LRange("l".into(), 0, -1)) else { + panic!("list missing"); + }; + assert_eq!(items[0], Value::BulkString(Some(BINARY.to_vec()))); + + store.execute(parse(&frame(&[b"HSET", b"h", b"f", BINARY])).unwrap()); + assert_eq!( + store.execute(Command::HGet("h".into(), "f".into())), + Value::BulkString(Some(BINARY.to_vec())) + ); + } + + #[test] + fn append_concatenates_bytes_rather_than_text() { + let store = KeyValueStore::new(); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); + store.execute(parse(&frame(&[b"APPEND", b"k", BINARY])).unwrap()); + + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing"); + }; + assert_eq!(got.len(), BINARY.len() * 2); + assert_eq!(&got[..BINARY.len()], BINARY); + assert_eq!(&got[BINARY.len()..], BINARY); + } + + #[test] + fn strlen_counts_bytes_not_characters() { + let store = KeyValueStore::new(); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); + assert_eq!( + store.execute(Command::Strlen("k".into())), + Value::Integer(BINARY.len() as i64) + ); + } + + #[test] + fn incr_on_a_binary_value_errors_like_any_non_numeric_value() { + // The bytes are stored faithfully; they are simply not a number. This must + // read as a type error, not as corruption. + let store = KeyValueStore::new(); + store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); + let Value::Error(e) = store.execute(Command::Incr("k".into())) else { + panic!("INCR on binary must error"); + }; + assert!(e.contains("not an integer"), "got {e:?}"); + } + + #[test] + fn a_binary_key_is_rejected() { + // Keys are matched by glob and checked against sync scopes as text, so a + // corrupted key would be silently unretrievable. + let err = parse(&frame(&[b"SET", BINARY, b"v"])).expect_err("binary key must be refused"); + assert!(err.starts_with("ERR "), "{err:?}"); + assert!(err.contains("must be text"), "{err:?}"); + } + + #[test] + fn binary_fields_and_members_are_rejected() { + for args in [ + vec![b"HSET".as_slice(), b"h", BINARY, b"v"], // hash field + vec![b"SADD".as_slice(), b"s", BINARY], // set member + vec![b"ZADD".as_slice(), b"z", b"1", BINARY], // zset member + vec![b"KEYS".as_slice(), BINARY], // glob pattern + ] { + let err = parse(&frame(&args)).expect_err("identifier must be refused"); + assert!(err.contains("must be text"), "{:?} -> {err:?}", args[0]); + } + } + + #[test] + fn the_error_names_which_argument_was_bad() { + // MSET k1 v1 v2 — index 3 is a key, so it is refused. + let err = + parse(&frame(&[b"MSET", b"k1", b"v1", BINARY, b"v2"])).expect_err("must be refused"); + assert!(err.contains("argument 3"), "got {err:?}"); + } + + #[test] + fn a_rejected_command_stores_nothing() { + let store = KeyValueStore::new(); + assert!(parse(&frame(&[b"SET", BINARY, b"v"])).is_err()); + assert_eq!(store.execute(Command::DbSize), Value::Integer(0)); + } + + #[test] + fn utf8_values_survive_unchanged() { + let store = KeyValueStore::new(); + for value in ["héllo ✓", "日本語", "\u{1F600}", "", "plain"] { + store.execute(Command::Set("k".into(), value.into(), Default::default())); + let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { + panic!("key missing after SET of {value:?}"); + }; + assert_eq!(String::from_utf8(got).unwrap(), value); + } + } +} + +// ── Snapshot compatibility ──────────────────────────────────────────────────── +// +// `SnapshotValue` held `String` up to 0.2.1 and holds `Blob` from 0.2.2. +// rmp-serde encodes those differently — msgpack `str` versus `bin` — so `Blob`'s +// deserializer accepts either. Without that, upgrading a server would silently +// start from an empty cache, or fail to boot. +#[cfg(test)] +mod snapshot_compat_tests { + // `super::*` already brings Command, Value and the snapshot types into + // scope from the store module's own imports. + use super::*; + use std::collections::HashMap; + + /// Mirror of the pre-0.2.2 `SnapshotValue`, used to produce a genuine old-format + /// payload rather than a hand-rolled byte string. Variant order matters: + /// rmp-serde encodes variants by index. + #[derive(Serialize)] + #[allow(dead_code)] // variants exist to fix the discriminant order, not to be built + enum LegacySnapshotValue { + Str(String), + Hash(HashMap), + List(Vec), + Set(Vec), + ZSet(Vec<(String, f64)>), + RateLimiter { + limit: u64, + window_ms: u64, + events: Vec, + }, + Json(String), + } + + #[derive(Serialize)] + struct LegacyEntry { + key: String, + value: LegacySnapshotValue, + expires_at_ms: Option, + } + + #[test] + fn a_pre_0_2_2_snapshot_still_restores() { + let legacy = vec![ + LegacyEntry { + key: "s".into(), + value: LegacySnapshotValue::Str("hello".into()), + expires_at_ms: None, + }, + LegacyEntry { + key: "l".into(), + value: LegacySnapshotValue::List(vec!["a".into(), "b".into()]), + expires_at_ms: None, + }, + LegacyEntry { + key: "h".into(), + value: LegacySnapshotValue::Hash(HashMap::from([( + "f".to_string(), + "v".to_string(), + )])), + expires_at_ms: None, + }, + ]; + let bytes = rmp_serde::to_vec(&legacy).expect("legacy snapshot must encode"); + + // Decode with the *current* types — this is what a restarted server does. + let entries: Vec = + rmp_serde::from_slice(&bytes).expect("a pre-0.2.2 snapshot must still decode"); + + let store = KeyValueStore::new(); + store.restore(entries); + + use crate::{cmd::Command, resp::Value}; + assert_eq!( + store.execute(Command::Get("s".into())), + Value::BulkString(Some(b"hello".to_vec())) + ); + assert_eq!( + store.execute(Command::HGet("h".into(), "f".into())), + Value::BulkString(Some(b"v".to_vec())) + ); + assert_eq!(store.execute(Command::LLen("l".into())), Value::Integer(2)); + } + + #[test] + fn binary_values_survive_a_snapshot_round_trip() { + let binary = vec![0xff, 0xfe, 0x00, 0x41, 0x80]; + let store = KeyValueStore::new(); + store.restore(vec![SnapshotEntry { + key: "b".into(), + value: SnapshotValue::Str(binary.clone().into()), + expires_at_ms: None, + }]); + + let bytes = rmp_serde::to_vec(&store.snapshot()).unwrap(); + let entries: Vec = rmp_serde::from_slice(&bytes).unwrap(); + + let restored = KeyValueStore::new(); + restored.restore(entries); + + use crate::{cmd::Command, resp::Value}; + assert_eq!( + restored.execute(Command::Get("b".into())), + Value::BulkString(Some(binary)), + "binary must survive snapshot and restore" + ); + } + + #[test] + fn a_binary_value_encodes_as_msgpack_bin_not_an_int_array() { + // Vec serializes as an array of integers by default, which would roughly + // double snapshot size for binary payloads. Blob emits a compact `bin`. + let store = KeyValueStore::new(); + store.restore(vec![SnapshotEntry { + key: "b".into(), + value: SnapshotValue::Str(vec![0xffu8; 1000].into()), + expires_at_ms: None, + }]); + let bytes = rmp_serde::to_vec(&store.snapshot()).unwrap(); + assert!( + bytes.len() < 1200, + "1000 bytes encoded to {} — likely an int array, not msgpack bin", + bytes.len() + ); + } +} diff --git a/core-engine/tests/binary_values.rs b/core-engine/tests/binary_values.rs deleted file mode 100644 index 4c4cc39..0000000 --- a/core-engine/tests/binary_values.rs +++ /dev/null @@ -1,140 +0,0 @@ -//! Values are byte-transparent; identifiers are text. -//! -//! A stored value may be arbitrary bytes — compressed blobs, protobuf, images — -//! and must come back exactly as it went in. Keys, hash fields, set and -//! sorted-set members and glob patterns are looked up and matched as text, so a -//! non-UTF-8 one is refused rather than lossily converted. - -use core_engine::{cmd::Command, resp::Value, store::KeyValueStore}; - -/// Invalid UTF-8 in any position: a lone continuation byte and a truncated -/// sequence, plus an embedded NUL and a byte that is legal only inside one. -const BINARY: &[u8] = &[0xff, 0xfe, 0x00, 0x41, 0x80, 0xc3]; - -fn parse(raw: &[u8]) -> Result { - let (v, _) = Value::parse(raw).unwrap(); - Command::from_value(v) -} - -/// Build a RESP array frame from raw byte arguments. -fn frame(args: &[&[u8]]) -> Vec { - let mut out = format!("*{}\r\n", args.len()).into_bytes(); - for a in args { - out.extend_from_slice(format!("${}\r\n", a.len()).as_bytes()); - out.extend_from_slice(a); - out.extend_from_slice(b"\r\n"); - } - out -} - -#[test] -fn a_binary_value_round_trips_byte_for_byte() { - let store = KeyValueStore::new(); - store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); - - let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { - panic!("key missing after SET"); - }; - assert_eq!(got, BINARY, "value must survive unchanged"); -} - -#[test] -fn binary_values_work_in_lists_and_hashes() { - let store = KeyValueStore::new(); - - store.execute(parse(&frame(&[b"RPUSH", b"l", BINARY, b"plain"])).unwrap()); - let Value::Array(Some(items)) = store.execute(Command::LRange("l".into(), 0, -1)) else { - panic!("list missing"); - }; - assert_eq!(items[0], Value::BulkString(Some(BINARY.to_vec()))); - - store.execute(parse(&frame(&[b"HSET", b"h", b"f", BINARY])).unwrap()); - assert_eq!( - store.execute(Command::HGet("h".into(), "f".into())), - Value::BulkString(Some(BINARY.to_vec())) - ); -} - -#[test] -fn append_concatenates_bytes_rather_than_text() { - let store = KeyValueStore::new(); - store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); - store.execute(parse(&frame(&[b"APPEND", b"k", BINARY])).unwrap()); - - let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { - panic!("key missing"); - }; - assert_eq!(got.len(), BINARY.len() * 2); - assert_eq!(&got[..BINARY.len()], BINARY); - assert_eq!(&got[BINARY.len()..], BINARY); -} - -#[test] -fn strlen_counts_bytes_not_characters() { - let store = KeyValueStore::new(); - store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); - assert_eq!( - store.execute(Command::Strlen("k".into())), - Value::Integer(BINARY.len() as i64) - ); -} - -#[test] -fn incr_on_a_binary_value_errors_like_any_non_numeric_value() { - // The bytes are stored faithfully; they are simply not a number. This must - // read as a type error, not as corruption. - let store = KeyValueStore::new(); - store.execute(parse(&frame(&[b"SET", b"k", BINARY])).unwrap()); - let Value::Error(e) = store.execute(Command::Incr("k".into())) else { - panic!("INCR on binary must error"); - }; - assert!(e.contains("not an integer"), "got {e:?}"); -} - -#[test] -fn a_binary_key_is_rejected() { - // Keys are matched by glob and checked against sync scopes as text, so a - // corrupted key would be silently unretrievable. - let err = parse(&frame(&[b"SET", BINARY, b"v"])).expect_err("binary key must be refused"); - assert!(err.starts_with("ERR "), "{err:?}"); - assert!(err.contains("must be text"), "{err:?}"); -} - -#[test] -fn binary_fields_and_members_are_rejected() { - for args in [ - vec![b"HSET".as_slice(), b"h", BINARY, b"v"], // hash field - vec![b"SADD".as_slice(), b"s", BINARY], // set member - vec![b"ZADD".as_slice(), b"z", b"1", BINARY], // zset member - vec![b"KEYS".as_slice(), BINARY], // glob pattern - ] { - let err = parse(&frame(&args)).expect_err("identifier must be refused"); - assert!(err.contains("must be text"), "{:?} -> {err:?}", args[0]); - } -} - -#[test] -fn the_error_names_which_argument_was_bad() { - // MSET k1 v1 v2 — index 3 is a key, so it is refused. - let err = parse(&frame(&[b"MSET", b"k1", b"v1", BINARY, b"v2"])).expect_err("must be refused"); - assert!(err.contains("argument 3"), "got {err:?}"); -} - -#[test] -fn a_rejected_command_stores_nothing() { - let store = KeyValueStore::new(); - assert!(parse(&frame(&[b"SET", BINARY, b"v"])).is_err()); - assert_eq!(store.execute(Command::DbSize), Value::Integer(0)); -} - -#[test] -fn utf8_values_survive_unchanged() { - let store = KeyValueStore::new(); - for value in ["héllo ✓", "日本語", "\u{1F600}", "", "plain"] { - store.execute(Command::Set("k".into(), value.into(), Default::default())); - let Value::BulkString(Some(got)) = store.execute(Command::Get("k".into())) else { - panic!("key missing after SET of {value:?}"); - }; - assert_eq!(String::from_utf8(got).unwrap(), value); - } -} diff --git a/core-engine/tests/snapshot_compat.rs b/core-engine/tests/snapshot_compat.rs deleted file mode 100644 index 59f814e..0000000 --- a/core-engine/tests/snapshot_compat.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! A snapshot written before values became bytes must still load. -//! -//! `SnapshotValue` held `String` up to 0.2.1 and holds `Blob` from 0.2.2. -//! rmp-serde encodes those differently — msgpack `str` versus `bin` — so -//! `Blob`'s deserializer accepts either. Without that, upgrading a server would -//! silently start from an empty cache, or fail to boot. - -use core_engine::store::{KeyValueStore, SnapshotEntry, SnapshotValue}; -use serde::Serialize; -use std::collections::HashMap; - -/// Mirror of the pre-0.2.2 `SnapshotValue`, used to produce a genuine old-format -/// payload rather than a hand-rolled byte string. Variant order matters: -/// rmp-serde encodes variants by index. -#[derive(Serialize)] -#[allow(dead_code)] // variants exist to fix the discriminant order, not to be built -enum LegacySnapshotValue { - Str(String), - Hash(HashMap), - List(Vec), - Set(Vec), - ZSet(Vec<(String, f64)>), - RateLimiter { - limit: u64, - window_ms: u64, - events: Vec, - }, - Json(String), -} - -#[derive(Serialize)] -struct LegacyEntry { - key: String, - value: LegacySnapshotValue, - expires_at_ms: Option, -} - -#[test] -fn a_pre_0_2_2_snapshot_still_restores() { - let legacy = vec![ - LegacyEntry { - key: "s".into(), - value: LegacySnapshotValue::Str("hello".into()), - expires_at_ms: None, - }, - LegacyEntry { - key: "l".into(), - value: LegacySnapshotValue::List(vec!["a".into(), "b".into()]), - expires_at_ms: None, - }, - LegacyEntry { - key: "h".into(), - value: LegacySnapshotValue::Hash(HashMap::from([("f".to_string(), "v".to_string())])), - expires_at_ms: None, - }, - ]; - let bytes = rmp_serde::to_vec(&legacy).expect("legacy snapshot must encode"); - - // Decode with the *current* types — this is what a restarted server does. - let entries: Vec = - rmp_serde::from_slice(&bytes).expect("a pre-0.2.2 snapshot must still decode"); - - let store = KeyValueStore::new(); - store.restore(entries); - - use core_engine::{cmd::Command, resp::Value}; - assert_eq!( - store.execute(Command::Get("s".into())), - Value::BulkString(Some(b"hello".to_vec())) - ); - assert_eq!( - store.execute(Command::HGet("h".into(), "f".into())), - Value::BulkString(Some(b"v".to_vec())) - ); - assert_eq!(store.execute(Command::LLen("l".into())), Value::Integer(2)); -} - -#[test] -fn binary_values_survive_a_snapshot_round_trip() { - let binary = vec![0xff, 0xfe, 0x00, 0x41, 0x80]; - let store = KeyValueStore::new(); - store.restore(vec![SnapshotEntry { - key: "b".into(), - value: SnapshotValue::Str(binary.clone().into()), - expires_at_ms: None, - }]); - - let bytes = rmp_serde::to_vec(&store.snapshot()).unwrap(); - let entries: Vec = rmp_serde::from_slice(&bytes).unwrap(); - - let restored = KeyValueStore::new(); - restored.restore(entries); - - use core_engine::{cmd::Command, resp::Value}; - assert_eq!( - restored.execute(Command::Get("b".into())), - Value::BulkString(Some(binary)), - "binary must survive snapshot and restore" - ); -} - -#[test] -fn a_binary_value_encodes_as_msgpack_bin_not_an_int_array() { - // Vec serializes as an array of integers by default, which would roughly - // double snapshot size for binary payloads. Blob emits a compact `bin`. - let store = KeyValueStore::new(); - store.restore(vec![SnapshotEntry { - key: "b".into(), - value: SnapshotValue::Str(vec![0xffu8; 1000].into()), - expires_at_ms: None, - }]); - let bytes = rmp_serde::to_vec(&store.snapshot()).unwrap(); - assert!( - bytes.len() < 1200, - "1000 bytes encoded to {} — likely an int array, not msgpack bin", - bytes.len() - ); -} diff --git a/sdks/recached-react/package-lock.json b/sdks/recached-react/package-lock.json index 0e6ba7f..c025e16 100644 --- a/sdks/recached-react/package-lock.json +++ b/sdks/recached-react/package-lock.json @@ -1,13 +1,13 @@ { "name": "@recached/react", - "version": "0.1.4", + "version": "0.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@recached/react", - "version": "0.1.4", - "license": "MIT", + "version": "0.2.2", + "license": "Apache-2.0", "devDependencies": { "@types/react": "^18", "typescript": "^5.9.3" diff --git a/sdks/recached-react/src/index.ts b/sdks/recached-react/src/index.ts index 8cf35ff..8b50b09 100644 --- a/sdks/recached-react/src/index.ts +++ b/sdks/recached-react/src/index.ts @@ -1,4 +1,4 @@ export { RecachedProvider, useRecached } from './context'; -export { useKey, useKeyJSON } from './useKey'; +export { useKey, useKeyBytes, useKeyJSON } from './useKey'; export { useKeys, type KeyValuePair } from './useKeys'; export { usePubSub } from './usePubSub'; diff --git a/sdks/recached-react/src/useKey.ts b/sdks/recached-react/src/useKey.ts index 7bc9a4b..f4bd644 100644 --- a/sdks/recached-react/src/useKey.ts +++ b/sdks/recached-react/src/useKey.ts @@ -8,7 +8,9 @@ import { useRecached } from './context'; * deleted — whether the mutation originated locally, from another tab via * BroadcastChannel, or from another client via the server WebSocket. * - * Returns `null` when the key does not exist or has expired. + * Returns `null` when the key does not exist, has expired, or holds a value that + * is not valid UTF-8 — a binary value has no string form, so read it with + * {@link useKeyBytes} instead. * * Built on React 18's `useSyncExternalStore` — safe with concurrent features. * @@ -28,7 +30,43 @@ export function useKey(key: string): string | null { const cache = useRecached(); return useSyncExternalStore( (cb) => cache.onMutation(cb), - () => cache.get(key), + // `get` throws on a binary value. This runs as a `getSnapshot`, so letting + // it propagate would take down the render tree over a value the component + // simply cannot display — report it as absent and let `useKeyBytes` read it. + () => { + try { + return cache.get(key); + } catch { + return null; + } + }, + () => null, + ); +} + +/** + * Reactively read a key as raw bytes. + * + * Behaves identically to {@link useKey} but returns the value's bytes, so it + * works for binary values a backend wrote — compressed payloads, protobuf, + * images. Text values come back as their UTF-8 bytes. + * + * ```tsx + * function Thumbnail() { + * const bytes = useKeyBytes('thumb:42'); + * const src = useMemo( + * () => (bytes ? URL.createObjectURL(new Blob([bytes])) : null), + * [bytes], + * ); + * return src ? : null; + * } + * ``` + */ +export function useKeyBytes(key: string): Uint8Array | null { + const cache = useRecached(); + return useSyncExternalStore( + (cb) => cache.onMutation(cb), + () => cache.getBytes(key), () => null, ); } diff --git a/sdks/recached-react/src/useKeys.ts b/sdks/recached-react/src/useKeys.ts index a09409a..fd29ebe 100644 --- a/sdks/recached-react/src/useKeys.ts +++ b/sdks/recached-react/src/useKeys.ts @@ -2,8 +2,9 @@ import { useEffect, useRef, useSyncExternalStore } from 'react'; import { useRecached } from './context'; /** A key/value pair from the local store. Collection-typed keys have `null` - * values — read those with typed accessors. */ -export type KeyValuePair = [key: string, value: string | null]; + * values — read those with typed accessors. A value that is not valid UTF-8 + * arrives as a `Uint8Array` rather than a mangled string. */ +export type KeyValuePair = [key: string, value: string | Uint8Array | null]; const EMPTY: KeyValuePair[] = []; diff --git a/sdks/recached-react/src/usePubSub.ts b/sdks/recached-react/src/usePubSub.ts index dbe8473..f1c26da 100644 --- a/sdks/recached-react/src/usePubSub.ts +++ b/sdks/recached-react/src/usePubSub.ts @@ -5,7 +5,9 @@ import { useRecached } from './context'; * Subscribe to a Recached pub/sub channel for the lifetime of the component. * * Sends `SUBSCRIBE` to the server on mount and `UNSUBSCRIBE` on unmount. - * The `handler` is called with each incoming message string. + * The `handler` is called with each incoming message. A publisher may send + * binary, in which case the payload arrives as a `Uint8Array` rather than a + * string — narrow the type before treating it as text. * * ```tsx * function Notifications() { @@ -20,7 +22,10 @@ import { useRecached } from './context'; * @param handler Called with each message payload. Identity need not be stable * across renders — the hook captures the latest ref internally. */ -export function usePubSub(channel: string, handler: (msg: string) => void): void { +export function usePubSub( + channel: string, + handler: (msg: string | Uint8Array) => void, +): void { const cache = useRecached(); const handlerRef = useRef(handler); handlerRef.current = handler; diff --git a/sdks/recached-vue/package-lock.json b/sdks/recached-vue/package-lock.json index b03d473..1eef94d 100644 --- a/sdks/recached-vue/package-lock.json +++ b/sdks/recached-vue/package-lock.json @@ -1,13 +1,13 @@ { "name": "@recached/vue", - "version": "0.1.4", + "version": "0.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@recached/vue", - "version": "0.1.4", - "license": "MIT", + "version": "0.2.2", + "license": "Apache-2.0", "devDependencies": { "@vue/runtime-core": "^3", "typescript": "^5.9.3", diff --git a/sdks/recached-vue/src/index.ts b/sdks/recached-vue/src/index.ts index 0bcad01..835f1e0 100644 --- a/sdks/recached-vue/src/index.ts +++ b/sdks/recached-vue/src/index.ts @@ -1,4 +1,4 @@ export { RecachedPlugin, useRecached, CACHE_KEY } from './plugin'; -export { useKey, useKeyJSON } from './useKey'; +export { useKey, useKeyBytes, useKeyJSON } from './useKey'; export { useKeys, type KeyValuePair } from './useKeys'; export { usePubSub } from './usePubSub'; diff --git a/sdks/recached-vue/src/useKey.ts b/sdks/recached-vue/src/useKey.ts index e582b1c..d6ab9f6 100644 --- a/sdks/recached-vue/src/useKey.ts +++ b/sdks/recached-vue/src/useKey.ts @@ -30,14 +30,42 @@ import { useRecached } from './plugin'; export function useKey(key: string): Ref { const cache = useRecached(); const value = ref(null); + // `get` throws on a binary value, which has no string form. Report it as + // absent rather than propagating out of a reactive update; read those with + // {@link useKeyBytes}. + const read = (): string | null => { + try { + return cache.get(key); + } catch { + return null; + } + }; const unsub = cache.onMutation(() => { - value.value = cache.get(key); + value.value = read(); }); - value.value = cache.get(key); + value.value = read(); onUnmounted(unsub); return value; } +/** + * Reactively read a key as raw bytes. + * + * Behaves identically to {@link useKey} but returns the value's bytes, so it + * works for binary values a backend wrote. Text values come back as their + * UTF-8 bytes. + */ +export function useKeyBytes(key: string): Ref { + const cache = useRecached(); + const value = ref(null); + const unsub = cache.onMutation(() => { + value.value = cache.getBytes(key); + }); + value.value = cache.getBytes(key); + onUnmounted(unsub); + return value as Ref; +} + /** * Reactively read a JSON-parsed value from the Recached store. * diff --git a/sdks/recached-vue/src/useKeys.ts b/sdks/recached-vue/src/useKeys.ts index 634a48e..70767d7 100644 --- a/sdks/recached-vue/src/useKeys.ts +++ b/sdks/recached-vue/src/useKeys.ts @@ -2,8 +2,9 @@ import { ref, onUnmounted, type Ref } from 'vue'; import { useRecached } from './plugin'; /** A key/value pair from the local store. Collection-typed keys have `null` - * values — read those with typed accessors. */ -export type KeyValuePair = [key: string, value: string | null]; + * values — read those with typed accessors. A value that is not valid UTF-8 + * arrives as a `Uint8Array` rather than a mangled string. */ +export type KeyValuePair = [key: string, value: string | Uint8Array | null]; /** * Live query: reactively read every key matching a glob pattern. diff --git a/sdks/recached-vue/src/usePubSub.ts b/sdks/recached-vue/src/usePubSub.ts index a5438f1..5514ef0 100644 --- a/sdks/recached-vue/src/usePubSub.ts +++ b/sdks/recached-vue/src/usePubSub.ts @@ -5,7 +5,9 @@ import { useRecached } from './plugin'; * Subscribe to a Recached pub/sub channel for the lifetime of the component. * * Sends `SUBSCRIBE` to the server on setup and `UNSUBSCRIBE` on `onUnmounted`. - * The `handler` is called with each incoming message string. + * The `handler` is called with each incoming message. A publisher may send + * binary, in which case the payload arrives as a `Uint8Array` rather than a + * string — narrow the type before treating it as text. * * ```vue *