diff --git a/CHANGELOG.md b/CHANGELOG.md index be2161b8..43f340ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,15 @@ signals a backwards-compatible change. retired `0x17 0x00`, whose payload also carried `inserted`, still decodes, with the counter dropped, and is never encoded. The `sketchlib-golden-bytes` submodule gains the `bloom_bits_*` fixtures. +- **Breaking: `SpaceSaving` no longer keeps a `total`.** `total()` is removed; + no Space-Saving readout read it, and after a merge it cannot be read off the + counts. A caller that needs the stream's total weight keeps a separate exact + count. The decode check that `total` covered the counted weight is gone. +- **ASAPv1: Space-Saving encodes as `0x18 0x01`, CountMinSketchWithSpaceSaving + as `0x1e 0x01` and CountSketchWithSpaceSaving as `0x1f 0x01`.** Each payload + drops the summary's `total`. The retired `0x18 0x00` / `0x1e 0x00` / + `0x1f 0x00` still decode, with `total` dropped, and are never encoded. The + `sketchlib-golden-bytes` submodule gains the `*_nototal_*` fixtures. - **Count Sketch estimates no longer allocate.** The per-row estimates a median is taken over are collected on the stack instead of in a fresh `Vec`: up to 20 rows for `Vector2D` (the fast and regular paths), diff --git a/asapv1_golden b/asapv1_golden index a73d634f..89b0c479 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit a73d634f6341b6e70e79fe64a9e7b9de82e60130 +Subproject commit 89b0c4798ab5e370bac8fab4b9250e16a67b4578 diff --git a/docs/api/api_count_min_sketch_with_space_saving.md b/docs/api/api_count_min_sketch_with_space_saving.md index c11f1c9d..d43ee91e 100644 --- a/docs/api/api_count_min_sketch_with_space_saving.md +++ b/docs/api/api_count_min_sketch_with_space_saving.md @@ -71,12 +71,14 @@ fn to_asapv1(&self) -> Result, RmpEncodeError> fn from_asapv1(bytes: &[u8]) -> Result ``` -The **ASAPv1** envelope, kind `0x1e 0x00` (spec §3.24), for +The **ASAPv1** envelope, kind `0x1e 0x01` (spec §3.24), for `CountMinSketchWithSpaceSaving, Mode, H>` with `T` `i32`, `i64` or `f64`, `Mode` `FastPath` or `RegularPath`, and `H: HashProfile`. The metadata is Count-Min's plus `capacity` and `key_type`; the payload is the matrix followed by the -summary's triples and running scalars. A matrix holding `p != 1` does not -serialize. There is no delta kind. +summary's triples and its ceiling `discarded_max`. The retired kind `0x1e 0x00`, +whose summary also carried a `total`, still decodes with that field dropped and +is never encoded. A matrix holding `p != 1` does not serialize. There is no +delta kind. ## Examples diff --git a/docs/api/api_count_sketch_with_space_saving.md b/docs/api/api_count_sketch_with_space_saving.md index 612bb5ff..87153d8c 100644 --- a/docs/api/api_count_sketch_with_space_saving.md +++ b/docs/api/api_count_sketch_with_space_saving.md @@ -69,10 +69,11 @@ fn to_asapv1(&self) -> Result, RmpEncodeError> fn from_asapv1(bytes: &[u8]) -> Result ``` -The **ASAPv1** envelope, kind `0x1f 0x00` (spec §3.25), for +The **ASAPv1** envelope, kind `0x1f 0x01` (spec §3.25), for `CountSketchWithSpaceSaving, Mode, H>` with `T` `i32`, `i64` or `f64`, `Mode` `FastPath` or `RegularPath`, and `H: HashProfile`. Same layout as -[CountMinSketchWithSpaceSaving](./api_count_min_sketch_with_space_saving.md). There is no delta kind. +[CountMinSketchWithSpaceSaving](./api_count_min_sketch_with_space_saving.md); the retired `0x1f 0x00` still +decodes, its `total` dropped, and is never encoded. There is no delta kind. ## Examples diff --git a/docs/api/api_space_saving.md b/docs/api/api_space_saving.md index b679c38b..f78593f0 100644 --- a/docs/api/api_space_saving.md +++ b/docs/api/api_space_saving.md @@ -67,7 +67,6 @@ fn is_guaranteed(&self, value: &DataInput) -> bool fn top_k(&self, k: usize) -> Vec<(OwnedDataInput, u64, u64)> fn entries(&self) -> Vec<(OwnedDataInput, u64, u64)> fn min_count(&self) -> u64 -fn total(&self) -> u64 fn len(&self) -> usize fn capacity(&self) -> usize ``` @@ -111,8 +110,11 @@ a key both sides evicted cannot be recovered. `merge_equivalence_battery` does not apply for that reason. The `min_count` a one-sided key picks up is weight the stream never carried, so -after a merge the counts sum to more than `total` and `estimate / total` is no -longer a frequency. +after a merge the counts can sum to more than the weight both streams recorded. + +The summary keeps no total weight: no readout above needs it, and after a merge +it cannot be read off the counts. A caller that needs the stream's total weight +keeps a separate exact count beside the summary. ## Serialization @@ -121,17 +123,19 @@ fn to_asapv1(&self) -> Result, RmpEncodeError> fn from_asapv1(bytes: &[u8]) -> Result ``` -These produce/consume the **ASAPv1** wire envelope (kind `0x18 0x00`) — see the +These produce/consume the **ASAPv1** wire envelope (kind `0x18 0x01`) — see the [ASAPv1 wire format spec](../asapv1_wire_format.md). The impl is bounded on `H: HashProfile`, so a summary built with an unprofiled hasher cannot serialize at all. -A summary travels as `capacity`, `total`, the unmonitored ceiling, and one +A summary travels as `capacity`, the unmonitored ceiling, and one `(key, count, error)` triple per monitored key. The bucket and counter lists and the key index all follow from the triples and are rebuilt on load, so no arena index reaches the wire and no crafted state can point one out of bounds or into a loop. `capacity` and the key type are ASAPv1 metadata; the payload is -`[keys, counts, errors, total, discarded_max]`. +`[keys, counts, errors, discarded_max]`. The retired kind `0x18 0x00`, whose +payload also carried a `total`, still decodes with that field dropped and is +never encoded. Keys are `OwnedDataInput`s, so the key type is a runtime property: the metadata's `key_type` names the **exact** variant (`"i32"` stays `"i32"`, never widened to diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 5341da28..ca389b8f 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -224,14 +224,17 @@ This registry is the master list of algorithms still to design payloads for, and | `0x16 0x00` | OctoSketch | - | TBD | reserved / not designed | | `0x17 0x00` | Bloom | Partitioned, with the `inserted` counter | Section 3.4 | retired: decoded (counter dropped), never encoded | | `0x17 0x01` | Bloom | Partitioned | Section 3.4 | implemented | -| `0x18 0x00` | Space-Saving | Stream-Summary | Section 3.5 | implemented | +| `0x18 0x00` | Space-Saving | Stream-Summary with `total` | Section 3.5 | retired: decoded (`total` dropped), never encoded | +| `0x18 0x01` | Space-Saving | Stream-Summary | Section 3.5 | implemented | | `0x19 0x00` | Count-Sketch-with-L2-norm (`CountSketchWithL2Norm`) | Count Sketch with L2 accumulators | Section 3.19 | implemented | | `0x1a 0x00` | UnivMon-Q (`UnivMonQ`) | Quantile pyramid | Section 3.20 | implemented | | `0x1b 0x00` | CountMinSketchFold | - | TBD | reserved / not designed | | `0x1c 0x00` | CountSketchFold | - | TBD | reserved / not designed | | `0x1d 0x00` | - | - | TBD | retired / permanently reserved, never reassign | -| `0x1e 0x00` | Count-Min-with-Space-Saving (CountMinSketchWithSpaceSaving, `Unstable`) | Count-Min + Space-Saving candidates | Section 3.24 | implemented | -| `0x1f 0x00` | Count-Sketch-with-Space-Saving (CountSketchWithSpaceSaving, `Unstable`) | Count Sketch + Space-Saving candidates | Section 3.25 | implemented | +| `0x1e 0x00` | Count-Min-with-Space-Saving (CountMinSketchWithSpaceSaving, `Unstable`) | Count-Min + Space-Saving candidates with `total` | Section 3.24 | retired: decoded (`total` dropped), never encoded | +| `0x1e 0x01` | Count-Min-with-Space-Saving (CountMinSketchWithSpaceSaving, `Unstable`) | Count-Min + Space-Saving candidates | Section 3.24 | implemented | +| `0x1f 0x00` | Count-Sketch-with-Space-Saving (CountSketchWithSpaceSaving, `Unstable`) | Count Sketch + Space-Saving candidates with `total` | Section 3.25 | retired: decoded (`total` dropped), never encoded | +| `0x1f 0x01` | Count-Sketch-with-Space-Saving (CountSketchWithSpaceSaving, `Unstable`) | Count Sketch + Space-Saving candidates | Section 3.25 | implemented | **Mapping notes** (mismatches between `apis.md` and Go's `magic_ids.go`): @@ -490,7 +493,7 @@ On the **fast** path one hash serves the whole matrix wherever it can: the layou In the two packed layouts the rows are **bit fields of one hash**, not independent draws. `rows <= 20` is declared and enforced on **both** doors whichever layout a geometry selects, so one geometry rule covers the format. `with_dimensions` is free to build a wider matrix in memory; such a sketch is rejected on **both** sides, so the format never emits bytes it would refuse to read back. -The same `1 <= rows <= 20` bound is declared by **every matrix-shaped payload**, under whichever name that payload's rows carry: Bloom's slices (`0x17 0x01`, §3.4, as `BLOOM_MAX_SLICES`), Count Sketch (`0x04 0x00`, §3.6), CountMinSketchWithHeap (`0x03 0x00`), CountSketchWithHeap (`0x0a 0x00`), CountMinSketchWithSpaceSaving (`0x1e 0x00`) and CountSketchWithSpaceSaving (`0x1f 0x00`) with their base matrix, CountSketchWithL2Norm (`0x19 0x00`), Coco's table (`0x0c 0x00`), Elastic's light layer (`0x0b 0x00`), Hydra's grid and its matrix counters (`0x07 0x01` / `0x07 0x02`), and every UnivMon (`0x10 0x00`) and Two-tier UnivMon (`0x11 0x00`) layer. +The same `1 <= rows <= 20` bound is declared by **every matrix-shaped payload**, under whichever name that payload's rows carry: Bloom's slices (`0x17 0x01`, §3.4, as `BLOOM_MAX_SLICES`), Count Sketch (`0x04 0x00`, §3.6), CountMinSketchWithHeap (`0x03 0x00`), CountSketchWithHeap (`0x0a 0x00`), CountMinSketchWithSpaceSaving (`0x1e 0x01`) and CountSketchWithSpaceSaving (`0x1f 0x01`) with their base matrix, CountSketchWithL2Norm (`0x19 0x00`), Coco's table (`0x0c 0x00`), Elastic's light layer (`0x0b 0x00`), Hydra's grid and its matrix counters (`0x07 0x01` / `0x07 0x02`), and every UnivMon (`0x10 0x00`) and Two-tier UnivMon (`0x11 0x00`) layer. **CountSketchWithL2Norm never draws per row**, and neither do the UnivMon layers built from it. One `hash128_seeded(seed_index, key)` supplies every row: the column bits at `r * ceil_log2(cols)` and the sign at bit `127 - r`. It still declares `rows <= 20`, and it carries the second bound the packed layout makes binding — `(ceil_log2(cols) + 1) * rows <= 128`, the bits one 128-bit hash has to give out (§3.19). @@ -573,10 +576,10 @@ This mirrors Count-Min, whose wire covers i32/i64/f64 counters while the in-memo 4. `len(words) == rows * ceil(cols / 64)` exactly, and that product must not overflow. 5. No bit may be set in a row's **trailing padding**, positions `cols .. 64*ceil(cols/64)`. A query cannot reach those bits, but `count_ones` sums them, so a crafted payload could skew every rate the filter reports — `fill_ratio`, `estimated_fpp`, `is_empty` — while every membership answer still looked normal. No encoder can produce them, so the rule never rejects real bytes, and it makes the decoded form canonical: a round trip re-serializes byte-identically. -### 3.5: Space-Saving payload (`0x18 0x00`) +### 3.5: Space-Saving payload (`0x18 0x01`) `SpaceSaving` monitors a fixed number of keys in a Stream-Summary: a doubly linked list of count buckets, a counter arena, and a key index into it. -**None of that structure reaches the wire.** Every link and index is derivable from the `(key, count, error)` triples the summary answers with, so the payload carries the triples plus the two running scalars and the decoder rebuilds the arenas. The structure is also the reason to do it this way: with no arena index on the wire, no crafted payload can point one out of bounds or into a cycle. +**None of that structure reaches the wire.** Every link and index is derivable from the `(key, count, error)` triples the summary answers with, so the payload carries the triples plus the ceiling `discarded_max` and the decoder rebuilds the arenas. The structure is also the reason to do it this way: with no arena index on the wire, no crafted payload can point one out of bounds or into a cycle. The counter budget `capacity` and the `key_type` are configuration and live in the metadata (Section 2), so the payload is: @@ -585,14 +588,17 @@ The counter budget `capacity` and the `key_type` are configuration and live in t | 0 | `keys` | array | one monitored key per counter; element type = `key_type`; homogeneous | | 1 | `counts` | array | u64 recorded count, parallel to `keys` | | 2 | `errors` | array | u64 error allowance, parallel to `keys`; `errors[i] <= counts[i]` | -| 3 | `total` | u64 | total weight recorded, monitored or displaced | -| 4 | `discarded_max` | u64 | the largest count known to have left the summary | +| 3 | `discarded_max` | u64 | the largest count known to have left the summary | The three arrays are parallel and equal-length; the number of monitored keys is `len(keys)` (derived, so not stored). `min_count` is **not** stored either, and it is recomputed on load. It is `discarded_max` on its own while the summary still has a spare counter, and `max(discarded_max, smallest count still held)` only once every counter is in use — `len(keys) == capacity` with a non-empty bucket list. A summary holding two counters against a capacity of `u32::MAX` reports `min_count == 0` however large its smallest held count is: no key has been turned away, so nothing bounds a key the summary does not monitor. `discarded_max`, by contrast, is **not** derivable from the triples — it records what an eviction or a merge dropped, and a merged summary can hold fewer keys than its capacity and still be missing keys the other side had already discarded. A payload carrying only the triples would decode such a summary with no ceiling at all, silently reporting `upper_bound == 0` for exactly the keys the ceiling exists to bound. So it is carried. +**No `total`.** The weight a summary recorded is a sum over the stream, not a Space-Saving readout: `estimate`, `error`, `upper_bound`, `min_count`, `is_guaranteed` and `top_k` read the triples and `discarded_max` alone. It is not derivable from the state either — after a merge the counts include the `min_count` each side added for keys the other did not hold, and a trim drops counted weight — so a caller that needs it keeps a separate exact count beside the summary. + +**The retired `0x18 0x00`.** It carried a u64 `total` at position 3, before `discarded_max` (five fields in all), and its decode required `sum(counts[i] - errors[i]) <= total` (saturating). A decoder still accepts it, reads `total` as an integer (Section 4), drops it without that check, and the result re-encodes as `0x18 0x01`. Nothing encodes `0x18 0x00`; per Q-VER the id is never reused. + **Metadata vs payload.** `capacity` is the summary's one sizing parameter, chosen at construction, so per the config-to-metadata rule it belongs in the descriptor (like HLL's `precision` or Count-Min's `rows`/`cols`). `key_type` is a structural param in the Count-Min `counter_type` / KLL `item_type` sense: it fixes the element type of the payload's `keys` array, and the payload cannot be read without it. Space-Saving carries the hash-spec group — it hashes, and a consumer must reproduce that hash to query a key — but carries **no seed-index key**. It hashes every key at seed index `0` unconditionally: that is a fixed part of the algorithm, not a profile choice, so neither `canonical_seed_index` nor `matrix_seed_index` would describe it truthfully, and a metadata field that can be wrong is worse than none. @@ -635,7 +641,7 @@ This is required, not cosmetic: `entries()` order follows the counter arena and **Decode rules.** Fail **closed** on each, with an error and never a panic: -1. `kind_id` is `0x18 0x00`. +1. `kind_id` is `0x18 0x01`, or the retired `0x18 0x00` with its five-field payload; any other id is rejected. 2. The hash-spec group matches the **target hasher's** `HashProfile`. `capacity` and `key_type` are properties of the *stored* summary rather than of the target type, so they are echoed back into the expected metadata rather than pinned — the same treatment Count-Min gives `rows`/`cols`. 3. `key_type` is one of the fourteen names above; anything else is rejected. The `keys` array is then read **as** that type, so every relabel of a non-empty `keys` fails except one integer name as another (Section 4). That one decodes whenever every key is in the new name's range, since a msgpack integer carries its value and not its type, and the summary then answers `0` for every key it holds (see "Why a key is never widened"). Only the producer can get this right. An empty `keys` array requires `key_type` `"u64"`. 4. `keys`, `counts` and `errors` have equal length. @@ -644,10 +650,9 @@ This is required, not cosmetic: `entries()` order follows the counter arena and 7. No key appears twice, judged by heap key order: two bit-identical `NaN`s are one key twice. 8. `capacity` **never sizes an allocation.** The counter arena and the key index both reserve from `len(keys)`, so a payload declaring `capacity = 2^32 - 1` with two counters costs two counters. A decoder must not preallocate from a declared size. 9. `discarded_max <= min(counts)`. The ceiling records what left the summary, and the algorithm only ever evicts a *smallest* counter, so a ceiling above a count still held describes no reachable state — and it would make `upper_bound` read lower than the count beside it. The comparison uses `0` as the smallest count of an empty summary, so an empty summary carrying a non-zero `discarded_max` is rejected too. -10. `sum(counts[i] - errors[i]) <= total`, with the sum taken in **saturating** `u64` arithmetic: each term is a saturating subtraction (rule 6 has already forbidden a negative one) and the running total saturates at `u64::MAX` rather than wrapping. Each counter's `count - error` is weight the summary claims to have actually recorded, so a `total` under that sum contradicts its own triples and would make every frequency the summary reports unbounded from below. The saturation is part of the contract, not an implementation detail: a set of triples whose exact sum exceeds `u64::MAX` is **accepted** whenever `total == u64::MAX`, so a decoder computing the sum in unbounded integers would reject bytes this one takes. Mirror the saturation. -11. Entries are in the emitted order above; any other order is rejected. +10. Entries are in the emitted order above; any other order is rejected. -Rules 9 and 10 are hard rejections, not repairs: a Go decoder must mirror them. +Rule 9 is a hard rejection, not a repair: a Go decoder must mirror it. The encode side refuses a summary holding one key twice, judged as rule 7 judges it: `NaN` never equals itself, so a `NaN` key offered twice seats two counters, and that state has no encoding. @@ -1396,7 +1401,7 @@ Rules 3 through 7 are enforced on the **encode** side too, so the format never e 2. The state's metadata equals the delta's, key for key, `key_type` excepted for CountMinSketchWithHeap and CountSketchWithHeap. 3. An integer add that overflows rejects the delta: a `counter_type` cell, or DDSketch's total count (§3.8 rule 6). -### 3.24: CountMinSketchWithSpaceSaving payload (`0x1e 0x00`) +### 3.24: CountMinSketchWithSpaceSaving payload (`0x1e 0x01`) `CountMinSketchWithSpaceSaving` is a Count-Min sketch (§3.2) paired with a Space-Saving summary (§3.5). Every update reaches both. A top-k query ranks the summary's monitored keys by their Count-Min estimate: the summary nominates the candidates and the matrix counts them. Both are **inlined**, so the payload is Count-Min's `counts` followed by §3.5's payload, its `counts` renamed `summary_counts`: @@ -1406,10 +1411,9 @@ Rules 3 through 7 are enforced on the **encode** side too, so the format never e | 1 | `keys` | array | one monitored key per counter; element type = `key_type`; homogeneous (§3.5) | | 2 | `summary_counts` | array | u64 recorded count, parallel to `keys` | | 3 | `errors` | array | u64 error allowance, parallel to `keys`; `errors[i] <= summary_counts[i]` | -| 4 | `total` | u64 | total weight the summary recorded | -| 5 | `discarded_max` | u64 | the largest count known to have left the summary | +| 4 | `discarded_max` | u64 | the largest count known to have left the summary | -The summary's arenas and key index are rebuilt from the triples, and `min_count` is recomputed, exactly as in §3.5. +The summary's arenas and key index are rebuilt from the triples, and `min_count` is recomputed, exactly as in §3.5. The summary carries no `total` (§3.5). The retired `0x1e 0x00` carried a u64 `total` at position 4, before `discarded_max`; a decoder still accepts it, reads `total` and drops it, and the result re-encodes as `0x1e 0x01`. Nothing encodes `0x1e 0x00`. **Metadata vs payload.** The descriptor keys, in canonical wire order: @@ -1432,25 +1436,27 @@ Wire counter types are Count-Min's: **`"i32"`, `"i64"` and `"f64"`**, pinned on **Decode rules.** Fail **closed** on each, with an error and never a panic: -1. `kind_id` is `0x1e 0x00`; any other id is rejected — including `0x1f 0x00` (a CountSketchWithSpaceSaving, whose metadata schema is this one), `0x03 0x00` (a CountMinSketchWithHeap), `0x02 0x00` (a Count-Min) and `0x18 0x00` (a Space-Saving). +1. `kind_id` is `0x1e 0x01`, or the retired `0x1e 0x00` with its six-field payload; any other id is rejected — including `0x1f 0x01` / `0x1f 0x00` (a CountSketchWithSpaceSaving, whose metadata schema is this one), `0x03 0x00` (a CountMinSketchWithHeap), `0x02 0x00` (a Count-Min) and `0x18 0x01` / `0x18 0x00` (a Space-Saving). 2. The metadata's hash spec, `counter_type` and `mode` must equal the target type's own, with `rows` / `cols` / `capacity` / `key_type` echoed back. Every key is required: a map missing one, or carrying an unknown one (`admission_rate` included), does not decode. 3. `rows` and `cols` are both non-zero and `rows <= 20` (`MATRIX_MAX_ROWS`, §3.2), checked **before** the matrix is built. 4. `key_type` is one of §3.5's fourteen names; anything else is rejected before the payload is read. The `keys` array is then read **as** that type, with §3.5 rule 3's relabel limits. An empty summary requires `key_type` `"u64"`. 5. `keys`, `summary_counts` and `errors` have equal length. 6. `len(counts) == rows * cols` exactly, checked **before** the allocation. -7. §3.5's rules 5, 6, 7, 8, 9 and 10 hold over `capacity`, `keys`, `summary_counts`, `errors`, `total` and `discarded_max`. Duplication (rule 7) is judged **before any entry is seated**, by heap key order (§3.5), as §3.7's rule 7 judges it: two bit-identical `NaN`s are one key twice, and `+0.0` and `-0.0` are two keys. `capacity` never sizes an allocation. +7. §3.5's rules 5, 6, 7, 8 and 9 hold over `capacity`, `keys`, `summary_counts`, `errors` and `discarded_max`. Duplication (rule 7) is judged **before any entry is seated**, by heap key order (§3.5), as §3.7's rule 7 judges it: two bit-identical `NaN`s are one key twice, and `+0.0` and `-0.0` are two keys. `capacity` never sizes an allocation. 8. Summary entries are in the emitted order above; any other order is rejected. Rules 3, 6 and 7's duplicate check are enforced on the **encode** side too, as are §3.5's mixed-variant and 128-bit-key refusals and the `u32` bounds on `capacity` and `cols`. A summary seats a `NaN` key anew on every insert, since `NaN` never finds itself; a summary holding the same `NaN` twice has no encoding and fails to serialize. -### 3.25: CountSketchWithSpaceSaving payload (`0x1f 0x00`) +### 3.25: CountSketchWithSpaceSaving payload (`0x1f 0x01`) `CountSketchWithSpaceSaving` is a Count Sketch (§3.6) paired with a Space-Saving summary, and its payload is §3.24's with the Count Sketch matrix as `counts`: | Pos | Field | Type | Notes | | ----- | ------- | ------ | ------- | | 0 | `counts` | array | the Count Sketch matrix, packed **row-major**, `rows*cols` cells; element type = `counter_type`, **signed** | -| 1-5 | `keys`, `summary_counts`, `errors`, `total`, `discarded_max` | | §3.24's | +| 1-4 | `keys`, `summary_counts`, `errors`, `discarded_max` | | §3.24's | + +The retired `0x1f 0x00` carried §3.24's retired `total` at position 4; a decoder still accepts it, drops `total`, and the result re-encodes as `0x1f 0x01`. Nothing encodes `0x1f 0x00`. **Metadata vs payload.** §3.24's schema and canonical key order. Wire counter types are Count Sketch's: **`"i32"`, `"i64"` and `"f64"`** (§3.6), pinned on decode; `mode` fixes the sign derivation as well as the column (§3.6). @@ -1462,7 +1468,7 @@ A top-k query ranks the summary's keys by the Count Sketch estimate, the median **Decode rules.** §3.24's rules 2 through 8 apply verbatim, with one difference: -1. `kind_id` is `0x1f 0x00`; any other id is rejected — including `0x1e 0x00` (a CountMinSketchWithSpaceSaving), `0x0a 0x00` (a CountSketchWithHeap) and `0x04 0x00` (a Count Sketch). +1. `kind_id` is `0x1f 0x01`, or the retired `0x1f 0x00` with its six-field payload; any other id is rejected — including `0x1e 0x01` / `0x1e 0x00` (a CountMinSketchWithSpaceSaving), `0x0a 0x00` (a CountSketchWithHeap) and `0x04 0x00` (a Count Sketch). The same encode-side checks apply. @@ -1533,7 +1539,7 @@ A `"f32"` key is float32 (`0xca`); a decoder rejects any other type there (§3.5 *Key and item arrays.* `keys` and `items` take the element type the metadata names — `key_type` for the `OwnedDataInput` keys of §3.5, §3.7, §3.10, §3.15, §3.16, §3.23, §3.24, §3.25 and Hydra's `cell_key_type` (§3.9), per §3.5's table; `item_type` for KLL's `items` (§3.3) and UniformSampling's `values` (§3.13). Four payloads name no such key and are fixed by their kind_id instead: Coco's `keys` (§3.12) and Elastic's `flow_ids` (§3.11) are nil-or-`str`, SetAggregator's `values` (§3.21) and SetAggregatorDelta's `added` / `removed` (§3.22) are `str`, and Coco's `values` are unsigned integers. -*Scalars.* Float scalars are listed above. Booleans are msgpack `true` / `false` (`0xc3` / `0xc2`) — `evictions` and `stale_copies` (§3.11), `candidate_complete` (§3.15, §3.16), `ever_evicted` (§3.20). Every other scalar is an integer under the family/width rule: the retired Bloom `0x17 0x00`'s `inserted` (§3.4), `total` / `discarded_max` (§3.5, §3.24, §3.25), `offset` / `negative_offset` (§3.8, **signed**), `zero_count` (§3.8), `total_seen` / `rng_state` (§3.13), `bucket_size` / `update_mode` (§3.15, §3.16), `count` / `source_id` / `next_sequence` (§3.20). +*Scalars.* Float scalars are listed above. Booleans are msgpack `true` / `false` (`0xc3` / `0xc2`) — `evictions` and `stale_copies` (§3.11), `candidate_complete` (§3.15, §3.16), `ever_evicted` (§3.20). Every other scalar is an integer under the family/width rule: the retired Bloom `0x17 0x00`'s `inserted` (§3.4), `discarded_max` (§3.5, §3.24, §3.25) and the retired Space-Saving kinds' `total`, `offset` / `negative_offset` (§3.8, **signed**), `zero_count` (§3.8), `total_seen` / `rng_state` (§3.13), `bucket_size` / `update_mode` (§3.15, §3.16), `count` / `source_id` / `next_sequence` (§3.20). *Nil.* A nullable slot is msgpack **nil** (`0xc0`), never an empty string and never a zero: an unoccupied Coco bucket, a free Elastic heavy bucket, an absent UnivMon-Q `min` / `max`. @@ -1618,7 +1624,7 @@ Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketc - **Q-ORDER**: a payload whose in-memory container order does not survive a rebuild **pins an emitted order**, as a cross-language contract. Heaps, arenas and hash maps all have this property, so the rule reaches Space-Saving's triples (§3.5, §3.24, §3.25), the top-k heap entries (§3.7), KMV's digests (§3.14), UniformSampling's entries (§3.13), UnivMon-Q's candidates and occurrences (§3.20), and every layer- and bucket-indexed array. Two sketches holding the same state then emit the same bytes whatever order they were built in, and a decoded sketch re-serializes byte-identically. The heap key comparator is the same everywhere: descending count, ties by heap key order (§3.5). - **Q-CAP**: **a declared capacity never sizes an allocation.** Every structure is built from the run the payload actually carries, and the declared bound is checked against that run first; a payload naming a capacity of `2^32 - 1` with two entries costs two entries. This holds for every `capacity` (§3.5, §3.24, §3.25), every `heap_size`, KMV's `k`, UniformSampling's `total_seen`, UnivMon-Q's `ordered_samples`, and Hydra's grid and counter dimensions. - **Q-KEYTYPE**: `key_type` names the **exact `OwnedDataInput` variant** and is never widened, wherever heap keys reach the wire — Space-Saving (§3.5), CountMinSketchWithHeap and CountSketchWithHeap (§3.7, §3.10), CountMinSketchWithSpaceSaving and CountSketchWithSpaceSaving (§3.24, §3.25), the UnivMon family (§3.15, §3.16) and Hydra's UnivMon counter (§3.9). A container whose keys mix variants has no single name and **fails to serialize**; `OwnedDataInput::I128` / `U128` have no msgpack integer form and are not wire types. An empty container emits `"u64"`, and a decoder rejects an empty container under any other name, so its `key_type` has one value rather than one per producer. That canonicalizes the key naming and nothing else: a Two-tier UnivMon pyramid still has more than one encoding per state, since layouts differing only in an `elephant_layers` at or above `layers` build identical layers, and an all-elephant pyramid carries its unused mouse dimensions on the wire regardless (§3.16). What holds everywhere is the round trip: a decoded container re-serializes byte-identically. -- **Q-DERIVED**: state a decoder can recompute exactly is **not carried**, and state it cannot is. Recomputed: DDSketch's `count` from its stores and `zero_count` (§3.8), `ExponentialHistogram`'s `l2_mass` and `merge_norm` (§3.17), Elastic's `heavy_buckets` from the declared bucket count (§3.11), every heap's digest index and array order, and every summary a sketch answers from its own state. Carried: Elastic's `stale_copies`, UnivMon's `update_mode`, `candidate_complete` and `bucket_size`, UnivMon-Q's `ever_evicted`, `source_id` and `next_sequence`, UniformSampling's `priorities`, `total_seen` and `rng_state`, and CountSketchWithL2Norm's `l2`. Where a cached derived field exists in memory, the encoder rejects a value that disagrees with what the decoder would recompute. State the sketch's own readouts do not need is not carried either: DDSketch's `sum` / `min` / `max` were (`0x05 0x00`) and are now a separate exact aggregate, since nothing a DDSketch answers reads them (§3.8), and so was Bloom's `inserted` counter (`0x17 0x00`), now a separate exact count, since no membership answer reads it (§3.4). +- **Q-DERIVED**: state a decoder can recompute exactly is **not carried**, and state it cannot is. Recomputed: DDSketch's `count` from its stores and `zero_count` (§3.8), `ExponentialHistogram`'s `l2_mass` and `merge_norm` (§3.17), Elastic's `heavy_buckets` from the declared bucket count (§3.11), every heap's digest index and array order, and every summary a sketch answers from its own state. Carried: Elastic's `stale_copies`, UnivMon's `update_mode`, `candidate_complete` and `bucket_size`, UnivMon-Q's `ever_evicted`, `source_id` and `next_sequence`, UniformSampling's `priorities`, `total_seen` and `rng_state`, and CountSketchWithL2Norm's `l2`. Where a cached derived field exists in memory, the encoder rejects a value that disagrees with what the decoder would recompute. State the sketch's own readouts do not need is not carried either: DDSketch's `sum` / `min` / `max` were (`0x05 0x00`) and are now a separate exact aggregate, since nothing a DDSketch answers reads them (§3.8), and so was Bloom's `inserted` counter (`0x17 0x00`), now a separate exact count, since no membership answer reads it (§3.4). Space-Saving's `total` was carried too (`0x18 0x00`, and inside `0x1e 0x00` / `0x1f 0x00`); no Space-Saving readout reads it, so it is no longer sketch state, and a caller that needs it keeps a separate exact count (§3.5). - **Q-NIL**: an absent slot is msgpack **nil** (`0xc0`), never a sentinel value. An empty string is a reachable key in Coco (§3.12) and a reachable flow id in Elastic (§3.11), so `""` cannot double as "free"; UnivMon-Q's absent extrema are nil for the same reason (§3.20). Go must model these as nullable, never as a zero value standing in for absent. - **Q-RNG**: RNG state is carried **resumable** when the generator is cheap to reproduce in another language, and opaque otherwise. UniformSampling's `rng_state` is one SplitMix64 word with published constants, so a decoded sampler continues the same draw sequence (§3.13); KLL's `coin` carries the compaction RNG's position and its `compaction_seed` is opaque state Go preserves without interpreting (Q-KLL-SEED). - **Q-UNIVMON-SCOPE**: every layer of `0x10 0x00` and `0x11 0x00` is a CountSketchWithL2Norm with no tag, and another per-layer counter needs a new structural param or a new id. `UnivMonPool` is a `Vec` free list of scratch `UnivMon`s — not of pyramids — with nothing an observer could query, so it is not a wire kind and holds no id. diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 77c8d157..143df1fb 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -110,8 +110,8 @@ Converted: bit grid packed into `u64`s; the wire covers the geometries `Bloom::with_capacity` produces. The retired `0x17 0x00` also carried an insert count; it still decodes, with the count dropped, and is never encoded. -- **Space-Saving** (`src/sketches/space_saving/wire.rs`, kind `0x18 0x00`) — the - payload is the `(key, count, error)` triples plus `total` and `discarded_max`; the +- **Space-Saving** (`src/sketches/space_saving/wire.rs`, kind `0x18 0x01`) — the + payload is the `(key, count, error)` triples plus `discarded_max`; the Stream-Summary's links and arenas are rebuilt on decode. - **CountMinSketchWithHeap** (`src/sketches/countminsketch_heap/wire.rs`, `0x03 0x00`) and **CountSketchWithHeap** (`src/sketches/countsketch_heap/wire.rs`, `0x0a 0x00`) — a base diff --git a/docs/tests.md b/docs/tests.md index 84150206..ba410558 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -415,11 +415,11 @@ Shared wire tests: [`src/sketches/countminsketch_space_saving/summary_wire.rs`]( | test_name | test_description | what_is_tested | | --- | --- | --- | -| `every_update_reaches_matrix_and_summary` | An insert reaches the matrix and the summary. | On a `3x1024` sketch with capacity `8`, inserts key `1` once and key `2` with weight `7`, and verifies the Count-Min estimates `1` and `7`, the summary estimates `1` and `7`, a summary total of `8`, and the geometry `(3, 1024, 8)`. | +| `every_update_reaches_matrix_and_summary` | An insert reaches the matrix and the summary. | On a `3x1024` sketch with capacity `8`, inserts key `1` once and key `2` with weight `7`, and verifies the Count-Min estimates `1` and `7`, the summary estimates `1` and `7` and the geometry `(3, 1024, 8)`. | | `top_k_ranks_the_candidates_by_their_matrix_count` | The summary nominates and the matrix counts. | On a `3x1024` sketch with capacity `2`, inserts `1` with weight `5`, then `2` and `3` once; verifies the matrix estimates `5, 1, 1`, that the summary holds `3` at `2` and has dropped `2`, and that `top_k(2)` is `[(1, 5), (3, 1)]`. | | `top_k_returns_at_most_capacity_keys_whatever_k` | `k` and `capacity` are independent. | On a fast-path `3x1024` sketch with capacity `3`, inserts keys `1..=5` with weight equal to the key, and verifies `top_k(10)` holds `3` keys, `top_k(1)` is `[(5, 5)]` and `top_k(0)` is empty. | | `top_k_breaks_count_ties_by_key_order` | A count tie falls to the key. | Inserts keys `9, 4, 7` with weight `2` each and verifies `top_k(3)` is `[(4, 2), (7, 2), (9, 2)]`. | -| `merge_keeps_a_candidate_no_local_heap_held` | A merged summary keeps a key that every one-entry heap dropped. | Two sources each insert a local key with weight `5` and the shared key `3` with weight `3`. Verifies the merged `CountMinSketchWithHeap` with heap size `1` estimates `3` at `6` but does not hold it, and the merged `CountMinSketchWithSpaceSaving` with capacity `3` returns `[(3, 6)]` from `top_k(1)` with a summary total of `16`. | +| `merge_keeps_a_candidate_no_local_heap_held` | A merged summary keeps a key that every one-entry heap dropped. | Two sources each insert a local key with weight `5` and the shared key `3` with weight `3`. Verifies the merged `CountMinSketchWithHeap` with heap size `1` estimates `3` at `6` but does not hold it, and the merged `CountMinSketchWithSpaceSaving` with capacity `3` returns `[(3, 6)]` from `top_k(1)`. | | `merge_adds_the_matrices` | Merge adds the matrices cell by cell. | Verifies the merged `2x64` matrix is the cell-wise sum of the two sides and `top_k(2)` is `[(1, 7), (2, 1)]`. | | `merge_rejects_a_capacity_mismatch` | Summaries of different capacity do not merge. | Verifies merging a capacity-`5` sketch into a capacity-`4` one panics with `capacity mismatch`. | | `merge_rejects_a_dimension_mismatch` | Matrices of different shape do not merge. | Verifies merging a `3x64` sketch into a `2x64` one panics with `dimension mismatch`. | @@ -427,11 +427,11 @@ Shared wire tests: [`src/sketches/countminsketch_space_saving/summary_wire.rs`]( | `summary_weight_rounds_up_and_drops_non_positive_weights` | `CandidateCounter::summary_weight` is the weight rounded up, or `0`. | Verifies `7i32 -> 7`, `-7i32 -> 0`, `0i64 -> 0`, `i64::MAX` unchanged, `i128::MAX -> u64::MAX`, `-1i128 -> 0`, `0.25 -> 1`, `2.0 -> 2`, `-0.5 -> 0`, `NaN -> 0` and `+inf -> u64::MAX`. | | `f64_counters_rank_by_their_fractional_estimate` | An `f64` sketch ranks by the unrounded matrix estimate. | Inserts key `1` with weight `0.25` and key `2` with `1.5`; verifies the summary records `1` for key `1`, and `top_k(2)` is key `2` at `1.5` then `0.25`. | | `a_non_positive_weight_reaches_the_matrix_alone` | A weight of zero or below leaves the summary unchanged. | Inserts weights `0` and `-3`, and verifies the summary is empty and the matrix estimates `-3`. | -| `round_trip_keeps_matrix_summary_and_bytes` | The matrix, the summary and the bytes survive a round trip. | A `2x8` sketch with capacity `3` fed `1:9, 2:5, 3:7, 4:2` (so `4` displaces `2`) encodes under `0x1e 0x00` with `rows 2`, `cols 8`, `capacity 3`, `i64`, `regular` and `u64`; verifies the decode holds the matrix, the entries `1:9/0, 3:7/0, 4:7/5`, total `23`, `discarded_max` `5`, the same `top_k(3)`, and re-serializes identically. | +| `round_trip_keeps_matrix_summary_and_bytes` | The matrix, the summary and the bytes survive a round trip. | A `2x8` sketch with capacity `3` fed `1:9, 2:5, 3:7, 4:2` (so `4` displaces `2`) encodes under `0x1e 0x01` with `rows 2`, `cols 8`, `capacity 3`, `i64`, `regular` and `u64`; verifies the decode holds the matrix, the entries `1:9/0, 3:7/0, 4:7/5`, `discarded_max` `5`, the same `top_k(3)`, and re-serializes identically. | | `every_wire_counter_type_and_mode_round_trips` | The `i32` fast-path and `f64` regular-path forms round trip, with string and byte keys. | Verifies an `i32` fast-path sketch with string keys and an `f64` sketch with a non-UTF-8 byte key each carry their `counter_type`, `mode` and `key_type`, decode to the same `top_k` or estimate, and re-serialize identically. | | `an_empty_summary_has_one_encoding` | An empty summary emits `key_type` `"u64"`. | Verifies a fresh `2x3` sketch with capacity `4` emits `"u64"` and decodes empty with capacity `4`. | | `an_empty_summary_must_carry_the_u64_key_type` | An empty summary carries `key_type` `"u64"`, for both kinds. | Verifies an empty CountMinSketchWithSpaceSaving and an empty CountSketchWithSpaceSaving relabelled `"string"` and `"f64"` each fail to decode with an "empty key set" complaint. | -| `foreign_kind_ids_are_rejected` | Only `0x1e 0x00` decodes. | Verifies `CountMinSketchWithHeap`, `CountSketchWithSpaceSaving`, `CountMinSketch` and `SpaceSaving` envelopes each fail with `kind_id mismatch`. | +| `foreign_kind_ids_are_rejected` | Only `0x1e 0x01` and the retired `0x1e 0x00` decode. | Verifies `CountMinSketchWithHeap`, `CountSketchWithSpaceSaving`, `CountMinSketch` and `SpaceSaving` envelopes each fail with `kind_id mismatch`. | | `counter_type_mode_and_profile_are_pinned` | The target fixes `counter_type`, `mode` and the hash profile. | Verifies `i64` regular bytes fail to decode as `f64`, `i32` and fast-path targets, and that a relabelled `hash_profile_id` fails with `metadata mismatch`. | | `metadata_keys_are_exactly_the_schema` | An extra key and a missing key both fail. | Verifies a map carrying `admission_rate` and a map missing `capacity` each fail to decode as `SummaryTopKMetadata`. | | `the_crafted_baseline_decodes` | The crafted envelope the decode-rule tests break is valid. | Verifies the `2x3`, capacity-`4`, `u64`-keyed crafted envelope decodes with three entries and its matrix. | @@ -442,7 +442,7 @@ Shared wire tests: [`src/sketches/countminsketch_space_saving/summary_wire.rs`]( | `counts_and_errors_describe_a_reachable_counter` | Decode rule 7, counters. | Verifies a summary count of `0` fails with `at zero` and an error of `5` against a count of `4` fails. | | `a_key_appears_once` | Decode rule 7, duplicates, judged on the wire representation. | Verifies keys `[7, 8, 7]` fail with `twice`, two bit-identical `f64` NaN keys fail with `twice`, and `0.0` beside `-0.0` decodes. | | `discarded_max_sits_at_or_under_the_smallest_count` | Decode rule 7, `discarded_max`. | Verifies `discarded_max` `5` over a smallest count of `4` fails, `4` decodes, and an empty summary with `discarded_max` `1` fails. | -| `total_covers_the_recorded_weight_saturating` | Decode rule 7, `total`. | Verifies a `total` of `15` under the `16` the triples record fails, and two counts of `u64::MAX` with `total` `u64::MAX` decode. | +| `the_retired_kind_with_total_decodes_and_drops_it` | The retired `0x1e 0x00` decodes with its `total` dropped. | Verifies the crafted baseline's six-field payload under `0x1e 0x00` decodes for a `total` of `16`, `15` (under the `16` the triples record, which the retired rule rejected), `0` and `u64::MAX`, and re-encodes as the five-field `0x1e 0x01` baseline; and that the six-field payload under `0x1e 0x01` and the five-field one under `0x1e 0x00` are both rejected. | | `entries_are_in_emitted_order` | Decode rule 8. | Verifies ascending counts fail, a count tie in descending key order fails, `i64` keys `-1` before `0` at a tie fail, and `0` before `-1` decodes. | | `a_declared_capacity_never_sizes_an_allocation` | `capacity` is echoed, not allocated. | Verifies `capacity` `u32::MAX` with three entries decodes, keeps that capacity and re-serializes identically. | | `states_with_no_encoding_fail_to_serialize` | The encode side refuses what the decode side would. | Verifies a `p = 0.5` matrix, mixed `u64` / `i64` keys, an `i128` key, two NaN counters, a capacity of `2^40` and an unfilled `Vector2D::unfilled` matrix each fail to serialize. | @@ -460,9 +460,9 @@ Wire tests: [`src/sketches/countsketch_space_saving/wire.rs`](../src/sketches/co | `merge_rejects_a_capacity_mismatch` | Summaries of different capacity do not merge. | Verifies merging a capacity-`3` sketch into a capacity-`4` one panics with `capacity mismatch`. | | `merge_rejects_a_dimension_mismatch` | Matrices of different shape do not merge. | Verifies merging a `2x32` sketch into a `2x64` one panics with `dimension mismatch`. | | `a_negative_weight_reaches_the_matrix_alone` | A negative weight moves the cells and not the summary. | On an `f64` sketch, inserts key `1` with `-2.5` and key `2` with `0.5`; verifies key `1` estimates `-2.5` and is not in the summary, key `2` is recorded at `1`, and `top_k(4)` is `[(2, 0.5)]`. | -| `round_trip_keeps_matrix_summary_and_bytes` | The matrix, the summary and the bytes survive a round trip. | A `3x8` sketch with capacity `2` fed `1:9, 2:5, 3:-4, 4:2` encodes under `0x1f 0x00`; verifies the decode holds the matrix, the entries `1:9/0, 4:7/5`, the same `top_k(2)`, and re-serializes identically. | +| `round_trip_keeps_matrix_summary_and_bytes` | The matrix, the summary and the bytes survive a round trip. | A `3x8` sketch with capacity `2` fed `1:9, 2:5, 3:-4, 4:2` encodes under `0x1f 0x01`; verifies the decode holds the matrix, the entries `1:9/0, 4:7/5`, the same `top_k(2)`, and re-serializes identically. | | `f64_counters_round_trip_on_the_fast_path` | An `f64` fast-path sketch round trips and pins its counter type and mode. | Verifies the decode estimates `-7` at `2.5`, re-serializes identically, and fails as an `f64` regular-path or `i64` fast-path target. | -| `foreign_kind_ids_are_rejected` | Only `0x1f 0x00` decodes. | Verifies `CountMinSketchWithSpaceSaving`, `CountSketchWithHeap` and `CountSketch` envelopes each fail to decode. | +| `foreign_kind_ids_are_rejected` | Only `0x1f 0x01` and the retired `0x1f 0x00` decode. | Verifies `CountMinSketchWithSpaceSaving`, `CountSketchWithHeap` and `CountSketch` envelopes each fail to decode. | | `a_sampled_matrix_fails_to_serialize` | A nested matrix holds `p = 1`. | Verifies a sketch whose matrix holds `p = 0.25` fails to serialize. | ### CountMinSketchFold @@ -574,44 +574,44 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `an_unmonitored_key_never_exceeds_the_minimum_count` | Dropped keys stay under the `min_count` ceiling. | Verifies every key the summary evicted has a true count at or below `min_count`, and that `upper_bound` and `error` for such a key both report that ceiling. | | `the_true_heavy_hitters_are_reported_exactly_and_in_order` | `top_k` returns the real heaviest keys in descending order. | Compares `top_k` against exact `FreqTruth` ordering and verifies the reported counts are exact and carry zero error, a true heavy hitter never having been displaced. | | `a_guaranteed_key_really_outranks_everything_dropped` | `is_guaranteed` is a claim about the real stream. | Verifies every key accepted by `is_guaranteed` has a true count strictly above the largest true count among the keys the summary dropped. | -| `an_arrival_displaces_the_minimum_and_inherits_its_count` | Eviction hands the victim's count to the arrival as its error. | Drives a hand-built sequence through a two-counter summary and verifies the new key takes the minimum slot at `min + 1` with the old minimum as its error allowance, while `total` still counts every arrival. | +| `an_arrival_displaces_the_minimum_and_inherits_its_count` | Eviction hands the victim's count to the arrival as its error. | Drives a hand-built sequence through a two-counter summary and verifies the new key takes the minimum slot at `min + 1` with the old minimum as its error allowance. | | `the_stream_summary_lists_stay_well_formed_under_eviction` | The bucket and counter lists stay traversable under sustained eviction. | Across capacities `1`, `2`, `17`, `256`, and `1_024`, verifies residency equals `min(capacity, distinct)`, the bucket walk reaches every counter exactly once in descending count order, and `entries` agrees with the walk. | -| `a_weighted_arrival_matches_repeating_it` | `update_weighted` matches that many single inserts. | Builds two summaries over the same stream, one key-at-a-time and one weighted per key, and verifies equal totals, equal residency, and equal per-key estimates. | +| `a_weighted_arrival_matches_repeating_it` | `update_weighted` matches that many single inserts. | Builds two summaries over the same stream, one key-at-a-time and one weighted per key, and verifies equal residency, and equal per-key estimates. | | `a_summary_larger_than_the_domain_is_exact` | A summary with room for the whole domain is exact. | With capacity above the distinct count, verifies every estimate equals the true count, every error is `0`, and `min_count` stays `0`. | -| `a_merge_keeps_the_total_and_never_reads_low` | Merge conserves the inserted mass and stays one-sided. | Partitions the stream into two shards, merges their summaries, and verifies the merged total equals the whole stream length, residency stays within capacity, and no monitored key reads below either side's truth. | -| `an_asapv1_round_trip_preserves_every_answer` | An ASAPv1 round trip preserves state and the rebuilt key index. | Round-trips through `to_asapv1` / `from_asapv1` and verifies residency, capacity, total, `min_count`, and per-key estimate/error all match, then confirms the decoded summary still takes inserts against its rebuilt index. | -| `an_empty_summary_answers` | An empty summary answers every query without panicking. | Verifies `len`/`total`/`min_count` are zero, point queries return `0`, `is_guaranteed` is false, and `top_k`/`entries` are empty. | -| `a_weighted_arrival_matches_repeating_it_under_eviction` | A weighted arrival matches repeating it once the summary is evicting. | Feeds the same Zipf stream key by key into two 64-counter summaries, one arrival at a time and one `update_weighted` per key, verifies the run saturates all 64 counters, and that `total`, `min_count`, and the whole sorted key/count/error entry list agree. | -| `update_batch_matches_repeated_updates_and_a_zero_weight_is_inert` | `update_batch` is the loop it replaces, and a zero weight records nothing. | Verifies a 128-counter summary filled by `update_batch` matches one filled one at a time on `total` and on `top_k(usize::MAX)`, then that `update_weighted` at weight `0` for a resident key and for an absent one leaves `total`, residency, and `top_k` untouched. | -| `a_zero_capacity_summary_still_answers` | A summary asked for no counters still holds one. | Verifies `with_capacity(0)` reports a capacity of `1`, and that after ten distinct keys residency is `1`, `total` is `10`, and the single held counter reads `10` with an error of `9`. | +| `a_merge_never_reads_low` | Merge stays one-sided. | Partitions the stream into two shards, merges their summaries, and verifies residency stays within capacity and no monitored key reads below either side's truth. | +| `an_asapv1_round_trip_preserves_every_answer` | An ASAPv1 round trip preserves state and the rebuilt key index. | Round-trips through `to_asapv1` / `from_asapv1` and verifies residency, capacity, `min_count`, and per-key estimate/error all match, then confirms the decoded summary still takes inserts against its rebuilt index. | +| `an_empty_summary_answers` | An empty summary answers every query without panicking. | Verifies `len`/`min_count` are zero, point queries return `0`, `is_guaranteed` is false, and `top_k`/`entries` are empty. | +| `a_weighted_arrival_matches_repeating_it_under_eviction` | A weighted arrival matches repeating it once the summary is evicting. | Feeds the same Zipf stream key by key into two 64-counter summaries, one arrival at a time and one `update_weighted` per key, verifies the run saturates all 64 counters, and that `min_count` and the whole sorted key/count/error entry list agree. | +| `update_batch_matches_repeated_updates_and_a_zero_weight_is_inert` | `update_batch` is the loop it replaces, and a zero weight records nothing. | Verifies a 128-counter summary filled by `update_batch` matches one filled one at a time on `top_k(usize::MAX)`, then that `update_weighted` at weight `0` for a resident key and for an absent one leaves residency and `top_k` untouched. | +| `a_zero_capacity_summary_still_answers` | A summary asked for no counters still holds one. | Verifies `with_capacity(0)` reports a capacity of `1`, and that after ten distinct keys residency is `1` and the single held counter reads `10` with an error of `9`. | | `a_merge_into_an_under_full_summary_keeps_the_ceiling_honest` | A merge carries the peer's ceiling even when there are counters to spare. | Merges a one-counter summary that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, then verifies residency is `1` and below capacity, `upper_bound` for the dropped key 7 still covers its true `10`, and `is_guaranteed` is false for the key that survived. | -| `a_merge_chain_stays_one_sided_against_the_truth` | A chain of merges over asymmetric capacities stays one-sided. | Shards the stream three ways by position into summaries of capacity 64, 512, and 7, merges them in turn, and verifies the merged `total` equals the stream length, residency stays within capacity, every true key's `upper_bound` covers its true count, every monitored key reads at or above its truth, and every key `is_guaranteed` accepts truly outranks the largest dropped key. | +| `a_merge_chain_stays_one_sided_against_the_truth` | A chain of merges over asymmetric capacities stays one-sided. | Shards the stream three ways by position into summaries of capacity 64, 512, and 7, merges them in turn, and verifies residency stays within capacity, every true key's `upper_bound` covers its true count, every monitored key reads at or above its truth, and every key `is_guaranteed` accepts truly outranks the largest dropped key. | | `a_merge_picks_the_same_survivors_every_time` | The merge depends on what the summaries hold, not on arrival order. | Merges two 200-counter summaries over keys `0..150` and `100..250`, verifies the union fills the capacity at 200 with exactly the 50 shared keys at count `2`, that five repeats give an identical key/count/error list, and that reversing both arrival orders gives the same list. | -| `a_round_trip_carries_the_merged_ceiling` | The ceiling a merge established survives the wire. | Merges a 4-counter summary of the stream's second half into a 32-counter summary of its first half, round-trips through ASAPv1, and verifies `min_count`, `total`, and residency are unchanged, every key's estimate matches, and every true key's decoded `upper_bound` still covers its true count. | +| `a_round_trip_carries_the_merged_ceiling` | The ceiling a merge established survives the wire. | Merges a 4-counter summary of the stream's second half into a 32-counter summary of its first half, round-trips through ASAPv1, and verifies `min_count` and residency are unchanged, every key's estimate matches, and every true key's decoded `upper_bound` still covers its true count. | | `a_fresh_summary_is_well_formed` | A fresh summary passes the structural check. | For `with_capacity(4)`, verifies `validate()` succeeds, `min_count` is `0`, and `capacity` is `4`. | | `a_capacity_of_zero_floors_at_one` | A requested capacity of zero floors at one counter. | Verifies `with_capacity(0)` reports a capacity of `1`, and that after two distinct keys `validate()` passes, residency is `1`, and the survivor reads `2`. | -| `a_weighted_arrival_displaces_the_minimum_and_starts_above_it` | A weighted eviction starts the arrival above the count it displaced. | In a two-counter summary holding `1` at 5 and `2` at 2, `add(3, 4)` is verified to evict key 2, leaving key 3 at `6` with error `2`, key 2 at `0`, key 1 at `5`, `min_count` at `5`, `total` at `11`, and `validate()` passing. | +| `a_weighted_arrival_displaces_the_minimum_and_starts_above_it` | A weighted eviction starts the arrival above the count it displaced. | In a two-counter summary holding `1` at 5 and `2` at 2, `add(3, 4)` is verified to evict key 2, leaving key 3 at `6` with error `2`, key 2 at `0`, key 1 at `5`, `min_count` at `5`, and `validate()` passing. | | `a_weighted_raise_passes_every_bucket_below_its_destination` | A multi-hop raise walks the bucket list rather than skipping it. | Over counters at 1, 2, 3, and 4, `add(1, 9)` lifts key 1 from the bottom to the top; verifies the descending walk becomes `[(1,10), (4,4), (3,3), (2,2)]` and `validate()` passes. | -| `counts_saturate_and_keep_the_bucket_order` | Counts saturate at `u64::MAX` instead of wrapping. | Raises a counter seeded at `u64::MAX - 2` twice past the ceiling, verifies `validate()` after each raise, both saturated keys read `u64::MAX`, `total` is `u64::MAX`, and the three-counter walk stays descending, ending at the small counter's `7`. | +| `counts_saturate_and_keep_the_bucket_order` | Counts saturate at `u64::MAX` instead of wrapping. | Raises a counter seeded at `u64::MAX - 2` twice past the ceiling, verifies `validate()` after each raise, both saturated keys read `u64::MAX`, and the three-counter walk stays descending, ending at the small counter's `7`. | | `an_eviction_from_a_saturated_counter_stays_sound` | Evicting a saturated counter keeps the structure sound. | A one-counter summary holding `u64::MAX` is evicted by a new key; verifies `validate()` passes, residency is `1`, the arrival reads `u64::MAX` with an error of `u64::MAX`, and the evicted key's `upper_bound` is `u64::MAX`. | -| `a_merge_saturates_instead_of_wrapping` | Merge saturates rather than wrapping past the ceiling. | Merges a three-counter peer into a two-counter summary already at `u64::MAX` and `u64::MAX - 1`; verifies `validate()` passes, residency stays `2`, `total` is `u64::MAX`, both survivors read `u64::MAX`, and all four keys' `upper_bound` is `u64::MAX`. | +| `a_merge_saturates_instead_of_wrapping` | Merge saturates rather than wrapping past the ceiling. | Merges a three-counter peer into a two-counter summary already at `u64::MAX` and `u64::MAX - 1`; verifies `validate()` passes, residency stays `2`, both survivors read `u64::MAX`, and all four keys' `upper_bound` is `u64::MAX`. | | `a_merge_carries_the_ceiling_into_an_under_full_summary` | A merge into an empty summary still raises its ceiling. | Merging a one-counter peer that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary is verified to leave residency `1` below capacity, `min_count` at or above `10`, `upper_bound(7)` at or above `10`, `is_guaranteed(8)` false, and `validate()` passing. | | `a_truncating_merge_keeps_the_keys_the_encoder_emits_first` | A merge that must truncate a count tie cuts it by key order alone. | Merges a 3-counter peer holding keys 40, 50, and 60 at 2 into a 4-counter summary holding key 1 at 9 and keys 10, 20, and 30 at 3; verifies `validate()` passes, the six tied keys each reach an `upper_bound` of `5`, the survivors are keys 1, 10, 20, and 30 while the swapped merge keeps 1, 10, and 20, and that the encoder emits those survivors in the same order. | | `a_chain_of_merges_keeps_the_ceiling_above_everything_dropped` | Two merges compound their ceilings. | Merges two one-counter peers (7 at 10 against 8 at 20, then 9 at 5 against 10 at 7) into a 5-counter summary; verifies `validate()` after each merge, residency below capacity, `upper_bound` covering keys 7 and 9 at their true 10 and 5, and `min_count` at or above `15`. | | `a_key_that_re_enters_after_a_merge_never_reads_low` | A key re-entering a merged summary comes back above the ceiling. | After a merge drops key 7 at its true 12, one further insert is verified to seat it at `min_count + 1` with `min_count` as its error, at or above its true `13`, with `validate()` passing. | -| `randomized_operations_keep_the_structure_sound` | Randomized weighted inserts keep the structure sound at every step. | Drives 4,000 `update_weighted` calls of weight 1, 3, 11, or 97 over a 96-key domain at capacities 1, 2, 7, 64, and 257, calling `validate()` after every step, then checks residency equals `min(capacity, distinct)`, `total` equals the inserted mass, the walk is the right length and descending, and every truth is bracketed by its counter's estimate and error. | -| `randomized_merges_keep_the_structure_sound` | Randomized merges keep the structure sound and one-sided. | For five capacity/domain pairs, merges two independently fuzzed summaries into a fuzzed one and then inserts 200 more keys, calling `validate()` after each merge and after the inserts, and checks the error sandwich against the combined truth, `total` equal to the combined mass, and residency within capacity. | -| `a_decoded_summary_rebuilds_both_link_directions` | Decoding rebuilds the bucket and counter links, not just the counts. | Rebuilds a fuzzed 48-counter summary from its triples and verifies the decoded summary passes `validate()`, matches residency, `min_count`, and `total`, still brackets every truth, and walks the same key/count set. | -| `a_crafted_state_fails_closed` | `rebuild` refuses state the algorithm could not have produced. | Verifies a zero capacity, entries over the capacity, a counter at `0`, an error above its count, a repeated key, a `discarded_max` of `u64::MAX` over the lowest count of `3`, and a `total` of `0` under a counter's `9` are each rejected with the matching complaint. | +| `randomized_operations_keep_the_structure_sound` | Randomized weighted inserts keep the structure sound at every step. | Drives 4,000 `update_weighted` calls of weight 1, 3, 11, or 97 over a 96-key domain at capacities 1, 2, 7, 64, and 257, calling `validate()` after every step, then checks residency equals `min(capacity, distinct)`, the walk is the right length and descending, and every truth is bracketed by its counter's estimate and error. | +| `randomized_merges_keep_the_structure_sound` | Randomized merges keep the structure sound and one-sided. | For five capacity/domain pairs, merges two independently fuzzed summaries into a fuzzed one and then inserts 200 more keys, calling `validate()` after each merge and after the inserts, and checks the error sandwich against the combined truth, and residency within capacity. | +| `a_decoded_summary_rebuilds_both_link_directions` | Decoding rebuilds the bucket and counter links, not just the counts. | Rebuilds a fuzzed 48-counter summary from its triples and verifies the decoded summary passes `validate()`, matches residency and `min_count`, still brackets every truth, and walks the same key/count set. | +| `a_crafted_state_fails_closed` | `rebuild` refuses state the algorithm could not have produced. | Verifies a zero capacity, entries over the capacity, a counter at `0`, an error above its count, a repeated key, and a `discarded_max` of `u64::MAX` over the lowest count of `3` are each rejected with the matching complaint. | | `a_declared_capacity_is_not_allocated_on_decode` | A huge declared capacity is not allocated on decode. | A state declaring a capacity of `1 << 40` with a single entry is verified to rebuild, pass `validate()`, report that capacity, and hold one counter. | | `a_non_utf8_byte_key_is_monitored_and_queried` | A byte array that is not UTF-8 is a key like any other. | Inserts `[0xff, 0x00, 0xfe]` three times and `[0x00, 0x80]` at weight 2 into a 4-counter summary, and verifies `validate()` passes, the repeats share one counter, both keys are held as their raw bytes, each estimates its weight, and an unseen byte key reads `0`. | | `a_byte_key_and_a_string_key_are_separate_counters` | `Bytes(b"abc")` and `Str("abc")` are two keys, not one. | Seats both in a 4-counter summary at weights 5 and 2 and verifies residency is `2`, the byte key reads `5`, and the string key reads `2` through `Str` and `String` alike. | | `an_evicted_byte_key_keeps_its_bound` | An evicted byte key still reads under the ceiling. | In a 2-counter summary holding a byte key at 3 and another at 2, a third byte key is verified to displace the smallest, leaving the evicted key at `estimate` `0` with an `upper_bound` at or above its true `2`, a `min_count` of `3`, and the newcomer at `3` carrying an error of `2`. | | `a_merge_pairs_byte_keys_by_their_bytes` | A merge pairs byte keys by their bytes. | Merges a peer holding the shared byte key at 2 and its own at 7 into a summary holding the shared key at 5 and its own at 3, and verifies `validate()` passes, all three byte keys survive as their raw bytes, and the shared key reads `7`. | -| `a_rebuild_keeps_a_byte_key` | A rebuild carries raw bytes and re-hashes them to the digest a query reaches. | Rebuilds a summary holding `[0xff, 0x00, 0xfe]` at 9 and five `0x00` bytes at 4 from its triples, and verifies the rebuilt summary passes `validate()`, holds the same raw byte keys, answers both `DataInput::Bytes` queries at their weights, and reports the same `total`. | -| `space_saving_envelope_structure_and_round_trip` | The envelope frames a summary under kind_id `0x18 0x00`, and the bytes are stable across a round trip. | For a 48-counter summary saturated by 30,000 weighted draws over 400 distinct keys, verifies the bytes open with the ASAPv1 magic, `envelope::VERSION`, a `kind_id_len` of `2`, and `0x18 0x00`, that re-serializing the decoded summary reproduces them exactly, and that residency, capacity, `total`, `min_count`, and `estimate`/`error`/`upper_bound`/`is_guaranteed` agree across all 400 keys. | +| `a_rebuild_keeps_a_byte_key` | A rebuild carries raw bytes and re-hashes them to the digest a query reaches. | Rebuilds a summary holding `[0xff, 0x00, 0xfe]` at 9 and five `0x00` bytes at 4 from its triples, and verifies the rebuilt summary passes `validate()`, holds the same raw byte keys, and answers both `DataInput::Bytes` queries at their weights. | +| `space_saving_envelope_structure_and_round_trip` | The envelope frames a summary under kind_id `0x18 0x01`, and the bytes are stable across a round trip. | For a 48-counter summary saturated by 30,000 weighted draws over 400 distinct keys, verifies the bytes open with the ASAPv1 magic, `envelope::VERSION`, a `kind_id_len` of `2`, and `0x18 0x01`, that re-serializing the decoded summary reproduces them exactly, and that residency, capacity, `min_count`, and `estimate`/`error`/`upper_bound`/`is_guaranteed` agree across all 400 keys. | | `space_saving_merged_ceiling_survives_the_wire` | The ceiling a merge leaves behind travels in `floor`, which the triples do not determine. | Merges a one-counter summary that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, then verifies the decoded `min_count` and the `upper_bound` on the dropped key 7 both match the source and still cover its true `10`, with identical re-serialized bytes; a payload carrying only the triples decodes to a `min_count` of `0` here. | -| `space_saving_empty_round_trip` | A summary monitoring nothing has one encoding. | Verifies an empty 16-counter summary reports the pinned `EMPTY_KEY_TYPE` of `u64` in its metadata and decodes back to residency `0`, capacity `16`, `min_count` `0`, `total` `0`, and identical re-serialized bytes. | +| `space_saving_empty_round_trip` | A summary monitoring nothing has one encoding. | Verifies an empty 16-counter summary reports the pinned `EMPTY_KEY_TYPE` of `u64` in its metadata and decodes back to residency `0`, capacity `16`, `min_count` `0`, and identical re-serialized bytes. | | `space_saving_empty_must_carry_the_u64_key_type` | An empty summary carries `key_type` `"u64"`. | Verifies an empty summary relabelled `"string"` and `"i64"` fails to decode with an "empty key set" complaint. | | `space_saving_every_key_type_round_trips_and_keeps_its_variant` | The `key_type` names the exact `OwnedDataInput` variant and is never widened. | Across all 14 wire key types - `i8`, `i16`, `i32`, `i64`, `isize`, `u8`, `u16`, `u32`, `u64`, `usize`, `f32`, `f64`, `string` reached from `Str` and `String` alike, and `bytes` - verifies the metadata carries the expected name, the bytes re-serialize identically, and every decoded key still answers its original `DataInput` with the weight it was given; a decoder widening `i32` to `i64` keeps the digest but stops comparing equal, so `estimate` would read zero. | | `space_saving_emitted_order_is_independent_of_seat_order` | The emitted order is descending count with ties broken by `key_order`, not the arena's seat order. | Rebuilds the same four triples - carrying two count ties that only the key separates - in one order and in reverse, and verifies both serialize to the same bytes. | @@ -621,11 +621,12 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `space_saving_refuses_byte_keys_mixed_with_string_keys` | A `Bytes` key and a `String` key are different key types, so a summary holding both has no `key_type`. | Seats `Bytes(b"abc")` and `Str("abc")` in one summary, verifies they take two counters, and that serializing fails with a "mix variants" complaint. | | `space_saving_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the summary rather than a hardcoded profile. | For a four-counter summary over a hasher declaring its own `HashProfile`, verifies the bytes round-trip with `estimate` intact and re-serialize identically, differ from the standard-profile summary's bytes over the same keys, and are rejected by a standard-profile decode. | | `space_saving_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the eight `SpaceSavingMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `SpaceSavingMetadata`. | -| `space_saving_rejects_a_crafted_envelope` | Every structural rule is pinned by the complaint it fails with. | Verifies thirteen crafted envelopes are each refused for their own reason: a zero capacity, entries over the capacity, `keys` longer than `counts`, `counts` longer than `errors`, a counter at zero, an error of `4` against a count of `3`, the same key twice, counts out of descending order, a count tie out of key order, an unknown `key_type` of `u256`, a `discarded_max` of `u64::MAX` over the lowest count of `3`, a `total` of `0` under a counter's `9`, and a foreign kind_id. | +| `space_saving_decodes_the_retired_kind_with_total` | The retired `0x18 0x00` decodes with its `total` dropped, and each kind reads only its own payload shape. | Verifies a five-field payload under `0x18 0x00` decodes for a `total` of `13`, `0` (under the `12` the counters record, which the retired rule rejected) and `u64::MAX`, answers the same estimates and errors, and re-encodes as the four-field `0x18 0x01` envelope; and that the five-field payload under `0x18 0x01` and the four-field one under `0x18 0x00` are both rejected. | +| `space_saving_rejects_a_crafted_envelope` | Every structural rule is pinned by the complaint it fails with. | Verifies twelve crafted envelopes are each refused for their own reason: a zero capacity, entries over the capacity, `keys` longer than `counts`, `counts` longer than `errors`, a counter at zero, an error of `4` against a count of `3`, the same key twice, counts out of descending order, a count tie out of key order, an unknown `key_type` of `u256`, a `discarded_max` of `u64::MAX` over the lowest count of `3`, and a foreign kind_id. | | `space_saving_rejects_signed_order_and_repeated_nan_keys` | A count tie is ordered by two's-complement bits, and a `NaN` key held twice has no encoding. | Verifies `i64` keys `[0, 1, i64::MIN, -1]` at one count decode and re-encode byte-identically while `[i64::MIN, -1, 0, 1]` fails to decode, that a summary rebuilt with two `NaN` keys fails to serialize, and that a payload carrying two `NaN` keys fails to decode. | | `space_saving_refuses_a_capacity_the_metadata_cannot_carry` | A capacity past the metadata's `u32` field fails the encode rather than truncating. | Rebuilds a one-counter summary declaring a capacity of `1 << 40`, verifies it reports that capacity, and that serializing fails with a message naming the "exceeds the u32 metadata field" rule. | | `space_saving_does_not_allocate_a_declared_capacity` | A declared capacity is metadata, never an allocation size. | Verifies an envelope declaring `u32::MAX` counters with two entries decodes, reports that capacity, holds two counters with key 7 reading `9`, a `min_count` of `0`, and a two-entry `top_k`. | -| `the_default_summary_holds_the_default_capacity` | `SpaceSaving::default()` carries the documented counter budget and fills it. | Verifies `SPACE_SAVING_DEFAULT_CAPACITY` is `1_024`, that a `SpaceSaving::default()` reports that capacity and is empty, and that after 60,000 `zipf(1.1)` draws over a 2,048-key domain at seed `9_001` residency is `1_024`, `total()` is `60_000`, and `top_k(usize::MAX)` returns `1_024` entries. | +| `the_default_summary_holds_the_default_capacity` | `SpaceSaving::default()` carries the documented counter budget and fills it. | Verifies `SPACE_SAVING_DEFAULT_CAPACITY` is `1_024`, that a `SpaceSaving::default()` reports that capacity and is empty, and that after 60,000 `zipf(1.1)` draws over a 2,048-key domain at seed `9_001` residency is `1_024` and `top_k(usize::MAX)` returns `1_024` entries. | | `a_seat_above_the_ceiling_saturates` | A key seated above an enormous ceiling stops at `u64::MAX` rather than wrapping. | In a 4-counter summary whose ceiling is `u64::MAX - 3` (merged in from a one-counter peer where key 9 reached `u64::MAX - 4` before key 8 displaced it), `add(5, 10)` is verified to leave `validate()` passing, key 5 at `u64::MAX` with an error of `u64::MAX - 3`, and key 8 still at `u64::MAX - 3`. | | `a_merge_saturates_a_shared_keys_count` | A shared key's paired counts saturate instead of wrapping. | Merges a one-counter peer holding key 2 at `6` into a one-counter summary holding it at `u64::MAX`, and verifies `validate()` passes, residency is `1`, and key 2 reads `u64::MAX`. | | `a_merge_saturates_a_shared_keys_error` | A shared key's paired errors saturate, so the merge never claims a count it cannot support. | For the same saturated overlap, verifies `validate()` passes, key 2's `error` is `u64::MAX`, and its `estimate` less its `error` is at most `2` - the number of times key 2 truly arrived. | @@ -634,13 +635,13 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `a_merge_saturates_the_ceiling` | The ceiling a merge leaves behind saturates rather than wrapping to a tiny bound. | Merges a 4-counter summary ceilinged at `10` (key 6 dropped, key 5 held) into one ceilinged at `u64::MAX - 3`, and verifies `validate()` passes, `min_count` is `u64::MAX`, and `upper_bound` for the dropped key 9 is at least the `u64::MAX - 4` it truly reached. | | `a_rebuild_carries_a_ceiling_no_counter_holds` | An under-full summary's ceiling lives only in `discarded_max`, and a rebuild carries it. | Merges a one-counter peer that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, checks residency is below capacity with `min_count` at `30`, then rebuilds it from its triples and verifies the rebuilt summary passes `validate()`, still reports `min_count` `30`, and covers key 7's true `10` in its `upper_bound`. | | `a_key_that_only_ties_the_ceiling_is_not_guaranteed` | `is_guaranteed` is strict: tying the ceiling is not outranking it. | In a 2-counter summary holding keys 1 and 2 at 3 each, the arrival of key 3 is verified to leave `validate()` passing, key 1 at `3` with error `0`, `min_count` at `3`, and `is_guaranteed` false for both key 1 and key 3; two further key-1 inserts then take it to `5` against a `min_count` of `4`, at which point `is_guaranteed` accepts it. | -| `clear_resets_every_answer` | `clear` leaves a summary that answers like a fresh one, ceiling and total included. | Clears a 2-counter summary that held key 1 at 5 and key 2 at 3 with key 3 evicting, and verifies `validate()` passes, `is_empty` holds, `len` is `0`, `capacity` is still `2`, `total`, `min_count`, `upper_bound`, and `error` are all `0`, `top_k(4)` and `entries` are empty, and that one further insert seats key 4 at `1` with a `total` of `1`. | +| `clear_resets_every_answer` | `clear` leaves a summary that answers like a fresh one, ceiling included. | Clears a 2-counter summary that held key 1 at 5 and key 2 at 3 with key 3 evicting, and verifies `validate()` passes, `is_empty` holds, `len` is `0`, `capacity` is still `2`, `min_count`, `upper_bound`, and `error` are all `0`, `top_k(4)` and `entries` are empty, and that one further insert seats key 4 at `1`. | | `colliding_keys_stay_distinct` | Keys sharing a digest are separate counters, each answering for itself. | Under a `OneDigest` hasher that files every key under one digest, inserts keys 10, 20, and 30 at weights 3, 5, and 1 into a 4-counter summary and verifies `validate()` passes, residency is `3`, the held keys are exactly `[10, 20, 30]`, each estimates its own weight, and an unseen key 40 reads `0`. | | `an_eviction_under_collision_keeps_the_index_straight` | An eviction drops the victim's key from the index before it seats the arrival under the same digest. | Under the `OneDigest` hasher, a 2-counter summary holding key 10 at 3 and key 20 at 2 takes key 30; verifies `validate()` passes, residency is `2`, the held keys are `[10, 30]`, key 30 reads `3` with an error of `2`, key 10 still reads `3`, and the evicted key 20 reads `0`. | | `a_merge_under_collision_pairs_by_key` | A merge pairs the two sides by key, not by the digest slot they share. | Under the `OneDigest` hasher, merges a 4-counter peer holding key 10 at 2 and key 30 at 7 into one holding key 20 at 3 and key 10 at 5, and verifies `validate()` passes, all three keys survive, the shared key 10 reads `7`, and keys 20 and 30 read `3` and `7`. | | `a_merge_under_collision_breaks_ties_by_key` | With one digest for everything, key order is the only tie-break left and it still fixes the survivors. | Under the `OneDigest` hasher, merges two 2-counter summaries holding keys 30 and 40 against keys 10 and 20, all at count 1, and verifies the survivors are `[10, 20]` and that swapping which side is merged into gives the same list. | | `a_decoded_summary_under_collision_keeps_its_keys` | A decode compares keys rather than digests, so it seats all of them and still catches a repeat. | Under the `OneDigest` hasher, rebuilds a 4-counter summary holding keys 10, 20, and 30 at weights 5, 3, and 9 from its triples and verifies the decoded summary passes `validate()`, holds all three keys, and estimates each at its weight; then that a `SpaceSavingState` listing key 10 twice fails `rebuild` with a "same key twice" complaint. | -| `known_state_matches_golden` | Known triples, seated out of emitted order, serialize to the golden bytes. | Rebuilds the `i64` capacity-8 and `bytes` capacity-3 summaries of `space_saving_{i64_c8, bytes_c3}_matches_golden` from their triples in reverse and verifies `to_asapv1` equals each golden. | +| `known_state_matches_golden` | Known triples, seated out of emitted order, serialize to the golden bytes. | Rebuilds the `i64` capacity-8 and `bytes` capacity-3 summaries of `space_saving_nototal_{i64_c8, bytes_c3}_matches_golden` from their triples in reverse and verifies `to_asapv1` equals each `space_saving_nototal_*` golden. | ### Bloom @@ -2064,7 +2065,7 @@ Test file: [`tests/e2e/wire.rs`](../tests/e2e/wire.rs) | `a_ddsketch_envelope_round_trips_and_names_its_kind` | A DDSketch envelope names kind_id `0x05 0x01` and preserves its quantile answers exactly. | Fills `DdSketch::new(0.01)` with `1.0 + v` over `uniform_u64(20_000, 1_000_000, 0x1DE7_0001)`, verifies the envelope's `kind_id_len` of `2` and bytes `0x05 0x01`, and that the decode matches the source on `get_count()`, `alpha()`, and `get_value_at_quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`. | | `a_hydra_envelope_round_trips_and_names_the_counter_it_carries` | A Hydra envelope's kind_id names the counter it carries, and the schema and cells survive. | Builds `Hydra::with_schema(4, 512, ["region", "user"], HydraCounter::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))`, updates each of the nine region/user pairs 40 times with `DataInput::Str("event")`, verifies the `kind_id_len` of `2` and bytes `0x07 0x01`, an unchanged `schema()`, and equal `query_key` answers under `HydraQuery::Frequency` for all nine fully specified keys. | | `a_kll_dynamic_envelope_round_trips_and_names_its_kind` | A KllDynamic envelope names kind_id `0x06 0x01` and re-encodes byte-identically. | Fills `KllDynamic::::with_k_and_seed(200, 0x5EED_9001)` from `uniform_u64(20_000, 1_000_000, 0xC0DE_0001)`, verifies the `kind_id_len` of `2` and bytes `0x06 0x01`, an unchanged `count()`, bit-identical `quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`, and that the decode re-serializes to the same bytes. | -| `a_space_saving_envelope_round_trips_and_names_its_kind` | A Space-Saving envelope names kind_id `0x18 0x00` and preserves every estimate. | Fills `SpaceSaving::::with_capacity(256)` from a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the `kind_id_len` of `2` and bytes `0x18 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | +| `a_space_saving_envelope_round_trips_and_names_its_kind` | A Space-Saving envelope names kind_id `0x18 0x01` and preserves every estimate. | Fills `SpaceSaving::::with_capacity(256)` from a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the `kind_id_len` of `2` and bytes `0x18 0x01`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | | `an_eh_bucket_sketch_envelope_round_trips_and_names_its_kind` | An EhBucketSketch envelope names kind_id `0x14 0x00` and keeps the payload sketch it wraps. | Fills an `EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512))` with the first 5,000 draws of the Zipf stream, verifies the `kind_id_len` of `2` and bytes `0x14 0x00`, an unchanged `sketch_type()`, and equal `query` answers for keys `0..512`. | | `an_elastic_envelope_round_trips_and_names_its_kind` | An Elastic envelope names kind_id `0x0b 0x00` and preserves every per-key answer. | Fills `ElasticSketch::::with_heavy_and_light(64, 3, 1_024)` with the keys `flow-{k}` of the 20,000-draw Zipf stream, verifies the `kind_id_len` of `2` and bytes `0x0b 0x00`, and that `query` on the decode equals the source for all 1,024 domain keys. | | `an_envelope_is_refused_by_every_decoder_but_its_own` | One sketch's envelope never decodes as another's, and malformed buffers are errors. | Verifies a `DdSketch::new(0.01)` envelope over `1..1_000` fails to decode as `Bloom` and as `SpaceSaving`, that a one-key `Bloom` envelope fails to decode as `DdSketch`, and that both an empty buffer and the first half of the DDSketch envelope are refused. | @@ -2092,7 +2093,7 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `cmsheap_i64_regular_2x3_delta_matches_golden` | A known CountMinSketchWithHeap delta serializes to its golden bytes and, merged, replaces the state's heap. | Verifies `CountMinSketchWithHeapDelta::::new(2, 3, [0, 1, 2, 4, 5], [127, -1, -33, -32769, 4294967296], heap)` with heap size `5` holding `i64` keys `{7: 128, -129: 128, -1: 4295032832}` serializes to `asapv1_golden/cmsheap_i64_regular_2x3_delta.hex`, that the fixture decodes and re-serializes identically, and that merged into the `string`-keyed `cmsheap_i64_regular_2x3_strkeys` it gives `[127, 0, 94, 128, -32469, 4295032832]` and exactly the delta's heap. | | `cmsheap_i64_regular_2x3_tuplekeys_matches_golden` | CountMinSketchWithHeap keys made of typed columns travel as `bytes` tuple keys. | Verifies the `2x3` Count-Min matrix with heap size `5` holding the tuple keys `["us-east", 443, true]: 300`, `["us-east", 80, null]: 300`, `[null, -1, 2.5]: 65536`, `["", i64::MIN, -0.0]: 1` and `["é", 0, NaN, false]: 127`, each `TupleKey::to_bytes`, serializes to `asapv1_golden/cmsheap_i64_regular_2x3_tuplekeys.hex`, that the decode holds that matrix and those entries, re-serializes identically, reads every key back as a canonical tuple key, and finds the `-0.0` tuple under the key built with `0.0`. | | `cmsheap_i64_regular_2x3_{strkeys, i64tie, strtie, empty}_matches_golden` | The `i64` regular-path CountMinSketchWithHeap fixtures serialize to their golden bytes. | For each, verifies the `2x3` Count-Min matrix with the fixture's heap (string keys at heap size `5`; an `i64` five-way set with a four-way tie; a string five-way set with a four-way tie; empty at heap size `4`) seated out of emitted order serializes to `asapv1_golden/cmsheap_i64_regular_2x3_.hex`, and that the decode holds that matrix and those entries and re-serializes identically. | -| `cmsspacesaving_i64_regular_2x3_strkeys_matches_golden` | A CountMinSketchWithSpaceSaving serializes to its golden bytes. | Verifies the `2x3` Count-Min i64 matrix beside a capacity-`4` summary fed `hot:300, warm:128, cold:1, b:7, aa:6` (so `aa` displaces `cold`) serializes to `asapv1_golden/cmsspacesaving_i64_regular_2x3_strkeys.hex`, and that the decode holds that matrix, the entries `hot:300/0, warm:128/0, aa:7/1, b:7/0`, total `442` and ceiling `7`, and re-serializes identically. | +| `cmsspacesaving_nototal_i64_regular_2x3_strkeys_matches_golden` | A CountMinSketchWithSpaceSaving serializes to its golden bytes, and the retired fixture re-encodes to them. | Verifies the `2x3` Count-Min i64 matrix beside a capacity-`4` summary fed `hot:300, warm:128, cold:1, b:7, aa:6` (so `aa` displaces `cold`) serializes to `asapv1_golden/cmsspacesaving_nototal_i64_regular_2x3_strkeys.hex` (`0x1e 0x01`), that the decode holds that matrix, the entries `hot:300/0, warm:128/0, aa:7/1, b:7/0` and ceiling `7`, and re-serializes identically, and that the retired `cmsspacesaving_i64_regular_2x3_strkeys.hex` (`0x1e 0x00`) decodes and re-encodes to the same bytes. | | `cs_f64_fast_2x4_delta_matches_golden` | A known f64 fast-path Count Sketch delta serializes to its golden bytes and merges into its state golden. | Verifies `CountSketchDelta::::new(2, 4, [1, 3, 5, 7], [-0.75, 2.25, 0.5, 1e300])` serializes to `asapv1_golden/cs_f64_fast_2x4_delta.hex`, that the fixture decodes and re-serializes identically, and that merged into `cs_f64_fast_2x4` it gives `[0.0, -0.75, 1.5, 0.0, 127.0, -32768.0, 0.125, 0.0]`. | | `cs_f64_fast_2x4_matches_golden` | A known f64 Count Sketch matrix serializes to its golden bytes. | Verifies `CountSketch::, FastPath>::from_storage` over `[[0.0, -0.0, 1.5, -2.25], [127.0, -32768.5, 0.125, -1e300]]` serializes to `asapv1_golden/cs_f64_fast_2x4.hex`, and that decoding it recovers the eight cells and re-serializes identically. | | `cs_i32_regular_2x4_matches_golden` | The i32 counter type reaches the metadata, and an i32 golden is not an i64 sketch. | Verifies `CountSketch::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` cast to `i32` serializes to `asapv1_golden/cs_i32_regular_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, that the i32 and i64 regular fixtures have equal length yet differ, and that the i32 fixture fails to decode as `CountSketch, RegularPath>`. | @@ -2105,7 +2106,7 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `csheap_i64_regular_2x4_i64tie_matches_golden` | A CountSketchWithHeap count tie emits by two's-complement key bits. | Verifies the `2x4` Count Sketch matrix with heap size `6` holding `i64` keys `{2: 9, -1: -3, 1: -3, 0: -3, i64::MIN: -3, -129: -40}`, seated out of emitted order, serializes to `asapv1_golden/csheap_i64_regular_2x4_i64tie.hex`, whose `-3` tie runs `0, 1, i64::MIN, -1`, and that the decode holds that matrix and those entries and re-serializes identically. | | `csheap_i64_regular_2x4_strkeys_matches_golden` | A CountSketchWithHeap serializes to its golden bytes. | Verifies the `2x4` Count Sketch matrix with heap size `5` holding `{"alpha": 4294967296, "beta": 127, "delta": 127, "gamma": -33}` serializes to `asapv1_golden/csheap_i64_regular_2x4_strkeys.hex`, and that the decode holds that matrix and those entries and re-serializes identically. | | `csheap_i64_regular_2x4_tuplekeys_matches_golden` | CountSketchWithHeap keys made of typed columns travel as `bytes` tuple keys. | Verifies the `2x4` Count Sketch matrix with heap size `6` holding four of the same tuple keys at `-33`, `127`, `65536` and `127` serializes to `asapv1_golden/csheap_i64_regular_2x4_tuplekeys.hex`, and that the decode holds that matrix and those entries, reads every key back as a canonical tuple key, and re-serializes identically. | -| `csspacesaving_f64_fast_2x4_i64keys_matches_golden` | A CountSketchWithSpaceSaving serializes to its golden bytes. | Verifies the `2x4` Count Sketch f64 matrix beside a capacity-`3` summary fed `-1:65536, i64::MIN:5, 128:5, 7:1, i64::MIN:1` (so `7` displaces `128`) serializes to `asapv1_golden/csspacesaving_f64_fast_2x4_i64keys.hex`, and that the decode holds that matrix, the entries `-1:65536/0, 7:6/5, i64::MIN:6/0`, total `65548` and ceiling `6`, and re-serializes identically. | +| `csspacesaving_nototal_f64_fast_2x4_i64keys_matches_golden` | A CountSketchWithSpaceSaving serializes to its golden bytes, and the retired fixture re-encodes to them. | Verifies the `2x4` Count Sketch f64 matrix beside a capacity-`3` summary fed `-1:65536, i64::MIN:5, 128:5, 7:1, i64::MIN:1` (so `7` displaces `128`) serializes to `asapv1_golden/csspacesaving_nototal_f64_fast_2x4_i64keys.hex` (`0x1f 0x01`), that the decode holds that matrix, the entries `-1:65536/0, 7:6/5, i64::MIN:6/0` and ceiling `6`, and re-serializes identically, and that the retired `csspacesaving_f64_fast_2x4_i64keys.hex` (`0x1f 0x00`) decodes and re-encodes to the same bytes. | | `ddsketch_buckets_{positive, signed, empty}_a001_matches_golden` | The `0x05 0x01` DDSketch fixtures decode to their known stores and round-trip. | For the positive and signed fixtures, verifies the decoded stores and zero count and identical re-serialized bytes; for the empty one, that a fresh sketch serializes to it. | | `ddsketch_retired_fixtures_decode_to_their_bucket_twins` | Each retired `0x05 0x00` / `0x05 0x80` fixture decodes and re-encodes as its `ddsketch_buckets_*` twin. | Decodes the positive, signed, empty and admission state fixtures and the delta fixture of the retired kinds and verifies each re-encodes to its twin's bytes. | | `ddsketch_buckets_signed_a001_admission_matches_golden` | A sampled DDSketch state closes its metadata with `admission_rate`. | Verifies the `ddsketch_buckets_signed_a001` state at rate `0.125` serializes to its fixture, which decodes to the same stores, rate `0.125` and count `18`. | @@ -2126,8 +2127,9 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `kmv_k8_matches_golden` | A KMV of known digests serializes to its golden bytes. | Verifies `Kmv::new(8)` fed the digests `[1, 127, 128, 65535, 4294967296, 0xfedcba9876543210]` out of order through `update_with_hash` serializes to `asapv1_golden/kmv_k8.hex`, and that the decode holds `k` `8` and those digests and re-serializes identically. | | `set_aggregator_empty_matches_golden` | An empty set serializes to its golden bytes, which round-trip unchanged. | Verifies an empty `SetAggregator` serializes to `asapv1_golden/set_aggregator_empty.hex`, and that decoding the fixture yields an empty set and re-serializes to the same bytes. | | `set_aggregator_strings_matches_golden` | A set's members serialize in ascending UTF-8 byte order to the golden bytes. | Verifies a `SetAggregator` of 11 members, among them `""`, a 31-byte fixstr, a 32-byte str8, a proper-prefix chain and two-, three- and four-byte code points, serializes to `asapv1_golden/set_aggregator_strings.hex`, and that decoding the fixture recovers the set and re-serializes to the same bytes. | -| `space_saving_bytes_c3_matches_golden` | A full `bytes`-keyed Space-Saving summary serializes to its golden bytes. | Runs the `space_saving_i64_c8_matches_golden` checks for capacity `3` holding `(ff 00, 9, 0)`, `("", 4, 1)`, `("ab", 4, 0)` with `total` `20` and `discarded_max` `3`, where `min_count()` is `4`, against `asapv1_golden/space_saving_bytes_c3.hex`. | -| `space_saving_i64_c8_matches_golden` | An `i64`-keyed Space-Saving summary serializes to its golden bytes, its count tie by two's-complement bits. | Verifies a summary of capacity `8` holding `(-1, 70000, 0)`, `(300, 9, 2)`, `(0, 5, 0)`, `(1, 5, 1)`, `(i64::MIN, 5, 4)`, `(-129, 5, 0)` with `total` `70030` and `discarded_max` `4`, decodes from `asapv1_golden/space_saving_i64_c8.hex` with that capacity, total, `min_count()` `4` and those triples and re-serializes identically. | +| `space_saving_nototal_bytes_c3_matches_golden` | A full `bytes`-keyed Space-Saving summary serializes to its golden bytes. | Runs the `space_saving_nototal_i64_c8_matches_golden` checks for capacity `3` holding `(ff 00, 9, 0)`, `("", 4, 1)`, `("ab", 4, 0)` with `discarded_max` `3`, where `min_count()` is `4`, against `asapv1_golden/space_saving_nototal_bytes_c3.hex`. | +| `space_saving_nototal_i64_c8_matches_golden` | An `i64`-keyed Space-Saving summary serializes to its golden bytes, its count tie by two's-complement bits. | Verifies a summary of capacity `8` holding `(-1, 70000, 0)`, `(300, 9, 2)`, `(0, 5, 0)`, `(1, 5, 1)`, `(i64::MIN, 5, 4)`, `(-129, 5, 0)` with `discarded_max` `4`, decodes from `asapv1_golden/space_saving_nototal_i64_c8.hex` (`0x18 0x01`) with that capacity, `min_count()` `4` and those triples and re-serializes identically. | +| `space_saving_retired_fixtures_decode_to_their_nototal_twins` | The retired `0x18 0x00` fixtures still decode, their `total` dropped. | Verifies `space_saving_i64_c8.hex` and `space_saving_bytes_c3.hex` decode and re-encode as `space_saving_nototal_i64_c8.hex` and `space_saving_nototal_bytes_c3.hex`. | | `uniform_sampling_r0p25_matches_golden` | A UniformSampling reservoir serializes to its golden bytes (`experimental`). | Verifies a sampler at rate `0.25` with `total_seen` `20`, `rng_state` `0x0123456789abcdef` and entries `(3, -0.0)`, `(3, 0.0)`, `(2^63, 1e300)`, `(u64::MAX, NaN)`, decodes from `asapv1_golden/uniform_sampling_r0p25.hex` with that rate, `total_seen` and the samples' bits and re-serializes identically. | | `univmon_optimized_l5_e2_3x8_1x4_h6_matches_golden` | A two-tier UnivMon pyramid serializes to its golden bytes. | Verifies `UnivMonTwoTier::new(6, 2, 3, 8, 1, 4, 5)` with counters set by `apply_delta`, the heaps `{"alpha": 40, "beta": 7, "gamma": 7}`, `{"delta": 3}`, `{}`, `{"eps": 1}`, `{}` and `bucket_size` `70` serializes to `asapv1_golden/univmon_optimized_l5_e2_3x8_1x4_h6.hex`, and that the decode reports that layout, every layer's cells and heap, and re-serializes identically. | | `univmon_q_i32_l3_c8_r3_matches_golden` | A UnivMon-Q golden decodes to its state and re-serializes byte-identically. | Verifies `asapv1_golden/univmon_q_i32_l3_c8_r3.hex` decodes to the config `levels` `3`, `width` `8`, `cols_halving_period` `1`, `depth` `3`, `counter_bits` `32`, `candidates` `2`, `ordered_samples` `3`, `seed_index` `5`, `source_id()` `7`, `count()` `12`, `min()` `-2.5` and `max()` `5.0`, and re-serializes to the same bytes. | diff --git a/src/asapv1/kind.rs b/src/asapv1/kind.rs index 7c6180ad..a8461698 100644 --- a/src/asapv1/kind.rs +++ b/src/asapv1/kind.rs @@ -47,8 +47,18 @@ pub(crate) const BLOOM: &[u8] = &[0x17, 0x01]; /// Retired Bloom kind that also carried the `inserted` counter: decoded, with /// the counter dropped, and never encoded. pub(crate) const BLOOM_WITH_INSERTED: &[u8] = &[0x17, 0x00]; -pub(crate) const SPACE_SAVING: &[u8] = &[0x18, 0x00]; +/// Space-Saving: the answer triples and the ceiling (spec §3.5). +pub(crate) const SPACE_SAVING: &[u8] = &[0x18, 0x01]; +/// Retired Space-Saving kind that also carried `total`: decoded, with `total` +/// dropped, and never encoded. +pub(crate) const SPACE_SAVING_WITH_TOTAL: &[u8] = &[0x18, 0x00]; pub(crate) const COUNT_SKETCH_WITH_L2_NORM: &[u8] = &[0x19, 0x00]; pub(crate) const UNIVMON_Q: &[u8] = &[0x1a, 0x00]; -pub(crate) const COUNT_MIN_WITH_SPACE_SAVING: &[u8] = &[0x1e, 0x00]; -pub(crate) const COUNT_SKETCH_WITH_SPACE_SAVING: &[u8] = &[0x1f, 0x00]; +/// Count-Min with a Space-Saving summary, without its `total` (spec §3.24). +pub(crate) const COUNT_MIN_WITH_SPACE_SAVING: &[u8] = &[0x1e, 0x01]; +/// Retired kind whose summary also carried `total`: decoded, never encoded. +pub(crate) const COUNT_MIN_WITH_SPACE_SAVING_WITH_TOTAL: &[u8] = &[0x1e, 0x00]; +/// Count Sketch with a Space-Saving summary, without its `total` (spec §3.25). +pub(crate) const COUNT_SKETCH_WITH_SPACE_SAVING: &[u8] = &[0x1f, 0x01]; +/// Retired kind whose summary also carried `total`: decoded, never encoded. +pub(crate) const COUNT_SKETCH_WITH_SPACE_SAVING_WITH_TOTAL: &[u8] = &[0x1f, 0x00]; diff --git a/src/sketches/countminsketch_space_saving.rs b/src/sketches/countminsketch_space_saving.rs index 5c2b9efe..6aca99e0 100644 --- a/src/sketches/countminsketch_space_saving.rs +++ b/src/sketches/countminsketch_space_saving.rs @@ -280,7 +280,6 @@ mod tests { assert_eq!(sk.estimate(&DataInput::U64(2)), 7); assert_eq!(sk.space_saving().estimate(&DataInput::U64(1)), 1); assert_eq!(sk.space_saving().estimate(&DataInput::U64(2)), 7); - assert_eq!(sk.space_saving().total(), 8); assert_eq!((sk.rows(), sk.cols(), sk.capacity()), (3, 1024, 8)); } @@ -345,7 +344,6 @@ mod tests { assert_eq!(heap_a.estimate(&DataInput::U64(3)), 6); assert!(heap_a.heap().find(&DataInput::U64(3)).is_none()); assert_eq!(ranked(&ss_a.top_k(1)), vec![(3, 6)]); - assert_eq!(ss_a.space_saving().total(), 16); } #[test] diff --git a/src/sketches/countminsketch_space_saving/summary_wire.rs b/src/sketches/countminsketch_space_saving/summary_wire.rs index 64afc5cc..a37d5fe3 100644 --- a/src/sketches/countminsketch_space_saving/summary_wire.rs +++ b/src/sketches/countminsketch_space_saving/summary_wire.rs @@ -1,6 +1,8 @@ -//! ASAPv1 metadata and payload shared by CountMinSketchWithSpaceSaving (`0x1e 0x00`) and -//! CountSketchWithSpaceSaving (`0x1f 0x00`): `[counts, keys, summary_counts, errors, total, -//! discarded_max]` (`docs/asapv1_wire_format.md` §3.24, §3.25). +//! ASAPv1 metadata and payload shared by CountMinSketchWithSpaceSaving (`0x1e 0x01`) and +//! CountSketchWithSpaceSaving (`0x1f 0x01`): `[counts, keys, summary_counts, errors, +//! discarded_max]` (`docs/asapv1_wire_format.md` §3.24, §3.25). The retired `0x1e 0x00` / +//! `0x1f 0x00` also carried the summary's `total` before `discarded_max`; they still +//! decode, with `total` dropped, and are never encoded. use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::de::DeserializeOwned; @@ -61,10 +63,29 @@ pub(crate) fn summary_topk_metadata( } } -/// The positional payload `[counts, keys, summary_counts, errors, total, +/// The positional payload `[counts, keys, summary_counts, errors, /// discarded_max]`. #[derive(Debug, Serialize, Deserialize)] pub(crate) struct SummaryTopKPayload { + #[serde( + deserialize_with = "exact_vec", + bound(deserialize = "C: Deserialize<'de>") + )] + pub(crate) counts: Vec, + #[serde( + deserialize_with = "exact_vec", + bound(deserialize = "K: Deserialize<'de>") + )] + pub(crate) keys: Vec, + pub(crate) summary_counts: Vec, + pub(crate) errors: Vec, + pub(crate) discarded_max: u64, +} + +/// The retired payload `[counts, keys, summary_counts, errors, total, +/// discarded_max]`: decoded only, and `total` is dropped. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct SummaryTopKPayloadWithTotal { #[serde( deserialize_with = "exact_vec", bound(deserialize = "C: Deserialize<'de>") @@ -81,6 +102,24 @@ pub(crate) struct SummaryTopKPayload { pub(crate) discarded_max: u64, } +/// Reads the current payload, or the retired one with `total` dropped. +fn read_payload( + payload: &[u8], + with_total: bool, +) -> Result, RmpDecodeError> { + if !with_total { + return envelope::decode_block(payload); + } + let retired: SummaryTopKPayloadWithTotal = envelope::decode_block(payload)?; + Ok(SummaryTopKPayload { + counts: retired.counts, + keys: retired.keys, + summary_counts: retired.summary_counts, + errors: retired.errors, + discarded_max: retired.discarded_max, + }) +} + /// Packs the payload with `keys` typed by `key_type`; a key of another variant /// fails the encode. fn encode_payload( @@ -88,7 +127,6 @@ fn encode_payload( key_type: &str, counts: &[C], entries: &[(&OwnedDataInput, u64, u64)], - total: u64, discarded_max: u64, ) -> Result, RmpEncodeError> { fn pack( @@ -96,7 +134,6 @@ fn encode_payload( key_type: &str, counts: &[C], entries: &[(&OwnedDataInput, u64, u64)], - total: u64, discarded_max: u64, wire: impl Fn(&OwnedDataInput) -> Option, ) -> Result, RmpEncodeError> { @@ -114,7 +151,6 @@ fn encode_payload( keys, summary_counts: entries.iter().map(|entry| entry.1).collect(), errors: entries.iter().map(|entry| entry.2).collect(), - total, discarded_max, }) } @@ -126,7 +162,6 @@ fn encode_payload( key_type, counts, entries, - total, discarded_max, |key| match key { OwnedDataInput::$variant($value) => Some($wire), @@ -157,11 +192,14 @@ fn encode_payload( } } -/// One kind's identity on the wire: its name in errors, its kind_id, and the -/// `counter_type` and `mode` its target type pins. +/// One kind's identity on the wire: its name in errors, its kind_id, the +/// retired kind_id it still decodes, and the `counter_type` and `mode` its +/// target type pins. pub(crate) struct SummaryKind { pub(crate) name: &'static str, pub(crate) kind_id: &'static [u8], + /// The retired id whose payload also carried the summary's `total`. + pub(crate) retired_kind_id_with_total: &'static [u8], pub(crate) counter_type: &'static str, pub(crate) mode: &'static str, } @@ -208,14 +246,7 @@ pub(crate) fn encode( capacity, key_type, ))?; - let payload = encode_payload( - sketch, - key_type, - counts, - &entries, - summary.total(), - summary.discarded_max(), - )?; + let payload = encode_payload(sketch, key_type, counts, &entries, summary.discarded_max())?; Ok(envelope::encode(kind.kind_id, &metadata, &payload)) } @@ -228,8 +259,9 @@ pub(crate) struct DecodedSummaryTopK { pub(crate) summary: SpaceSaving, } -/// Decodes a `kind` envelope under §3.24's decode rules. Neither the geometry -/// nor `capacity` sizes an allocation. +/// Decodes a `kind` envelope, or its retired id with `total` dropped, under +/// §3.24's decode rules. Neither the geometry nor `capacity` sizes an +/// allocation. pub(crate) fn decode( kind: &SummaryKind, bytes: &[u8], @@ -238,10 +270,11 @@ pub(crate) fn decode( let complain = |e: String| RmpDecodeError::Uncategorized(format!("ASAPv1 {sketch}: {e}")); let (kind_id, metadata, payload) = envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; - if kind_id != kind.kind_id { + let with_total = kind_id == kind.retired_kind_id_with_total; + if kind_id != kind.kind_id && !with_total { return Err(complain(format!( - "kind_id mismatch: stored {kind_id:?}, expected {:?}", - kind.kind_id + "kind_id mismatch: stored {kind_id:?}, expected {:?} (or the retired {:?})", + kind.kind_id, kind.retired_kind_id_with_total ))); } let meta: SummaryTopKMetadata = envelope::decode_block(metadata)?; @@ -262,21 +295,13 @@ pub(crate) fn decode( macro_rules! unpack { ($ty:ty, $into:expr) => {{ - let p: SummaryTopKPayload = envelope::decode_block(payload)?; + let p: SummaryTopKPayload = read_payload(payload, with_total)?; let keys: Vec = p.keys.into_iter().map($into).collect(); - ( - p.counts, - keys, - p.summary_counts, - p.errors, - p.total, - p.discarded_max, - ) + (p.counts, keys, p.summary_counts, p.errors, p.discarded_max) }}; } - let (counts, keys, summary_counts, errors, total, discarded_max) = match meta.key_type.as_str() - { + let (counts, keys, summary_counts, errors, discarded_max) = match meta.key_type.as_str() { "i8" => unpack!(i8, OwnedDataInput::I8), "i16" => unpack!(i16, OwnedDataInput::I16), "i32" => unpack!(i32, OwnedDataInput::I32), @@ -334,7 +359,7 @@ pub(crate) fn decode( .zip(errors) .map(|((key, count), error)| (key, count, error)) .collect(); - let summary = SpaceSaving::from_parts(meta.capacity as usize, total, discarded_max, entries) + let summary = SpaceSaving::from_parts(meta.capacity as usize, discarded_max, entries) .map_err(complain)?; Ok(DecodedSummaryTopK { rows, @@ -398,7 +423,6 @@ mod tests { keys: Vec, summary_counts: Vec, errors: Vec, - total: u64, discarded_max: u64, } @@ -412,7 +436,6 @@ mod tests { keys: vec![7, 8, 9], summary_counts: vec![9, 4, 4], errors: vec![0, 1, 0], - total: 16, discarded_max: 0, } } @@ -433,7 +456,6 @@ mod tests { keys: self.keys.iter().collect::>(), summary_counts: self.summary_counts.clone(), errors: self.errors.clone(), - total: self.total, discarded_max: self.discarded_max, }) .expect("payload"); @@ -446,7 +468,7 @@ mod tests { let sketch = populated(); let encoded = sketch.to_asapv1().expect("serialize"); assert!(encoded.starts_with(b"ASAPv1")); - assert_eq!(&encoded[7..10], &[2u8, 0x1e, 0x00]); + assert_eq!(&encoded[7..10], &[2u8, 0x1e, 0x01]); let meta = metadata_of(&encoded); assert_eq!(meta.metadata_version, 1); @@ -470,7 +492,6 @@ mod tests { (OwnedDataInput::U64(4), 7, 5), ] ); - assert_eq!(decoded.space_saving().total(), 23); assert_eq!(decoded.space_saving().discarded_max(), 5); assert_eq!(decoded.capacity(), 3); assert_eq!(decoded.top_k(3), sketch.top_k(3)); @@ -698,7 +719,6 @@ mod tests { keys: Vec::new(), summary_counts: c.summary_counts, errors: c.errors, - total: c.total, discarded_max: c.discarded_max, } } @@ -744,7 +764,6 @@ mod tests { keys: vec![f64::NAN, f64::NAN, 1.0], summary_counts: vec![9, 9, 4], errors: vec![0, 0, 0], - total: 22, ..craft_with_keys() }; assert!(decode_error(&nan.bytes()).contains("twice")); @@ -754,7 +773,6 @@ mod tests { keys: vec![0.0f64, -0.0, 1.0], summary_counts: vec![9, 9, 4], errors: vec![0, 0, 0], - total: 22, ..craft_with_keys() }; assert!(Target::from_asapv1(&zeros.bytes()).is_ok()); @@ -777,16 +795,47 @@ mod tests { assert!(decode_error(&empty.bytes()).contains("ceiling of 1")); } + /// The retired `0x1e 0x00` decodes with its `total` dropped, whatever that + /// total was (`15` sits under the 16 the counters account for, which the + /// retired rule rejected), and re-encodes as `0x1e 0x01`. Each kind reads + /// only its own payload shape. #[test] - fn total_covers_the_recorded_weight_saturating() { - let mut c = craft(); - c.total = 15; - assert!(decode_error(&c.bytes()).contains("total of 15")); + fn the_retired_kind_with_total_decodes_and_drops_it() { + let c = craft(); + let retired = |kind_id: &[u8], total: u64| { + let metadata = rmp_serde::to_vec_named(&summary_topk_metadata::( + c.rows, c.cols, "i64", "regular", c.capacity, c.key_type, + )) + .expect("metadata"); + let payload = rmp_serde::to_vec(&SummaryTopKPayloadWithTotal { + counts: c.counts.clone(), + keys: c.keys.clone(), + summary_counts: c.summary_counts.clone(), + errors: c.errors.clone(), + total, + discarded_max: c.discarded_max, + }) + .expect("payload"); + envelope::encode(kind_id, &metadata, &payload) + }; + for total in [16, 15, 0, u64::MAX] { + let decoded = Target::from_asapv1(&retired( + kind::COUNT_MIN_WITH_SPACE_SAVING_WITH_TOTAL, + total, + )) + .expect("retired decodes"); + assert_eq!(decoded.to_asapv1().expect("re-encode"), c.bytes()); + } - let mut c = craft(); - c.summary_counts = vec![u64::MAX, u64::MAX, 4]; - c.total = u64::MAX; - assert!(Target::from_asapv1(&c.bytes()).is_ok()); + assert!(decode_error(&retired(kind::COUNT_MIN_WITH_SPACE_SAVING, 16)).contains("array")); + let current = c.bytes(); + let (_, metadata, payload) = envelope::split(¤t).expect("split"); + let current_as_retired = envelope::encode( + kind::COUNT_MIN_WITH_SPACE_SAVING_WITH_TOTAL, + metadata, + payload, + ); + assert!(Target::from_asapv1(¤t_as_retired).is_err()); } #[test] @@ -872,7 +921,7 @@ mod tests { let huge = Target::from_parts( CountMinSketch::with_dimensions(2, 3), - SpaceSaving::from_parts(1 << 40, 3, 0, vec![(OwnedDataInput::U64(1), 3, 0)]) + SpaceSaving::from_parts(1 << 40, 0, vec![(OwnedDataInput::U64(1), 3, 0)]) .expect("sparse"), ); assert!( diff --git a/src/sketches/countminsketch_space_saving/wire.rs b/src/sketches/countminsketch_space_saving/wire.rs index b5f931d0..a786343f 100644 --- a/src/sketches/countminsketch_space_saving/wire.rs +++ b/src/sketches/countminsketch_space_saving/wire.rs @@ -1,4 +1,4 @@ -//! ASAPv1 wire serialization for CountMinSketchWithSpaceSaving (kind_id `0x1e 0x00`, +//! ASAPv1 wire serialization for CountMinSketchWithSpaceSaving (kind_id `0x1e 0x01`, //! `docs/asapv1_wire_format.md` §3.24). The encoding is shared with //! CountSketchWithSpaceSaving and lives in [`super::summary_wire`]. @@ -22,11 +22,12 @@ where const KIND: SummaryKind = SummaryKind { name: "CountMinSketchWithSpaceSaving", kind_id: kind::COUNT_MIN_WITH_SPACE_SAVING, + retired_kind_id_with_total: kind::COUNT_MIN_WITH_SPACE_SAVING_WITH_TOTAL, counter_type: T::COUNTER_TYPE, mode: Mode::MODE, }; - /// Serializes the sketch into an ASAPv1 envelope (kind_id `0x1e 0x00`). Fails + /// Serializes the sketch into an ASAPv1 envelope (kind_id `0x1e 0x01`). Fails /// when the matrix samples its updates or the state breaks a §3.24 decode /// rule, and on mixed-variant or 128-bit keys. pub fn to_asapv1(&self) -> Result, RmpEncodeError> { @@ -44,7 +45,8 @@ where ) } - /// Deserializes a sketch from an ASAPv1 envelope. The hash spec, counter + /// Deserializes a sketch from an ASAPv1 envelope: kind_id `0x1e 0x01`, or + /// the retired `0x1e 0x00` with the summary's `total` dropped. The hash spec, counter /// type and mode are pinned against this target; `rows`, `cols`, /// `capacity` and `key_type` are echoed back. pub fn from_asapv1(bytes: &[u8]) -> Result { diff --git a/src/sketches/countsketch_space_saving/wire.rs b/src/sketches/countsketch_space_saving/wire.rs index 907bf3fd..39c1eafc 100644 --- a/src/sketches/countsketch_space_saving/wire.rs +++ b/src/sketches/countsketch_space_saving/wire.rs @@ -1,4 +1,4 @@ -//! ASAPv1 wire serialization for CountSketchWithSpaceSaving (kind_id `0x1f 0x00`, +//! ASAPv1 wire serialization for CountSketchWithSpaceSaving (kind_id `0x1f 0x01`, //! `docs/asapv1_wire_format.md` §3.25), through the encoding it shares with //! CountMinSketchWithSpaceSaving. @@ -24,11 +24,12 @@ where const KIND: SummaryKind = SummaryKind { name: "CountSketchWithSpaceSaving", kind_id: kind::COUNT_SKETCH_WITH_SPACE_SAVING, + retired_kind_id_with_total: kind::COUNT_SKETCH_WITH_SPACE_SAVING_WITH_TOTAL, counter_type: T::COUNTER_TYPE, mode: Mode::MODE, }; - /// Serializes the sketch into an ASAPv1 envelope (kind_id `0x1f 0x00`). Fails + /// Serializes the sketch into an ASAPv1 envelope (kind_id `0x1f 0x01`). Fails /// when the matrix samples its updates or the state breaks a §3.24 decode /// rule, and on mixed-variant or 128-bit keys. pub fn to_asapv1(&self) -> Result, RmpEncodeError> { @@ -44,7 +45,8 @@ where ) } - /// Deserializes a sketch from an ASAPv1 envelope. The hash spec, counter + /// Deserializes a sketch from an ASAPv1 envelope: kind_id `0x1f 0x01`, or + /// the retired `0x1f 0x00` with the summary's `total` dropped. The hash spec, counter /// type and mode are pinned against this target; `rows`, `cols`, /// `capacity` and `key_type` are echoed back. pub fn from_asapv1(bytes: &[u8]) -> Result { @@ -79,7 +81,7 @@ mod tests { sketch.update_weighted(&DataInput::U64(key), weight); } let encoded = sketch.to_asapv1().expect("serialize"); - assert_eq!(&encoded[7..10], &[2u8, 0x1f, 0x00]); + assert_eq!(&encoded[7..10], &[2u8, 0x1f, 0x01]); let decoded = Target::from_asapv1(&encoded).expect("decode"); assert_eq!( diff --git a/src/sketches/space_saving.rs b/src/sketches/space_saving.rs index fa4a5b7a..ade6c8f9 100644 --- a/src/sketches/space_saving.rs +++ b/src/sketches/space_saving.rs @@ -77,7 +77,6 @@ pub struct SpaceSaving { bucket_head: usize, bucket_tail: usize, index: Index, - total: u64, /// Largest count known to have left the summary, so the ceiling on any key /// that is no longer monitored. discarded_max: u64, @@ -102,7 +101,6 @@ impl SpaceSaving { bucket_head: NIL, bucket_tail: NIL, index: Index::with_capacity_and_hasher(capacity, DigestBuildHasher::default()), - total: 0, discarded_max: 0, _hasher: PhantomData, } @@ -126,12 +124,6 @@ impl SpaceSaving { self.monitored.is_empty() } - /// Total weight recorded, monitored or displaced. - #[inline(always)] - pub fn total(&self) -> u64 { - self.total - } - /// The ceiling on the true count of any key the summary does not monitor. /// /// The larger of the smallest count still held, once every counter is in @@ -156,7 +148,6 @@ impl SpaceSaving { self.bucket_head = NIL; self.bucket_tail = NIL; self.index.clear(); - self.total = 0; self.discarded_max = 0; } @@ -176,7 +167,6 @@ impl SpaceSaving { if weight == 0 { return; } - self.total = self.total.saturating_add(weight); let digest = H::hash64_seeded(0, value); if let Some(cid) = self.find(digest, value) { @@ -303,8 +293,8 @@ impl SpaceSaving { /// concatenated streams — a key evicted on both sides cannot be recovered. /// /// The `min_count` a one-sided key picks up is added weight that the stream - /// never carried, so the merged counts sum to more than [`Self::total`] and - /// an estimate divided by the total is no longer a frequency. + /// never carried, so the merged counts can sum to more than the weight both + /// streams recorded. pub fn merge(&mut self, other: &Self) { let mine_min = self.min_count(); let theirs_min = other.min_count(); @@ -358,9 +348,7 @@ impl SpaceSaving { flat.truncate(self.capacity); } - let total = self.total.saturating_add(other.total); self.clear(); - self.total = total; self.discarded_max = discarded_max; for entry in flat { self.seat(entry.digest, entry.key, entry.count, entry.error); @@ -558,7 +546,6 @@ fn key_order(key: &OwnedDataInput) -> (u8, u128, &[u8]) { /// the key index all follow from them. struct SpaceSavingState { capacity: usize, - total: u64, discarded_max: u64, entries: Vec<(OwnedDataInput, u64, u64)>, } @@ -569,7 +556,6 @@ impl SpaceSaving { fn rebuilt(&self) -> Result { Self::rebuild(SpaceSavingState { capacity: self.capacity, - total: self.total, discarded_max: self.discarded_max, entries: self .monitored @@ -583,8 +569,7 @@ impl SpaceSaving { impl SpaceSaving { /// Rebuilds a summary from decoded triples, rejecting any that no run of /// the algorithm could have produced: a duplicate or zero-count key, an - /// error above its count, a ceiling above the lowest count, or a total - /// under the weight the counters account for. + /// error above its count, or a ceiling above the lowest count. fn rebuild(state: SpaceSavingState) -> Result { if state.capacity == 0 { return Err("space-saving capacity is zero".to_string()); @@ -618,7 +603,6 @@ impl SpaceSaving { bucket_head: NIL, bucket_tail: NIL, index: Index::with_capacity_and_hasher(entries.len(), DigestBuildHasher::default()), - total: state.total, discarded_max: state.discarded_max, _hasher: PhantomData, }; @@ -642,17 +626,6 @@ impl SpaceSaving { summary.discarded_max )); } - let recorded = summary - .monitored - .iter() - .map(|c| summary.buckets[c.bucket].count.saturating_sub(c.error)) - .fold(0u64, u64::saturating_add); - if recorded > summary.total { - return Err(format!( - "space-saving carries a total of {} under the {recorded} its counters account for", - summary.total - )); - } Ok(summary) } } @@ -938,7 +911,6 @@ mod tests { assert_eq!(summary.error(&DataInput::I64(3)), 2); assert_eq!(summary.estimate(&DataInput::I64(1)), 5); assert_eq!(summary.min_count(), 5); - assert_eq!(summary.total(), 11); } #[test] @@ -972,7 +944,6 @@ mod tests { assert_eq!(summary.estimate(&DataInput::I64(1)), u64::MAX); assert_eq!(summary.estimate(&DataInput::I64(2)), u64::MAX); assert_eq!(summary.estimate(&DataInput::I64(3)), 7); - assert_eq!(summary.total(), u64::MAX); let walked = walk(&summary); assert_eq!(walked.len(), 3); for pair in walked.windows(2) { @@ -1014,7 +985,6 @@ mod tests { left.validate().expect("after a saturating merge"); assert_eq!(left.len(), 2); - assert_eq!(left.total(), u64::MAX); for (key, count) in walk(&left) { assert_eq!(count, u64::MAX, "key {key} wrapped past the ceiling"); } @@ -1225,8 +1195,8 @@ mod tests { assert!(summary.is_guaranteed(&DataInput::I64(1))); } - /// `clear` leaves a summary that answers like a fresh one, ceiling and - /// total included. + /// `clear` leaves a summary that answers like a fresh one, ceiling + /// included. #[test] fn clear_resets_every_answer() { let mut summary: SpaceSaving = SpaceSaving::with_capacity(2); @@ -1238,7 +1208,6 @@ mod tests { } summary.update(&DataInput::I64(3)); assert!(summary.min_count() > 0); - assert!(summary.total() > 0); summary.clear(); @@ -1246,7 +1215,6 @@ mod tests { assert!(summary.is_empty()); assert_eq!(summary.len(), 0); assert_eq!(summary.capacity(), 2); - assert_eq!(summary.total(), 0, "clear left the recorded weight behind"); assert_eq!(summary.min_count(), 0, "clear left the ceiling behind"); assert_eq!(summary.upper_bound(&DataInput::I64(1)), 0); assert_eq!(summary.error(&DataInput::I64(1)), 0); @@ -1256,7 +1224,6 @@ mod tests { summary.update(&DataInput::I64(4)); summary.validate().expect("after refilling"); assert_eq!(summary.estimate(&DataInput::I64(4)), 1); - assert_eq!(summary.total(), 1); } #[test] @@ -1430,7 +1397,6 @@ mod tests { let (summary, truth) = fuzzed(capacity, 96, 4_000, seed); assert_sound_against(&summary, &truth); assert_eq!(summary.len(), capacity.min(truth.len())); - assert_eq!(summary.total(), truth.values().sum::()); let walked = walk(&summary); assert_eq!(walked.len(), summary.len()); for pair in walked.windows(2) { @@ -1461,7 +1427,6 @@ mod tests { *truth.entry(*key).or_default() += *count; } assert_sound_against(&left, &truth); - assert_eq!(left.total(), truth.values().sum::()); assert!(left.len() <= left.capacity()); let mut state = seed; @@ -1484,7 +1449,6 @@ mod tests { decoded.validate().expect("decoded summary"); assert_eq!(decoded.len(), summary.len()); assert_eq!(decoded.min_count(), summary.min_count()); - assert_eq!(decoded.total(), summary.total()); assert_sound_against(&decoded, &truth); let mut walked = walk(&decoded); @@ -1498,7 +1462,6 @@ mod tests { fn a_crafted_state_fails_closed() { let over_capacity = SpaceSavingState { capacity: 1, - total: 4, discarded_max: 0, entries: vec![ (OwnedDataInput::I64(1), 2, 0), @@ -1509,7 +1472,6 @@ mod tests { ( SpaceSavingState { capacity: 0, - total: 0, discarded_max: 0, entries: Vec::new(), }, @@ -1519,7 +1481,6 @@ mod tests { ( SpaceSavingState { capacity: 4, - total: 1, discarded_max: 0, entries: vec![(OwnedDataInput::I64(1), 0, 0)], }, @@ -1528,7 +1489,6 @@ mod tests { ( SpaceSavingState { capacity: 4, - total: 1, discarded_max: 0, entries: vec![(OwnedDataInput::I64(1), 3, 4)], }, @@ -1537,7 +1497,6 @@ mod tests { ( SpaceSavingState { capacity: 4, - total: 2, discarded_max: 0, entries: vec![ (OwnedDataInput::I64(1), 2, 0), @@ -1549,21 +1508,11 @@ mod tests { ( SpaceSavingState { capacity: 4, - total: 3, discarded_max: u64::MAX, entries: vec![(OwnedDataInput::I64(1), 3, 0)], }, "ceiling of 18446744073709551615 above its lowest count of 3", ), - ( - SpaceSavingState { - capacity: 4, - total: 0, - discarded_max: 0, - entries: vec![(OwnedDataInput::I64(1), 9, 0)], - }, - "total of 0 under the 9", - ), ]; for (state, expected) in cases { let problem = SpaceSaving::::rebuild(state) @@ -1579,7 +1528,6 @@ mod tests { fn a_declared_capacity_is_not_allocated_on_decode() { let state = SpaceSavingState { capacity: 1 << 40, - total: 3, discarded_max: 0, entries: vec![(OwnedDataInput::I64(1), 3, 0)], }; @@ -1706,7 +1654,6 @@ mod tests { assert_eq!(bytes_of(&decoded), bytes_of(&summary)); assert_eq!(decoded.estimate(&DataInput::Bytes(RAW)), 9); assert_eq!(decoded.estimate(&DataInput::Bytes(&[0x00; 5])), 4); - assert_eq!(decoded.total(), summary.total()); } } @@ -1871,7 +1818,6 @@ mod collisions { let repeated = SpaceSavingState { capacity: 4, - total: 9, discarded_max: 0, entries: vec![ (OwnedDataInput::I64(10), 5, 0), diff --git a/src/sketches/space_saving/parts.rs b/src/sketches/space_saving/parts.rs index b93e935a..e2905ec1 100644 --- a/src/sketches/space_saving/parts.rs +++ b/src/sketches/space_saving/parts.rs @@ -36,16 +36,14 @@ impl SpaceSaving { } /// Rebuilds a summary from its wire state, rejecting a state §3.5's decode - /// rules 5 to 10 reject. `capacity` never sizes an allocation. + /// rules 5 to 9 reject. `capacity` never sizes an allocation. pub(crate) fn from_parts( capacity: usize, - total: u64, discarded_max: u64, entries: Vec<(OwnedDataInput, u64, u64)>, ) -> Result { Self::rebuild(SpaceSavingState { capacity, - total, discarded_max, entries, }) diff --git a/src/sketches/space_saving/wire.rs b/src/sketches/space_saving/wire.rs index 40455bae..1627f390 100644 --- a/src/sketches/space_saving/wire.rs +++ b/src/sketches/space_saving/wire.rs @@ -5,16 +5,18 @@ //! constant, the `key_type` mapping, and the `to_asapv1` / //! `from_asapv1` impls) while the algorithm lives in the parent //! module file. Being a descendant module, it reads the summary's private -//! `monitored` / `buckets` / `total` / `discarded_max` fields and reuses the private +//! `monitored` / `buckets` / `discarded_max` fields and reuses the private //! `rebuild` entry point directly, without widening any field visibility. See //! `docs/asapv1_wire_format.md` §3.5. //! -//! Space-Saving is one algorithm — a single kind_id `0x18 0x00`. Its structural +//! Space-Saving is one algorithm — a single kind_id `0x18 0x01`. Its structural //! parameters are the counter `capacity` and the `key_type`, both in the //! metadata, so the payload is the answer triples split into parallel arrays -//! plus the two running scalars: `[keys, counts, errors, total, discarded_max]`. The -//! bucket list, the counter arena and the key index are all derived and are -//! rebuilt on load, so no arena index reaches the wire. +//! plus the ceiling: `[keys, counts, errors, discarded_max]`. The bucket list, +//! the counter arena and the key index are all derived and are rebuilt on load, +//! so no arena index reaches the wire. The retired `0x18 0x00` also carried a +//! `total` before `discarded_max`; it still decodes, with `total` dropped, and +//! is never encoded. //! //! ## `key_type` names the exact `OwnedDataInput` variant //! @@ -44,11 +46,12 @@ //! in, and re-serializing a decoded summary reproduces its bytes exactly. use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use crate::asapv1::envelope; use crate::asapv1::kind; -use crate::asapv1::kind::SPACE_SAVING; +use crate::asapv1::kind::{SPACE_SAVING, SPACE_SAVING_WITH_TOTAL}; use crate::asapv1::wire_key::{WireBytes, WireString, exact_vec}; use crate::{HashProfile, OwnedDataInput, SketchHasher}; @@ -119,12 +122,26 @@ fn space_saving_metadata(capacity: u32, key_type: &str) -> Space } /// Space-Saving payload (ASAPv1 §3.5), a msgpack **array** (`to_vec`, -/// positional): `[keys, counts, errors, total, discarded_max]`. The three arrays are +/// positional): `[keys, counts, errors, discarded_max]`. The three arrays are /// parallel and equal-length; `keys`'s element type is fixed by the metadata -/// `key_type`. `total` is the recorded weight, `discarded_max` the largest count known -/// to have left the summary. +/// `key_type`. `discarded_max` is the largest count known to have left the +/// summary. #[derive(Debug, Serialize, Deserialize)] struct SpaceSavingPayload { + #[serde( + deserialize_with = "exact_vec", + bound(deserialize = "K: Deserialize<'de>") + )] + keys: Vec, + counts: Vec, + errors: Vec, + discarded_max: u64, +} + +/// The retired `0x18 0x00` payload, `[keys, counts, errors, total, +/// discarded_max]`: decoded only, and `total` is dropped. +#[derive(Debug, Serialize, Deserialize)] +struct SpaceSavingPayloadWithTotal { #[serde( deserialize_with = "exact_vec", bound(deserialize = "K: Deserialize<'de>") @@ -136,6 +153,23 @@ struct SpaceSavingPayload { discarded_max: u64, } +/// Reads the current payload, or the retired one with `total` dropped. +fn read_payload( + payload: &[u8], + with_total: bool, +) -> Result, RmpDecodeError> { + if !with_total { + return envelope::decode_block(payload); + } + let retired: SpaceSavingPayloadWithTotal = envelope::decode_block(payload)?; + Ok(SpaceSavingPayload { + keys: retired.keys, + counts: retired.counts, + errors: retired.errors, + discarded_max: retired.discarded_max, + }) +} + /// The `key_type` the payload will be written in, taken from the first key in /// emitted order. Rejects a 128-bit key outright; the homogeneity of the rest /// is enforced by [`encode_payload`]. @@ -181,7 +215,6 @@ fn check_wire_entries<'a>( fn encode_payload( key_type: &str, entries: &[(&OwnedDataInput, u64, u64)], - total: u64, discarded_max: u64, ) -> Result, RmpEncodeError> { let counts: Vec = entries.iter().map(|entry| entry.1).collect(); @@ -200,7 +233,6 @@ fn encode_payload( keys, counts, errors, - total, discarded_max, }) }}; @@ -231,7 +263,6 @@ fn encode_payload( keys, counts, errors, - total, discarded_max, }) } @@ -247,7 +278,6 @@ fn encode_payload( keys, counts, errors, - total, discarded_max, }) } @@ -257,18 +287,20 @@ fn encode_payload( } } -/// Reads the payload with `keys` typed by the metadata `key_type` and assembles -/// the state the in-memory rebuild consumes. Rejects an unknown `key_type` and +/// Reads the payload (the retired one when `with_total`) with `keys` typed by +/// the metadata `key_type` and assembles the state the in-memory rebuild +/// consumes. Rejects an unknown `key_type` and /// parallel arrays of unequal length; a `keys` array whose msgpack types do not /// match `key_type` fails to decode. fn decode_payload( key_type: &str, capacity: usize, payload: &[u8], + with_total: bool, ) -> Result { macro_rules! unpack { ($variant:ident, $ty:ty) => {{ - let decoded: SpaceSavingPayload<$ty> = envelope::decode_block(payload)?; + let decoded: SpaceSavingPayload<$ty> = read_payload(payload, with_total)?; ( decoded .keys @@ -277,13 +309,12 @@ fn decode_payload( .collect::>(), decoded.counts, decoded.errors, - decoded.total, decoded.discarded_max, ) }}; } - let (keys, counts, errors, total, discarded_max) = match key_type { + let (keys, counts, errors, discarded_max) = match key_type { "i8" => unpack!(I8, i8), "i16" => unpack!(I16, i16), "i32" => unpack!(I32, i32), @@ -297,7 +328,7 @@ fn decode_payload( "f32" => unpack!(F32, f32), "f64" => unpack!(F64, f64), "string" => { - let decoded: SpaceSavingPayload = envelope::decode_block(payload)?; + let decoded: SpaceSavingPayload = read_payload(payload, with_total)?; ( decoded .keys @@ -306,12 +337,11 @@ fn decode_payload( .collect::>(), decoded.counts, decoded.errors, - decoded.total, decoded.discarded_max, ) } "bytes" => { - let decoded: SpaceSavingPayload = envelope::decode_block(payload)?; + let decoded: SpaceSavingPayload = read_payload(payload, with_total)?; ( decoded .keys @@ -320,7 +350,6 @@ fn decode_payload( .collect::>(), decoded.counts, decoded.errors, - decoded.total, decoded.discarded_max, ) } @@ -343,7 +372,6 @@ fn decode_payload( Ok(SpaceSavingState { capacity, - total, discarded_max, entries: keys .into_iter() @@ -359,7 +387,7 @@ fn decode_payload( // directly. impl SpaceSaving { /// Serializes the summary into an ASAPv1 MessagePack envelope - /// (kind_id `0x18 0x00`). The metadata is derived from the hasher's + /// (kind_id `0x18 0x01`). The metadata is derived from the hasher's /// [`HashProfile`], so it truthfully describes how the keys were hashed. /// /// Fails when the monitored keys mix [`OwnedDataInput`] variants, when any of @@ -377,11 +405,13 @@ impl SpaceSaving { .map_err(RmpEncodeError::Syntax)?; let key_type = wire_key_type(&entries)?; let metadata = rmp_serde::to_vec_named(&space_saving_metadata::(capacity, key_type))?; - let payload = encode_payload(key_type, &entries, self.total, self.discarded_max)?; + let payload = encode_payload(key_type, &entries, self.discarded_max)?; Ok(envelope::encode(kind::SPACE_SAVING, &metadata, &payload)) } - /// Deserializes a summary from an ASAPv1 MessagePack envelope. `capacity` + /// Deserializes a summary from an ASAPv1 MessagePack envelope: kind_id + /// `0x18 0x01`, or the retired `0x18 0x00` with its `total` read and + /// dropped. `capacity` /// and `key_type` are structural (they are properties of the stored /// summary, not of the target), so they are echoed back into the expected /// metadata; the hash spec is pinned against this target's profile. @@ -392,9 +422,10 @@ impl SpaceSaving { pub fn from_asapv1(bytes: &[u8]) -> Result { let (kind_id, metadata, payload) = envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; - if kind_id != kind::SPACE_SAVING { + let with_total = kind_id == kind::SPACE_SAVING_WITH_TOTAL; + if kind_id != kind::SPACE_SAVING && !with_total { return Err(RmpDecodeError::Uncategorized(format!( - "Space-Saving kind_id mismatch: stored {kind_id:?}, expected {SPACE_SAVING:?}" + "Space-Saving kind_id mismatch: stored {kind_id:?}, expected {SPACE_SAVING:?} (or the retired {SPACE_SAVING_WITH_TOTAL:?})" ))); } let meta: SpaceSavingMetadata = envelope::decode_block(metadata)?; @@ -403,7 +434,7 @@ impl SpaceSaving { "ASAPv1 Space-Saving envelope: metadata mismatch".to_string(), )); } - let state = decode_payload(&meta.key_type, meta.capacity as usize, payload)?; + let state = decode_payload(&meta.key_type, meta.capacity as usize, payload, with_total)?; check_wire_entries(state.entries.iter().map(|entry| (&entry.0, entry.1))) .map_err(RmpDecodeError::Uncategorized)?; Self::rebuild(state).map_err(RmpDecodeError::Uncategorized) @@ -453,7 +484,7 @@ mod tests { assert!(bytes.starts_with(envelope::MAGIC)); assert_eq!(bytes[6], envelope::VERSION); assert_eq!(bytes[7], 2, "kind_id_len"); - assert_eq!(&bytes[8..10], &[0x18, 0x00], "Space-Saving kind_id"); + assert_eq!(&bytes[8..10], &[0x18, 0x01], "Space-Saving kind_id"); let meta = metadata_of(&bytes); assert_eq!(meta.metadata_version, 1); @@ -463,7 +494,6 @@ mod tests { let (_, _, payload) = envelope::split(&bytes).expect("split"); let emitted: SpaceSavingPayload = from_slice(payload).expect("payload"); assert_eq!(emitted.counts.len(), summary.len()); - assert_eq!(emitted.total, summary.total()); for pair in emitted.counts.windows(2) { assert!(pair[0] >= pair[1], "the emitted counts are not descending"); } @@ -481,7 +511,6 @@ mod tests { assert_eq!(decoded.len(), summary.len()); assert_eq!(decoded.capacity(), summary.capacity()); - assert_eq!(decoded.total(), summary.total()); assert_eq!(decoded.min_count(), summary.min_count()); let mut monitored = 0; @@ -556,7 +585,6 @@ mod tests { assert_eq!(decoded.len(), 0); assert_eq!(decoded.capacity(), 16); assert_eq!(decoded.min_count(), 0); - assert_eq!(decoded.total(), 0); assert_eq!(decoded.to_asapv1().expect("re-serialize"), bytes); } @@ -702,14 +730,12 @@ mod tests { let seated = SpaceSaving::::rebuild(SpaceSavingState { capacity: 4, - total: 28, discarded_max: 0, entries, }) .expect("rebuild"); let reseated = SpaceSaving::::rebuild(SpaceSavingState { capacity: 4, - total: 28, discarded_max: 0, entries: reversed, }) @@ -908,7 +934,6 @@ mod tests { keys: Vec, counts: Vec, errors: Vec, - total: u64, discarded_max: u64, ) -> Vec { let metadata = @@ -918,13 +943,61 @@ mod tests { keys, counts, errors, - total, discarded_max, }) .expect("payload"); envelope::encode(kind::SPACE_SAVING, &metadata, &payload) } + /// The retired `0x18 0x00` decodes with its `total` dropped, whatever that + /// total was, and re-encodes as `0x18 0x01`. Each kind reads only its own + /// payload shape. + #[test] + fn space_saving_decodes_the_retired_kind_with_total() { + let metadata = rmp_serde::to_vec_named(&space_saving_metadata::(4, "u64")) + .expect("metadata"); + let retired_payload = |total: u64| { + rmp_serde::to_vec(&SpaceSavingPayloadWithTotal:: { + keys: vec![7, 8], + counts: vec![9, 4], + errors: vec![0, 1], + total, + discarded_max: 0, + }) + .expect("payload") + }; + let current = crafted(4, vec![7, 8], vec![9, 4], vec![0, 1], 0); + // `0` sits under the 12 the counters account for, which the retired + // decode rule 10 rejected; there is no such rule now. + for total in [13, 0, u64::MAX] { + let retired = envelope::encode( + kind::SPACE_SAVING_WITH_TOTAL, + &metadata, + &retired_payload(total), + ); + let decoded = + SpaceSaving::::from_asapv1(&retired).expect("retired decodes"); + assert_eq!(decoded.estimate(&DataInput::U64(7)), 9); + assert_eq!(decoded.error(&DataInput::U64(8)), 1); + assert_eq!(decoded.to_asapv1().expect("re-encode"), current); + } + + let mislabelled = [ + envelope::encode(kind::SPACE_SAVING, &metadata, &retired_payload(13)), + envelope::encode( + kind::SPACE_SAVING_WITH_TOTAL, + &metadata, + envelope::split(¤t).expect("split").2, + ), + ]; + for bytes in mislabelled { + assert!( + SpaceSaving::::from_asapv1(&bytes).is_err(), + "a payload read under the other kind's shape must be rejected" + ); + } + } + #[test] fn space_saving_rejects_a_crafted_envelope() { let relabelled = |key_type: &str| { @@ -935,7 +1008,6 @@ mod tests { keys: vec![1], counts: vec![1], errors: vec![0], - total: 1, discarded_max: 0, }) .expect("payload"); @@ -945,40 +1017,34 @@ mod tests { let (foreign_meta, foreign_payload) = relabelled("u64"); let cases: Vec<(Vec, &str)> = vec![ + (crafted(0, vec![1], vec![2], vec![0], 0), "capacity is zero"), ( - crafted(0, vec![1], vec![2], vec![0], 2, 0), - "capacity is zero", - ), - ( - crafted(1, vec![1, 2], vec![2, 2], vec![0, 0], 4, 0), + crafted(1, vec![1, 2], vec![2, 2], vec![0, 0], 0), "over a capacity of 1", ), ( - crafted(4, vec![1, 2], vec![2], vec![0, 0], 2, 0), + crafted(4, vec![1, 2], vec![2], vec![0, 0], 0), "2 keys against 1 counts", ), ( - crafted(4, vec![1, 2], vec![2, 2], vec![0], 4, 0), + crafted(4, vec![1, 2], vec![2, 2], vec![0], 0), "2 counts and 1 errors", ), + (crafted(4, vec![1], vec![0], vec![0], 0), "counter at zero"), ( - crafted(4, vec![1], vec![0], vec![0], 0, 0), - "counter at zero", - ), - ( - crafted(4, vec![1], vec![3], vec![4], 3, 0), + crafted(4, vec![1], vec![3], vec![4], 0), "error of 4 against a count of 3", ), ( - crafted(4, vec![1, 1], vec![2, 1], vec![0, 0], 3, 0), + crafted(4, vec![1, 1], vec![2, 1], vec![0, 0], 0), "the same key appears twice", ), ( - crafted(4, vec![1, 2], vec![2, 3], vec![0, 0], 5, 0), + crafted(4, vec![1, 2], vec![2, 3], vec![0, 0], 0), "not in descending count, ties by key", ), ( - crafted(4, vec![2, 1], vec![2, 2], vec![0, 0], 4, 0), + crafted(4, vec![2, 1], vec![2, 2], vec![0, 0], 0), "not in descending count, ties by key", ), ( @@ -986,13 +1052,9 @@ mod tests { "is not an ASAPv1 wire key type", ), ( - crafted(4, vec![1], vec![3], vec![0], 3, u64::MAX), + crafted(4, vec![1], vec![3], vec![0], u64::MAX), "ceiling of 18446744073709551615 above its lowest count of 3", ), - ( - crafted(4, vec![1], vec![9], vec![0], 0, 0), - "total of 0 under the 9", - ), ( envelope::encode(&[0x02, 0x00], &foreign_meta, &foreign_payload), "kind_id mismatch", @@ -1019,12 +1081,10 @@ mod tests { rmp_serde::to_vec_named(&space_saving_metadata::(4, key_type)) .expect("metadata"); let errors = vec![0; keys.len()]; - let total = counts.iter().sum(); let payload = rmp_serde::to_vec(&SpaceSavingPayload { keys, counts, errors, - total, discarded_max: 0, }) .expect("payload"); @@ -1043,7 +1103,6 @@ mod tests { let nan = |count: u64| (OwnedDataInput::F64(f64::NAN), count, 0); let twice = SpaceSaving::::rebuild(SpaceSavingState { capacity: 4, - total: 5, discarded_max: 0, entries: vec![nan(3), nan(2)], }) @@ -1060,7 +1119,6 @@ mod tests { keys: vec![f64::NAN, f64::NAN], counts: vec![3, 2], errors: vec![0, 0], - total: 5, discarded_max: 0, }) .expect("payload"); @@ -1080,7 +1138,6 @@ mod tests { fn space_saving_refuses_a_capacity_the_metadata_cannot_carry() { let summary = SpaceSaving::::rebuild(SpaceSavingState { capacity: 1 << 40, - total: 3, discarded_max: 0, entries: vec![(OwnedDataInput::U64(1), 3, 0)], }) @@ -1101,7 +1158,7 @@ mod tests { /// counters would be tens of gigabytes if it sized the arena. #[test] fn space_saving_does_not_allocate_a_declared_capacity() { - let bytes = crafted(u32::MAX, vec![7, 8], vec![9, 4], vec![0, 1], 13, 0); + let bytes = crafted(u32::MAX, vec![7, 8], vec![9, 4], vec![0, 1], 0); let decoded = SpaceSaving::::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.capacity(), u32::MAX as usize); assert_eq!(decoded.len(), 2); @@ -1120,12 +1177,11 @@ mod tests { fn known_state_matches_golden() { use crate::asapv1::envelope::hex; - let summary = |capacity, total, discarded_max, emitted: Vec<(OwnedDataInput, u64, u64)>| { + let summary = |capacity, discarded_max, emitted: Vec<(OwnedDataInput, u64, u64)>| { let mut entries = emitted; entries.reverse(); SpaceSaving::::rebuild(SpaceSavingState { capacity, - total, discarded_max, entries, }) @@ -1142,20 +1198,20 @@ mod tests { ] .map(|(key, count, error)| (OwnedDataInput::I64(key), count, error)); let golden = hex(include_str!( - "../../../asapv1_golden/space_saving_i64_c8.hex" + "../../../asapv1_golden/space_saving_nototal_i64_c8.hex" )); assert_eq!( - summary(8, 70030, 4, i64_keys.to_vec()).to_asapv1().unwrap(), + summary(8, 4, i64_keys.to_vec()).to_asapv1().unwrap(), golden ); let byte_keys = [(&b"\xff\x00"[..], 9, 0), (b"", 4, 1), (b"ab", 4, 0)] .map(|(key, count, error)| (OwnedDataInput::Bytes(key.to_vec()), count, error)); let golden = hex(include_str!( - "../../../asapv1_golden/space_saving_bytes_c3.hex" + "../../../asapv1_golden/space_saving_nototal_bytes_c3.hex" )); assert_eq!( - summary(3, 20, 3, byte_keys.to_vec()).to_asapv1().unwrap(), + summary(3, 3, byte_keys.to_vec()).to_asapv1().unwrap(), golden ); } diff --git a/tests/PBT_COVERAGE.md b/tests/PBT_COVERAGE.md index 00858f4d..9058f5f0 100644 --- a/tests/PBT_COVERAGE.md +++ b/tests/PBT_COVERAGE.md @@ -384,23 +384,23 @@ Compared against: an exact `HashMap` count. Every bound is deterministic. - structure - `len() == min(distinct, capacity)`; `len() <= capacity` - every `error <= min_count` - - `sum(counts) == total() == arrivals` - - full summary: `min_count >= smallest counter`; `min_count * capacity <= total()` + - `sum(counts) == arrivals` + - full summary: `min_count >= smallest counter`; `min_count * capacity <= arrivals` - `top_k(k)`: `min(k, len)` entries, non-increasing, each matching the table; nothing above the cut left out - a guaranteed key's truth `>` every dropped key's truth - `min_count` non-decreasing over the stream - domain `<= capacity` - exact: `count == truth`; `error == 0`; `upper_bound == truth`; order-free - - `update_weighted(k, w) == w x update(k)`: counters, total, `min_count` + - `update_weighted(k, w) == w x update(k)`: counters, `min_count` - `update_batch == one at a time` - `clear` then a different stream `== fresh` - merge - - bounds hold on `xs ++ ys` after merging two capacities; `total()` adds - - `merge(s, empty) == s`: counters, len, total, `min_count` + - bounds hold on `xs ++ ys` after merging two capacities + - `merge(s, empty) == s`: counters, len, `min_count` - `merge(a, b) == merge(b, a)`; domain `capacity + 3` - after two narrow merges and a tail: `upper_bound(k) >= truth(k)` for every key - wire - - ASAPv1 at `edge_dimension` keeps `total()`, `min_count`, 64 `(estimate, error)`; after merging a 1-counter summary whose ceiling no counter holds + - ASAPv1 at `edge_dimension` keeps `min_count`, 64 `(estimate, error)`; after merging a 1-counter summary whose ceiling no counter holds ## topk_wrappers.rs — 15 diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 12fda029..ccc6dbe3 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -81,10 +81,17 @@ const GOLDEN_CMSHEAP_TUPLE: &str = include_str!("../asapv1_golden/cmsheap_i64_regular_2x3_tuplekeys.hex"); const GOLDEN_CSHEAP_TUPLE: &str = include_str!("../asapv1_golden/csheap_i64_regular_2x4_tuplekeys.hex"); +// The retired CMSSpaceSaving / CSSpaceSaving kinds `1e 00` / `1f 00`, decode-only. const GOLDEN_CMSSS_STR: &str = include_str!("../asapv1_golden/cmsspacesaving_i64_regular_2x3_strkeys.hex"); const GOLDEN_CSSS_I64: &str = include_str!("../asapv1_golden/csspacesaving_f64_fast_2x4_i64keys.hex"); +// The current kinds `1e 01` / `1f 01`, whose summary carries no `total`. +const GOLDEN_CMSSS_NOTOTAL_STR: &str = + include_str!("../asapv1_golden/cmsspacesaving_nototal_i64_regular_2x3_strkeys.hex"); +const GOLDEN_CSSS_NOTOTAL_I64: &str = + include_str!("../asapv1_golden/csspacesaving_nototal_f64_fast_2x4_i64keys.hex"); + // The retired DDSketch kinds `05 00` / `05 80`, decode-only. const GOLDEN_DD_POSITIVE: &str = include_str!("../asapv1_golden/ddsketch_positive_a001.hex"); const GOLDEN_DD_SIGNED: &str = include_str!("../asapv1_golden/ddsketch_signed_a001.hex"); @@ -136,8 +143,15 @@ const GOLDEN_BLOOM_FAST: &str = include_str!("../asapv1_golden/bloom_fast_5x32.h const GOLDEN_BLOOM_BITS_REGULAR: &str = include_str!("../asapv1_golden/bloom_bits_regular_3x128.hex"); const GOLDEN_BLOOM_BITS_FAST: &str = include_str!("../asapv1_golden/bloom_bits_fast_5x32.hex"); + +// The retired Space-Saving kind `18 00`, decode-only. const GOLDEN_SPACE_SAVING_I64: &str = include_str!("../asapv1_golden/space_saving_i64_c8.hex"); const GOLDEN_SPACE_SAVING_BYTES: &str = include_str!("../asapv1_golden/space_saving_bytes_c3.hex"); +// The current Space-Saving kind `18 01`. +const GOLDEN_SPACE_SAVING_NOTOTAL_I64: &str = + include_str!("../asapv1_golden/space_saving_nototal_i64_c8.hex"); +const GOLDEN_SPACE_SAVING_NOTOTAL_BYTES: &str = + include_str!("../asapv1_golden/space_saving_nototal_bytes_c3.hex"); #[cfg(feature = "experimental")] const GOLDEN_UNIFORM: &str = include_str!("../asapv1_golden/uniform_sampling_r0p25.hex"); const GOLDEN_PYRAMID: &str = @@ -833,20 +847,20 @@ fn csheap_i64_regular_2x4_tuplekeys_matches_golden() { // --------------------------------------------------------------------------- // CountMinSketchWithSpaceSaving / CountSketchWithSpaceSaving: the matrix set directly and the summary built -// by weighted inserts, whose triples do not depend on the hash. +// by weighted inserts, whose triples do not depend on the hash. Each retired +// `1e 00` / `1f 00` fixture decodes, its `total` dropped, and re-encodes as its +// `*_nototal_*` twin. // --------------------------------------------------------------------------- -/// Asserts `summary` has `capacity`, `total` and `ceiling` and holds exactly -/// `entries` as `(key, count, error)`. +/// Asserts `summary` has `capacity` and `ceiling` and holds exactly `entries` +/// as `(key, count, error)`. fn assert_summary( summary: &SpaceSaving, capacity: usize, - total: u64, ceiling: u64, entries: &[(DataInput<'_>, u64, u64)], ) { assert_eq!(summary.capacity(), capacity); - assert_eq!(summary.total(), total); assert_eq!(summary.min_count(), ceiling); assert_eq!(summary.len(), entries.len()); for (key, count, error) in entries { @@ -858,9 +872,9 @@ fn assert_summary( /// `aa` displaces `cold` (count 1) and carries it as error; the `7` tie falls /// to the keys, so `"aa"` precedes `"b"`. #[test] -fn cmsspacesaving_i64_regular_2x3_strkeys_matches_golden() { +fn cmsspacesaving_nototal_i64_regular_2x3_strkeys_matches_golden() { type Sketch = CountMinSketchWithSpaceSaving, RegularPath>; - let want = decode_hex(GOLDEN_CMSSS_STR); + let want = decode_hex(GOLDEN_CMSSS_NOTOTAL_STR); let mut summary = SpaceSaving::with_capacity(4); for (key, weight) in [ ("hot", 300), @@ -887,16 +901,19 @@ fn cmsspacesaving_i64_regular_2x3_strkeys_matches_golden() { (DataInput::Str("aa"), 7, 1), (DataInput::Str("b"), 7, 0), ]; - assert_summary(decoded.space_saving(), 4, 442, 7, &entries); + assert_summary(decoded.space_saving(), 4, 7, &entries); assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); + + let retired = Sketch::from_asapv1(&decode_hex(GOLDEN_CMSSS_STR)).expect("decode retired"); + assert_eq!(retired.to_asapv1().expect("re-encode"), want); } /// `7` displaces `128` (count 5) and carries it as error; the `6` tie falls to /// the keys' two's-complement bits, so `7` precedes `i64::MIN`. #[test] -fn csspacesaving_f64_fast_2x4_i64keys_matches_golden() { +fn csspacesaving_nototal_f64_fast_2x4_i64keys_matches_golden() { type Sketch = CountSketchWithSpaceSaving, FastPath>; - let want = decode_hex(GOLDEN_CSSS_I64); + let want = decode_hex(GOLDEN_CSSS_NOTOTAL_I64); let mut summary = SpaceSaving::with_capacity(3); for (key, weight) in [(-1, 65536), (i64::MIN, 5), (128, 5), (7, 1), (i64::MIN, 1)] { summary.update_weighted(&DataInput::I64(key), weight); @@ -916,8 +933,11 @@ fn csspacesaving_f64_fast_2x4_i64keys_matches_golden() { (DataInput::I64(7), 6, 5), (DataInput::I64(i64::MIN), 6, 0), ]; - assert_summary(decoded.space_saving(), 3, 65548, 6, &entries); + assert_summary(decoded.space_saving(), 3, 6, &entries); assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); + + let retired = Sketch::from_asapv1(&decode_hex(GOLDEN_CSSS_I64)).expect("decode retired"); + assert_eq!(retired.to_asapv1().expect("re-encode"), want); } // --------------------------------------------------------------------------- @@ -1847,19 +1867,20 @@ fn bloom_retired_fixtures_decode_to_their_bits_twins() { // --------------------------------------------------------------------------- // Space-Saving: golden decodes to the known triples and round-trips. Encoding -// the known triples is checked in `src/sketches/space_saving/wire.rs`. +// the known triples is checked in `src/sketches/space_saving/wire.rs`. Each +// retired `18 00` fixture decodes, its `total` dropped, and re-encodes as its +// `space_saving_nototal_*` twin. // --------------------------------------------------------------------------- fn assert_space_saving_golden( golden: &str, capacity: usize, - total: u64, emitted: &[(OwnedDataInput, u64, u64)], min_count: u64, ) { let want = decode_hex(golden); let decoded: SpaceSaving = SpaceSaving::from_asapv1(&want).expect("decode"); - assert_eq!((decoded.capacity(), decoded.total()), (capacity, total)); + assert_eq!(decoded.capacity(), capacity); assert_eq!(decoded.min_count(), min_count); let mut held = decoded.entries(); held.sort_by(|a, b| format!("{a:?}").cmp(&format!("{b:?}"))); @@ -1871,7 +1892,7 @@ fn assert_space_saving_golden( /// The count-5 tie emits by two's-complement bits: `0, 1, i64::MIN, -129`. #[test] -fn space_saving_i64_c8_matches_golden() { +fn space_saving_nototal_i64_c8_matches_golden() { let emitted = [ (-1, 70000, 0), (300, 9, 2), @@ -1881,15 +1902,29 @@ fn space_saving_i64_c8_matches_golden() { (-129, 5, 0), ] .map(|(key, count, error)| (OwnedDataInput::I64(key), count, error)); - assert_space_saving_golden(GOLDEN_SPACE_SAVING_I64, 8, 70030, &emitted, 4); + assert_space_saving_golden(GOLDEN_SPACE_SAVING_NOTOTAL_I64, 8, &emitted, 4); } /// A full summary: `min_count` is its smallest count. `b""` precedes `b"ab"`. #[test] -fn space_saving_bytes_c3_matches_golden() { +fn space_saving_nototal_bytes_c3_matches_golden() { let emitted = [(&b"\xff\x00"[..], 9, 0), (b"", 4, 1), (b"ab", 4, 0)] .map(|(key, count, error)| (OwnedDataInput::Bytes(key.to_vec()), count, error)); - assert_space_saving_golden(GOLDEN_SPACE_SAVING_BYTES, 3, 20, &emitted, 4); + assert_space_saving_golden(GOLDEN_SPACE_SAVING_NOTOTAL_BYTES, 3, &emitted, 4); +} + +/// Each retired `18 00` fixture decodes to its twin's triples and re-encodes +/// as the twin. +#[test] +fn space_saving_retired_fixtures_decode_to_their_nototal_twins() { + for (retired, twin) in [ + (GOLDEN_SPACE_SAVING_I64, GOLDEN_SPACE_SAVING_NOTOTAL_I64), + (GOLDEN_SPACE_SAVING_BYTES, GOLDEN_SPACE_SAVING_NOTOTAL_BYTES), + ] { + let decoded: SpaceSaving = + SpaceSaving::from_asapv1(&decode_hex(retired)).expect("decode retired"); + assert_eq!(decoded.to_asapv1().expect("re-encode"), decode_hex(twin)); + } } // --------------------------------------------------------------------------- diff --git a/tests/e2e/heavy_hitters.rs b/tests/e2e/heavy_hitters.rs index 76f6f22d..7ee9212e 100644 --- a/tests/e2e/heavy_hitters.rs +++ b/tests/e2e/heavy_hitters.rs @@ -215,7 +215,6 @@ fn an_arrival_displaces_the_minimum_and_inherits_its_count() { assert_eq!(summary.estimate(&DataInput::I64(3)), 3); assert_eq!(summary.error(&DataInput::I64(3)), 2); assert_eq!(summary.min_count(), 3); - assert_eq!(summary.total(), 6); } /// The bucket and counter lists stay well formed under sustained eviction: @@ -291,7 +290,6 @@ fn a_weighted_arrival_matches_repeating_it() { weighted.update_weighted(&DataInput::I64(key), counts[&key]); } - assert_eq!(weighted.total(), singles.total()); assert_eq!(weighted.len(), singles.len()); for (key, count) in &counts { assert_eq!( @@ -319,10 +317,10 @@ fn a_summary_larger_than_the_domain_is_exact() { } } -/// Merging keeps the total, respects capacity, and never reports a shared key -/// below what either side saw. +/// Merging respects capacity and never reports a shared key below what either +/// side saw. #[test] -fn a_merge_keeps_the_total_and_never_reads_low() { +fn a_merge_never_reads_low() { let stream = stream(); let (left_stream, right_stream): (Vec, Vec) = stream.iter().partition(|key| *key % 2 == 0); @@ -334,7 +332,6 @@ fn a_merge_keeps_the_total_and_never_reads_low() { left.merge(&right); - assert_eq!(left.total(), stream.len() as u64); assert!(left.len() <= left.capacity()); let mut checked: HashSet = HashSet::new(); @@ -371,7 +368,6 @@ fn an_asapv1_round_trip_preserves_every_answer() { assert_eq!(decoded.len(), summary.len()); assert_eq!(decoded.capacity(), summary.capacity()); - assert_eq!(decoded.total(), summary.total()); assert_eq!(decoded.min_count(), summary.min_count()); for (key, _) in truth.pairs() { let probe = DataInput::I64(key); @@ -394,7 +390,6 @@ fn an_empty_summary_answers() { assert!(summary.is_empty()); assert_eq!(summary.len(), 0); assert_eq!(summary.min_count(), 0); - assert_eq!(summary.total(), 0); assert_eq!(summary.estimate(&DataInput::I64(1)), 0); assert_eq!(summary.upper_bound(&DataInput::I64(1)), 0); assert!(!summary.is_guaranteed(&DataInput::I64(1))); @@ -424,7 +419,6 @@ fn a_weighted_arrival_matches_repeating_it_under_eviction() { } assert_eq!(weighted.len(), 64, "the run should have evicted heavily"); - assert_eq!(weighted.total(), singles.total()); assert_eq!(weighted.min_count(), singles.min_count()); let mut left: Vec<(i64, u64, u64)> = weighted @@ -452,13 +446,11 @@ fn update_batch_matches_repeated_updates_and_a_zero_weight_is_inert() { bulk.update_batch(&values); let one_at_a_time = filled(128, &stream); - assert_eq!(bulk.total(), one_at_a_time.total()); assert_eq!(bulk.top_k(usize::MAX), one_at_a_time.top_k(usize::MAX)); let before = bulk.top_k(usize::MAX); bulk.update_weighted(&DataInput::I64(stream[0]), 0); bulk.update_weighted(&DataInput::I64(i64::MIN), 0); - assert_eq!(bulk.total(), one_at_a_time.total()); assert_eq!(bulk.len(), one_at_a_time.len()); assert_eq!(bulk.top_k(usize::MAX), before); } @@ -472,7 +464,6 @@ fn a_zero_capacity_summary_still_answers() { summary.update(&DataInput::I64(key)); } assert_eq!(summary.len(), 1); - assert_eq!(summary.total(), 10); let held = summary.top_k(4); assert_eq!(held.len(), 1); assert_eq!(held[0].1, 10); @@ -522,7 +513,6 @@ fn a_merge_chain_stays_one_sided_against_the_truth() { merged.merge(&filled(512, &shards[1])); merged.merge(&filled(7, &shards[2])); - assert_eq!(merged.total(), stream.len() as u64); assert!(merged.len() <= merged.capacity()); let mut dropped_max = 0; @@ -614,7 +604,6 @@ fn a_round_trip_carries_the_merged_ceiling() { let decoded: SpaceSaving = SpaceSaving::from_asapv1(&bytes).expect("deserialize"); assert_eq!(decoded.min_count(), merged.min_count()); - assert_eq!(decoded.total(), merged.total()); assert_eq!(decoded.len(), merged.len()); for (key, count) in truth_of(&stream).pairs() { let probe = DataInput::I64(key); @@ -642,7 +631,6 @@ fn the_default_summary_holds_the_default_capacity() { } assert_eq!(summary.len(), SPACE_SAVING_DEFAULT_CAPACITY); - assert_eq!(summary.total(), stream.len() as u64); assert_eq!( summary.top_k(usize::MAX).len(), SPACE_SAVING_DEFAULT_CAPACITY diff --git a/tests/e2e/wire.rs b/tests/e2e/wire.rs index 08bb32c6..f9843e5c 100644 --- a/tests/e2e/wire.rs +++ b/tests/e2e/wire.rs @@ -134,7 +134,7 @@ fn a_space_saving_envelope_round_trips_and_names_its_kind() { summary.update(&DataInput::U64(k)); } let bytes = summary.to_asapv1().expect("encode"); - assert_kind(&bytes, 0x18, 0x00, "SpaceSaving"); + assert_kind(&bytes, 0x18, 0x01, "SpaceSaving"); let decoded = SpaceSaving::::from_asapv1(&bytes).expect("decode"); for k in 0..1_024u64 { assert_eq!( diff --git a/tests/pbt/space_saving.rs b/tests/pbt/space_saving.rs index 75e6bb3b..59af9402 100644 --- a/tests/pbt/space_saving.rs +++ b/tests/pbt/space_saving.rs @@ -216,8 +216,7 @@ proptest! { let s = summary_of(capacity, &stream); let held: u64 = s.entries().iter().map(|(_, count, _)| count).sum(); - prop_assert_eq!(s.total(), stream.len() as u64); - prop_assert_eq!(held, s.total()); + prop_assert_eq!(held, stream.len() as u64); } /// The ceiling is pinned from both sides: it covers every counter the @@ -238,9 +237,9 @@ proptest! { s.min_count(), capacity, lowest ); prop_assert!( - u128::from(s.min_count()) * capacity as u128 <= u128::from(s.total()), + u128::from(s.min_count()) * capacity as u128 <= stream.len() as u128, "ceiling {} over {} counters exceeds the total {}", - s.min_count(), capacity, s.total() + s.min_count(), capacity, stream.len() ); } @@ -321,7 +320,6 @@ proptest! { let reordered = summary_of(capacity, &shuffled); prop_assert_eq!(monitored(&reordered), monitored(&s), "capacity {}", capacity); - prop_assert_eq!(reordered.total(), s.total()); prop_assert_eq!(reordered.min_count(), s.min_count()); } @@ -344,7 +342,6 @@ proptest! { "capacity {} over {} arrivals", capacity, stream.len() ); prop_assert_eq!(monitored(&expanded), exact, "capacity {}", capacity); - prop_assert_eq!(weighted.total(), expanded.total()); prop_assert_eq!(weighted.min_count(), expanded.min_count()); } @@ -362,7 +359,6 @@ proptest! { monitored(&bulk), monitored(&one_at_a_time), "capacity {} over {} arrivals", capacity, stream.len() ); - prop_assert_eq!(bulk.total(), one_at_a_time.total()); prop_assert_eq!(bulk.min_count(), one_at_a_time.min_count()); } @@ -379,7 +375,6 @@ proptest! { prop_assert!(reused.is_empty()); prop_assert_eq!(reused.len(), 0); - prop_assert_eq!(reused.total(), 0); prop_assert_eq!(reused.min_count(), 0); for key in &stream { @@ -391,7 +386,6 @@ proptest! { monitored(&reused), monitored(&fresh), "capacity {} after {} discarded arrivals", capacity, used.len() ); - prop_assert_eq!(reused.total(), fresh.total()); prop_assert_eq!(reused.min_count(), fresh.min_count()); } @@ -431,7 +425,6 @@ proptest! { let held = monitored(&merged); prop_assert!(merged.len() <= merged.capacity()); - prop_assert_eq!(merged.total(), concatenated.len() as u64); for (key, t) in &truth { let bound = merged.upper_bound(&DataInput::I64(*key)); @@ -466,7 +459,6 @@ proptest! { "capacity {} over {} arrivals", capacity, stream.len() ); prop_assert_eq!(merged.len(), base.len()); - prop_assert_eq!(merged.total(), base.total()); prop_assert_eq!(merged.min_count(), base.min_count()); } @@ -486,7 +478,6 @@ proptest! { monitored(&ab), monitored(&ba), "capacity {} over {} and {} arrivals", capacity, left.len(), right.len() ); - prop_assert_eq!(ab.total(), ba.total()); prop_assert_eq!(ab.min_count(), ba.min_count()); } @@ -519,7 +510,6 @@ proptest! { let held = monitored(&s); prop_assert!(s.len() <= s.capacity()); - prop_assert_eq!(s.total(), whole.len() as u64); for (key, t) in truth_of(&whole) { let bound = s.upper_bound(&DataInput::I64(key)); @@ -561,7 +551,6 @@ proptest! { round_trip!( Ss, sketch, - |s: &Ss| s.total(), |s: &Ss| s.min_count(), |s: &Ss| (0..PROBE_KEYS as u64) .map(|k| (s.estimate(&DataInput::U64(k)), s.error(&DataInput::U64(k))))