From 2477e9748b69c504ce76a6995b37228fec086c64 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 28 Oct 2026 22:18:03 +0000 Subject: [PATCH] feat(hll)!: HIP drops kxq0/kxq1 from the wire under kind 01 04 kxq0 and kxq1 are the sums of 2^-register over the registers below 32 and of 32 or more. Every term is a power of two, so recomputing them from the registers on decode gives the incrementally kept values bit for bit, and a decoded sketch keeps counting exactly as the encoded one would. The HIP payload is now the registers and hip_est under the new kind 01 04; the retired 01 03 still decodes, its two sums dropped, and is never encoded. The golden submodule gains the hll_hip_est_* fixtures. Part of #250. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + asapv1_golden | 2 +- docs/api/api_eh_bucket_sketch.md | 8 +- docs/api/api_hyperloglog.md | 12 +- docs/asapv1_wire_format.md | 29 +- docs/message_pack_format.md | 2 +- docs/tests.md | 16 +- src/asapv1/envelope.rs | 4 +- src/asapv1/kind.rs | 6 +- src/asapv1/wire_key.rs | 9 +- src/sketch_framework/eh_bucket_sketch/wire.rs | 2 +- src/sketches/hll.rs | 45 ++- src/sketches/hll/wire.rs | 302 ++++++++++++++---- tests/asapv1_golden.rs | 21 +- tests/hll_custom_precision.rs | 14 +- tests/pbt/hll.rs | 12 +- 16 files changed, 381 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d34f8cfd..c58a26f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,13 @@ signals a backwards-compatible change. dropped, and are never encoded; `EhBucketSketch` and `ExponentialHistogram` buckets nested under the retired ids decode too. The `sketchlib-golden-bytes` submodule gains the `*_no_l2` fixtures. +- **ASAPv1: HLL HIP encodes as `0x01 0x04`.** The payload is the registers + and `hip_est`; `hip_kxq0` / `hip_kxq1`, sums of `2^-register` over the + registers, are no longer carried and are recomputed on decode, bit for bit, + so a decoded sketch keeps counting exactly as the encoded one would. The + retired `0x01 0x03` still decodes, with the two sums dropped, and is never + encoded. The `sketchlib-golden-bytes` submodule gains the `hll_hip_est_*` + fixtures. - **Count Sketch estimates no longer allocate.** The per-row estimates a median is taken over are collected on the stack instead of in a fresh `Vec`: up to 20 rows for `Vector2D` (the fast and regular paths), diff --git a/asapv1_golden b/asapv1_golden index 26b0f3d6..7286b2fb 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit 26b0f3d67aeb3c99512131857d6dd839c47473ae +Subproject commit 7286b2fb75088ab2195f2cfdc70ca45f85919cb4 diff --git a/docs/api/api_eh_bucket_sketch.md b/docs/api/api_eh_bucket_sketch.md index 42acb0f3..ccbe29d1 100644 --- a/docs/api/api_eh_bucket_sketch.md +++ b/docs/api/api_eh_bucket_sketch.md @@ -83,10 +83,10 @@ The ten encoded nested kind_ids. The retired `0x05 0x00`, `0x10 0x00` and | `UnivMon` | `0x10 0x01` (retired `0x10 0x00` decodes) | UnivMon | default | Each id is pinned to one algorithm: `Hll` is Ertl-MLE, so `0x01 0x01` (Classic) -and `0x01 0x03` (HIP) are rejected, and `Kll` is compact, so `0x06 0x01` -(dynamic) is rejected. The dispatch is the same in every build: a decoder built -without `experimental` rejects `0x0d 0x00` with an error naming the variant and -the feature, and its encoder can never emit it. +and `0x01 0x04` (HIP), with the retired `0x01 0x03`, are rejected, and `Kll` is +compact, so `0x06 0x01` (dynamic) is rejected. The dispatch is the same in every +build: a decoder built without `experimental` rejects `0x0d 0x00` with an error +naming the variant and the feature, and its encoder can never emit it. An unrecognized kind_id is rejected. ## Examples diff --git a/docs/api/api_hyperloglog.md b/docs/api/api_hyperloglog.md index 7183a8b4..45067ae9 100644 --- a/docs/api/api_hyperloglog.md +++ b/docs/api/api_hyperloglog.md @@ -84,11 +84,15 @@ These produce/consume the **ASAPv1** wire envelope — see the `Variant: HllWireVariant` and `H: HashProfile`, so every (variant × precision × hasher) combination serializes — **HLL is fully wire-covered**. Each variant maps to its own `kind_id` (Classic → `0x01 0x01`, -Ertl-MLE → `0x01 0x02`, HIP → `0x01 0x03`); `precision` and the +Ertl-MLE → `0x01 0x02`, HIP → `0x01 0x04`); `precision` and the `HashProfile`-derived hash spec live in the envelope metadata, and the payload is -the register bytes (plus three HIP scalars for the HIP variant). `HyperLogLogHip` -is a non-generic struct that hashes through the default functions, so it is -wire-eligible under the **standard** hash profile only. +the register bytes (plus the running estimate `hip_est` for the HIP variant). +HIP's `kxq0` / `kxq1` sums are recomputed from the registers on decode, bit for +bit, so a decoded HIP sketch keeps counting exactly as the encoded one would. +`HyperLogLogHip::from_asapv1` also reads the retired `0x01 0x03` payload, which +carried those two sums, and drops them; `to_asapv1` never writes it. +`HyperLogLogHip` is a non-generic struct that hashes through the default +functions, so it is wire-eligible under the **standard** hash profile only. ## Deltas diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index a957e54c..1f08b2d0 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -73,7 +73,7 @@ erDiagram SerializedSketch ||--|| METADATA : carries-one SerializedSketch ||--|| PAYLOAD : carries-one PAYLOAD ||--o| HLL_PAYLOAD : example-kind-0x0101-0x0102 - PAYLOAD ||--o| HLL_HIP_PAYLOAD : example-kind-0x0103 + PAYLOAD ||--o| HLL_HIP_PAYLOAD : example-kind-0x0104 PAYLOAD ||--o| COUNTMIN_PAYLOAD : example-kind-0x0200 PAYLOAD ||--o| KLL_PAYLOAD : example-kind-0x0600-0x0601 ENVELOPE { @@ -102,8 +102,6 @@ erDiagram } HLL_HIP_PAYLOAD { bin registers - f64 hip_kxq0 - f64 hip_kxq1 f64 hip_est } COUNTMIN_PAYLOAD { @@ -185,7 +183,8 @@ This registry is the master list of algorithms still to design payloads for, and | `0x01 0x00` | HLL | Unspecified | - | reserved | | `0x01 0x01` | HLL | Classic ("Regular") | Section 3.1 | implemented | | `0x01 0x02` | HLL | Ertl-MLE ("Datafusion") | Section 3.1 | implemented | -| `0x01 0x03` | HLL | HIP | Section 3.1 | implemented | +| `0x01 0x03` | HLL | HIP with `hip_kxq0` / `hip_kxq1` | Section 3.1 | retired: decoded (`hip_kxq0` / `hip_kxq1` dropped), never encoded | +| `0x01 0x04` | HLL | HIP | Section 3.1 | implemented | | `0x01 0x81` | HLL | Classic delta | Section 3.23 | implemented | | `0x01 0x82` | HLL | Ertl-MLE delta | Section 3.23 | implemented | | `0x02 0x00` | Count-Min | Count-Min | Section 3.2 | implemented | @@ -441,9 +440,9 @@ Per sketch. **Raw state only**, a **positional msgpack array** in the order its - No field derivable from another (no HLL `precision`; no CMS `l1`/`l2`, which are `sum(count)` / `sum(count^2)` and are recomputed on decode; no CountSketchWithL2Norm or UnivMon-layer `l2`, which is each row's `sum(counts[row]^2)` saturated at `i64::MAX` and is recomputed on decode, §3.19). Nothing derivable rides the wire. - msgpack array (positional), never a keyed map. The exact msgpack types are in "Wire encoding rules". -### 3.1: HLL payload (`0x01 0x01` / `0x01 0x02` / `0x01 0x03`) +### 3.1: HLL payload (`0x01 0x01` / `0x01 0x02` / `0x01 0x04`) -The variant is in `kind_id`, precision is in the metadata (and equals `log2(register count)`), so the only real state is the register bytes (plus three running scalars for HIP). +The variant is in `kind_id`, precision is in the metadata (and equals `log2(register count)`), so the only real state is the register bytes (plus the running estimate for HIP). **Classic / Ertl-MLE** (`0x01 0x01`, `0x01 0x02`), identical layout: @@ -451,18 +450,20 @@ The variant is in `kind_id`, precision is in the metadata (and equals `log2(regi | ----- | ------- | ------ | ------- | | 0 | `registers` | bin | one byte per register; length is `2^precision` | -**HIP** (`0x01 0x03`): +**HIP** (`0x01 0x04`): | Pos | Field | Type | Notes | | ----- | ------- | ------ | ------- | | 0 | `registers` | bin | one byte per register | -| 1 | `hip_kxq0` | f64 | HIP running estimate state | -| 2 | `hip_kxq1` | f64 | | -| 3 | `hip_est` | f64 | | +| 1 | `hip_est` | f64 | the running HIP estimate | + +**`hip_est` is carried, `kxq0` / `kxq1` are not.** HIP adds `2^precision / (kxq0 + kxq1)` to `hip_est` each time an insert raises a register, so the estimate depends on the order the registers rose in and no walk of the registers reproduces it. `kxq0` and `kxq1` are the sums of `2^-r` over the registers `r` below 32 and over those of 32 or more; a decoder recomputes them from the registers. Every term is a power of two, so every partial sum is a multiple of the smallest term: `kxq0` is exact for precision up to 22 and `kxq1` at every precision. Within that bound the recomputed values equal the ones the encoder kept incrementally, bit for bit, and a decoded sketch that keeps inserting reaches the same `hip_est` as one that never left memory. + +**The retired `0x01 0x03`.** It carried `hip_kxq0` and `hip_kxq1` as two f64 fields between `registers` and `hip_est` (four fields in all). A decoder still accepts it, reads the two sums as float64 (Section 4) and drops them for the recomputed ones, and the result re-encodes as `0x01 0x04`. Nothing encodes `0x01 0x03`; per Q-VER the id is never reused. HIP has no delta kind. **Decode rules** (all fail closed, per Section 1's decoder rules): -1. `kind_id` is the variant's own — `0x01 0x01` Classic, `0x01 0x02` Ertl-MLE, `0x01 0x03` HIP. Each decoder owns exactly one id and rejects its two siblings. +1. `kind_id` is the variant's own — `0x01 0x01` Classic, `0x01 0x02` Ertl-MLE, `0x01 0x04` HIP (or the retired `0x01 0x03` with its four-field payload). Each decoder owns its own ids and rejects its siblings'. 2. The metadata map must equal the target type's own, `precision` included. Precision is a type parameter of the register storage rather than a property of the stored sketch, so it is **pinned**, not echoed back: p12 bytes do not decode into a p14 sketch. 3. `len(registers) == 2^precision` exactly. 4. No register byte exceeds `RANK_HASH_BITS + 1`, which is `65 - precision` — **53** at p12, **51** at p14, **49** at p16. An insert writes `leading_zeros((hash << precision) + INDEX_MASK) + 1`: the shift shoves the `precision` bucket-index bits off the top and the `+ INDEX_MASK` refills them, so the run being counted is only the `RANK_HASH_BITS = 64 - precision` bits the index leaves, and the largest rank reachable is one past that. A higher byte names a rank no hash could have produced, and it would inflate every estimate the sketch reports. @@ -1192,7 +1193,7 @@ The `kind_id` is carried rather than a name, because Section 1's registry is alr The retired `0x10 0x00` and `0x19 0x00` (§3.15, §3.19), which also carried `l2`, still decode into the `UnivMon` and `CountSketchWithL2Norm` arms through those variants' own decoders, `l2` dropped and recomputed; the encoder writes only `0x10 0x01` and `0x19 0x01`. They are the same arms, not further entries among the ten. -**Each id names one algorithm, and its siblings are not it.** `0x01 0x01`, `0x01 0x03` and `0x06 0x01` are rejected as unknown here: the arms hold `HyperLogLog` and the compact `Kll` and nothing else. `CountMinSketch` and `CountSketch` both hold `Vector2D` on the `FastPath`, so their descriptors carry `counter_type = "i32"` and `mode = "fast"`; `i32` is never widened, since a nested sketch must decode back into the arm it was stored as (Q-CS). +**Each id names one algorithm, and its siblings are not it.** `0x01 0x01`, `0x01 0x03`, `0x01 0x04` and `0x06 0x01` are rejected as unknown here: the arms hold `HyperLogLog` and the compact `Kll` and nothing else. `CountMinSketch` and `CountSketch` both hold `Vector2D` on the `FastPath`, so their descriptors carry `counter_type = "i32"` and `mode = "fast"`; `i32` is never widened, since a nested sketch must decode back into the arm it was stored as (Q-CS). **Feature-gating contract (cross-language; Go must honour it).** @@ -1517,7 +1518,7 @@ Golden byte-vectors lock it. **Float fields.** Every float field is msgpack **float64** (`0xcb`). A decoder rejects an integer or a float32 in a float field, and a float in an integer field. The float fields are: - metadata: `alpha` (§3.8), `sample_rate` (§3.13), `admission_rate` (Section 2); -- HLL HIP's `hip_kxq0` / `hip_kxq1` / `hip_est` (§3.1) and the retired DDSketch `0x05 0x00`'s `sum` / `min` / `max` (§3.8); +- HLL HIP's `hip_est` and the retired `0x01 0x03`'s `hip_kxq0` / `hip_kxq1` (§3.1) and the retired DDSketch `0x05 0x00`'s `sum` / `min` / `max` (§3.8); - `counts`, `heap_counts` and `deltas` elements where `counter_type` reads `"f64"` (§3.2, §3.6, §3.7, §3.10, §3.23, §3.24, §3.25); - `items` / `values` elements where `item_type` or `cell_item_type` reads `"f64"` (§3.3, §3.9, §3.13); - `keys` elements where `key_type` or `cell_key_type` reads `"f64"` (§3.5, §3.7, §3.9, §3.10, §3.15, §3.16, §3.23, §3.24, §3.25). @@ -1635,7 +1636,7 @@ Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketc - **Q-ORDER**: a payload whose in-memory container order does not survive a rebuild **pins an emitted order**, as a cross-language contract. Heaps, arenas and hash maps all have this property, so the rule reaches Space-Saving's triples (§3.5, §3.24, §3.25), the top-k heap entries (§3.7), KMV's digests (§3.14), UniformSampling's entries (§3.13), UnivMon-Q's candidates and occurrences (§3.20), and every layer- and bucket-indexed array. Two sketches holding the same state then emit the same bytes whatever order they were built in, and a decoded sketch re-serializes byte-identically. The heap key comparator is the same everywhere: descending count, ties by heap key order (§3.5). - **Q-CAP**: **a declared capacity never sizes an allocation.** Every structure is built from the run the payload actually carries, and the declared bound is checked against that run first; a payload naming a capacity of `2^32 - 1` with two entries costs two entries. This holds for every `capacity` (§3.5, §3.24, §3.25), every `heap_size`, KMV's `k`, UniformSampling's `total_seen`, UnivMon-Q's `ordered_samples`, and Hydra's grid and counter dimensions. - **Q-KEYTYPE**: `key_type` names the **exact `OwnedDataInput` variant** and is never widened, wherever heap keys reach the wire — Space-Saving (§3.5), CountMinSketchWithHeap and CountSketchWithHeap (§3.7, §3.10), CountMinSketchWithSpaceSaving and CountSketchWithSpaceSaving (§3.24, §3.25), the UnivMon family (§3.15, §3.16) and Hydra's UnivMon counter (§3.9). A container whose keys mix variants has no single name and **fails to serialize**; `OwnedDataInput::I128` / `U128` have no msgpack integer form and are not wire types. An empty container emits `"u64"`, and a decoder rejects an empty container under any other name, so its `key_type` has one value rather than one per producer. That canonicalizes the key naming and nothing else: a Two-tier UnivMon pyramid still has more than one encoding per state, since layouts differing only in an `elephant_layers` at or above `layers` build identical layers, and an all-elephant pyramid carries its unused mouse dimensions on the wire regardless (§3.16). What holds everywhere is the round trip: a decoded container re-serializes byte-identically. -- **Q-DERIVED**: state a decoder can recompute exactly is **not carried**, and state it cannot is. Recomputed: DDSketch's `count` from its stores and `zero_count` (§3.8), CountSketchWithL2Norm's and every UnivMon layer's `l2` from its counters (§3.19; the retired `0x19 0x00`, `0x10 0x00`, `0x11 0x00` and `0x07 0x04` carried it), `ExponentialHistogram`'s `l2_mass` and `merge_norm` (§3.17), Elastic's `heavy_buckets` from the declared bucket count (§3.11), every heap's digest index and array order, and every summary a sketch answers from its own state. Carried: Elastic's `stale_copies`, UnivMon's `update_mode`, `candidate_complete` and `bucket_size`, UnivMon-Q's `ever_evicted`, `source_id` and `next_sequence`, and UniformSampling's `priorities`, `total_seen` and `rng_state`. Where a cached derived field exists in memory, the encoder rejects a value that disagrees with what the decoder would recompute. State the sketch's own readouts do not need is not carried either: DDSketch's `sum` / `min` / `max` were (`0x05 0x00`) and are now a separate exact aggregate, since nothing a DDSketch answers reads them (§3.8), and so was Bloom's `inserted` counter (`0x17 0x00`), now a separate exact count, since no membership answer reads it (§3.4). Space-Saving's `total` was carried too (`0x18 0x00`, and inside `0x1e 0x00` / `0x1f 0x00`); no Space-Saving readout reads it, so it is no longer sketch state, and a caller that needs it keeps a separate exact count (§3.5). +- **Q-DERIVED**: state a decoder can recompute exactly is **not carried**, and state it cannot is. Recomputed: DDSketch's `count` from its stores and `zero_count` (§3.8), HLL HIP's `kxq0` / `kxq1` from its registers (§3.1), CountSketchWithL2Norm's and every UnivMon layer's `l2` from its counters (§3.19; the retired `0x19 0x00`, `0x10 0x00`, `0x11 0x00` and `0x07 0x04` carried it), `ExponentialHistogram`'s `l2_mass` and `merge_norm` (§3.17), Elastic's `heavy_buckets` from the declared bucket count (§3.11), every heap's digest index and array order, and every summary a sketch answers from its own state. Carried: Elastic's `stale_copies`, UnivMon's `update_mode`, `candidate_complete` and `bucket_size`, UnivMon-Q's `ever_evicted`, `source_id` and `next_sequence`, and UniformSampling's `priorities`, `total_seen` and `rng_state`. Where a cached derived field exists in memory and an API can set it, the encoder rejects a value that disagrees with what the decoder would recompute; HIP's `kxq0` / `kxq1` are private, kept only by `update_with_hash`, and exact by construction (§3.1), so there is nothing to reject. State the sketch's own readouts do not need is not carried either: DDSketch's `sum` / `min` / `max` were (`0x05 0x00`) and are now a separate exact aggregate, since nothing a DDSketch answers reads them (§3.8), and so was Bloom's `inserted` counter (`0x17 0x00`), now a separate exact count, since no membership answer reads it (§3.4). Space-Saving's `total` was carried too (`0x18 0x00`, and inside `0x1e 0x00` / `0x1f 0x00`); no Space-Saving readout reads it, so it is no longer sketch state, and a caller that needs it keeps a separate exact count (§3.5). - **Q-NIL**: an absent slot is msgpack **nil** (`0xc0`), never a sentinel value. An empty string is a reachable key in Coco (§3.12) and a reachable flow id in Elastic (§3.11), so `""` cannot double as "free"; UnivMon-Q's absent extrema are nil for the same reason (§3.20). Go must model these as nullable, never as a zero value standing in for absent. - **Q-RNG**: RNG state is carried **resumable** when the generator is cheap to reproduce in another language, and opaque otherwise. UniformSampling's `rng_state` is one SplitMix64 word with published constants, so a decoded sampler continues the same draw sequence (§3.13); KLL's `coin` carries the compaction RNG's position and its `compaction_seed` is opaque state Go preserves without interpreting (Q-KLL-SEED). - **Q-UNIVMON-SCOPE**: every layer of `0x10 0x01` and `0x11 0x01` (and of the retired `0x10 0x00` / `0x11 0x00`) is a CountSketchWithL2Norm with no tag, and another per-layer counter needs a new structural param or a new id. `UnivMonPool` is a `Vec` free list of scratch `UnivMon`s — not of pyramids — with nothing an observer could query, so it is not a wire kind and holds no id. diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index ecdda52f..caa7ee0b 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -83,7 +83,7 @@ beside `wire.rs`, declared `pub(crate)` so the other user can import it by path: Converted: - **HLL** (`src/sketches/hll/wire.rs`) — all variants (Classic → `0x01 0x01`, - Ertl-MLE → `0x01 0x02`, HIP → `0x01 0x03`) × precisions (P12/P14/P16) × + Ertl-MLE → `0x01 0x02`, HIP → `0x01 0x04`) × precisions (P12/P14/P16) × `H: HashProfile`. HLL is **fully wire-covered**. (HIP is a non-generic struct hashed through the default functions, so it is wire-eligible under the standard profile only.) diff --git a/docs/tests.md b/docs/tests.md index 66258e1b..2272d958 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -132,8 +132,11 @@ Delta tests: [`src/sketches/hll/delta.rs`](../src/sketches/hll/delta.rs) | `hllds_p12_round_trip_serialization` | P12 HIP HyperLogLog round trip preserves bytes and estimate stability. | Applies the same `100_000`-value serialization round-trip checks for `HyperLogLogHipP12`: non-empty bytes, byte-for-byte reserialization equality, and bounded estimate drift. | | `hll_correctness_test` | Register update logic matches expected bucket/index behavior for all HLL variants. | Runs fixed hashed inserts against Classic, Ertl, and HIP variants; asserts exact expected register values at specific bucket indices and confirms an untouched bucket remains zero. | | `hll_envelope_structure_and_kind_id_guard` | The envelope frames an Ertl-MLE sketch under kind_id `0x01 0x02`, and a Classic decoder refuses it. | For 1,000 inserts into `HyperLogLog`, verifies the bytes open with the ASAPv1 magic, `envelope::VERSION`, a `kind_id_len` of `2`, and `0x01 0x02`, that the decoded registers match the source, and that a `HyperLogLog` decode fails. | -| `hll_hip_round_trip_preserves_state` | The HIP running scalars travel beside the registers. | For 1,000 inserts into `HyperLogLogHip`, verifies the bytes carry kind_id `0x01 0x03` and that the decoded sketch matches the source on the registers and on `kxq0`, `kxq1`, and `est`. | -| `known_state_matches_golden` | Known stores serialize to the golden bytes. | Builds the positive and signed `alpha = 0.01` states through `DdSketchState` and verifies they serialize to `ddsketch_buckets_positive_a001` and `ddsketch_buckets_signed_a001`. | +| `hll_hip_round_trip_preserves_state` | The HIP running estimate travels beside the registers, and the derived sums come back bit for bit. | For 1,000 inserts into `HyperLogLogHip`, verifies the bytes carry kind_id `0x01 0x04` and that the decoded sketch matches the source on the registers and, bit for bit, on `kxq0`, `kxq1`, and `est`. | +| `known_state_matches_golden` | Known register and HIP state serializes to the `asapv1_golden/` bytes and decodes back to it; the retired HIP fixtures decode with `kxq0` / `kxq1` recomputed. | Builds P12 `Classic` and `Ertl` sketches with registers `0 -> 1`, `1 -> 7`, `100 -> 42`, `4095 -> 3`, and `HyperLogLogHipImpl` sketches at P12 (`est` `3.0`) and P14 (registers `0 -> 1`, `1 -> 7`, `8192 -> 42`, `16383 -> 51`; `est` `4.125`), verifies each `to_asapv1` equals `hll_classic_p12`, `hll_ertl_mle_p12`, `hll_hip_est_p12` and `hll_hip_est_p14`, and that decoding each golden gives back those registers and estimates. Decoding the retired `hll_hip_p12` / `hll_hip_p14` (`0x01 0x03`) gives the same registers and estimates, with `kxq0` / `kxq1` equal to the sums of `2^-r` over the registers rather than the fixtures' `1.5` / `2.5` and `16380.5` / `0.25`. | +| `hip_encodes_registers_and_estimate_only` | The HIP payload is the register bin and `hip_est`, nothing else. | For 500 inserts into `HyperLogLogHipP12`, verifies the kind_id is `0x01 0x04` and the payload decodes as a two-element `(bin, f64)` array holding the registers and the estimate. | +| `retired_hip_kind_decodes_with_kxq_recomputed` | The retired `0x01 0x03` payload still decodes, its `kxq0` / `kxq1` dropped. | Decodes `asapv1_golden/hll_hip_p12.hex` and verifies `kxq0` / `kxq1` equal the sums over the registers (split at rank 32), `est` is `3.0`, and the sketch re-encodes under `0x01 0x04`. | +| `hip_decode_then_continue_matches_an_unserialized_sketch` | Recomputing `kxq0` / `kxq1` on decode changes nothing downstream: a decoded HIP sketch keeps counting exactly as one that never left memory. | At P12, P14 and P16, over SplitMix64 hash streams in which every 7th hash is placed at a rank of 32 or more (so `kxq1` is non-zero), encodes after 0, 1,000 or 40,000 updates, decodes, feeds both sketches the same next 3,000 or 40,000 hashes, and verifies registers, `kxq0`, `kxq1` and `est` are bit-identical, and that the incremental `kxq0` / `kxq1` equal the sums over the registers. | | `hll_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a P14 `Ertl` sketch over a hasher declaring its own `HashProfile`, verifies the registers round-trip, the bytes differ from the standard-profile sketch's over the same 1,000 inserts, and a standard-profile decode rejects them. | | `hll_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the eight `HllMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HllMetadata`. | | `hll_precision_cross_rejection` | The metadata `precision` pins the register storage the bytes belong to. | Verifies a populated `HyperLogLogP12` envelope fails to decode as the P14 `HyperLogLog`. | @@ -257,7 +260,7 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | `merge_delta_adds_the_zero_count` | A zero-count delta adds to the zero count and the total. | Merges a count of `2` at bucket `10`, then a zero-count delta of `3`, and verifies the zero count is `3`, the total `5`, and `q = 0` reads `0.0`. | | `merge_delta_rejects_without_changing_the_state` | A rejected delta leaves the sketch unchanged. | Verifies a delta whose `alpha` is one ulp above `0.01` fails on an `alpha = 0.01` sketch, and that on a sketch holding `u64::MAX - 1` samples a delta whose negative count overflows the total and a delta whose `zero_count` overflows it each fail, leaving the count and the serialized bytes as they were. | | `merge_delta_rejects_a_store_span_past_i32` | A delta that would grow a store past `i32` fails instead of overflowing. | Verifies a delta at index `-10` onto a decoded state whose positive store is `[1]` at offset `i32::MAX - 1` fails, leaving the serialized bytes as they were. | -| `known_state_matches_golden` | Known stores and scalars serialize to the golden bytes. | Builds the positive and signed `alpha = 0.01` states through `DdSketchState` and verifies `to_asapv1` equals `asapv1_golden/ddsketch_positive_a001.hex` and `ddsketch_signed_a001.hex`. | +| `known_state_matches_golden` | Known stores serialize to the golden bytes. | Builds the positive and signed `alpha = 0.01` states through `DdSketchState` and verifies they serialize to `ddsketch_buckets_positive_a001` and `ddsketch_buckets_signed_a001`. | ### CountMinSketchWithHeap @@ -1165,7 +1168,7 @@ Wire tests: [`src/sketch_framework/eh_bucket_sketch/wire.rs`](../src/sketch_fram | `eh_bucket_sketch_decodes_retired_l2_carrying_kind_ids` | Buckets nested under the retired `0x19 0x00` and `0x10 0x00` decode into their variants. | Nests the retired `count_l2hh_2x4_seed7` and `univmon_str_l3_2x4_h5` golden blocks under their own ids, verifies each decodes and re-encodes under `0x19 0x01` / `0x10 0x01`, and that the retired ids share their current ids' names. | | `eh_bucket_sketch_rejects_an_experimental_kind_id` | An experimental variant's kind_id is refused without the feature. | Verifies crafted triples carrying the `CocoSketch`, `ElasticSketch`, and `UniformSampling` kind_ids each fail to decode, with a message naming the variant and "experimental" in builds without the feature. | | `eh_bucket_sketch_rejects_an_unknown_kind_id` | An unknown kind_id is refused. | Verifies a Count-Min triple relabelled `0xff 0xff` fails with a "not a wire variant" complaint. | -| `eh_bucket_sketch_rejects_sibling_algorithm_kind_ids` | The nested ids are pinned to one algorithm each. | Verifies an HLL triple relabelled `0x01 0x01` or `0x01 0x03` fails to decode, and that a KLL triple relabelled `0x06 0x01` fails too. | +| `eh_bucket_sketch_rejects_sibling_algorithm_kind_ids` | The nested ids are pinned to one algorithm each. | Verifies an HLL triple relabelled `0x01 0x01`, `0x01 0x03` or `0x01 0x04` fails to decode, and that a KLL triple relabelled `0x06 0x01` fails too. | | `eh_bucket_sketch_rejects_a_mismatched_kind_id_and_descriptor` | A kind_id that does not match the blocks it carries is refused by the variant's own decoder. | Verifies an HLL triple relabelled with the Count-Min kind_id fails to decode. | | `eh_bucket_sketch_rejects_foreign_kind_ids` | A Count-Min envelope and an `ExponentialHistogram` envelope are not union envelopes. | Verifies both fail to decode as an `EhBucketSketch`. | | `eh_bucket_sketch_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes `metadata_version` plus a `bogus_field` as a named map and verifies it does not decode as `EhSketchListMetadata`. | @@ -2131,7 +2134,8 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `hll_classic_p14_delta_matches_golden` | A known P14 Classic delta serializes to its golden bytes and merges by `max`. | Verifies `HyperLogLogDelta::::new` over indices `[0, 2, 200, 8192, 16383]` and registers `[51, 1, 9, 7, 1]` serializes to `asapv1_golden/hll_classic_p14_delta.hex`, that the decoded fixture re-serializes identically, and that merged into `hll_classic_p14` it gives registers `[0] = 51`, `[1] = 7`, `[2] = 1`, `[200] = 9`, `[8192] = 42`, `[16383] = 51` and zero elsewhere. | | `hll_ertl_mle_p12_matches_golden` | The P12 Ertl-MLE golden decodes to the same register pattern and re-serializes identically. | Verifies `HyperLogLogP12::::from_asapv1` on `asapv1_golden/hll_ertl_mle_p12.hex` gives the shared P12 pattern and that `to_asapv1` reproduces the fixture. | | `hll_ertl_mle_p14_delta_matches_golden` | The same delta under the Ertl-MLE kind_id. | Runs the `hll_classic_p14_delta_matches_golden` checks for `Ertl` against `hll_ertl_mle_p14_delta` and `hll_ertl_mle_p14`. | -| `hll_hip_p12_matches_golden` | The P12 HIP golden round-trips. | Verifies `HyperLogLogHipP12::from_asapv1` decodes `asapv1_golden/hll_hip_p12.hex` and `to_asapv1` reproduces it. | +| `hll_hip_p12_matches_golden` | The P12 HIP golden round-trips. | Verifies `HyperLogLogHipP12::from_asapv1` decodes `asapv1_golden/hll_hip_est_p12.hex` (`0x01 0x04`) and `to_asapv1` reproduces it. | +| `hll_hip_retired_fixtures_decode_to_their_est_twins` | The retired HIP fixtures still decode and re-encode under the current kind. | Verifies `hll_hip_p12` and `hll_hip_p14` (`0x01 0x03`) decode and re-encode as `hll_hip_est_p12` and `hll_hip_est_p14`. | | `kll_f64_k200_matches_golden` | A deterministic f64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `Kll::::with_k_and_seed(200, 42)` fed `1..=50` (below the level-0 capacity, so no compaction fires) serializes to `asapv1_golden/kll_f64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | | `kll_i64_k200_matches_golden` | A deterministic i64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `Kll::::with_k_and_seed(200, 42)` fed `1..=50` serializes to `asapv1_golden/kll_i64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | | `kmv_k8_matches_golden` | A KMV of known digests serializes to its golden bytes. | Verifies `Kmv::new(8)` fed the digests `[1, 127, 128, 65535, 4294967296, 0xfedcba9876543210]` out of order through `update_with_hash` serializes to `asapv1_golden/kmv_k8.hex`, and that the decode holds `k` `8` and those digests and re-serializes identically. | @@ -2181,7 +2185,7 @@ Test file: [`tests/hll_custom_precision.rs`](../tests/hll_custom_precision.rs) | --- | --- | --- | | `custom_precision_indexing_and_iteration` | A macro-generated register storage indexes, slices and iterates. | On an `HllBucketListP10` built by `impl_hll_registers!` outside the crate, sets index `0` to `7` and index `NUM_REGISTERS - 1` to `3`, verifies both read back, that `storage[0..2]` is `[7, 0]`, that `copy_from_slice` of `[1, 2]` into `1..3` lands in `as_slice()`, and that iterating the whole storage sums to `13`. | | `custom_precision_merges` | A merge at a custom precision estimates the union. | Fills `HyperLogLogImpl` with items `key-{i:08}` for `i` in `0..60_000` and a second with `40_000..100_000`, merges them, and verifies the estimate is within 2% relative of the 100,000-item union. | -| `custom_precision_round_trips_through_the_asapv1_wire_format` | The ASAPv1 envelope carries a custom precision and refuses a mismatched register count. | Over 5,000 distinct `key-{i:08}` items at `lg_k = 13`, verifies `to_asapv1` / `from_asapv1` preserves the registers and `estimate()` for both `Classic` and `Ertl`, preserves `estimate()` for `HyperLogLogHipImpl` so that its running `kxq0`, `kxq1` and `est` travel rather than being recomputed, and that a fresh `lg_k = 18` envelope fails to decode as `HllBucketListP13`. | +| `custom_precision_round_trips_through_the_asapv1_wire_format` | The ASAPv1 envelope carries a custom precision and refuses a mismatched register count. | Over 5,000 distinct `key-{i:08}` items at `lg_k = 13`, verifies `to_asapv1` / `from_asapv1` preserves the registers and `estimate()` for both `Classic` and `Ertl`, preserves `estimate()` for `HyperLogLogHipImpl` and, after 2,000 more inserts into both, still matches it (so the recomputed `kxq0` / `kxq1` count as the originals do), and that a fresh `lg_k = 18` envelope fails to decode as `HllBucketListP13`. | | `custom_precisions_estimate_correctly` | Every macro-generated precision reports the right constants and estimates in band. | Through a shared `check_precision` helper, verifies for each precision that `PRECISION` is `lg_k`, `NUM_REGISTERS` is `1 << lg_k`, `RANK_HASH_BITS` is `64 - lg_k` and `INDEX_MASK` is `NUM_REGISTERS - 1`, that a default storage has that length, is non-empty and is zeroed, that at least one register is set after the inserts and none exceeds `RANK_HASH_BITS + 1`, and that the `Classic`, `Ertl` and HIP estimates all fall inside the tolerance; invoked at `lg_k` `4` with 100 items at `0.60`, `8` with 1,000 at `0.30`, `10` with 10,000 at `0.15`, `13` with 50,000 at `0.06` and `18` with 200,000 at `0.02`. | | `large_precision_allocates_on_the_heap` | Register storage is allocated on the heap, so a 4 MiB precision constructs on a test thread. | Verifies a fresh `HyperLogLogImpl` — `1 << 22` registers, past the 2 MiB test-thread stack — reports `NUM_REGISTERS` registers that are all zero, and that an `HllBucketListP18` sketch over 1,000 `key-{i:08}` items survives an ASAPv1 round trip with an identical register slice. | diff --git a/src/asapv1/envelope.rs b/src/asapv1/envelope.rs index 56202422..597a8ee0 100644 --- a/src/asapv1/envelope.rs +++ b/src/asapv1/envelope.rs @@ -209,7 +209,8 @@ mod tests { type HllClassic = HyperLogLogP12; let classic = hex(include_str!("../../asapv1_golden/hll_classic_p12.hex")); - let hip = hex(include_str!("../../asapv1_golden/hll_hip_p12.hex")); + let hip = hex(include_str!("../../asapv1_golden/hll_hip_est_p12.hex")); + let hip_with_kxq = hex(include_str!("../../asapv1_golden/hll_hip_p12.hex")); let classic_delta = hex(include_str!( "../../asapv1_golden/hll_classic_p14_delta.hex" )); @@ -302,6 +303,7 @@ mod tests { let cases: Vec = vec![ case!("HLL classic", classic, HllClassic), case!("HLL HIP", hip, HyperLogLogHipP12), + case!("HLL HIP retired 01 03", hip_with_kxq, HyperLogLogHipP12), case!("HLL classic delta", classic_delta, HyperLogLogDelta), case!("CMS", cms, Cms), case!("CMS delta", cms_delta, CountMinSketchDelta), diff --git a/src/asapv1/kind.rs b/src/asapv1/kind.rs index f98c66c0..6da45c61 100644 --- a/src/asapv1/kind.rs +++ b/src/asapv1/kind.rs @@ -3,7 +3,11 @@ pub(crate) const HLL_FAMILY: u8 = 0x01; pub(crate) const HLL_CLASSIC: &[u8] = &[HLL_FAMILY, 0x01]; pub(crate) const HLL_ERTL: &[u8] = &[HLL_FAMILY, 0x02]; -pub(crate) const HLL_HIP: &[u8] = &[HLL_FAMILY, 0x03]; +/// HLL HIP: the registers and the running estimate (spec §3.1). +pub(crate) const HLL_HIP: &[u8] = &[HLL_FAMILY, 0x04]; +/// Retired HLL HIP kind that also carried `kxq0` / `kxq1`: decoded, with +/// those two recomputed from the registers, and never encoded. +pub(crate) const HLL_HIP_WITH_KXQ: &[u8] = &[HLL_FAMILY, 0x03]; pub(crate) const HLL_CLASSIC_DELTA: &[u8] = &[HLL_FAMILY, 0x81]; pub(crate) const HLL_ERTL_DELTA: &[u8] = &[HLL_FAMILY, 0x82]; pub(crate) const COUNT_MIN: &[u8] = &[0x02, 0x00]; diff --git a/src/asapv1/wire_key.rs b/src/asapv1/wire_key.rs index 46532450..0af3dbad 100644 --- a/src/asapv1/wire_key.rs +++ b/src/asapv1/wire_key.rs @@ -356,7 +356,8 @@ mod tests { .expect("arity"); let dd = &positive.to_asapv1().unwrap(); - let hip = &hex(include_str!("../../asapv1_golden/hll_hip_p12.hex")); + let hip = &hex(include_str!("../../asapv1_golden/hll_hip_est_p12.hex")); + let hip_with_kxq = &hex(include_str!("../../asapv1_golden/hll_hip_p12.hex")); let cms = &hex(include_str!("../../asapv1_golden/cms_f64_fast_2x3.hex")); let cms_delta = &hex(include_str!( "../../asapv1_golden/cms_f64_fast_2x3_delta.hex" @@ -389,9 +390,11 @@ mod tests { let rows = vec![ row("DDSketch alpha", dd, 0.25, F32, dd_ok), - row("HLL hip_kxq0", hip, 1.5, F32, hip_ok), - row("HLL hip_kxq1", hip, 2.5, F32, hip_ok), row("HLL hip_est", hip, 3.0, Int, hip_ok), + row("HLL hip_est", hip, 3.0, F32, hip_ok), + row("HLL retired hip_kxq0", hip_with_kxq, 1.5, F32, hip_ok), + row("HLL retired hip_kxq1", hip_with_kxq, 2.5, F32, hip_ok), + row("HLL retired hip_est", hip_with_kxq, 3.0, Int, hip_ok), row("CMS f64 count", cms, 1.5, F32, cms_ok), row("CMS f64 delta", cms_delta, 2.5, Int, delta_ok), row("CS f64 count", cs, -2.25, F32, cs_ok), diff --git a/src/sketch_framework/eh_bucket_sketch/wire.rs b/src/sketch_framework/eh_bucket_sketch/wire.rs index 68f61b47..ef96d1ca 100644 --- a/src/sketch_framework/eh_bucket_sketch/wire.rs +++ b/src/sketch_framework/eh_bucket_sketch/wire.rs @@ -551,7 +551,7 @@ pub(crate) mod tests { #[test] fn eh_bucket_sketch_rejects_sibling_algorithm_kind_ids() { let hll = EhBucketSketch::Hll(HyperLogLog::::default()); - for sibling in [[0x01u8, 0x01], [0x01, 0x03]] { + for sibling in [[0x01u8, 0x01], [0x01, 0x03], [0x01, 0x04]] { let triple = relabelled(&hll, &sibling); assert!( EhBucketSketch::from_asapv1(&envelope_for(&triple)).is_err(), diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 4d6f5695..e725579f 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -95,6 +95,14 @@ pub(crate) fn classic_estimate(registers: &[u8]) -> f64 { est } +/// `2^-exponent`, built from its bits so it is exact by construction: biased +/// exponent `1023 - exponent`, zero mantissa. A register holds at most +/// `65 - p` (at most 64), well inside the normal range. +#[inline] +fn inverse_power_of_two(exponent: u8) -> f64 { + f64::from_bits((1023 - u64::from(exponent)) << 52) +} + /// Marker type selecting the classic HyperLogLog estimation algorithm. #[derive(Clone, Copy, Debug, Default)] pub struct Classic; @@ -106,12 +114,41 @@ pub struct Ertl; #[derive(Clone, Debug)] pub struct HyperLogLogHipImpl { registers: Registers, + /// `sum(2^-r)` over the registers `r` below 32. Derived from the + /// registers ([`hip_kxq`]) and kept so an update costs O(1). kxq0: f64, + /// `sum(2^-r)` over the registers `r` of 32 or more; see `kxq0`. kxq1: f64, + /// The running HIP estimate. It depends on the arrival order, so it is + /// state, not derivable from the registers. est: f64, admission_rate: f64, } +/// `(kxq0, kxq1)` for a register slice: the sums of `2^-r` over the registers +/// `r` below 32 and over those of 32 or more, the split +/// `HyperLogLogHipImpl::update_with_hash` keeps them in. +/// +/// Every term is a power of two, so every partial sum, and every step of the +/// incremental subtract-then-add, is a multiple of the smallest term. `kxq0` +/// (terms `2^0..2^-31`, sum at most `2^p`) is then exact for `p <= 22`, and +/// `kxq1` (terms `2^-32..2^-(65 - p)`, sum at most `2^(p - 32)`) at every +/// precision. Within that bound this equals the incrementally kept values bit +/// for bit, whatever the order. +fn hip_kxq(registers: &[u8]) -> (f64, f64) { + let mut kxq0 = 0.0; + let mut kxq1 = 0.0; + for ®ister in registers { + let term = inverse_power_of_two(register); + if register < 32 { + kxq0 += term; + } else { + kxq1 += term; + } + } + (kxq0, kxq1) +} + /// HyperLogLog with 12-bit precision. pub type HyperLogLogP12 = HyperLogLogImpl; @@ -411,14 +448,14 @@ impl HyperLogLogHipImpl { registers[bucket_num] = leading_zero; self.est += Registers::NUM_REGISTERS as f64 / (self.kxq0 + self.kxq1); if old_value < 32 { - self.kxq0 -= 1.0 / ((1_u64 << old_value) as f64); + self.kxq0 -= inverse_power_of_two(old_value); } else { - self.kxq1 -= 1.0 / ((1_u64 << old_value) as f64); + self.kxq1 -= inverse_power_of_two(old_value); } if new_value < 32 { - self.kxq0 += 1.0 / ((1_u64 << new_value) as f64); + self.kxq0 += inverse_power_of_two(new_value); } else { - self.kxq1 += 1.0 / ((1_u64 << new_value) as f64); + self.kxq1 += inverse_power_of_two(new_value); } } } diff --git a/src/sketches/hll/wire.rs b/src/sketches/hll/wire.rs index cbe6ecb8..294ad9ee 100644 --- a/src/sketches/hll/wire.rs +++ b/src/sketches/hll/wire.rs @@ -49,9 +49,20 @@ pub(crate) struct HllPayloadPlain { pub(crate) registers: Vec, } -/// HIP payload: the register bin plus the three HIP running scalars. +/// HIP payload (`01 04`): the register bin plus the running HIP estimate. +/// `kxq0` / `kxq1` are derived from the registers and not carried. #[derive(Debug, Serialize, Deserialize)] pub(crate) struct HllPayloadHip { + #[serde(with = "serde_bytes")] + pub(crate) registers: Vec, + #[serde(deserialize_with = "exact")] + pub(crate) hip_est: f64, +} + +/// The retired HIP payload (`01 03`), decode-only: the register bin, then +/// `kxq0`, `kxq1` and the estimate. The two sums are read and dropped. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct HllPayloadHipWithKxq { #[serde(with = "serde_bytes")] pub(crate) registers: Vec, #[serde(deserialize_with = "exact")] @@ -253,27 +264,41 @@ impl HyperLogLogHipImpl { let metadata = encode_metadata::(self.admission_rate)?; let payload = rmp_serde::to_vec(&HllPayloadHip { registers, - hip_kxq0: self.kxq0, - hip_kxq1: self.kxq1, hip_est: self.est, })?; Ok(envelope::encode(kind::HLL_HIP, &metadata, &payload)) } - /// Deserializes a sketch from an ASAPv1 MessagePack envelope. + /// Deserializes a sketch from an ASAPv1 MessagePack envelope: the current + /// `01 04`, or the retired `01 03`, whose `kxq0` / `kxq1` are dropped. + /// Both are recomputed from the registers either way. pub fn from_asapv1(bytes: &[u8]) -> Result { + let (stored_kind, _, _) = envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; + let retired = stored_kind == kind::HLL_HIP_WITH_KXQ; + let expected_kind = if retired { + kind::HLL_HIP_WITH_KXQ + } else { + kind::HLL_HIP + }; let (payload, admission_rate) = validated_hll_payload::( bytes, - kind::HLL_HIP, + expected_kind, Registers::PRECISION as u32, )?; - let payload: HllPayloadHip = envelope::decode_block(payload)?; - let registers = registers_from_bytes::(&payload.registers)?; + let (registers, est) = if retired { + let payload: HllPayloadHipWithKxq = envelope::decode_block(payload)?; + (payload.registers, payload.hip_est) + } else { + let payload: HllPayloadHip = envelope::decode_block(payload)?; + (payload.registers, payload.hip_est) + }; + let registers = registers_from_bytes::(®isters)?; + let (kxq0, kxq1) = super::hip_kxq(registers.as_slice()); Ok(Self { registers, - kxq0: payload.hip_kxq0, - kxq1: payload.hip_kxq1, - est: payload.hip_est, + kxq0, + kxq1, + est, admission_rate, }) } @@ -456,14 +481,11 @@ mod tests { admission_rate: 1.0, } } - fn hip( - regs: &[u8], - kxq0: f64, - kxq1: f64, - est: f64, - ) -> HyperLogLogHipImpl { + fn hip(regs: &[u8], est: f64) -> HyperLogLogHipImpl { + let registers: R = registers_from_bytes(regs).expect("registers"); + let (kxq0, kxq1) = super::super::hip_kxq(registers.as_slice()); HyperLogLogHipImpl { - registers: registers_from_bytes(regs).expect("registers"), + registers, kxq0, kxq1, est, @@ -492,41 +514,49 @@ mod tests { let decoded = HyperLogLogImpl::::from_asapv1(&ertl).unwrap(); assert_eq!(decoded.registers.as_slice(), p12.as_slice()); - for (golden, regs, state) in [ - ( - hex(include_str!("../../../asapv1_golden/hll_hip_p12.hex")), - &p12, - (1.5, 2.5, 3.0), - ), - ( - hex(include_str!("../../../asapv1_golden/hll_hip_p14.hex")), - &p14, - (16380.5, 0.25, 4.125), - ), - ] { - let (kxq0, kxq1, est) = state; - let encoded = if regs.len() == 4096 { - hip::(regs, kxq0, kxq1, est) - .to_asapv1() - .unwrap() - } else { - hip::(regs, kxq0, kxq1, est) - .to_asapv1() - .unwrap() - }; - assert_eq!(encoded, golden); - } - let hip_p14 = hex(include_str!("../../../asapv1_golden/hll_hip_p14.hex")); + // The current `01 04` fixtures: the registers and the estimate. + let hip_p12 = hex(include_str!("../../../asapv1_golden/hll_hip_est_p12.hex")); + let hip_p14 = hex(include_str!("../../../asapv1_golden/hll_hip_est_p14.hex")); + assert_eq!( + hip::(&p12, 3.0).to_asapv1().unwrap(), + hip_p12 + ); + assert_eq!( + hip::(&p14, 4.125).to_asapv1().unwrap(), + hip_p14 + ); + let decoded = HyperLogLogHipImpl::::from_asapv1(&hip_p12).unwrap(); + assert_eq!(decoded.registers.as_slice(), p12.as_slice()); + assert_eq!(decoded.est, 3.0); let decoded = HyperLogLogHipImpl::::from_asapv1(&hip_p14).unwrap(); assert_eq!(decoded.registers.as_slice(), p14.as_slice()); + assert_eq!(decoded.est, 4.125); + + // The retired `01 03` fixtures carry `kxq0` / `kxq1` values no + // register state produces; decoding drops them for the sums over the + // registers and keeps the estimate. + let retired = hex(include_str!("../../../asapv1_golden/hll_hip_p12.hex")); + let decoded = HyperLogLogHipImpl::::from_asapv1(&retired).unwrap(); + assert_eq!(decoded.registers.as_slice(), p12.as_slice()); assert_eq!( (decoded.kxq0, decoded.kxq1, decoded.est), - (16380.5, 0.25, 4.125) + ( + 4092.0 + 0.5 + 2f64.powi(-7) + 2f64.powi(-3), + 2f64.powi(-42), + 3.0 + ) + ); + let retired = hex(include_str!("../../../asapv1_golden/hll_hip_p14.hex")); + let decoded = HyperLogLogHipImpl::::from_asapv1(&retired).unwrap(); + assert_eq!(decoded.registers.as_slice(), p14.as_slice()); + assert_eq!( + (decoded.kxq0, decoded.kxq1, decoded.est), + ( + 16380.0 + 0.5 + 2f64.powi(-7), + 2f64.powi(-42) + 2f64.powi(-51), + 4.125 + ) ); - let hip_p12 = hex(include_str!("../../../asapv1_golden/hll_hip_p12.hex")); - let decoded = HyperLogLogHipImpl::::from_asapv1(&hip_p12).unwrap(); - assert_eq!(decoded.registers.as_slice(), p12.as_slice()); - assert_eq!((decoded.kxq0, decoded.kxq1, decoded.est), (1.5, 2.5, 3.0)); } #[test] @@ -563,9 +593,9 @@ mod tests { let decoded = HyperLogLogHip::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.registers.as_slice(), sketch.registers.as_slice()); - assert_eq!(decoded.kxq0, sketch.kxq0); - assert_eq!(decoded.kxq1, sketch.kxq1); - assert_eq!(decoded.est, sketch.est); + assert_eq!(decoded.kxq0.to_bits(), sketch.kxq0.to_bits()); + assert_eq!(decoded.kxq1.to_bits(), sketch.kxq1.to_bits()); + assert_eq!(decoded.est.to_bits(), sketch.est.to_bits()); } // A test-only custom hasher: it hashes exactly like `DefaultXxHasher` @@ -758,21 +788,28 @@ mod tests { // The HIP payload carries the same bin and is held to the same bound. let metadata = rmp_serde::to_vec_named(&standard_hll_metadata(14)).expect("meta"); - let payload = rmp_serde::to_vec(&HllPayloadHip { + let current = rmp_serde::to_vec(&HllPayloadHip { + registers: bin(p14, P14_MAX + 1), + hip_est: 0.0, + }) + .expect("payload"); + let retired = rmp_serde::to_vec(&HllPayloadHipWithKxq { registers: bin(p14, P14_MAX + 1), hip_kxq0: p14 as f64, hip_kxq1: 0.0, hip_est: 0.0, }) .expect("payload"); - let over = envelope::encode(kind::HLL_HIP, &metadata, &payload); - let problem = HyperLogLogHip::from_asapv1(&over) - .expect_err("a register past the maximum rank must be rejected") - .to_string(); - assert!( - problem.contains("exceeds the maximum rank 51"), - "got {problem}" - ); + for (kind_id, payload) in [(kind::HLL_HIP, current), (kind::HLL_HIP_WITH_KXQ, retired)] { + let over = envelope::encode(kind_id, &metadata, &payload); + let problem = HyperLogLogHip::from_asapv1(&over) + .expect_err("a register past the maximum rank must be rejected") + .to_string(); + assert!( + problem.contains("exceeds the maximum rank 51"), + "got {problem}" + ); + } } /// The encode path holds the same bound: `apply_delta` takes an arbitrary @@ -855,4 +892,151 @@ mod tests { "HIP bytes must be rejected by a Classic decoder" ); } + + /// `(kxq0, kxq1)` summed from scratch over `registers`: `2^-r` for each + /// register `r`, split at `r < 32` exactly as `update_with_hash` splits it. + fn summed_kxq(registers: &[u8]) -> (f64, f64) { + let low: f64 = registers + .iter() + .filter(|&&r| r < 32) + .map(|&r| 1.0 / ((1_u64 << r) as f64)) + .sum(); + let high: f64 = registers + .iter() + .filter(|&&r| r >= 32) + .map(|&r| 1.0 / ((1_u64 << r) as f64)) + .sum(); + (low, high) + } + + /// The HIP payload: the register bin, then the running estimate, and + /// nothing else. `kxq0` / `kxq1` are derived from the registers and are + /// not carried. + #[test] + fn hip_encodes_registers_and_estimate_only() { + let mut sketch = HyperLogLogHipP12::new(); + for value in 0..500 { + sketch.update(&DataInput::U64(value)); + } + let bytes = sketch.to_asapv1().expect("serialize"); + let (kind_id, _, payload) = envelope::split(&bytes).expect("split"); + assert_eq!(kind_id, [0x01, 0x04]); + let (registers, est): (serde_bytes::ByteBuf, f64) = + rmp_serde::from_slice(payload).expect("two-field payload"); + assert_eq!(registers.as_slice(), sketch.registers.as_slice()); + assert_eq!(est.to_bits(), sketch.est.to_bits()); + } + + /// The retired `01 03` payload still decodes: its `kxq0` / `kxq1` are read + /// and dropped, recomputed from the registers, and the result re-encodes + /// under the current kind. The golden carries `kxq0 = 1.5, kxq1 = 2.5`, + /// which no register state produces, so a kept value would show. + #[test] + fn retired_hip_kind_decodes_with_kxq_recomputed() { + let retired = envelope::hex(include_str!("../../../asapv1_golden/hll_hip_p12.hex")); + assert_eq!(&retired[8..10], [0x01, 0x03]); + let decoded = HyperLogLogHipP12::from_asapv1(&retired).expect("decode retired"); + let (kxq0, kxq1) = summed_kxq(decoded.registers.as_slice()); + assert_eq!(decoded.kxq0.to_bits(), kxq0.to_bits()); + assert_eq!(decoded.kxq1.to_bits(), kxq1.to_bits()); + assert_eq!(decoded.est, 3.0); + let reencoded = decoded.to_asapv1().expect("re-encode"); + assert_eq!(&reencoded[8..10], [0x01, 0x04]); + } + + /// Each kind id fixes its payload length: the two-field `01 04` payload + /// under the retired id, or the four-field `01 03` payload under the + /// current one, is rejected. + #[test] + fn hip_payload_must_match_its_kind_field_count() { + let current = envelope::hex(include_str!("../../../asapv1_golden/hll_hip_est_p12.hex")); + let retired = envelope::hex(include_str!("../../../asapv1_golden/hll_hip_p12.hex")); + let (_, current_metadata, current_payload) = envelope::split(¤t).expect("split"); + let (_, retired_metadata, retired_payload) = envelope::split(&retired).expect("split"); + let two_fields_as_retired = + envelope::encode(kind::HLL_HIP_WITH_KXQ, current_metadata, current_payload); + let four_fields_as_current = + envelope::encode(kind::HLL_HIP, retired_metadata, retired_payload); + assert!(HyperLogLogHipP12::from_asapv1(&two_fields_as_retired).is_err()); + assert!(HyperLogLogHipP12::from_asapv1(&four_fields_as_current).is_err()); + } + + /// The hash that raises register `bucket` to rank `rank` (`1..=R + 1`, + /// `R` the rank hash bits). + fn hash_placing(bucket: usize, rank: u8) -> u64 { + let rank_bits = Registers::RANK_HASH_BITS as u32; + let rank_part = if u32::from(rank) > rank_bits { + 0 + } else { + 1u64 << (rank_bits - u32::from(rank)) + }; + ((bucket as u64) << rank_bits) | rank_part + } + + /// The hashes a test stream feeds: a SplitMix64 run, so most ranks are + /// small, with every 7th hash placed at a rank of 32 or more so the + /// `kxq1` side fills too. + fn hip_stream(len: usize, seed: u64) -> Vec { + let max_rank = (Registers::RANK_HASH_BITS + 1) as u64; + let mut state = seed; + (0..len) + .map(|i| { + state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^= z >> 31; + if i % 7 == 0 { + let bucket = (z as usize) % Registers::NUM_REGISTERS; + let rank = 32 + (z >> 40) % (max_rank - 31); + hash_placing::(bucket, rank as u8) + } else { + z + } + }) + .collect() + } + + /// Decoding recomputes `kxq0` / `kxq1` bit for bit, so a decoded sketch + /// that keeps updating holds the same `est` as one that never went + /// through the wire. Every term is a power of two between `2^-65` and + /// `2^0`, and each side's sum stays within 53 bits of its smallest term, + /// so the incremental and the summed values are the same `f64`. + fn assert_decode_then_continue_is_exact() { + for (seed, before, after) in [(1, 0, 3_000), (2, 1_000, 3_000), (3, 40_000, 40_000)] { + let stream = hip_stream::(before + after, seed); + let mut kept = HyperLogLogHipImpl::::new(); + kept.update_batch_with_hashes(&stream[..before]); + let bytes = kept.to_asapv1().expect("serialize"); + let mut restored = + HyperLogLogHipImpl::::from_asapv1(&bytes).expect("decode"); + assert_eq!(restored.kxq0.to_bits(), kept.kxq0.to_bits()); + assert_eq!(restored.kxq1.to_bits(), kept.kxq1.to_bits()); + kept.update_batch_with_hashes(&stream[before..]); + restored.update_batch_with_hashes(&stream[before..]); + assert!(kept.kxq1 > 0.0, "the stream reaches ranks of 32 or more"); + assert_eq!(restored.registers.as_slice(), kept.registers.as_slice()); + assert_eq!(restored.kxq0.to_bits(), kept.kxq0.to_bits()); + assert_eq!(restored.kxq1.to_bits(), kept.kxq1.to_bits()); + assert_eq!( + restored.est.to_bits(), + kept.est.to_bits(), + "p = {}, {before} then {after} updates", + Registers::PRECISION + ); + let (kxq0, kxq1) = summed_kxq(kept.registers.as_slice()); + assert_eq!( + (kept.kxq0.to_bits(), kept.kxq1.to_bits()), + (kxq0.to_bits(), kxq1.to_bits()) + ); + } + } + + #[test] + fn hip_decode_then_continue_matches_an_unserialized_sketch() { + use crate::structures::fixed_structure::HllRegistersP16; + assert_decode_then_continue_is_exact::(); + assert_decode_then_continue_is_exact::(); + assert_decode_then_continue_is_exact::(); + } } diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index c0cd2017..4490e076 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -44,8 +44,12 @@ fn decode_hex(s: &str) -> Vec { const GOLDEN_CLASSIC: &str = include_str!("../asapv1_golden/hll_classic_p12.hex"); const GOLDEN_ERTL: &str = include_str!("../asapv1_golden/hll_ertl_mle_p12.hex"); -const GOLDEN_HIP: &str = include_str!("../asapv1_golden/hll_hip_p12.hex"); -const GOLDEN_HIP_P14: &str = include_str!("../asapv1_golden/hll_hip_p14.hex"); +// The retired HIP kind `01 03`, decode-only. +const GOLDEN_HIP_WITH_KXQ: &str = include_str!("../asapv1_golden/hll_hip_p12.hex"); +const GOLDEN_HIP_WITH_KXQ_P14: &str = include_str!("../asapv1_golden/hll_hip_p14.hex"); +// The current HIP kind `01 04`: registers and estimate. +const GOLDEN_HIP: &str = include_str!("../asapv1_golden/hll_hip_est_p12.hex"); +const GOLDEN_HIP_P14: &str = include_str!("../asapv1_golden/hll_hip_est_p14.hex"); const GOLDEN_CLASSIC_P14: &str = include_str!("../asapv1_golden/hll_classic_p14.hex"); const GOLDEN_ERTL_P14: &str = include_str!("../asapv1_golden/hll_ertl_mle_p14.hex"); const GOLDEN_CLASSIC_P14_DELTA: &str = include_str!("../asapv1_golden/hll_classic_p14_delta.hex"); @@ -232,6 +236,19 @@ fn hll_hip_p14_matches_golden() { assert_eq!(native.to_asapv1().expect("re-serialize"), want); } +/// Each retired `01 03` fixture decodes, its `kxq0` / `kxq1` dropped and +/// recomputed from the registers, and re-encodes as its `hll_hip_est_*` twin. +#[test] +fn hll_hip_retired_fixtures_decode_to_their_est_twins() { + let p12 = HyperLogLogHipP12::from_asapv1(&decode_hex(GOLDEN_HIP_WITH_KXQ)).expect("decode"); + assert_eq!(p12.to_asapv1().expect("re-encode"), decode_hex(GOLDEN_HIP)); + let p14 = HyperLogLogHipP14::from_asapv1(&decode_hex(GOLDEN_HIP_WITH_KXQ_P14)).expect("decode"); + assert_eq!( + p14.to_asapv1().expect("re-encode"), + decode_hex(GOLDEN_HIP_P14) + ); +} + // --------------------------------------------------------------------------- // HLL delta: build known delta -> serialize == golden, round-trips, and merged // into the matching P14 state golden gives the expected registers. diff --git a/tests/hll_custom_precision.rs b/tests/hll_custom_precision.rs index 778b8fc0..50e05558 100644 --- a/tests/hll_custom_precision.rs +++ b/tests/hll_custom_precision.rs @@ -173,7 +173,7 @@ fn large_precision_allocates_on_the_heap() { /// The ASAPv1 envelope writes `precision` into the metadata and validates the /// register length on the way back in. -/// Covers all three variants, including HIP's extra running scalars. +/// Covers all three variants, including HIP's running estimate. #[test] fn custom_precision_round_trips_through_the_asapv1_wire_format() { let mut classic = HyperLogLogImpl::::new(); @@ -200,8 +200,16 @@ fn custom_precision_round_trips_through_the_asapv1_wire_format() { let bytes = hip.to_asapv1().expect("hip encode"); let restored = HyperLogLogHipImpl::::from_asapv1(&bytes).expect("hip decode"); - // HIP's estimate is a running scalar, so this also checks that kxq0/kxq1/est - // survived the round-trip rather than being recomputed from registers. + // HIP's estimate is a running scalar carried on the wire; kxq0/kxq1 are + // recomputed from the registers, so a restored sketch must keep counting + // exactly as the original does. + assert_eq!(restored.estimate(), hip.estimate()); + let (mut hip, mut restored) = (hip, restored); + for i in 0..2_000u64 { + let value = DataInput::String(item(5_000 + i as usize)); + hip.update(&value); + restored.update(&value); + } assert_eq!(restored.estimate(), hip.estimate()); // A payload whose register count does not match the target precision is diff --git a/tests/pbt/hll.rs b/tests/pbt/hll.rs index a91e0c04..82aef969 100644 --- a/tests/pbt/hll.rs +++ b/tests/pbt/hll.rs @@ -442,18 +442,18 @@ macro_rules! hip_laws { const M: usize = <$buckets>::NUM_REGISTERS; const P: u32 = <$buckets>::PRECISION as u32; - /// The registers, read back from the ASAPv1 payload. The struct - /// holds them privately behind the `kxq0`/`kxq1` accumulators, and - /// those accumulators are what the increment law checks, so the - /// envelope is the one view of the registers that stays independent - /// of it. + /// The registers, read back from the ASAPv1 payload (the + /// register bin, then `hip_est`). The struct holds them privately + /// behind the `kxq0`/`kxq1` accumulators, and those accumulators + /// are what the increment law checks, so the envelope is the one + /// view of the registers that stays independent of it. fn registers_of(sketch: &S) -> Vec { let bytes = sketch.to_asapv1().expect("encode"); let at = 8 + bytes[7] as usize; let metadata_len = u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) as usize; let payload = &bytes[at + 8 + metadata_len..]; - let (registers, _, _, _): (serde_bytes::ByteBuf, f64, f64, f64) = + let (registers, _): (serde_bytes::ByteBuf, f64) = rmp_serde::from_slice(payload).expect("decode"); registers.into_vec() }