From d3a5ddc361bc7172488ea43457f4e869df2604f5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 28 Oct 2026 00:43:55 +0000 Subject: [PATCH 1/3] ddsketch: drop sum/min/max from the state (new ASAPv1 kind 0x05 0x01) The DDSketch state is now the two bucket stores and the zero count. No DDSketch readout needs sum/min/max: they only clamped quantiles to [min, max]. A delta names buckets, not values, so a delta-fed receiver could only fold in bucket representatives and ended with different scalars than the producer. Sum, minimum and maximum belong to a separate exact aggregate. - Remove the sum/min/max fields and the sum()/min()/max() accessors. - Quantiles read every rank, the extremes included, from its bucket's representative; q = 0 / 1 are within alpha of the true extremes. - merge_delta and apply_delta update buckets and count only, so deltas rebuild the producer bit for bit. - ASAPv1: encode 0x05 0x01 / 0x05 0x81 (five-field payload); decode the retired 0x05 0x00 / 0x05 0x80 with the scalars dropped. EhBucketSketch nests 0x05 0x01 and accepts 0x05 0x00. - EhBucketSketch "min" / "max" on a DDSketch read quantiles 0 / 1. - Golden fixtures for the new kinds (submodule bump); docs and tests. --- CHANGELOG.md | 13 + asapv1_golden | 2 +- docs/api/api_ddsketch.md | 39 ++- docs/api/api_eh_bucket_sketch.md | 2 +- docs/api/api_octo.md | 5 +- docs/asapv1_wire_format.md | 47 ++- docs/ddsketch_signed_values.md | 4 +- docs/message_pack_format.md | 6 +- docs/tests.md | 44 +-- src/asapv1/kind.rs | 12 +- src/asapv1/wire_key.rs | 3 - src/sketch_framework/eh_bucket_sketch.rs | 52 ++- src/sketch_framework/eh_bucket_sketch/wire.rs | 13 +- src/sketches/ddsketch.rs | 229 +++----------- src/sketches/ddsketch/delta.rs | 144 ++++++--- src/sketches/ddsketch/wire.rs | 299 ++++++++---------- tests/PBT_COVERAGE.md | 6 +- tests/asapv1_golden.rs | 126 ++++---- tests/ddsketch_signed.rs | 10 +- tests/e2e/quantiles.rs | 35 +- tests/e2e/wire.rs | 2 +- tests/pbt/ddsketch.rs | 47 +-- 22 files changed, 534 insertions(+), 606 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6e3a239..48fb9715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,19 @@ signals a backwards-compatible change. ### Changed - **`Kmv` builds without the `experimental` feature**, with its ASAPv1 codec. +- **Breaking: `DdSketch` no longer keeps `sum`, `min` or `max`.** The state is + the two bucket stores and the zero count. `sum()`, `min()` and `max()` are + removed; a caller that needs them keeps a separate exact aggregate. + Quantiles are no longer clamped to `[min, max]`: `q = 0` and `q = 1` read + the lowest and highest occupied buckets, within `alpha` of the true + extremes. `EhBucketSketch`'s `"min"` / `"max"` commands on a DDSketch read + those quantiles. A DDSketch delta (`merge_delta`, Octo's `apply_delta`) now + rebuilds the producer's state exactly. +- **ASAPv1: DDSketch encodes as `0x05 0x01`, its delta as `0x05 0x81`.** The + payload drops the three scalars (five fields). The retired `0x05 0x00` / + `0x05 0x80` still decode, with the scalars dropped, and are never encoded. + The `sketchlib-golden-bytes` submodule gains the `ddsketch_buckets_*` + fixtures. - **Count Sketch estimates no longer allocate.** The per-row estimates a median is taken over are collected on the stack instead of in a fresh `Vec`: up to 20 rows for `Vector2D` (the fast and regular paths), diff --git a/asapv1_golden b/asapv1_golden index b1fe1513..08440a11 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit b1fe151313b187c98ee8fe0175cb15761d44418a +Subproject commit 08440a1196027e5b75fe779462f0262fa1e34544 diff --git a/docs/api/api_ddsketch.md b/docs/api/api_ddsketch.md index adbc630c..bc2d2a48 100644 --- a/docs/api/api_ddsketch.md +++ b/docs/api/api_ddsketch.md @@ -34,10 +34,14 @@ the type-erased dispatch path. fn get_value_at_quantile(&self, q: f64) -> Option fn quantile_interpolated(&self, q: f64) -> Result, String> fn get_count(&self) -> u64 -fn min(&self) -> Option -fn max(&self) -> Option ``` +Every answer is a bucket's representative: within `alpha` of a sample, the +extremes included (`q <= 0` reads the lowest occupied bucket, `q >= 1` the +highest). The sketch keeps no sum, minimum or maximum: those are not DDSketch +readouts, and a caller that needs them keeps a separate exact aggregate beside +the sketch. + ## Merge ```rust @@ -51,24 +55,23 @@ fn to_asapv1(&self) -> Result, RmpEncodeError> fn from_asapv1(bytes: &[u8]) -> Result ``` -These produce/consume the **ASAPv1** wire envelope (kind `0x05 0x00`) — see the +These produce the **ASAPv1** wire envelope of kind `0x05 0x01`, and consume it +or the retired `0x05 0x00` (whose `sum` / `min` / `max` are dropped) — see the [ASAPv1 wire format spec](../asapv1_wire_format.md). DDSketch never hashes its inputs, so the metadata carries **no hash-spec group**: it is `metadata_version` plus `alpha`, the sketch's one construction parameter. `gamma` and its logs are derived from `alpha` and never reach the wire. -Every payload is `[counts, offset, sum, min, max, negative_counts, -negative_offset, zero_count]`: the positive store and the negative-magnitude -store carried verbatim (the bucket index of `counts[i]` is `offset + i`), the -three scalars the buckets do not determine, and the zero count. `count` is -**not** carried — it is the sum of both stores and `zero_count`, recovered -exactly on decode. `sum`, `min` and `max` come back exactly rather than as -α-bounded bucket estimates, so a decoded sketch re-serializes byte-identically. +Every payload is `[counts, offset, negative_counts, negative_offset, +zero_count]`: the positive store and the negative-magnitude store carried +verbatim (the bucket index of `counts[i]` is `offset + i`) and the zero count. +`count` is **not** carried — it is the sum of both stores and `zero_count`, +recovered exactly on decode, so a decoded sketch re-serializes byte-identically. Bucket indices are signed, so either offset may be negative. Decode rejects a `metadata_version` other than `1`, an `alpha` outside `(0, 1)`, a store span -past `i32`, a non-zero offset on an empty store, counts that overflow the total, -and scalars inconsistent with that total; `to_asapv1` enforces the same +past `i32`, a non-zero offset on an empty store, and counts that overflow the +total; `to_asapv1` enforces the same rules, so the format never emits bytes it would refuse to read back. ## Deltas @@ -91,12 +94,14 @@ fn from_asapv1(bytes: &[u8]) -> Result fn merge_delta(&mut self, delta: &DdSketchDelta) -> Result<(), String> ``` -A `DdSketchDelta` is the ASAPv1 delta kind `0x05 0x80`: the changed buckets of +A `DdSketchDelta` is the ASAPv1 delta kind `0x05 0x81` (the retired `0x05 0x80` +has the same payload and still decodes): the changed buckets of each store as strictly ascending bucket indices inside the range `alpha`'s mapping produces, the non-zero count added to each, and a count added to the zero count. `new` and the decoder refuse anything else. `merge_delta` adds each -count to its bucket, growing the store as insertion does, and folds the -bucket's representative into `sum`, `min` and `max`; a delta of another +count to its bucket, growing the store as insertion does, and the zero count to +the zero count, so the receiver holds the producer's buckets exactly; a delta of +another `alpha` or admission rate, one that overflows the total count, or one that would grow a store past `i32`, fails and leaves the sketch unchanged. @@ -111,8 +116,8 @@ fn with_admission_rate(self, p: f64) -> Self ``` `p` is the probability with which each update was admitted, `1` unless set. -`count` and `sum` cover admitted samples alone; a consumer divides them by `p`. -Quantiles, `min` and `max` need no rescaling. The ASAPv1 metadata carries `p` as +`count` covers admitted samples alone; a consumer divides it by `p`. Quantiles +need no rescaling. The ASAPv1 metadata carries `p` as its last key, `admission_rate`, when `p < 1`; a `p` outside `(0, 1]` fails to serialize. `merge` and `merge_delta` fail when the two rates differ. A sketch nested in another kind (`EhBucketSketch`, `Hydra`, CountMinSketchWithHeap, CountSketchWithHeap, Elastic) must diff --git a/docs/api/api_eh_bucket_sketch.md b/docs/api/api_eh_bucket_sketch.md index dd8e5419..191d465d 100644 --- a/docs/api/api_eh_bucket_sketch.md +++ b/docs/api/api_eh_bucket_sketch.md @@ -73,7 +73,7 @@ The ten nested kind_ids: | `CocoSketch` | `0x0c 0x00` | Coco | default | | `CountSketchWithL2Norm` | `0x19 0x00` | CountSketchWithL2Norm | default | | `CountSketch` | `0x04 0x00` | Count Sketch | default | -| `DdSketch` | `0x05 0x00` | DDSketch | default | +| `DdSketch` | `0x05 0x01` (the retired `0x05 0x00` still decodes) | DDSketch | default | | `ElasticSketch` | `0x0b 0x00` | Elastic | default | | `Hll` | `0x01 0x02` | HLL Ertl-MLE | default | | `Kll` | `0x06 0x00` | KLL compact | default | diff --git a/docs/api/api_octo.md b/docs/api/api_octo.md index 271dbac6..eb3bb95c 100644 --- a/docs/api/api_octo.md +++ b/docs/api/api_octo.md @@ -202,9 +202,8 @@ fn apply_delta(&mut self, delta: DdSketchBucketDelta) `bucket_index_for` exposes the logarithmic mapping so a worker can hold one-byte counters over the same bucket space; it returns `None` for any -value `add` would itself drop. `apply_delta` advances `count` exactly and -advances `sum`/`min`/`max` with the bucket's representative value — the -same α-bounded estimate a deserialize-and-recompute produces. +value `add` would itself drop. `apply_delta` adds the count to its bucket and +to `count`, exactly as the samples' `add`s would. A delta naming an index outside the store's own range is **dropped**, not allocated: the worker's index space follows its own α, so a delta from a diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 11bb1dd4..77c1020b 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -194,8 +194,10 @@ This registry is the master list of algorithms still to design payloads for, and | `0x03 0x80` | Count-Min-with-heap (CountMinSketchWithHeap) | CountMinSketchWithHeap delta | Section 3.23 | implemented | | `0x04 0x00` | Count Sketch | Count Sketch | Section 3.6 | implemented | | `0x04 0x80` | Count Sketch | Count Sketch delta | Section 3.23 | implemented | -| `0x05 0x00` | DDSketch | Logarithmic bucket stores | Section 3.8 | implemented | -| `0x05 0x80` | DDSketch | DDSketch delta | Section 3.23 | implemented | +| `0x05 0x00` | DDSketch | Logarithmic bucket stores with `sum` / `min` / `max` | Section 3.8 | retired: decoded (scalars dropped), never encoded | +| `0x05 0x01` | DDSketch | Logarithmic bucket stores | Section 3.8 | implemented | +| `0x05 0x80` | DDSketch | DDSketch delta of `0x05 0x00` | Section 3.23 | retired: decoded, never encoded | +| `0x05 0x81` | DDSketch | DDSketch delta | Section 3.23 | implemented | | `0x06 0x00` | KLL | Compact | Section 3.3 | implemented | | `0x06 0x01` | KLL dynamic | Dynamic | Section 3.3 | implemented | | `0x07 0x00` | Hydra | KLL counter | Section 3.9 | implemented | @@ -350,7 +352,7 @@ Only these kinds carry it. A consumer multiplies these estimates by `1/p` and re | HLL | cardinality | | Count-Min | a key's frequency; a row's total | | Count Sketch | a key's frequency | -| DDSketch | `count`, `sum` | +| DDSketch | `count` | HLL admits by key: all updates of a key are admitted or none are. @@ -729,41 +731,37 @@ Wire counter types are the base Count-Min's: **`"i32"`, `"i64"` and `"f64"`**. ` **A producer-side constraint: a `NaN` key.** `TopKHeap` finds a resident by digest and then compares with `PartialEq`, and `NaN` never equals itself, so offering the same `NaN` key twice seats **two** entries rather than rescoring one. That state is reachable in memory and has **no encoding**: the encode-side duplicate check reads the two keys' bits, sees one key twice, and fails with the same complaint the decoder would make. Nothing repairs it on either door — a producer that inserts float keys must keep `NaN` out of a heap it intends to serialize. -### 3.8: DDSketch payload (`0x05 0x00`) +### 3.8: DDSketch payload (`0x05 0x01`) -`DdSketch` maps the magnitude of a value `v` to the bucket index `floor(ln(|v|) / ln(gamma))`. Every payload carries all eight fields: +`DdSketch` maps the magnitude of a value `v` to the bucket index `floor(ln(|v|) / ln(gamma))`. The state is the buckets alone; every payload carries all five fields: | Pos | Field | Type | Notes | | ----- | ------- | ------ | ------- | | 0 | `counts` | array | positive store: dense per-bucket sample counts, u64; the bucket index of `counts[i]` is `offset + i`. Carried verbatim, growth padding included | | 1 | `offset` | int | absolute bucket index of `counts[0]`; **signed**. `0` when `counts` is empty | -| 2 | `sum` | f64 | sum of the values **as ingested**; `0.0` when empty | -| 3 | `min` | f64 | smallest value as ingested; `+inf` when empty | -| 4 | `max` | f64 | largest value as ingested; `-inf` when empty | -| 5 | `negative_counts` | array | negative store, by magnitude: u64; the bucket index of `negative_counts[i]` is `negative_offset + i`. Carried verbatim, growth padding included. Empty when the sketch holds no negative value | -| 6 | `negative_offset` | int | absolute bucket index of `negative_counts[0]`; **signed**. `0` when `negative_counts` is empty | -| 7 | `zero_count` | u64 | number of values counted as zero | +| 2 | `negative_counts` | array | negative store, by magnitude: u64; the bucket index of `negative_counts[i]` is `negative_offset + i`. Carried verbatim, growth padding included. Empty when the sketch holds no negative value | +| 3 | `negative_offset` | int | absolute bucket index of `negative_counts[0]`; **signed**. `0` when `negative_counts` is empty | +| 4 | `zero_count` | u64 | number of values counted as zero | **Metadata vs payload.** `alpha` shapes every bucket index and is chosen at construction, so per the config-to-metadata rule it is a structural param, like HLL's `precision`. `gamma`, `log_gamma` and `inv_log_gamma` are derived from it (`gamma = (1 + alpha) / (1 - alpha)`) and appear nowhere: the decoder re-derives them exactly as `DdSketch::new` does. The metadata is `metadata_version` then `alpha`. **DDSketch carries no hash-spec group.** It never hashes: it maps a raw `f64` through a logarithm and carries no hasher type parameter at all. The hash spec answers "how were keys hashed", and a sketch that does not hash has no truthful answer, so the group is omitted entirely. This is the Q-KLL precedent, and the reason DDSketch's metadata schema is not `HashProfile`-derived the way HLL's and Count-Min's are. -**"As ingested" is not always "exact".** On the `add` path the three scalars see the caller's own `f64`, so they are exact. On the `apply_delta` path they do not: a delta names a bucket index rather than a value, so the sketch folds in that bucket's **representative** — `sum` accumulates `representative * delta.value`, and `min` / `max` compare against the representative. A sketch fed only deltas reports representative-based extrema. For a signed sum, cancellation can amplify relative error; it does not inherit the per-value relative accuracy `alpha`. They are still carried rather than recomputed: a sketch fed by both paths mixes exact and representative contributions, and no walk of the buckets reproduces the mix. +**No `sum`, `min` or `max`.** Every DDSketch readout comes from the buckets: a quantile, `q = 0` and `q = 1` included, is its bucket's representative and within `alpha` of a sample. A sum, minimum or maximum of the values is not a DDSketch readout but a separate exact aggregate kept beside the sketch. Carrying them in the state would also break deltas: a delta names buckets, not values, so a receiver could only fold in bucket representatives and would end with different scalars than the producer. -**`sum` saturates.** `add`, `apply_delta` and `merge` set `sum` to the new total when it is finite, else to `f64::MAX` with the total's sign. +**The retired `0x05 0x00`.** It carried `sum` / `min` / `max` as three f64 fields between `offset` and `negative_counts` (eight fields in all). A decoder still accepts it, reads the three scalars as float64 (Section 4) and drops them, and the result re-encodes as `0x05 0x01`. Nothing encodes `0x05 0x00`; per Q-VER the id is never reused. **Decode rules.** Fail **closed** on each, with an error and never a panic: -1. `kind_id` is `0x05 0x00`; any other id is rejected. -2. The metadata map is exact: `metadata_version` and `alpha` are both required, `metadata_version` is `1`, and no other key is accepted except `admission_rate` (rule 8). `alpha` is a property of the *stored* sketch rather than of the target type, so it is echoed back into the expected block rather than pinned — the same treatment Count-Min gives `rows` / `cols`. +1. `kind_id` is `0x05 0x01`, or the retired `0x05 0x00` with its eight-field payload; any other id is rejected. +2. The metadata map is exact: `metadata_version` and `alpha` are both required, `metadata_version` is `1`, and no other key is accepted except `admission_rate` (rule 7). `alpha` is a property of the *stored* sketch rather than of the target type, so it is echoed back into the expected block rather than pinned — the same treatment Count-Min gives `rows` / `cols`. 3. `alpha` is finite and strictly inside `(0, 1)` — both endpoints excluded, and `NaN` and the infinities rejected with them. Checked **before** the payload is read, so a crafted accuracy never reaches the store. Outside that range the `gamma` derivation is non-positive or infinite and every bucket index is meaningless. 4. An empty `counts` sits at `offset == 0` and an empty `negative_counts` at `negative_offset == 0`. 5. Each populated store has its highest bucket index (`offset + len - 1`, `negative_offset + len - 1`) representable as an `i32`, checked in `i64` so the check itself cannot overflow. Applied from the declared offset and the array length alone, **before** the store is rebuilt. 6. The total sample count is the **checked** sum `sum(counts) + sum(negative_counts) + zero_count`; an overflowing total is rejected rather than wrapping into a count the quantile walk would disagree with. -7. The scalars are consistent with that total: a zero total carries `sum == 0.0`, `min == +inf` and `max == -inf`; a non-zero one carries finite `sum` / `min` / `max` with `min <= max`. If `sum(negative_counts) == 0` and `zero_count == 0`, also `0 < min` and `sum >= min`. -8. An optional `admission_rate` is accepted (Section 2, [admission rate](#admission-rate-admission_rate)). +7. An optional `admission_rate` is accepted (Section 2, [admission rate](#admission-rate-admission_rate)). -Rules 3, 5, 6 and 7 are enforced on the **encode** side too, so the format never emits bytes it would refuse to read back. +Rules 3, 4, 5 and 6 are enforced on the **encode** side too, so the format never emits bytes it would refuse to read back. ### 3.9: Hydra payload (`0x07 0x00`-`0x07 0x04`) @@ -1166,7 +1164,8 @@ The `kind_id` is carried rather than a name, because Section 1's registry is alr | `0x01 0x02` | HLL Ertl-MLE | `Hll(HyperLogLog)` | always | | `0x02 0x00` | Count-Min | `CountMinSketch(CountMinSketch, FastPath>)` | always | | `0x04 0x00` | Count Sketch | `CountSketch(CountSketch, FastPath>)` | always | -| `0x05 0x00` | DDSketch | `DdSketch(DdSketch)` | always | +| `0x05 0x01` | DDSketch | `DdSketch(DdSketch)` | always | +| `0x05 0x00` | DDSketch, retired: decoded into the same arm, never encoded | `DdSketch(DdSketch)` | always | | `0x06 0x00` | KLL compact | `Kll(Kll)` | always | | `0x0b 0x00` | Elastic | `ElasticSketch(ElasticSketch)` | always | | `0x0c 0x00` | Coco | `CocoSketch(CocoSketch)` | always | @@ -1328,7 +1327,7 @@ The encode side refuses a member that is not valid UTF-8. The encode side refuses a member that is not valid UTF-8 and a delta that breaks rule 5. -### 3.23: Delta payloads (`0x01 0x81`, `0x01 0x82`, `0x02 0x80`, `0x03 0x80`, `0x04 0x80`, `0x05 0x80`, `0x0a 0x80`) +### 3.23: Delta payloads (`0x01 0x81`, `0x01 0x82`, `0x02 0x80`, `0x03 0x80`, `0x04 0x80`, `0x05 0x81`, `0x0a 0x80`) A delta is a change to the state of its **state kind**, the kind_id with the variant's top bit cleared (Section 1). @@ -1363,7 +1362,7 @@ Apply: `counts[i] += delta`. Apply: the matrix as Count-Min's; the state's heap and `key_type` become the delta's. -**DDSketch** (`0x05 0x80`): +**DDSketch** (`0x05 0x81`; the retired `0x05 0x80` has the same payload and still decodes): | Pos | Field | Type | Notes | | ----- | ------- | ------ | ------- | @@ -1373,7 +1372,7 @@ Apply: the matrix as Count-Min's; the state's heap and `key_type` become the del | 3 | `negative_deltas` | array | one u64 count per index | | 4 | `zero_count` | u64 | added to the zero count | -Apply: for each `k` in `indices`, then each `k` in `negative_indices`, in payload order, the count `c` is added to bucket `k`, `sum += r·c` (saturating, §3.8), `min = min(min, r)` and `max = max(max, r)`, where `r = (1 + alpha)·gamma^k`, negated for the negative store. A non-zero `zero_count` is added to the zero count, then `min = min(min, 0.0)` and `max = max(max, 0.0)`. +Apply: for each `k` in `indices`, then each `k` in `negative_indices`, the count is added to bucket `k` of its store, and `zero_count` is added to the zero count. Nothing else changes, so a receiver fed deltas holds, bucket for bucket, the state the producer built from the same samples. **Decode rules.** Fail **closed** on each, with an error and never a panic: @@ -1497,7 +1496,7 @@ Golden byte-vectors lock it. **Float fields.** Every float field is msgpack **float64** (`0xcb`). A decoder rejects an integer or a float32 in a float field, and a float in an integer field. The float fields are: - metadata: `alpha` (§3.8), `sample_rate` (§3.13), `admission_rate` (Section 2); -- HLL HIP's `hip_kxq0` / `hip_kxq1` / `hip_est` (§3.1) and DDSketch's `sum` / `min` / `max` (§3.8); +- HLL HIP's `hip_kxq0` / `hip_kxq1` / `hip_est` (§3.1) and the retired DDSketch `0x05 0x00`'s `sum` / `min` / `max` (§3.8); - `counts`, `heap_counts` and `deltas` elements where `counter_type` reads `"f64"` (§3.2, §3.6, §3.7, §3.10, §3.23, §3.24, §3.25); - `items` / `values` elements where `item_type` or `cell_item_type` reads `"f64"` (§3.3, §3.9, §3.13); - `keys` elements where `key_type` or `cell_key_type` reads `"f64"` (§3.5, §3.7, §3.9, §3.10, §3.15, §3.16, §3.23, §3.24, §3.25). @@ -1615,7 +1614,7 @@ Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketc - **Q-ORDER**: a payload whose in-memory container order does not survive a rebuild **pins an emitted order**, as a cross-language contract. Heaps, arenas and hash maps all have this property, so the rule reaches Space-Saving's triples (§3.5, §3.24, §3.25), the top-k heap entries (§3.7), KMV's digests (§3.14), UniformSampling's entries (§3.13), UnivMon-Q's candidates and occurrences (§3.20), and every layer- and bucket-indexed array. Two sketches holding the same state then emit the same bytes whatever order they were built in, and a decoded sketch re-serializes byte-identically. The heap key comparator is the same everywhere: descending count, ties by heap key order (§3.5). - **Q-CAP**: **a declared capacity never sizes an allocation.** Every structure is built from the run the payload actually carries, and the declared bound is checked against that run first; a payload naming a capacity of `2^32 - 1` with two entries costs two entries. This holds for every `capacity` (§3.5, §3.24, §3.25), every `heap_size`, KMV's `k`, UniformSampling's `total_seen`, UnivMon-Q's `ordered_samples`, and Hydra's grid and counter dimensions. - **Q-KEYTYPE**: `key_type` names the **exact `OwnedDataInput` variant** and is never widened, wherever heap keys reach the wire — Space-Saving (§3.5), CountMinSketchWithHeap and CountSketchWithHeap (§3.7, §3.10), CountMinSketchWithSpaceSaving and CountSketchWithSpaceSaving (§3.24, §3.25), the UnivMon family (§3.15, §3.16) and Hydra's UnivMon counter (§3.9). A container whose keys mix variants has no single name and **fails to serialize**; `OwnedDataInput::I128` / `U128` have no msgpack integer form and are not wire types. An empty container emits `"u64"`, and a decoder rejects an empty container under any other name, so its `key_type` has one value rather than one per producer. That canonicalizes the key naming and nothing else: a Two-tier UnivMon pyramid still has more than one encoding per state, since layouts differing only in an `elephant_layers` at or above `layers` build identical layers, and an all-elephant pyramid carries its unused mouse dimensions on the wire regardless (§3.16). What holds everywhere is the round trip: a decoded container re-serializes byte-identically. -- **Q-DERIVED**: state a decoder can recompute exactly is **not carried**, and state it cannot is. Recomputed: DDSketch's `count` from its stores and `zero_count` (§3.8), `ExponentialHistogram`'s `l2_mass` and `merge_norm` (§3.17), Elastic's `heavy_buckets` from the declared bucket count (§3.11), every heap's digest index and array order, and every summary a sketch answers from its own state. Carried: DDSketch's `sum` / `min` / `max`, Elastic's `stale_copies`, UnivMon's `update_mode`, `candidate_complete` and `bucket_size`, UnivMon-Q's `ever_evicted`, `source_id` and `next_sequence`, UniformSampling's `priorities`, `total_seen` and `rng_state`, and CountSketchWithL2Norm's `l2`. Where a cached derived field exists in memory, the encoder rejects a value that disagrees with what the decoder would recompute. +- **Q-DERIVED**: state a decoder can recompute exactly is **not carried**, and state it cannot is. Recomputed: DDSketch's `count` from its stores and `zero_count` (§3.8), `ExponentialHistogram`'s `l2_mass` and `merge_norm` (§3.17), Elastic's `heavy_buckets` from the declared bucket count (§3.11), every heap's digest index and array order, and every summary a sketch answers from its own state. Carried: Elastic's `stale_copies`, UnivMon's `update_mode`, `candidate_complete` and `bucket_size`, UnivMon-Q's `ever_evicted`, `source_id` and `next_sequence`, UniformSampling's `priorities`, `total_seen` and `rng_state`, and CountSketchWithL2Norm's `l2`. Where a cached derived field exists in memory, the encoder rejects a value that disagrees with what the decoder would recompute. State the sketch's own readouts do not need is not carried either: DDSketch's `sum` / `min` / `max` were (`0x05 0x00`) and are now a separate exact aggregate, since nothing a DDSketch answers reads them (§3.8). - **Q-NIL**: an absent slot is msgpack **nil** (`0xc0`), never a sentinel value. An empty string is a reachable key in Coco (§3.12) and a reachable flow id in Elastic (§3.11), so `""` cannot double as "free"; UnivMon-Q's absent extrema are nil for the same reason (§3.20). Go must model these as nullable, never as a zero value standing in for absent. - **Q-RNG**: RNG state is carried **resumable** when the generator is cheap to reproduce in another language, and opaque otherwise. UniformSampling's `rng_state` is one SplitMix64 word with published constants, so a decoded sampler continues the same draw sequence (§3.13); KLL's `coin` carries the compaction RNG's position and its `compaction_seed` is opaque state Go preserves without interpreting (Q-KLL-SEED). - **Q-UNIVMON-SCOPE**: every layer of `0x10 0x00` and `0x11 0x00` is a CountSketchWithL2Norm with no tag, and another per-layer counter needs a new structural param or a new id. `UnivMonPool` is a `Vec` free list of scratch `UnivMon`s — not of pyramids — with nothing an observer could query, so it is not a wire kind and holds no id. diff --git a/docs/ddsketch_signed_values.md b/docs/ddsketch_signed_values.md index 71782c35..80ee7d40 100644 --- a/docs/ddsketch_signed_values.md +++ b/docs/ddsketch_signed_values.md @@ -4,13 +4,13 @@ `try_add` returns errors for NaN, infinities, or magnitudes above the mapping's maximum. `add` ignores those errors. Magnitudes below the mapping's minimum are counted as zero, including subnormal inputs. Such nonzero tiny observations are outside the relative-error guarantee. Original sample values are still compressed into buckets, not retained individually. -`DdSketch` answers the ceil(q*n) order statistic and keeps exact ingested extrema. For indexable nonzero order statistics the estimate has absolute error at most `alpha * abs(value)`, for either sign. +`DdSketch` answers the ceil(q*n) order statistic from its buckets, the extremes included; it keeps no ingested sum or extrema. For indexable nonzero order statistics the estimate has absolute error at most `alpha * abs(value)`, for either sign. If interpolation combines endpoints x and y with weight t, the absolute error is bounded by `alpha * ((1-t)*abs(x) + t*abs(y))`, excluding zero-mapped tiny values. Opposite signs can cancel, so the interpolated result need not have relative error alpha. Dividing quantiles additionally requires a finite nonzero true denominator and valid relative-error bounds for both operands. This change does not establish those planner preconditions. ## Wire and downstream upgrade -- Native ASAPv1: every state is metadata version 1 with the eight-field payload, negative counts, negative offset and zero count included. `DdSketchDelta` (`0x05 0x80`) carries both stores and a zero-count increment. +- Native ASAPv1: every state is metadata version 1 with the five-field payload of kind `0x05 0x01` (both stores and the zero count); the retired eight-field `0x05 0x00` still decodes. `DdSketchDelta` (`0x05 0x81`) carries both stores and a zero-count increment. - Octo: `DdSketchBucketDelta` now requires a `sign: DdSign` discriminator. Existing positive delta constructors should use `DdSign::Positive`. Worker residual keys are `(DdSign, i32)`. Both promotion and flush preserve all three categories; partial counts still need flushing before complete queries. Go implementations of the ASAP wire formats and downstream dependency pins are not changed by this Rust library patch. diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 58cefb9c..3d53a256 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -115,9 +115,9 @@ Converted: - **CountMinSketchWithHeap** (`src/sketches/countminsketch_heap/wire.rs`, `0x03 0x00`) and **CountSketchWithHeap** (`src/sketches/countsketch_heap/wire.rs`, `0x0a 0x00`) — a base matrix inlined ahead of the shared top-k heap entries. -- **DDSketch** (`src/sketches/ddsketch/wire.rs`, `0x05 0x00`) — both bucket - stores verbatim with their offsets, `sum`, `min`, `max` and `zero_count`; no - hash-spec group. +- **DDSketch** (`src/sketches/ddsketch/wire.rs`, `0x05 0x01`) — both bucket + stores verbatim with their offsets and `zero_count`; no hash-spec group. The + retired `0x05 0x00`, which also carried `sum`, `min` and `max`, still decodes. - **Elastic** (`src/sketches/elastic/wire.rs`, `0x0b 0x00`) — the heavy table's four parallel arrays, the `stale_copies` flag, and the inlined light Count-Min. - **Coco** (`src/sketches/coco/wire.rs`, `0x0c 0x00`) — the dense bucket table as diff --git a/docs/tests.md b/docs/tests.md index e8111f79..40ef2e46 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -133,7 +133,7 @@ Delta tests: [`src/sketches/hll/delta.rs`](../src/sketches/hll/delta.rs) | `hll_correctness_test` | Register update logic matches expected bucket/index behavior for all HLL variants. | Runs fixed hashed inserts against Classic, Ertl, and HIP variants; asserts exact expected register values at specific bucket indices and confirms an untouched bucket remains zero. | | `hll_envelope_structure_and_kind_id_guard` | The envelope frames an Ertl-MLE sketch under kind_id `0x01 0x02`, and a Classic decoder refuses it. | For 1,000 inserts into `HyperLogLog`, verifies the bytes open with the ASAPv1 magic, `envelope::VERSION`, a `kind_id_len` of `2`, and `0x01 0x02`, that the decoded registers match the source, and that a `HyperLogLog` decode fails. | | `hll_hip_round_trip_preserves_state` | The HIP running scalars travel beside the registers. | For 1,000 inserts into `HyperLogLogHip`, verifies the bytes carry kind_id `0x01 0x03` and that the decoded sketch matches the source on the registers and on `kxq0`, `kxq1`, and `est`. | -| `known_state_matches_golden` | Known register and HIP state serializes to the `asapv1_golden/` bytes and decodes back to it. | Builds P12 `Classic` and `Ertl` sketches with registers `0 -> 1`, `1 -> 7`, `100 -> 42`, `4095 -> 3`, and `HyperLogLogHipImpl` sketches at P12 (`kxq0` `1.5`, `kxq1` `2.5`, `est` `3.0`) and P14 (registers `0 -> 1`, `1 -> 7`, `8192 -> 42`, `16383 -> 51`; `kxq0` `16380.5`, `kxq1` `0.25`, `est` `4.125`), verifies each `to_asapv1` equals `hll_classic_p12`, `hll_ertl_mle_p12`, `hll_hip_p12` and `hll_hip_p14`, and that decoding each golden gives back those registers and HIP scalars. | +| `known_state_matches_golden` | Known stores serialize to the golden bytes. | Builds the positive and signed `alpha = 0.01` states through `DdSketchState` and verifies they serialize to `ddsketch_buckets_positive_a001` and `ddsketch_buckets_signed_a001`. | | `hll_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a P14 `Ertl` sketch over a hasher declaring its own `HashProfile`, verifies the registers round-trip, the bytes differ from the standard-profile sketch's over the same 1,000 inserts, and a standard-profile decode rejects them. | | `hll_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the eight `HllMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HllMetadata`. | | `hll_precision_cross_rejection` | The metadata `precision` pins the register storage the bytes belong to. | Verifies a populated `HyperLogLogP12` envelope fails to decode as the P14 `HyperLogLog`. | @@ -220,20 +220,20 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | test_name | test_description | what_is_tested | | --- | --- | --- | -| `quantile_interpolated_two_values` | The interpolated quantile of two values is linear between them. | On `[20, 40]` at `alpha = 0.01`, `q` in `{0, 0.5, 0.9, 1}` gives `20, 30, 38, 40` within `alpha` relative, and `get_value_at_quantile(0.9)` keeps its nearest-rank `40`. | -| `quantile_interpolated_one_value_and_empty` | A single value answers every `q`; an empty sketch answers `None`. | One value `20` gives `20` at `q` in `{0, 0.3, 1}`; an empty sketch gives `Ok(None)`. | +| `quantile_interpolated_two_values` | The interpolated quantile of two values is linear between them. | On `[20, 40]` at `alpha = 0.01`, `q` in `{0, 0.5, 0.9, 1}` answers within `alpha` of `20`, `30`, `38` and `40`, and the nearest-rank `get_value_at_quantile(0.9)` is within `alpha` of `40`. | +| `quantile_interpolated_one_value_and_empty` | A single value answers every `q` with one bucket; an empty sketch answers `None`. | One value `20` answers within `alpha` of `20` at `q = 0`, and the same value at `q` in `{0.3, 1}`; an empty sketch answers `None` at `0.5`. | | `quantile_interpolated_refuses_q_outside_the_unit_interval` | `q` outside `[0, 1]` is an error. | `-0.1`, `1.1`, NaN and `±inf` give `Err`, on a filled and on an empty sketch. | | `quantile_interpolated_signed_values` | Signed values interpolate across both stores and the zero bucket. | On `[-40, -20, -5, 0, 10, 30, 70]` for 101 `q`, the error is within `alpha ((1 - f) abs(lower) + f abs(upper))` of the exact interpolated value, and within `alpha` relative where both neighbours share a sign. | | `quantile_interpolated_matches_the_exact_value_on_a_large_stream` | A large signed stream meets the bound at every `q`. | 20,000 seeded values `±exp(U(-3, 6))` (20% negative) at `alpha = 0.01`, 201 `q` in `[0, 1]`: each answer is within `alpha ((1 - f) abs(lower) + f abs(upper))` of the exact interpolated quantile. | -| `insert_and_query_basic` | Basic insert/query preserves count semantics and quantile monotonicity. | Inserts mixed values `[0.0, -5.0, 1.0, 2.0, 3.0, 10.0, 50.0, 100.0, 1000.0]`, verifies non-positive values are ignored (`count == 7`), and checks queried quantiles at `{0.0, 0.5, 0.9, 0.99, 1.0}` are monotone and bounded by sketch min/max. | +| `insert_and_query_basic` | Basic insert/query preserves count semantics and quantile monotonicity. | Inserts mixed values `[0.0, -5.0, 1.0, 2.0, 3.0, 10.0, 50.0, 100.0, 1000.0]`, verifies the count is `9`, and that quantiles at `{0, 0.5, 0.9, 0.99, 1}` are non-decreasing and inside the stream's range widened by `alpha`. | | `empty_quantile_returns_none` | Empty sketch returns no quantiles and zero count. | For a new `DdSketch(alpha=0.01)`, asserts `get_value_at_quantile` returns `None` for `p in {0.0, 0.5, 1.0}` and `get_count() == 0`. | -| `merge_two_sketches_combines_counts_and_bounds` | Merge combines counts and preserves quantile boundary invariants. | Merges sketches built from `[1,2,3,4]` and `[5,10,20]`, then verifies merged `count=7`, `min=1`, `max=20`, exact boundary quantiles (`q0=1`, `q1=20`), and median lies within `[1,20]`. | -| `dds_serialization_round_trip` | Serialization round trip preserves count, sum, bounds, and selected quantiles. | Serializes/deserializes a populated sketch (`alpha=0.01`), verifies non-empty bytes, equal `count/sum/min/max`, and exact quantile matches at `{0.0, 0.1, 0.5, 0.9, 1.0}`. | -| `ddsketch_envelope_structure_and_round_trip` | The envelope frames a sketch under kind_id `0x05 0x00`, and the bytes are stable across a round trip. | For a sketch at `alpha = 0.01` over eight values, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x05 0x00`, that the metadata carries `metadata_version` `1` and that alpha, that the decoded store counts and offset match, that re-serializing reproduces the bytes, and that quantiles at `{0, 0.1, 0.5, 0.9, 1}` are unchanged. | -| `ddsketch_scalars_survive_exactly` | `sum`, `min`, and `max` are carried, not recomputed from the buckets. | Verifies the decoded sketch reports the source's `sum` and `alpha`, a `min` of exactly `0.25`, and a `max` of exactly `1000.0`, rather than the alpha-bounded bucket representatives a recomputation would give. | -| `ddsketch_payload_is_eight_fields_without_a_count` | A positive-only sketch carries all eight payload fields and no `count`. | Verifies the payload reads back as an eight-element array whose bucket counts sum to `get_count()`, with an empty negative store at offset `0` and a zero `zero_count`, and that the decoded sketch reports the same count. | -| `ddsketch_empty_round_trip` | A sketch that saw nothing round-trips. | Verifies a fresh `DdSketch(alpha=0.01)` decodes back with a `count` of `0`, `min` and `max` of `None`, an empty store, and identical re-serialized bytes. | -| `ddsketch_merged_round_trip` | A store grown on both sides of a merge round-trips. | Merges 200 values against the same 200 scaled by `0.001`, then verifies the decoded sketch matches the source on `count`, `sum`, `min`, and `max` and re-serializes to the same bytes. | +| `merge_two_sketches_combines_counts_and_bounds` | Merge combines counts, and the extreme quantiles read the extreme buckets. | Merges sketches built from `[1,2,3,4]` and `[5,10,20]`, verifies the count is `7`, that `q = 0` and `q = 1` are within `alpha` of `1` and `20`, and that the median lies between them. | +| `dds_serialization_round_trip` | Serialization round trip preserves count and selected quantiles. | Serializes/deserializes a populated sketch and verifies `count` and the quantiles at `{0, 0.1, 0.5, 0.9, 1}` are unchanged. | +| `ddsketch_envelope_structure_and_round_trip` | The envelope frames a sketch under kind_id `0x05 0x01`, and the bytes are stable across a round trip. | For a populated `alpha = 0.01` sketch, verifies the `ASAPv1` magic, kind_id `0x05 0x01`, metadata version `1` and `alpha`, the decoded positive store, identical re-serialized bytes, and unchanged quantiles. | +| `ddsketch_payload_is_five_fields` | The payload is the two stores and the zero count, with no `count` and no scalars. | Verifies the payload reads back as a five-element array whose bucket counts sum to `get_count()`, with an empty negative store at offset `0` and a zero `zero_count`, and that the decoded sketch reports the same count. | +| `ddsketch_decodes_the_retired_kind_with_scalars` | Bytes of the retired kind `0x05 0x00` still decode, their `sum` / `min` / `max` dropped. | Builds an eight-field `0x05 0x00` envelope around a populated sketch's stores, verifies it decodes to the same count and re-encodes to the sketch's own `0x05 0x01` bytes, and that a five-field payload under `0x05 0x00` is refused. | +| `ddsketch_empty_round_trip` | A sketch that saw nothing round-trips. | Verifies a fresh `DdSketch(alpha=0.01)` decodes back with a `count` of `0`, no quantile, an empty store, and identical re-serialized bytes. | +| `ddsketch_merged_round_trip` | A store grown on both sides of a merge round-trips. | Merges 200 values against the same 200 scaled by `0.001`, then verifies the decoded sketch matches the source on `count` and on the quantiles at `{0, 0.5, 1}` and re-serializes to the same bytes. | | `ddsketch_rejects_foreign_kind_id` | Another sketch's kind_id is refused even when the rest parses cleanly. | Verifies a `3x8` Count-Min envelope fails to decode as a `DdSketch`, and that an empty DDSketch envelope decodes but fails with its kind_id relabelled `0x02 0x00`. | | `dd_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the two `DdMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `DdMetadata`. | | `dd_metadata_rejects_a_missing_alpha_key` | `alpha` is required and can never be silently defaulted. | Encodes a named map holding only `metadata_version` and verifies it does not decode as `DdMetadata`. | @@ -243,8 +243,7 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | `ddsketch_rejects_a_store_span_past_i32` | The span rule fires from the offset and the array length, before the store is rebuilt. | Verifies a crafted positive store and a crafted negative store of ten buckets at offset `i32::MAX - 2` each fail with a "store span past i32" complaint, while the positive payload three buckets long decodes. | | `ddsketch_rejects_a_nonzero_offset_on_an_empty_store` | An empty store has exactly one encoding. | Verifies a crafted empty store at offset `42` fails with an "empty store must be at offset 0" complaint. | | `ddsketch_rejects_a_total_count_overflow` | Counts that overflow the recovered total are refused rather than wrapped. | Verifies a positive store `[u64::MAX, 1]`, a positive `[u64::MAX]` beside a negative `[1]`, and a positive `[u64::MAX]` beside a `zero_count` of `1` each fail with an "overflow the total sample count" complaint. | -| `ddsketch_rejects_inconsistent_scalars` | The scalars the buckets do not determine are still bounded by them. | Verifies six crafted payloads each fail on a scalar rule: a populated store carrying the empty-store sentinels, `min` above `max`, a non-positive `min`, a `sum` below the smallest ingested value, an empty store carrying a non-zero `sum`, and all-zero buckets carrying populated-store scalars. | -| `ddsketch_positive_scalar_checks_follow_the_counts` | `0 < min` and `sum >= min` apply only while the negative store sums to zero and `zero_count` is zero. | Verifies a negative `min` beside an all-zero negative store `[0, 0]` fails with a "scalars out of order" complaint, and that a negative `min` beside a negative `[1]` and a `min` of `0.0` beside a `zero_count` of `2` each decode and re-serialize identically. | +| `ddsketch_signed_payload_round_trips` | Signed stores and the zero count decode and re-serialize byte for byte. | Verifies a crafted payload with a negative store and one with a `zero_count` of `2` each decode and re-encode to the same bytes. | | `ddsketch_rejects_serializing_a_store_span_past_i32` | The encode side refuses a store its own decoder would reject. | Verifies a sketch whose store holds ten buckets at offset `i32::MAX - 2` fails to serialize. | | `ddsketch_rejects_serializing_an_overflowing_bucket_total` | The encode side enforces the total-count rule too. | Verifies a sketch whose store holds `[u64::MAX, 1]` fails to serialize. | | `representative_within_alpha_at_bucket_edges` | Every bucket's representative is within `alpha` of both of its edges. | For `alpha` in `{0.001, 0.01, 0.05, 0.1}` and bucket indices `{-100, -1, 0, 1, 7, 500}`, verifies `bin_representative(k)` lies in `[lower_bound(k), lower_bound(k + 1)]` and its relative error against each edge is at most `alpha + 1e-9`. | @@ -252,7 +251,8 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | `untrackable_extreme_is_dropped` | A finite value outside the indexable range is not recorded, so the dense store never spans the gap. | For an `alpha = 0.01` sketch over `1..=2000`, adds `max_indexable * 10.0` and `min_indexable / 10.0` from `ddsketch_indexable_bounds(0.01)` and verifies both `get_count()` and the store's bucket span are unchanged, then that `max_indexable / 2.0` is still recorded and raises the count by `1`. | | `decode_rejects_each_payload_rule` | A `0x05 0x80` payload breaking any delta rule fails to decode. | At `alpha = 0.01`, verifies indices `[-35418, 35448]` (`key(min_v)`, `key(max_v)`) in both stores decode, and that index `-35419`, index `35449`, and negative-store index `35449` each fail. | | `decode_rejects_another_kind_and_new_another_alpha` | A delta decodes only under its own kind_id, and `new` checks `alpha`. | Verifies a delta envelope decodes and fails with its kind_id relabelled `0x05 0x00`, and that `new` refuses `alpha` `1.0`. | -| `merge_delta_folds_the_scalars` | Applying a delta folds each representative, and `0.0` for zeros, into the scalars. | Merges a count of `2` at bucket `10` into an empty sketch and verifies `min`, `max` equal `r = 1.01 * gamma^10` and `sum` equals `2r`; then merges a `zero_count` of `3` and verifies `min` `0.0`, `zero_count` `3` and `get_count()` `5`. | +| `deltas_rebuild_the_producer_bit_for_bit` | Deltas of a producer's changed buckets rebuild its state exactly at a receiver. | Feeds three batches of signed values, zeros and tiny magnitudes to a producer, sends each batch's changed buckets as a `0x05 0x81` delta through the wire, and verifies the receiver's count, quantiles and non-zero cells equal the producer's after every batch. | +| `merge_delta_adds_the_zero_count` | A zero-count delta adds to the zero count and the total. | Merges a count of `2` at bucket `10`, then a zero-count delta of `3`, and verifies the zero count is `3`, the total `5`, and `q = 0` reads `0.0`. | | `merge_delta_rejects_without_changing_the_state` | A rejected delta leaves the sketch unchanged. | Verifies a delta whose `alpha` is one ulp above `0.01` fails on an `alpha = 0.01` sketch, and that on a sketch holding `u64::MAX - 1` samples a delta whose negative count overflows the total and a delta whose `zero_count` overflows it each fail, leaving the count and the serialized bytes as they were. | | `merge_delta_rejects_a_store_span_past_i32` | A delta that would grow a store past `i32` fails instead of overflowing. | Verifies a delta at index `-10` onto a decoded state whose positive store is `[1]` at offset `i32::MAX - 1` fails, leaving the serialized bytes as they were. | | `known_state_matches_golden` | Known stores and scalars serialize to the golden bytes. | Builds the positive and signed `alpha = 0.01` states through `DdSketchState` and verifies `to_asapv1` equals `asapv1_golden/ddsketch_positive_a001.hex` and `ddsketch_signed_a001.hex`. | @@ -1158,7 +1158,8 @@ Wire tests: [`src/sketch_framework/eh_bucket_sketch/wire.rs`](../src/sketch_fram | `eh_bucket_sketch_rejects_crafted_blocks` | Crafted blocks fail closed with an error, never a panic. | Verifies a Count-Min triple whose descriptor is cut in half, and one whose state is three `0xc1` bytes, each fail to decode. | | `eh_bucket_sketch_rejects_a_custom_hash_profile_descriptor` | A descriptor naming a custom hash profile is refused, since the variant's decoder pins the profile of the type it rebuilds. | Verifies a triple built from a custom-profile Count-Min fails to decode. | | `eh_bucket_sketch_query_agrees_after_decode` | A decoded union answers a query the way the original did. | For every populated variant, verifies the source and the decode return the same answer for that variant's sample input. | -| `nested_ddsketch_survives_an_asapv1_round_trip` | The nested DDSketch keeps its running state through an ASAPv1 round trip. | Feeds `10.0`, `20.0`, `30.0`, `40.0` into an `EhBucketSketch::DdSketch(DdSketch::new(0.01))`, round-trips it through `to_asapv1` / `from_asapv1`, and verifies the decode matches on `get_count`, `sum`, `min`, and `max` and that its `q = 0.5` query lands inside `[10.0, 40.0]`. | +| `nested_ddsketch_survives_an_asapv1_round_trip` | The nested DDSketch keeps its state through an ASAPv1 round trip. | Feeds `10.0`, `20.0`, `30.0` and `40.0` into an `EhBucketSketch::DdSketch`, round-trips it, and verifies the count and the inner sketch's bytes are unchanged and the restored median is in range. | +| `ddsketch_min_and_max_commands_are_within_alpha` | `"min"` and `"max"` on a DDSketch read quantiles 0 and 1. | Feeds `-3.5`, `10`, `20` and `1234.5` and verifies `"min"` and `"max"` are within `alpha` of `-3.5` and `1234.5`, and `"count"` is `4`. | ### ExponentialHistogramUnivMon @@ -1670,7 +1671,7 @@ Test file: [`tests/e2e/quantiles.rs`](../tests/e2e/quantiles.rs) | --- | --- | --- | | `ddsketch_satisfies_the_relative_value_error_contract` | DDSketch meets the relative-value-error guarantee against the order statistic of its rank convention. | Crosses `alpha in {0.001, 0.01, 0.05, 0.1}` x `n in {1_000, 20_000, 100_000}` x the six `dds_streams` shapes (`adversarial-bucket-edges` log-uniform on `gamma` edges, positive-filtered `normal(1000, 250)`, `exponential(1e-3)`, `uniform` over `[1e6, 1e7)`, `zipf(1.1)` over 8192 keys in `[1e6, 1e7]`, and `wide-dynamic-range` spanning nine decades). Asserts `DdSketch::get_count()` equals `n` (no sample dropped), then per alpha tallies `RelativeQuantileSpec::core(alpha)` on `get_value_at_quantile` over the q grid `[0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0]` and closes with `assert_none`: every estimate must be within `alpha + 8*f64::EPSILON*(1 + \|ln v\|)` of `sorted[ceil(q*n) - 1]`. Zero violations tolerated, since the guarantee is deterministic. | | `ddsketch_answers_the_ceil_nearest_rank_order_statistic` | `get_value_at_quantile` answers the `ceil(q*n)` order statistic. | Runs 8 `(n, q, idx)` probes — `(3, 0.4, 1)`, `(4, 0.34, 1)`, `(4, 0.3, 1)`, `(7, 0.2, 1)`, `(7, 0.6, 4)`, `(5, 0.5, 2)`, `(10, 0.25, 2)`, `(10, 0.15, 1)` — on values `100 * gamma^(3i)` at `ALPHA = 0.01`, so one rank of movement is several buckets. Asserts `DdRankConvention::CeilNearestRank.index(n, q)` equals `idx` and that the answer satisfies `RelativeQuantileSpec::core(ALPHA).check` against `sorted[idx]`. | -| `ddsketch_endpoints_are_exact` | The endpoints are the exact stored minimum and maximum. | For each of `alpha in {0.001, 0.01, 0.05, 0.1}` over 5_000 `log_uniform_f64` values (seed `4_242`, exponent range `3..30`), asserts `get_value_at_quantile(0.0) == Some(truth.min())`, `(1.0) == Some(truth.max())`, and `min()`/`max()` equal the exact extremes. | +| `ddsketch_endpoints_are_within_alpha` | The endpoints read the lowest and highest occupied buckets. | For each of `alpha in {0.001, 0.01, 0.05, 0.1}` over 5_000 log-uniform values, verifies `q = 0` and `q = 1` are within `alpha` of the exact minimum and maximum. | | `ddsketch_satisfies_the_relative_error_contract_at_bucket_boundaries` | Values on, just below and just above a bucket edge all stay inside `alpha`. | For each of the four alphas and `k in {-40, -7, 0, 1, 13, 60, 200}` computes `edge = gamma^k` and probes `edge`, `edge*(1 - f64::EPSILON)`, `edge*(1 + f64::EPSILON)`, `edge*sqrt(gamma)` and `edge*gamma*(1 - 1e-12)`. Each probe is the single sample of a fresh sketch, so the mapping is isolated from any rank effect; `get_value_at_quantile(0.5)` must satisfy `alpha + numerical_slack` against the probe itself. Probes the mapping refuses (`get_count() == 0`) are skipped rather than asserted on. | | `ddsketch_merge_and_delta_replay_preserve_the_relative_error_contract` | Shard merge and bucket-delta replay both leave the relative-error contract intact. | Per `alpha in {0.001, 0.01, 0.05, 0.1}` over `zipf_f64(40_000, 8_192, 1.1, 1e3, 1e7)`: a 4-way round-robin shard merge must report `get_count() == 40_000` exactly and satisfy the contract over the full q grid; a sketch rebuilt by replaying every non-zero `positive_store_counts()` entry as an `octo_delta::DdSketchBucketDelta` at `positive_store_offset() + i` must reproduce `get_count()` and satisfy the contract over the interior grid `DDS_QS[1..6] = [0.01, 0.1, 0.5, 0.9, 0.99]` (its endpoints are bucket representatives, so `q=0`/`q=1` are excluded). Both tallies use `assert_none`. | | `ddsketch_satisfies_the_relative_value_error_contract_at_extreme_accuracy_parameters` | The relative-value contract still holds at accuracy parameters three orders of magnitude either side of the usual range. | For `alpha in {1e-5, 1e-4, 0.3, 0.5, 0.9}` (seed `0x0DDA_0000 + i*7919`), takes the six `dds_streams(alpha, 20_000, seed)` shapes filtered to `[min, max]` from `ddsketch::ddsketch_indexable_bounds(alpha)`, skipping any shape left with fewer than 100 values. Asserts `get_count()` equals the surviving count, then `assert_none` on `RelativeQuantileSpec::core(alpha)` over the 7-point `DDS_QS` grid. | @@ -2056,7 +2057,7 @@ Test file: [`tests/e2e/wire.rs`](../tests/e2e/wire.rs) | --- | --- | --- | | `a_bloom_envelope_round_trips_on_both_paths_and_names_its_kind` | A Bloom envelope names kind_id `0x17 0x00` and preserves every answer on both hash paths. | Fills a `Bloom` and a `Bloom`, each `with_capacity(2_000, 0.01)`, with `DataInput::I64(0..2_000)`, verifies both envelopes carry a `kind_id_len` of `2` and the bytes `0x17 0x00`, that each decode reports all 2,000 members present, that the regular decode answers exactly as its source on the 2,000 non-members `10_000..12_000`, and that `inserted()` is unchanged. | | `a_coco_envelope_round_trips_and_names_its_kind` | A Coco envelope names kind_id `0x0c 0x00` and preserves every per-key estimate. | Fills `CocoSketch::::with_dimensions_and_seed(4, 512, 0xC0C0)` with the keys `flow-{k}` of a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the envelope's `kind_id_len` of `2` and bytes `0x0c 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | -| `a_ddsketch_envelope_round_trips_and_names_its_kind` | A DDSketch envelope names kind_id `0x05 0x00` and preserves its quantile answers exactly. | Fills `DdSketch::new(0.01)` with `1.0 + v` over `uniform_u64(20_000, 1_000_000, 0x1DE7_0001)`, verifies the envelope's `kind_id_len` of `2` and bytes `0x05 0x00`, and that the decode matches the source on `get_count()`, `alpha()`, and `get_value_at_quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`. | +| `a_ddsketch_envelope_round_trips_and_names_its_kind` | A DDSketch envelope names kind_id `0x05 0x01` and preserves its quantile answers exactly. | Fills `DdSketch::new(0.01)` with `1.0 + v` over `uniform_u64(20_000, 1_000_000, 0x1DE7_0001)`, verifies the envelope's `kind_id_len` of `2` and bytes `0x05 0x00`, and that the decode matches the source on `get_count()`, `alpha()`, and `get_value_at_quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`. | | `a_hydra_envelope_round_trips_and_names_the_counter_it_carries` | A Hydra envelope's kind_id names the counter it carries, and the schema and cells survive. | Builds `Hydra::with_schema(4, 512, ["region", "user"], HydraCounter::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))`, updates each of the nine region/user pairs 40 times with `DataInput::Str("event")`, verifies the `kind_id_len` of `2` and bytes `0x07 0x01`, an unchanged `schema()`, and equal `query_key` answers under `HydraQuery::Frequency` for all nine fully specified keys. | | `a_kll_dynamic_envelope_round_trips_and_names_its_kind` | A KllDynamic envelope names kind_id `0x06 0x01` and re-encodes byte-identically. | Fills `KllDynamic::::with_k_and_seed(200, 0x5EED_9001)` from `uniform_u64(20_000, 1_000_000, 0xC0DE_0001)`, verifies the `kind_id_len` of `2` and bytes `0x06 0x01`, an unchanged `count()`, bit-identical `quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`, and that the decode re-serializes to the same bytes. | | `a_space_saving_envelope_round_trips_and_names_its_kind` | A Space-Saving envelope names kind_id `0x18 0x00` and preserves every estimate. | Fills `SpaceSaving::::with_capacity(256)` from a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the `kind_id_len` of `2` and bytes `0x18 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | @@ -2100,14 +2101,15 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `csheap_i64_regular_2x4_strkeys_matches_golden` | A CountSketchWithHeap serializes to its golden bytes. | Verifies the `2x4` Count Sketch matrix with heap size `5` holding `{"alpha": 4294967296, "beta": 127, "delta": 127, "gamma": -33}` serializes to `asapv1_golden/csheap_i64_regular_2x4_strkeys.hex`, and that the decode holds that matrix and those entries and re-serializes identically. | | `csheap_i64_regular_2x4_tuplekeys_matches_golden` | CountSketchWithHeap keys made of typed columns travel as `bytes` tuple keys. | Verifies the `2x4` Count Sketch matrix with heap size `6` holding four of the same tuple keys at `-33`, `127`, `65536` and `127` serializes to `asapv1_golden/csheap_i64_regular_2x4_tuplekeys.hex`, and that the decode holds that matrix and those entries, reads every key back as a canonical tuple key, and re-serializes identically. | | `csspacesaving_f64_fast_2x4_i64keys_matches_golden` | A CountSketchWithSpaceSaving serializes to its golden bytes. | Verifies the `2x4` Count Sketch f64 matrix beside a capacity-`3` summary fed `-1:65536, i64::MIN:5, 128:5, 7:1, i64::MIN:1` (so `7` displaces `128`) serializes to `asapv1_golden/csspacesaving_f64_fast_2x4_i64keys.hex`, and that the decode holds that matrix, the entries `-1:65536/0, 7:6/5, i64::MIN:6/0`, total `65548` and ceiling `6`, and re-serializes identically. | -| `ddsketch_{positive, signed, empty}_a001_matches_golden` | The DDSketch fixtures decode to their known state and round-trip. | For the positive and signed fixtures, verifies `asapv1_golden/ddsketch__a001.hex` decodes to the fixture's stores, zero count and scalars and re-serializes to the same bytes; for the empty one, that `DdSketch::new(0.01)` serializes to it. | -| `ddsketch_signed_a001_admission_matches_golden` | A sampled DDSketch state closes its metadata with `admission_rate`. | Verifies the `ddsketch_signed_a001` state with `with_admission_rate(0.125)` serializes to `asapv1_golden/ddsketch_signed_a001_admission.hex` and re-serializes identically after decode, and that the decode reports `admission_rate()` `0.125`, `get_count()` `18` and `sum()` `2523.90625`, unscaled. | -| `ddsketch_signed_a001_delta_matches_golden` | A known DDSketch delta serializes to its golden bytes and merges into the signed state golden. | Verifies `DdSketchDelta::new(0.01, [-300, -100, -5, 7, 200, 400], [1, 127, 128, 256, 65536, 4294967296], [-208, -33, -1, 0, 130, 1000], [2, 3, 3, 1, 3, 1], 9)` serializes to `asapv1_golden/ddsketch_signed_a001_delta.hex`, that the fixture decodes and re-serializes identically, and that merged into `ddsketch_signed_a001` it gives the summed non-zero buckets of both stores, `zero_count` `16`, `get_count()` `4295033384`, `sum` folded in payload order from `r = 1.01 * gamma^k`, `min` `-r(1000)`, and `max` `r(400)`. | +| `ddsketch_buckets_{positive, signed, empty}_a001_matches_golden` | The `0x05 0x01` DDSketch fixtures decode to their known stores and round-trip. | For the positive and signed fixtures, verifies the decoded stores and zero count and identical re-serialized bytes; for the empty one, that a fresh sketch serializes to it. | +| `ddsketch_retired_fixtures_decode_to_their_bucket_twins` | Each retired `0x05 0x00` / `0x05 0x80` fixture decodes and re-encodes as its `ddsketch_buckets_*` twin. | Decodes the positive, signed, empty and admission state fixtures and the delta fixture of the retired kinds and verifies each re-encodes to its twin's bytes. | +| `ddsketch_buckets_signed_a001_admission_matches_golden` | A sampled DDSketch state closes its metadata with `admission_rate`. | Verifies the `ddsketch_buckets_signed_a001` state at rate `0.125` serializes to its fixture, which decodes to the same stores, rate `0.125` and count `18`. | +| `ddsketch_buckets_signed_a001_delta_matches_golden` | A known DDSketch delta serializes to its golden bytes and merges into the signed state golden. | Verifies the `0x05 0x81` delta serializes to and round-trips through its fixture, and that merged into `ddsketch_buckets_signed_a001` it gives the expected buckets, `zero_count = 16` and total count. | | `delta_result_empty_matches_golden` | An empty delta serializes to its golden bytes, which round-trip unchanged. | Verifies a `SetAggregatorDelta` with both sets empty serializes to `asapv1_golden/delta_result_empty.hex` (kind_id `0x08 0x80`), and that decoding the fixture yields two empty sets and re-serializes to the same bytes. | | `delta_result_strings_matches_golden` | A delta's two sides serialize in ascending UTF-8 byte order to the golden bytes. | Verifies a `SetAggregatorDelta` with `added` `{"😀", "queue", "été"}` and `removed` `{"中", "db", "", "cache"}` serializes to `asapv1_golden/delta_result_strings.hex`, and that decoding the fixture recovers both sets and re-serializes to the same bytes. | | `eh_w100_k2_cm_2x4_matches_golden` | An ExponentialHistogram of Count-Min buckets serializes to its golden bytes. | Verifies `ExponentialHistogram::new(2, 100, prototype)` over an all-zero `2x4` `i32` fast-path Count-Min, holding buckets `[[1,0,2,0],[0,3,0,0]]` of size `3` over `[10, 30]` and `[[0,0,0,1],[1,0,0,0]]` of size `1` over `[40, 40]`, serializes to `asapv1_golden/eh_w100_k2_cm_2x4.hex`, and that the decode reports `window` `100`, `k` `2`, both buckets' cells, sizes and times, the zero prototype, and re-serializes identically. | | `ehsketchlist_cm_i32_fast_2x4_matches_golden` | An EhBucketSketch Count-Min serializes to its golden bytes. | Verifies `EhBucketSketch::CountMinSketch` over `[[127, -1, 0, 65536], [-32768, 128, i32::MAX, i32::MIN]]` serializes to `asapv1_golden/ehsketchlist_cm_i32_fast_2x4.hex`, and that the decode holds those cells and re-serializes identically. | -| `ehsketchlist_ddsketch_a001_matches_golden` | An EhBucketSketch DDSketch carries the DDSketch's own blocks. | Verifies `EhBucketSketch::DdSketch` over the decoded `ddsketch_signed_a001` fixture serializes to `asapv1_golden/ehsketchlist_ddsketch_a001.hex`, and that the decode's inner DDSketch serializes to `ddsketch_signed_a001` and the whole re-serializes identically. | +| `ehsketchlist_ddsketch_buckets_a001_matches_golden` | An EhBucketSketch DDSketch carries the DDSketch's own `0x05 0x01` blocks. | Verifies `EhBucketSketch::DdSketch` over the signed state serializes to the fixture, round-trips with the inner bytes intact, and that the retired `ehsketchlist_ddsketch_a001` decodes and re-encodes as it. | | `hll_classic_p12_admission_matches_golden` | A sampled HLL state closes its metadata with `admission_rate`. | Verifies the `hll_classic_p12` state with `with_admission_rate(0.5)` serializes to `asapv1_golden/hll_classic_p12_admission.hex`, and that decoding the fixture recovers the P12 register pattern, `admission_rate()` `0.5`, and a byte-identical re-serialization. | | `hll_classic_p12_matches_golden` | The P12 Classic golden decodes to its known register pattern and re-serializes identically. | Verifies `HyperLogLogP12::::from_asapv1` on `asapv1_golden/hll_classic_p12.hex` gives 4,096 registers that are zero except `[0] = 1`, `[1] = 7`, `[100] = 42` and `[4095] = 3`, and that `to_asapv1` reproduces the fixture. | | `hll_classic_p14_delta_matches_golden` | A known P14 Classic delta serializes to its golden bytes and merges by `max`. | Verifies `HyperLogLogDelta::::new` over indices `[0, 2, 200, 8192, 16383]` and registers `[51, 1, 9, 7, 1]` serializes to `asapv1_golden/hll_classic_p14_delta.hex`, that the decoded fixture re-serializes identically, and that merged into `hll_classic_p14` it gives registers `[0] = 51`, `[1] = 7`, `[2] = 1`, `[200] = 9`, `[8192] = 42`, `[16383] = 51` and zero elsewhere. | diff --git a/src/asapv1/kind.rs b/src/asapv1/kind.rs index e1cb09fd..2d392865 100644 --- a/src/asapv1/kind.rs +++ b/src/asapv1/kind.rs @@ -12,8 +12,16 @@ pub(crate) const COUNT_MIN_WITH_HEAP: &[u8] = &[0x03, 0x00]; pub(crate) const COUNT_MIN_WITH_HEAP_DELTA: &[u8] = &[0x03, 0x80]; pub(crate) const COUNT_SKETCH: &[u8] = &[0x04, 0x00]; pub(crate) const COUNT_SKETCH_DELTA: &[u8] = &[0x04, 0x80]; -pub(crate) const DDSKETCH: &[u8] = &[0x05, 0x00]; -pub(crate) const DDSKETCH_DELTA: &[u8] = &[0x05, 0x80]; +/// DDSketch: the two bucket stores and the zero count (spec §3.8). +pub(crate) const DDSKETCH: &[u8] = &[0x05, 0x01]; +/// The DDSketch delta (spec §3.23). +pub(crate) const DDSKETCH_DELTA: &[u8] = &[0x05, 0x81]; +/// Retired DDSketch kind that also carried `sum` / `min` / `max`: decoded, +/// with those scalars dropped, and never encoded. +pub(crate) const DDSKETCH_WITH_SCALARS: &[u8] = &[0x05, 0x00]; +/// Retired delta of [`DDSKETCH_WITH_SCALARS`]: decoded, never encoded. Its +/// payload is the current delta's. +pub(crate) const DDSKETCH_WITH_SCALARS_DELTA: &[u8] = &[0x05, 0x80]; pub(crate) const KLL_FAMILY: u8 = 0x06; pub(crate) const KLL: &[u8] = &[KLL_FAMILY, 0x00]; pub(crate) const KLL_DYNAMIC: &[u8] = &[KLL_FAMILY, 0x01]; diff --git a/src/asapv1/wire_key.rs b/src/asapv1/wire_key.rs index 154e104b..46532450 100644 --- a/src/asapv1/wire_key.rs +++ b/src/asapv1/wire_key.rs @@ -389,9 +389,6 @@ mod tests { let rows = vec![ row("DDSketch alpha", dd, 0.25, F32, dd_ok), - row("DDSketch sum", dd, 3.0, Int, dd_ok), - row("DDSketch min", dd, 1.0, Int, dd_ok), - row("DDSketch max", dd, 2.0, F32, dd_ok), row("HLL hip_kxq0", hip, 1.5, F32, hip_ok), row("HLL hip_kxq1", hip, 2.5, F32, hip_ok), row("HLL hip_est", hip, 3.0, Int, hip_ok), diff --git a/src/sketch_framework/eh_bucket_sketch.rs b/src/sketch_framework/eh_bucket_sketch.rs index 0ca4b738..d28b8ba3 100644 --- a/src/sketch_framework/eh_bucket_sketch.rs +++ b/src/sketch_framework/eh_bucket_sketch.rs @@ -174,6 +174,11 @@ impl EhBucketSketch { } /// Queries the sketch using a sketch-specific interpretation of `key`. + /// + /// A DDSketch answers a numeric `key` as a quantile and the commands + /// `"count"`, `"min"` and `"max"`; `"min"` and `"max"` are quantiles 0 and + /// 1, the lowest and highest occupied buckets' representatives, within + /// `alpha` of the smallest and largest samples. pub fn query(&self, key: &DataInput) -> Result { match (self, key) { (EhBucketSketch::CountMinSketch(count_min), _) => Ok(count_min.estimate(key) as f64), @@ -203,14 +208,22 @@ impl EhBucketSketch { .ok_or("DDSketch has no samples"), (EhBucketSketch::DdSketch(dd), DataInput::Str(cmd)) => match *cmd { "count" => Ok(dd.get_count() as f64), - "min" => dd.min().ok_or("DDSketch has no samples"), - "max" => dd.max().ok_or("DDSketch has no samples"), + "min" => dd + .get_value_at_quantile(0.0) + .ok_or("DDSketch has no samples"), + "max" => dd + .get_value_at_quantile(1.0) + .ok_or("DDSketch has no samples"), _ => Err("Unsupported command for DDSketch"), }, (EhBucketSketch::DdSketch(dd), DataInput::String(cmd)) => match cmd.as_str() { "count" => Ok(dd.get_count() as f64), - "min" => dd.min().ok_or("DDSketch has no samples"), - "max" => dd.max().ok_or("DDSketch has no samples"), + "min" => dd + .get_value_at_quantile(0.0) + .ok_or("DDSketch has no samples"), + "max" => dd + .get_value_at_quantile(1.0) + .ok_or("DDSketch has no samples"), _ => Err("Unsupported command for DDSketch"), }, (EhBucketSketch::ElasticSketch(elastic), DataInput::String(s)) => { @@ -345,6 +358,22 @@ mod tests { assert!((10.0..=30.0).contains(&q50), "unexpected q50 {q50}"); } + /// `"min"` and `"max"` on a DDSketch read quantiles 0 and 1: the lowest and + /// highest buckets, within `alpha` of the smallest and largest samples. + #[test] + fn ddsketch_min_and_max_commands_are_within_alpha() { + let alpha = 0.01; + let mut dd = EhBucketSketch::DdSketch(DdSketch::new(alpha)); + for v in [-3.5f64, 10.0, 20.0, 1234.5] { + dd.update(&DataInput::F64(v)); + } + let min = dd.query(&DataInput::Str("min")).unwrap(); + let max = dd.query(&DataInput::String("max".to_string())).unwrap(); + assert!((min - -3.5).abs() <= (alpha + 1e-9) * 3.5, "{min}"); + assert!((max - 1234.5).abs() <= (alpha + 1e-9) * 1234.5, "{max}"); + assert_eq!(dd.query(&DataInput::Str("count")).unwrap(), 4.0); + } + /// An ASAPv1 round trip of a nested DDSketch carries the sketch's whole /// state. #[test] @@ -353,22 +382,15 @@ mod tests { for v in [10.0f64, 20.0, 30.0, 40.0] { dd.update(&DataInput::F64(v)); } - let before = match &dd { - EhBucketSketch::DdSketch(inner) => { - (inner.get_count(), inner.sum(), inner.min(), inner.max()) - } + let state = |sketch: &EhBucketSketch| match sketch { + EhBucketSketch::DdSketch(inner) => (inner.get_count(), inner.to_asapv1().unwrap()), _ => panic!("expected the DDSketch variant"), }; + let before = state(&dd); let bytes = dd.to_asapv1().expect("encode"); let restored = EhBucketSketch::from_asapv1(&bytes).expect("decode"); - let after = match &restored { - EhBucketSketch::DdSketch(inner) => { - (inner.get_count(), inner.sum(), inner.min(), inner.max()) - } - _ => panic!("expected the DDSketch variant"), - }; - assert_eq!(after, before, "the nested sketch lost its running state"); + assert_eq!(state(&restored), before, "the nested sketch lost its state"); let q50 = restored .query(&DataInput::F64(0.5)) diff --git a/src/sketch_framework/eh_bucket_sketch/wire.rs b/src/sketch_framework/eh_bucket_sketch/wire.rs index 0e1def59..bec8599c 100644 --- a/src/sketch_framework/eh_bucket_sketch/wire.rs +++ b/src/sketch_framework/eh_bucket_sketch/wire.rs @@ -25,7 +25,9 @@ //! //! All ten ids dispatch whatever features are on. A decoder built without //! `experimental` rejects `0x0d 0x00` by name with the feature named, and its -//! encoder can never emit it. +//! encoder can never emit it. A DDSketch bucket encodes as `0x05 0x01`; the +//! retired `0x05 0x00` still decodes into the same variant, its `sum` / `min` / +//! `max` dropped. use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; @@ -45,7 +47,7 @@ pub(crate) fn variant_name(kind_id: &[u8]) -> Option<&'static str> { kind::COCO => Some("Coco"), kind::COUNT_SKETCH_WITH_L2_NORM => Some("CountSketchWithL2Norm"), kind::COUNT_SKETCH => Some("CountSketch"), - kind::DDSKETCH => Some("DDSketch"), + kind::DDSKETCH | kind::DDSKETCH_WITH_SCALARS => Some("DDSketch"), kind::ELASTIC => Some("Elastic"), kind::HLL_ERTL => Some("HLL"), kind::KLL => Some("KLL"), @@ -188,9 +190,9 @@ fn rebuild_variant(triple: &SketchState) -> Result Ok(EhBucketSketch::CountSketch( crate::CountSketch::from_asapv1(&bytes)?, )), - kind::DDSKETCH => Ok(EhBucketSketch::DdSketch(crate::DdSketch::from_asapv1( - &bytes, - )?)), + kind::DDSKETCH | kind::DDSKETCH_WITH_SCALARS => Ok(EhBucketSketch::DdSketch( + crate::DdSketch::from_asapv1(&bytes)?, + )), kind::ELASTIC => Ok(EhBucketSketch::ElasticSketch( crate::ElasticSketch::from_asapv1(&bytes)?, )), @@ -432,6 +434,7 @@ pub(crate) mod tests { assert_eq!(variant_name(kind_id), Some(name), "{kind_id:02x?}"); } assert_eq!(variant_name(&[0xff, 0xff]), None); + assert_eq!(variant_name(kind::DDSKETCH_WITH_SCALARS), Some("DDSketch")); for sketch in populated_variants() { let emitted = sketch_state(&sketch).expect("state").kind_id; diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index e9b55fa0..328187e1 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -22,9 +22,9 @@ use crate::common::numerical::NumericalValue; use crate::common::structures::Vector1D; use crate::octo_delta::{DdSign, DdSketchBucketDelta}; -/// The ASAPv1 DDSketch delta (kind_id `0x05 0x80`). +/// The ASAPv1 DDSketch delta (kind_id `0x05 0x81`). mod delta; -/// ASAPv1 wire serialization (kind_id `0x05 0x00`). +/// ASAPv1 wire serialization (kind_id `0x05 0x01`). mod wire; pub use delta::DdSketchDelta; @@ -121,9 +121,13 @@ impl Buckets { /// Mergeable, relative-error quantile sketch using logarithmically-spaced buckets. /// -/// ASAPv1 serialization lives in the `wire` submodule (kind_id `0x05 0x00`); it -/// carries `alpha`, both bucket stores, the zero count and `sum` / `min` / -/// `max`, and recovers `count` by summing them. +/// ASAPv1 serialization lives in the `wire` submodule (kind_id `0x05 0x01`); it +/// carries `alpha`, both bucket stores and the zero count, and recovers `count` +/// by summing them. +/// +/// The state is the buckets alone. A sum, minimum or maximum of the values is +/// not a DDSketch readout: keep it with a separate exact aggregate next to the +/// sketch. #[derive(Debug)] pub struct DdSketch { alpha: f64, @@ -133,9 +137,6 @@ pub struct DdSketch { positive_store: Buckets, count: u64, - sum: f64, - min: f64, - max: f64, negative_store: Buckets, zero_count: u64, admission_rate: f64, @@ -146,9 +147,6 @@ pub struct DdSketch { struct DdSketchState { alpha: f64, positive_store: Buckets, - sum: f64, - min: f64, - max: f64, negative_store: Buckets, zero_count: u64, admission_rate: f64, @@ -169,9 +167,6 @@ impl TryFrom for DdSketch { inv_log_gamma: 1.0 / log_gamma, positive_store: state.positive_store, count, - sum: state.sum, - min: state.min, - max: state.max, negative_store: state.negative_store, zero_count: state.zero_count, admission_rate: state.admission_rate, @@ -216,9 +211,6 @@ impl DdSketch { inv_log_gamma, positive_store: Buckets::new(), count: 0, - sum: 0.0, - min: f64::INFINITY, - max: f64::NEG_INFINITY, negative_store: Buckets::new(), zero_count: 0, admission_rate: admit_all(), @@ -226,7 +218,7 @@ impl DdSketch { } /// Sets the probability `p` with which each sample was admitted. `count` - /// and `sum` cover admitted samples alone; a consumer divides them by `p`. + /// covers admitted samples alone; a consumer divides it by `p`. pub fn with_admission_rate(mut self, p: f64) -> Self { self.admission_rate = p; self @@ -237,21 +229,6 @@ impl DdSketch { self.admission_rate } - /// Advances the running `sum` by `delta`, saturating at `f64::MAX` instead - /// of reaching `+inf`. The wire format refuses a non-finite `sum` on a - /// populated store, so an unguarded `+=` would leave a legally-ingested - /// sketch that can never be serialized. `count` and the bucket store stay - /// exact, so only `sum` degrades, and only past `f64::MAX`. - #[inline(always)] - fn add_to_sum(&mut self, delta: f64) { - let next = self.sum + delta; - self.sum = if next.is_finite() { - next - } else { - f64::MAX.copysign(next) - }; - } - /// Adds a finite signed sample. Tiny magnitudes enter the zero bucket. /// Invalid or too-large values are ignored by this compatibility API; /// use [`Self::try_add`] to receive an explicit error. @@ -271,9 +248,6 @@ impl DdSketch { return Err("DDSketch value exceeds indexable range"); } self.count = self.count.checked_add(1).ok_or("DDSketch count overflow")?; - self.add_to_sum(v); - self.min = self.min.min(v); - self.max = self.max.max(v); if v.abs() < min_indexable { self.zero_count += 1; } else { @@ -323,10 +297,8 @@ impl DdSketch { /// Adds a promoted bucket count from an OctoSketch worker. /// - /// A delta carries only a bucket and a count, so `sum`, `min` and `max` are - /// advanced with the bucket's representative value - the same α-bounded - /// estimate a deserialize-and-recompute produces. Quantiles and `count` - /// stay exact with respect to the bucket store. + /// The bucket and `count` change exactly as the same samples' `add`s + /// would change them. pub fn apply_delta(&mut self, delta: DdSketchBucketDelta) { if delta.sign == DdSign::Zero { self.apply_zero_delta(delta.count); @@ -354,80 +326,20 @@ impl DdSketch { store.ensure(delta.index); let slot = (delta.index - store.offset) as usize; store.counts.as_mut_slice()[slot] += delta.count; - - let representative = self.bin_representative(delta.index) - * if delta.sign == DdSign::Negative { - -1.0 - } else { - 1.0 - }; self.count += delta.count; - self.add_to_sum(representative * delta.count as f64); - if representative < self.min { - self.min = representative; - } - if representative > self.max { - self.max = representative; - } } - /// Returns the estimated value at quantile `q` (in `[0, 1]`), or `None` if the sketch is empty. + /// Returns the estimated value at quantile `q`, or `None` if the sketch is + /// empty or `q` is NaN: the representative of the bucket holding the + /// order statistic at nearest rank `ceil(q * n)` (1-based). `q <= 0` reads + /// the lowest occupied bucket and `q >= 1` the highest, so every answer, + /// the extremes included, is within `alpha` relative of a sample. pub fn get_value_at_quantile(&self, q: f64) -> Option { if self.count == 0 || q.is_nan() { return None; } - if q <= 0.0 { - return Some(self.min); - } - if q >= 1.0 { - return Some(self.max); - } - - let rank = ((q * self.count as f64).ceil() as u64).max(1); - let mut seen = 0u64; - for (i, &count) in self - .negative_store - .counts - .as_slice() - .iter() - .enumerate() - .rev() - { - seen += count; - if seen >= rank { - return Some( - (-self.bin_representative(self.negative_store.offset + i as i32)) - .clamp(self.min, self.max), - ); - } - } - seen += self.zero_count; - if seen >= rank { - return Some(0.0); - } - - let slice = self.positive_store.counts.as_slice(); - let offset = self.positive_store.offset; - - for (i, &c) in slice.iter().enumerate() { - if c == 0 { - continue; - } - seen += c; - if seen >= rank { - let bin = offset + i as i32; - let mut v = self.bin_representative(bin); - if v < self.min { - v = self.min; - } - if v > self.max { - v = self.max; - } - return Some(v); - } - } - - Some(self.max) + let rank = ((q * self.count as f64).ceil() as u64).clamp(1, self.count); + Some(self.value_at_rank(rank - 1)) } /// Returns the quantile at `q` interpolated linearly between the two @@ -438,10 +350,9 @@ impl DdSketch { /// - `Ok(None)` for an empty sketch. /// - `Err` for `q` outside `[0, 1]`, NaN included; mapping such a `q` to /// `-inf` / `+inf` (as PromQL does) is the caller's job. - /// - Rank 0 and rank `n - 1` are the exactly retained minimum and - /// maximum; every other order statistic is its bucket's representative - /// (negative store, zero bucket or positive store), clamped to - /// `[min, max]`. + /// - Every order statistic, rank 0 and rank `n - 1` included, is its + /// bucket's representative (negative store, zero bucket or positive + /// store). /// /// Error: each neighbour is within `alpha` relative of the true order /// statistic when its magnitude is indexable, so when both neighbours are @@ -472,38 +383,29 @@ impl DdSketch { } /// The estimated order statistic at 0-based `rank` (`rank < count`): the - /// exact minimum and maximum at the ends, otherwise the representative - /// of the bucket holding it, clamped to `[min, max]`. + /// representative of the bucket holding it, `0.0` in the zero bucket. fn value_at_rank(&self, rank: u64) -> f64 { debug_assert!(rank < self.count); - if rank == 0 { - return self.min; - } - if rank == self.count - 1 { - return self.max; - } let mut seen = 0u64; let negative = self.negative_store.counts.as_slice(); for (i, &count) in negative.iter().enumerate().rev() { seen += count; if seen > rank { - let bin = self.negative_store.offset + i as i32; - return (-self.bin_representative(bin)).clamp(self.min, self.max); + return -self.bin_representative(self.negative_store.offset + i as i32); } } seen += self.zero_count; if seen > rank { - return 0.0_f64.clamp(self.min, self.max); + return 0.0; } let positive = self.positive_store.counts.as_slice(); for (i, &count) in positive.iter().enumerate() { seen += count; if seen > rank { - let bin = self.positive_store.offset + i as i32; - return self.bin_representative(bin).clamp(self.min, self.max); + return self.bin_representative(self.positive_store.offset + i as i32); } } - self.max + unreachable!("rank {rank} is below the bucket total {}", self.count) } /// Returns the total number of samples inserted so far. @@ -511,24 +413,6 @@ impl DdSketch { self.count } - /// Returns the minimum sample seen, or `None` if the sketch is empty. - pub fn min(&self) -> Option { - if self.count == 0 { - None - } else { - Some(self.min) - } - } - - /// Returns the maximum sample seen, or `None` if the sketch is empty. - pub fn max(&self) -> Option { - if self.count == 0 { - None - } else { - Some(self.max) - } - } - /// Returns the relative-accuracy parameter `alpha`. pub fn alpha(&self) -> f64 { self.alpha @@ -595,14 +479,6 @@ impl DdSketch { size_of::().saturating_add(slots.saturating_mul(size_of::())) } - /// Returns the running sum. Exact over the values passed to [`Self::add`]; - /// a bucket promoted through [`Self::apply_delta`] contributes its bucket - /// representative instead, and the total saturates at `f64::MAX` rather - /// than overflowing to `+inf`. - pub fn sum(&self) -> f64 { - self.sum - } - /// Returns the raw bucket-count slice. Each entry is the number of /// samples in the bucket whose absolute index is `positive_store_offset() + i`. pub fn positive_store_counts(&self) -> &[u64] { @@ -628,10 +504,6 @@ impl DdSketch { pub fn apply_zero_delta(&mut self, count: u64) { self.count += count; self.zero_count += count; - if count != 0 { - self.min = self.min.min(0.0); - self.max = self.max.max(0.0); - } } /// Merges another DDSketch into this one. Returns `Err` if the two sketches @@ -663,13 +535,6 @@ impl DdSketch { } self.count += other.count; - self.add_to_sum(other.sum); - if other.min < self.min { - self.min = other.min; - } - if other.max > self.max { - self.max = other.max; - } // Merge bucket vectors Self::merge_store(&mut self.positive_store, &other.positive_store); @@ -743,9 +608,6 @@ impl Clone for DdSketch { inv_log_gamma: self.inv_log_gamma, positive_store: self.positive_store.clone(), count: self.count, - sum: self.sum, - min: self.min, - max: self.max, negative_store: self.negative_store.clone(), zero_count: self.zero_count, admission_rate: self.admission_rate, @@ -795,15 +657,19 @@ mod tests { "q={q}: {actual} against {expected}" ); } - // The nearest-rank readout keeps its convention. - assert_eq!(sketch.get_value_at_quantile(0.9), Some(40.0)); + // The nearest-rank readout keeps its convention: rank ceil(0.9 * 2) = 2, + // the bucket holding 40. + let nearest = sketch.get_value_at_quantile(0.9).unwrap(); + assert!((nearest - 40.0).abs() <= (0.01 + 1e-9) * 40.0, "{nearest}"); } #[test] fn quantile_interpolated_one_value_and_empty() { let one = sketch_of(0.01, &[20.0]); - for q in [0.0, 0.3, 1.0] { - assert_eq!(one.quantile_interpolated(q).unwrap(), Some(20.0)); + let bucket = one.quantile_interpolated(0.0).unwrap().unwrap(); + assert!((bucket - 20.0).abs() <= (0.01 + 1e-9) * 20.0, "{bucket}"); + for q in [0.3, 1.0] { + assert_eq!(one.quantile_interpolated(q).unwrap(), Some(bucket)); } let empty = DdSketch::new(0.01); assert_eq!(empty.quantile_interpolated(0.5).unwrap(), None); @@ -895,8 +761,7 @@ mod tests { for &p in &ps { let q = s.get_value_at_quantile(p).expect("quantile"); assert!(q >= prev - 1e-12, "non-monotone at p={p}: {q} < {prev}"); - assert!(q <= s.max().unwrap() + 1e-12); - assert!(q >= s.min().unwrap() - 1e-12); + assert!((-5.0 * 1.01..=1000.0 * 1.01).contains(&q), "p={p}: {q}"); prev = q; } } @@ -929,18 +794,17 @@ mod tests { s1.merge(&s2).unwrap(); - // counts and bounds assert_eq!(s1.get_count(), (vals1.len() + vals2.len()) as u64); - assert_eq!(s1.min().unwrap(), 1.0); - assert_eq!(s1.max().unwrap(), 20.0); - // extreme quantiles should match bounds - assert_eq!(s1.get_value_at_quantile(0.0).unwrap(), 1.0); - assert_eq!(s1.get_value_at_quantile(1.0).unwrap(), 20.0); + // The extreme quantiles read the lowest and highest buckets, within + // alpha of the smallest and largest samples. + let lowest = s1.get_value_at_quantile(0.0).unwrap(); + let highest = s1.get_value_at_quantile(1.0).unwrap(); + assert!((lowest - 1.0).abs() <= (ALPHA + 1e-9) * 1.0, "{lowest}"); + assert!((highest - 20.0).abs() <= (ALPHA + 1e-9) * 20.0, "{highest}"); - // sanity: middle quantile is within [min, max] let mid = s1.get_value_at_quantile(0.5).unwrap(); - assert!((1.0..=20.0).contains(&mid)); + assert!((lowest..=highest).contains(&mid)); } #[test] @@ -957,12 +821,9 @@ mod tests { ); let decoded = DdSketch::from_asapv1(&encoded).expect("DDSketch deserialization fail"); - // `count` is summed back from the buckets; `sum`/`min`/`max` are carried - // on the wire, so every scalar and every quantile comes back exact. + // `count` is summed back from the buckets, and every quantile reads + // the same buckets, so both come back exact. assert_eq!(decoded.get_count(), s.get_count()); - assert_eq!(decoded.sum(), s.sum()); - assert_eq!(decoded.min(), s.min()); - assert_eq!(decoded.max(), s.max()); for q in [0.0, 0.1, 0.5, 0.9, 1.0] { assert_eq!( decoded.get_value_at_quantile(q), diff --git a/src/sketches/ddsketch/delta.rs b/src/sketches/ddsketch/delta.rs index d34308d4..3794efef 100644 --- a/src/sketches/ddsketch/delta.rs +++ b/src/sketches/ddsketch/delta.rs @@ -1,6 +1,7 @@ -//! The DDSketch delta, ASAPv1 kind `0x05 0x80` (`docs/asapv1_wire_format.md` +//! The DDSketch delta, ASAPv1 kind `0x05 0x81` (`docs/asapv1_wire_format.md` //! §3.23). Its metadata is the state's (§3.8); its payload is -//! `[indices, deltas, negative_indices, negative_deltas, zero_count]`. +//! `[indices, deltas, negative_indices, negative_deltas, zero_count]`. The +//! retired `0x05 0x80` has the same payload and still decodes. use std::ops::RangeInclusive; @@ -11,7 +12,7 @@ use crate::asapv1::admission_rate; use crate::asapv1::delta::check_cells; use crate::asapv1::envelope; use crate::asapv1::kind; -use crate::asapv1::kind::DDSKETCH_DELTA; +use crate::asapv1::kind::{DDSKETCH_DELTA, DDSKETCH_WITH_SCALARS_DELTA}; use super::wire::{check_alpha, decode_metadata, encode_metadata}; use super::{Buckets, DdSketch, GROW_CHUNK, ddsketch_indexable_bounds}; @@ -123,13 +124,15 @@ impl DdSketchDelta { Ok(envelope::encode(kind::DDSKETCH_DELTA, &metadata, &payload)) } - /// Deserializes a delta from an ASAPv1 MessagePack envelope. + /// Deserializes a delta from an ASAPv1 MessagePack envelope of kind + /// `0x05 0x81`, or of the retired `0x05 0x80`, whose payload is the same. pub fn from_asapv1(bytes: &[u8]) -> Result { let (kind_id, metadata, payload) = envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; - if kind_id != kind::DDSKETCH_DELTA { + if kind_id != DDSKETCH_DELTA && kind_id != DDSKETCH_WITH_SCALARS_DELTA { return Err(RmpDecodeError::Uncategorized(format!( - "DDSketch delta kind_id mismatch: stored {kind_id:?}, expected {DDSKETCH_DELTA:?}" + "DDSketch delta kind_id mismatch: stored {kind_id:?}, expected {DDSKETCH_DELTA:?} \ + (or the retired {DDSKETCH_WITH_SCALARS_DELTA:?})" ))); } let (alpha, admission_rate) = decode_metadata(metadata)?; @@ -148,11 +151,10 @@ impl DdSketchDelta { } impl DdSketch { - /// Adds a delta to the sketch. Each positive bucket, then each negative - /// one, adds its count `c` and folds the bucket representative `r` into the - /// scalars (`sum += r * c`, saturating; `min` / `max`); a non-zero - /// `zero_count` adds to the zero count and folds `0.0` into `min` / `max`. - /// Fails, changing nothing, when the delta's `alpha` or admission rate + /// Adds a delta to the sketch: each count to its positive or negative + /// bucket, and `zero_count` to the zero count. The result is the state the + /// same samples' `add`s would have built, bucket for bucket. Fails, + /// changing nothing, when the delta's `alpha` or admission rate /// differs, the total sample count overflows, or a store would span past /// `i32`. pub fn merge_delta(&mut self, delta: &DdSketchDelta) -> Result<(), String> { @@ -189,18 +191,9 @@ impl DdSketch { }; store.ensure(k); store.counts.as_mut_slice()[(k - store.offset) as usize] += c; - let magnitude = self.bin_representative(k); - let r = if negative { -magnitude } else { magnitude }; - self.add_to_sum(r * c as f64); - self.min = self.min.min(r); - self.max = self.max.max(r); } } - if delta.zero_count != 0 { - self.zero_count += delta.zero_count; - self.min = self.min.min(0.0); - self.max = self.max.max(0.0); - } + self.zero_count += delta.zero_count; self.count = count; Ok(()) } @@ -254,30 +247,108 @@ mod tests { fn decode_rejects_another_kind_and_new_another_alpha() { let mut bytes = delta_bytes((&[0], &[1]), (&[], &[]), 0); assert!(DdSketchDelta::from_asapv1(&bytes).is_ok()); - bytes[9] = 0x00; + // The retired delta kind has the same payload and still decodes. + bytes[9] = 0x80; + assert!(DdSketchDelta::from_asapv1(&bytes).is_ok()); + bytes[9] = 0x01; assert!(DdSketchDelta::from_asapv1(&bytes).is_err()); assert!(DdSketchDelta::new(1.0, vec![], vec![], vec![], vec![], 1).is_err()); } - /// A delta onto an empty sketch moves `min` / `max` to the representatives - /// it adds, and its zero count folds in `0.0`. + /// The changed buckets of a producer, sent as deltas after each batch of + /// updates, rebuild the producer's state bit for bit at the receiver. + #[test] + fn deltas_rebuild_the_producer_bit_for_bit() { + /// The cells of `after` that differ from `before`, as a delta. + fn diff(before: &DdSketch, after: &DdSketch) -> DdSketchDelta { + let cells = |store: &Buckets, earlier: &Buckets| { + let count_at = |s: &Buckets, k: i32| { + let i = i64::from(k) - i64::from(s.offset); + usize::try_from(i) + .ok() + .and_then(|i| s.counts.as_slice().get(i).copied()) + .unwrap_or(0) + }; + let mut indices = Vec::new(); + let mut deltas = Vec::new(); + if let Some((l, r)) = store.range() { + for k in l..=r { + let c = count_at(store, k) - count_at(earlier, k); + if c != 0 { + indices.push(k); + deltas.push(c); + } + } + } + (indices, deltas) + }; + let (indices, deltas) = cells(&after.positive_store, &before.positive_store); + let (negative_indices, negative_deltas) = + cells(&after.negative_store, &before.negative_store); + DdSketchDelta::new( + ALPHA, + indices, + deltas, + negative_indices, + negative_deltas, + after.zero_count - before.zero_count, + ) + .unwrap() + } + + let batches: [&[f64]; 3] = [ + &[3.0, 3.1, 250.0, -7.5], + &[0.0, 1e-9, 3.0, 9_000.0, -0.004], + &[42.0, -7.5, 0.0, 1.0], + ]; + let mut producer = DdSketch::new(ALPHA); + let mut receiver = DdSketch::new(ALPHA); + for batch in batches { + let before = producer.clone(); + for v in batch { + producer.add(v); + } + let wire = diff(&before, &producer).to_asapv1().unwrap(); + receiver + .merge_delta(&DdSketchDelta::from_asapv1(&wire).unwrap()) + .unwrap(); + assert_eq!(receiver.get_count(), producer.get_count()); + for q in [0.0, 0.25, 0.5, 0.75, 1.0] { + assert_eq!( + receiver.get_value_at_quantile(q), + producer.get_value_at_quantile(q) + ); + } + } + // Stores grown in a different order still hold the same cells. + let cells = |s: &DdSketch| { + let nonzero = |counts: &[u64], offset: i32| { + counts + .iter() + .enumerate() + .filter(|(_, c)| **c != 0) + .map(|(i, c)| (offset + i as i32, *c)) + .collect::>() + }; + ( + nonzero(s.positive_store_counts(), s.positive_store_offset()), + nonzero(s.negative_store_counts(), s.negative_store_offset()), + s.zero_count(), + ) + }; + assert_eq!(cells(&receiver), cells(&producer)); + } + + /// A zero-count-only delta adds to the zero count and the total. #[test] - fn merge_delta_folds_the_scalars() { + fn merge_delta_adds_the_zero_count() { let mut sketch = DdSketch::new(ALPHA); let delta = DdSketchDelta::new(ALPHA, vec![10], vec![2], vec![], vec![], 0).unwrap(); sketch.merge_delta(&delta).unwrap(); - let r = (1.0 + ALPHA) * ((1.0 + ALPHA) / (1.0 - ALPHA)).powf(10.0); - assert_eq!( - (sketch.min(), sketch.max(), sketch.sum()), - (Some(r), Some(r), 2.0 * r) - ); - let zeros = DdSketchDelta::new(ALPHA, vec![], vec![], vec![], vec![], 3).unwrap(); sketch.merge_delta(&zeros).unwrap(); - assert_eq!( - (sketch.min(), sketch.zero_count(), sketch.get_count()), - (Some(0.0), 3, 5) - ); + assert_eq!((sketch.zero_count(), sketch.get_count()), (3, 5)); + assert_eq!(sketch.get_value_at_quantile(0.0), Some(0.0)); } #[test] @@ -314,9 +385,6 @@ mod tests { counts: Vector1D::from_vec(vec![1]), offset: i32::MAX - 1, }, - sum: 1e300, - min: 1e300, - max: 1e300, negative_store: Buckets::new(), zero_count: 0, admission_rate: 1.0, diff --git a/src/sketches/ddsketch/wire.rs b/src/sketches/ddsketch/wire.rs index 65329506..ec73c8a7 100644 --- a/src/sketches/ddsketch/wire.rs +++ b/src/sketches/ddsketch/wire.rs @@ -4,9 +4,15 @@ //! ASAPv1 serialization (the metadata/payload DTOs, the kind_id constant, and //! the `to_asapv1` / `from_asapv1` impls) while the //! algorithm lives in the parent module file. Being a descendant module, it -//! reads the sketch's private `store` / `count` / `sum` / `min` / `max` fields -//! directly without widening any field visibility. See -//! `docs/asapv1_wire_format.md` §3.8. +//! reads the sketch's private stores and `count` directly without widening any +//! field visibility. See `docs/asapv1_wire_format.md` §3.8. +//! +//! ## Kind ids +//! +//! The sketch encodes as `0x05 0x01`: the two stores and the zero count. The +//! retired `0x05 0x00` also carried `sum` / `min` / `max`; it still decodes, +//! with those three scalars read and dropped, so bytes written before the +//! change stay readable. Nothing encodes it. //! //! ## DDSketch metadata has no hash spec //! @@ -32,7 +38,7 @@ use serde::{Deserialize, Serialize}; use crate::asapv1::admission_rate; use crate::asapv1::envelope; use crate::asapv1::kind; -use crate::asapv1::kind::DDSKETCH; +use crate::asapv1::kind::{DDSKETCH, DDSKETCH_WITH_SCALARS}; use crate::asapv1::wire_key::exact; use crate::common::structures::Vector1D; @@ -92,14 +98,25 @@ pub(super) fn decode_metadata(metadata: &[u8]) -> Result<(f64, f64), RmpDecodeEr Ok((meta.alpha, admission_rate)) } -/// DDSketch payload (ASAPv1 §3.8), a msgpack **array** (`to_vec`, positional). -/// Each store is carried verbatim including the growth padding, so a decoded -/// sketch re-serializes byte-identically; the bucket index of `counts[i]` is -/// `offset + i`. `sum` / `min` / `max` are the running scalars, which the -/// bucket counts do not determine. The total sample count is the sum of both -/// stores and `zero_count`, and so is not carried. +/// DDSketch payload (ASAPv1 §3.8, kind `0x05 0x01`), a msgpack **array** +/// (`to_vec`, positional). Each store is carried verbatim including the growth +/// padding, so a decoded sketch re-serializes byte-identically; the bucket +/// index of `counts[i]` is `offset + i`. The total sample count is the sum of +/// both stores and `zero_count`, and so is not carried. #[derive(Debug, Serialize, Deserialize)] struct DdPayload { + counts: Vec, + offset: i32, + negative_counts: Vec, + negative_offset: i32, + zero_count: u64, +} + +/// The payload of the retired kind `0x05 0x00`: [`DdPayload`]'s fields with +/// `sum` / `min` / `max` between `offset` and `negative_counts`. Decoded only; +/// the three scalars are dropped. +#[derive(Debug, Serialize, Deserialize)] +struct DdPayloadWithScalars { counts: Vec, offset: i32, #[serde(deserialize_with = "exact")] @@ -113,6 +130,18 @@ struct DdPayload { zero_count: u64, } +impl From for DdPayload { + fn from(p: DdPayloadWithScalars) -> Self { + Self { + counts: p.counts, + offset: p.offset, + negative_counts: p.negative_counts, + negative_offset: p.negative_offset, + zero_count: p.zero_count, + } + } +} + /// Checks a relative accuracy against the domain [`DdSketch::new`] accepts: /// finite and strictly inside `(0, 1)`. Outside it the `gamma` derivation is /// non-positive or infinite and every bucket index is meaningless. @@ -154,39 +183,7 @@ fn store_total(store: &Buckets) -> Option { .try_fold(0u64, |acc, &c| acc.checked_add(c)) } -/// Checks the running scalars against the total count. An empty sketch carries -/// `0.0` / `+inf` / `-inf`, the state [`DdSketch::new`] starts in. A populated -/// one carries finite scalars with `min <= max`; when every value is positive, -/// also `0 < min` and `sum >= min`. -fn check_scalars( - count: u64, - positive_only: bool, - sum: f64, - min: f64, - max: f64, -) -> Result<(), String> { - if count == 0 { - if sum == 0.0 && min == f64::INFINITY && max == f64::NEG_INFINITY { - return Ok(()); - } - return Err(format!( - "DDSketch empty store must carry sum=0, min=inf, max=-inf, got {sum}, {min}, {max}" - )); - } - if !(sum.is_finite() && min.is_finite() && max.is_finite()) { - return Err(format!( - "DDSketch scalars must be finite for a populated store: {sum}, {min}, {max}" - )); - } - if !(min <= max && (!positive_only || (min > 0.0 && sum >= min))) { - return Err(format!( - "DDSketch scalars out of order: sum={sum}, min={min}, max={max}" - )); - } - Ok(()) -} - -/// Checks a state against §3.8 rules 3 through 7 and returns its total sample +/// Checks a state against §3.8 rules 3 through 6 and returns its total sample /// count, the checked sum of both stores and `zero_count`. pub(super) fn check_state(state: &DdSketchState) -> Result { check_alpha(state.alpha)?; @@ -198,8 +195,6 @@ pub(super) fn check_state(state: &DdSketchState) -> Result { .and_then(|c| c.checked_add(negative)) .and_then(|c| c.checked_add(state.zero_count)) .ok_or_else(overflow)?; - let positive_only = negative == 0 && state.zero_count == 0; - check_scalars(count, positive_only, state.sum, state.min, state.max)?; Ok(count) } @@ -207,19 +202,16 @@ pub(super) fn check_state(state: &DdSketchState) -> Result { // so these impls read the private fields and construct the struct directly. impl DdSketch { /// Serializes the sketch into an ASAPv1 MessagePack envelope - /// (kind_id `0x05 0x00`). `alpha` lands in the metadata; the payload is the - /// two bucket stores, the zero count and the three scalars. + /// (kind_id `0x05 0x01`). `alpha` lands in the metadata; the payload is the + /// two bucket stores and the zero count. /// /// A sketch whose state the decoder would refuse — an out-of-range `alpha`, - /// a store spanning past `i32`, counts that overflow `u64`, or inconsistent - /// scalars — is an error rather than bytes that would be refused on decode. + /// a store spanning past `i32`, or counts that overflow `u64` — is an error + /// rather than bytes that would be refused on decode. pub fn to_asapv1(&self) -> Result, RmpEncodeError> { let state = DdSketchState { alpha: self.alpha, positive_store: self.positive_store.clone(), - sum: self.sum, - min: self.min, - max: self.max, negative_store: self.negative_store.clone(), zero_count: self.zero_count, admission_rate: self.admission_rate, @@ -230,9 +222,6 @@ impl DdSketch { let payload = rmp_serde::to_vec(&DdPayload { counts: state.positive_store.counts.into_vec(), offset: state.positive_store.offset, - sum: self.sum, - min: self.min, - max: self.max, negative_counts: state.negative_store.counts.into_vec(), negative_offset: state.negative_store.offset, zero_count: self.zero_count, @@ -240,19 +229,26 @@ impl DdSketch { Ok(envelope::encode(kind::DDSKETCH, &metadata, &payload)) } - /// Deserializes a DDSketch from an ASAPv1 MessagePack envelope. The index - /// mapping is rebuilt from the metadata `alpha` and the total sample count - /// is summed from the stores. Every inconsistency fails closed. + /// Deserializes a DDSketch from an ASAPv1 MessagePack envelope of kind + /// `0x05 0x01`, or of the retired `0x05 0x00`, whose `sum` / `min` / `max` + /// are read and dropped. The index mapping is rebuilt from the metadata + /// `alpha` and the total sample count is summed from the stores. Every + /// inconsistency fails closed. pub fn from_asapv1(bytes: &[u8]) -> Result { let (kind_id, metadata, payload) = envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; - if kind_id != kind::DDSKETCH { + if kind_id != DDSKETCH && kind_id != DDSKETCH_WITH_SCALARS { return Err(RmpDecodeError::Uncategorized(format!( - "DDSketch kind_id mismatch: stored {kind_id:?}, expected {DDSKETCH:?}" + "DDSketch kind_id mismatch: stored {kind_id:?}, expected {DDSKETCH:?} \ + (or the retired {DDSKETCH_WITH_SCALARS:?})" ))); } let (alpha, admission_rate) = decode_metadata(metadata)?; - let mut p: DdPayload = envelope::decode_block(payload)?; + let mut p: DdPayload = if kind_id == DDSKETCH { + envelope::decode_block(payload)? + } else { + envelope::decode_block::(payload)?.into() + }; p.counts.shrink_to_fit(); p.negative_counts.shrink_to_fit(); DdSketch::try_from(DdSketchState { @@ -261,9 +257,6 @@ impl DdSketch { counts: Vector1D::from_vec(p.counts), offset: p.offset, }, - sum: p.sum, - min: p.min, - max: p.max, negative_store: Buckets { counts: Vector1D::from_vec(p.negative_counts), offset: p.negative_offset, @@ -291,13 +284,10 @@ mod tests { } /// A payload with only a positive store. - fn positive(counts: Vec, offset: i32, sum: f64, min: f64, max: f64) -> DdPayload { + fn positive(counts: Vec, offset: i32) -> DdPayload { DdPayload { counts, offset, - sum, - min, - max, negative_counts: Vec::new(), negative_offset: 0, zero_count: 0, @@ -313,8 +303,8 @@ mod tests { envelope::encode(kind::DDSKETCH, &metadata, &payload) } - fn crafted(alpha: f64, counts: Vec, offset: i32, sum: f64, min: f64, max: f64) -> Vec { - crafted_payload(alpha, &positive(counts, offset, sum, min, max)) + fn crafted(alpha: f64, counts: Vec, offset: i32) -> Vec { + crafted_payload(alpha, &positive(counts, offset)) } /// The message a rejected decode fails with, so a test can pin *which* @@ -331,9 +321,9 @@ mod tests { let bytes = sketch.to_asapv1().expect("serialize"); assert!(bytes.starts_with(b"ASAPv1")); - assert_eq!(&bytes[7..10], &[2u8, 0x05, 0x00]); // kind_id_len=2, kind_id=[0x05,0x00] + assert_eq!(&bytes[7..10], &[2u8, 0x05, 0x01]); // kind_id_len=2, kind_id=[0x05,0x01] let (kind_id, metadata, _) = envelope::split(&bytes).expect("split"); - assert_eq!(kind_id, &[0x05, 0x00]); + assert_eq!(kind_id, &[0x05, 0x01]); let meta: DdMetadata = from_slice(metadata).expect("metadata"); assert_eq!(meta.metadata_version, 1); assert_eq!(meta.alpha, ALPHA); @@ -360,38 +350,53 @@ mod tests { } } - /// `sum` / `min` / `max` are not derivable from the buckets, so they are - /// carried and come back exactly — not as the α-bounded bucket - /// representatives a recomputation would produce. - #[test] - fn ddsketch_scalars_survive_exactly() { - let sketch = populated(); - let decoded = - DdSketch::from_asapv1(&sketch.to_asapv1().expect("serialize")).expect("decode"); - assert_eq!(decoded.sum(), sketch.sum()); - assert_eq!(decoded.min(), Some(0.25)); - assert_eq!(decoded.max(), Some(1000.0)); - assert_eq!(decoded.alpha(), sketch.alpha()); - } - - /// A positive-only sketch still carries all eight payload fields, and the - /// total sample count is not one of them. + /// The payload is the two stores and the zero count: five fields, with + /// neither the total sample count nor any running scalar. #[test] - fn ddsketch_payload_is_eight_fields_without_a_count() { - type Eight = (Vec, i32, f64, f64, f64, Vec, i32, u64); + fn ddsketch_payload_is_five_fields() { + type Five = (Vec, i32, Vec, i32, u64); let sketch = populated(); let bytes = sketch.to_asapv1().expect("serialize"); let (_, _, payload) = envelope::split(&bytes).expect("split"); - let eight: Eight = envelope::decode_block(payload).expect("8-element payload"); - assert_eq!(eight.0.iter().sum::(), sketch.get_count()); - assert_eq!((eight.5.len(), eight.6, eight.7), (0, 0, 0)); + let five: Five = envelope::decode_block(payload).expect("5-element payload"); + assert_eq!(five.0.iter().sum::(), sketch.get_count()); + assert_eq!((five.2.len(), five.3, five.4), (0, 0, 0)); assert_eq!( DdSketch::from_asapv1(&bytes).expect("decode").get_count(), sketch.get_count() ); } + /// Bytes of the retired kind `0x05 0x00` decode to the same stores, with + /// their `sum` / `min` / `max` dropped, and re-encode as `0x05 0x01`. + #[test] + fn ddsketch_decodes_the_retired_kind_with_scalars() { + let sketch = populated(); + let current = sketch.to_asapv1().expect("serialize"); + let metadata = encode_metadata(ALPHA, 1.0).unwrap(); + let payload = rmp_serde::to_vec(&DdPayloadWithScalars { + counts: sketch.positive_store_counts().to_vec(), + offset: sketch.positive_store_offset(), + sum: 1166.25, + min: 0.25, + max: 1000.0, + negative_counts: Vec::new(), + negative_offset: 0, + zero_count: 0, + }) + .unwrap(); + let retired = envelope::encode(kind::DDSKETCH_WITH_SCALARS, &metadata, &payload); + let decoded = DdSketch::from_asapv1(&retired).expect("decode the retired kind"); + assert_eq!(decoded.get_count(), sketch.get_count()); + assert_eq!(decoded.to_asapv1().unwrap(), current); + + // A retired payload must still have all eight fields. + let five = rmp_serde::to_vec(&positive(vec![3], 7)).unwrap(); + let short = envelope::encode(kind::DDSKETCH_WITH_SCALARS, &metadata, &five); + assert!(DdSketch::from_asapv1(&short).is_err()); + } + #[test] fn ddsketch_empty_round_trip() { let sketch = DdSketch::new(ALPHA); @@ -399,14 +404,13 @@ mod tests { let decoded = DdSketch::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.get_count(), 0); - assert_eq!(decoded.min(), None); - assert_eq!(decoded.max(), None); + assert_eq!(decoded.get_value_at_quantile(0.5), None); assert!(decoded.positive_store_counts().is_empty()); assert_eq!(decoded.to_asapv1().expect("re-serialize"), bytes); } /// A merged sketch — a store grown on both sides — round-trips with its - /// scalars and quantiles intact. + /// quantiles intact. #[test] fn ddsketch_merged_round_trip() { let mut left = DdSketch::new(ALPHA); @@ -420,9 +424,12 @@ mod tests { let bytes = left.to_asapv1().expect("serialize"); let decoded = DdSketch::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.get_count(), left.get_count()); - assert_eq!(decoded.sum(), left.sum()); - assert_eq!(decoded.min(), left.min()); - assert_eq!(decoded.max(), left.max()); + for q in [0.0, 0.5, 1.0] { + assert_eq!( + decoded.get_value_at_quantile(q), + left.get_value_at_quantile(q) + ); + } assert_eq!(decoded.to_asapv1().expect("re-serialize"), bytes); } @@ -447,6 +454,10 @@ mod tests { DdSketch::from_asapv1(&bytes).is_err(), "a foreign kind_id must be rejected" ); + // The delta kind is not a state. + bytes[8] = 0x05; + bytes[9] = 0x81; + assert!(DdSketch::from_asapv1(&bytes).is_err()); } /// Fail closed on an unexpected metadata key. @@ -491,7 +502,7 @@ mod tests { /// `metadata_version` is `1`; any other version is refused. #[test] fn ddsketch_rejects_another_metadata_version() { - let payload = rmp_serde::to_vec(&positive(vec![3], 7, 6.0, 1.5, 2.5)).unwrap(); + let payload = rmp_serde::to_vec(&positive(vec![3], 7)).unwrap(); for version in [0u8, 2] { let metadata = rmp_serde::to_vec_named(&DdMetadata { metadata_version: version, @@ -512,7 +523,7 @@ mod tests { #[test] fn ddsketch_rejects_alpha_outside_the_unit_interval() { for alpha in [0.0, 1.0, -0.5, 2.0, f64::NAN, f64::INFINITY] { - let bytes = crafted(alpha, vec![3], 7, 6.0, 1.5, 2.5); + let bytes = crafted(alpha, vec![3], 7); let err = decode_error(&bytes); assert!( err.contains("alpha") || err.contains("metadata mismatch"), @@ -520,7 +531,7 @@ mod tests { ); } // The neighbouring in-range value is the control case. - assert!(DdSketch::from_asapv1(&crafted(ALPHA, vec![3], 7, 6.0, 1.5, 2.5)).is_ok()); + assert!(DdSketch::from_asapv1(&crafted(ALPHA, vec![3], 7)).is_ok()); } /// `alpha` is checked before the payload is read: a bad `alpha` in front of @@ -537,18 +548,18 @@ mod tests { /// store is rebuilt — and never panics. #[test] fn ddsketch_rejects_a_store_span_past_i32() { - let err = decode_error(&crafted(ALPHA, vec![1; 10], i32::MAX - 2, 5.0, 1.0, 2.0)); + let err = decode_error(&crafted(ALPHA, vec![1; 10], i32::MAX - 2)); assert!( err.contains("store span past i32"), "an overflowing store span must be rejected as such, got {err}" ); - let mut negative = positive(vec![3], 7, 6.0, -1.0, 2.5); + let mut negative = positive(vec![3], 7); negative.negative_counts = vec![1; 10]; negative.negative_offset = i32::MAX - 2; let err = decode_error(&crafted_payload(ALPHA, &negative)); assert!(err.contains("store span past i32"), "negative store: {err}"); // The same payload one bucket shorter than the boundary decodes. - let ok = crafted(ALPHA, vec![1; 3], i32::MAX - 2, 5.0, 1.0, 2.0); + let ok = crafted(ALPHA, vec![1; 3], i32::MAX - 2); assert!(DdSketch::from_asapv1(&ok).is_ok()); } @@ -557,8 +568,7 @@ mod tests { /// different bytes. #[test] fn ddsketch_rejects_a_nonzero_offset_on_an_empty_store() { - let empty = || positive(Vec::new(), 0, 0.0, f64::INFINITY, f64::NEG_INFINITY); - let mut payload = empty(); + let mut payload = positive(Vec::new(), 0); payload.offset = 42; let err = decode_error(&crafted_payload(ALPHA, &payload)); assert!( @@ -571,15 +581,11 @@ mod tests { /// zero count — are rejected rather than wrapping. #[test] fn ddsketch_rejects_a_total_count_overflow() { - let mut negative = positive(vec![u64::MAX], 0, 5.0, -1.0, 2.0); + let mut negative = positive(vec![u64::MAX], 0); negative.negative_counts = vec![1]; - let mut zero = positive(vec![u64::MAX], 0, 5.0, 0.0, 2.0); + let mut zero = positive(vec![u64::MAX], 0); zero.zero_count = 1; - for payload in [ - positive(vec![u64::MAX, 1], 0, 5.0, 1.0, 2.0), - negative, - zero, - ] { + for payload in [positive(vec![u64::MAX, 1], 0), negative, zero] { let err = decode_error(&crafted_payload(ALPHA, &payload)); assert!( err.contains("overflow the total sample count"), @@ -588,50 +594,13 @@ mod tests { } } - /// The scalars the buckets do not determine are still bounded by them: an - /// empty store carries the sentinels, a populated one finite positive - /// bounds in order. + /// Signed stores and the zero count decode and re-serialize byte for byte. #[test] - fn ddsketch_rejects_inconsistent_scalars() { - let cases = [ - // Populated store with the empty-store sentinels. - (vec![3u64], 7i32, 6.0f64, f64::INFINITY, f64::NEG_INFINITY), - // min above max. - (vec![3], 7, 6.0, 9.0, 2.0), - // Non-positive min. - (vec![3], 7, 6.0, 0.0, 2.0), - // Sum below the smallest ingested value. - (vec![3], 7, 0.5, 1.5, 2.5), - // Empty store carrying a non-zero sum. - (Vec::new(), 0, 5.0, f64::INFINITY, f64::NEG_INFINITY), - // All-zero buckets carrying populated-store scalars. - (vec![0, 0], 7, 6.0, 1.5, 2.5), - ]; - for (counts, offset, sum, min, max) in cases { - let err = decode_error(&crafted(ALPHA, counts, offset, sum, min, max)); - assert!( - err.contains("DDSketch empty store must carry") - || err.contains("scalars out of order") - || err.contains("scalars must be finite"), - "sum={sum}, min={min}, max={max} must be rejected by a scalar rule, got {err}" - ); - } - } - - /// `0 < min` and `sum >= min` hold only while the negative store sums to - /// zero and `zero_count` is zero; an all-zero negative store keeps them. - #[test] - fn ddsketch_positive_scalar_checks_follow_the_counts() { - let mut zero_negative = positive(vec![3], 7, 6.0, -1.0, 2.5); - zero_negative.negative_counts = vec![0, 0]; - zero_negative.negative_offset = 5; - let err = decode_error(&crafted_payload(ALPHA, &zero_negative)); - assert!(err.contains("scalars out of order"), "got {err}"); - - let mut negative = positive(vec![3], 7, -0.5, -1.5, 2.5); + fn ddsketch_signed_payload_round_trips() { + let mut negative = positive(vec![3], 7); negative.negative_counts = vec![1]; negative.negative_offset = 5; - let mut zero = positive(vec![3], 7, 0.0, 0.0, 2.5); + let mut zero = positive(vec![3], 7); zero.zero_count = 2; for payload in [negative, zero] { let bytes = crafted_payload(ALPHA, &payload); @@ -668,13 +637,12 @@ mod tests { ); } - /// Known stores and scalars serialize to the `asapv1_golden/` bytes. + /// Known stores serialize to the `asapv1_golden/` bytes. #[test] fn known_state_matches_golden() { use crate::asapv1::envelope::hex; - let state = |positive: (&[u64], i32), negative: (&[u64], i32), zero_count, scalars| { - let (sum, min, max) = scalars; + let state = |positive: (&[u64], i32), negative: (&[u64], i32), zero_count| { let store = |(counts, offset): (&[u64], i32)| Buckets { counts: Vector1D::from_vec(counts.to_vec()), offset, @@ -682,9 +650,6 @@ mod tests { DdSketch::try_from(DdSketchState { alpha: 0.01, positive_store: store(positive), - sum, - min, - max, negative_store: store(negative), zero_count, admission_rate: 1.0, @@ -696,21 +661,15 @@ mod tests { (&[1, 0, 127, 128, 300, 65536, 4294967296], -40), (&[], 0), 0, - (2181071000.0, 0.453125, 0.5078125), ); let golden = hex(include_str!( - "../../../asapv1_golden/ddsketch_positive_a001.hex" + "../../../asapv1_golden/ddsketch_buckets_positive_a001.hex" )); assert_eq!(positive.to_asapv1().unwrap(), golden); - let signed = state( - (&[3, 0, 2], 310), - (&[5, 1], -208), - 7, - (2523.90625, -0.016, 515.0), - ); + let signed = state((&[3, 0, 2], 310), (&[5, 1], -208), 7); let golden = hex(include_str!( - "../../../asapv1_golden/ddsketch_signed_a001.hex" + "../../../asapv1_golden/ddsketch_buckets_signed_a001.hex" )); assert_eq!(signed.to_asapv1().unwrap(), golden); } diff --git a/tests/PBT_COVERAGE.md b/tests/PBT_COVERAGE.md index 5f100807..df856027 100644 --- a/tests/PBT_COVERAGE.md +++ b/tests/PBT_COVERAGE.md @@ -98,17 +98,17 @@ Compared against: the sorted stream. `DdSketch::get_value_at_quantile` targets t - placement - negative store, zero counter, positive store hold exactly the three class counts; `get_count() == arrivals` - - `sum() == stream sum`, bit for bit - first dense allocation: one 128-bucket chunk, seed value at its center; both signed stores - relative error `<= alpha` - `|quantile(q) - exact| <= alpha * |exact|`, across the sign boundary - `quantile(q)` non-decreasing in `q`, across the sign boundary - - single value `v`: `quantile(q) == v` for every `q`; `min`, `max`, `q=0`, `q=1` equal the tracked extremes + - single value `v`: `quantile(q)` within `alpha` of `v`, the same for every `q` + - `q=0`, `q=1` within `alpha` of the stream's minimum and maximum - merge - merged `quantile(q)` within `alpha` of the combined stream's exact value - different `alpha`: refused, receiver count unchanged; same `alpha`: merges - wire - - ASAPv1 keeps count, `alpha`, `sum`, quantiles; stream of NaN, ±inf, ±0, f64 extremes + - ASAPv1 keeps count, `alpha`, quantiles; stream of NaN, ±inf, ±0, f64 extremes ## elastic.rs — 12 diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 9c6d22bf..50833521 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -85,10 +85,22 @@ const GOLDEN_CMSSS_STR: &str = include_str!("../asapv1_golden/cmsspacesaving_i64_regular_2x3_strkeys.hex"); const GOLDEN_CSSS_I64: &str = include_str!("../asapv1_golden/csspacesaving_f64_fast_2x4_i64keys.hex"); +// The retired DDSketch kinds `05 00` / `05 80`, decode-only. const GOLDEN_DD_POSITIVE: &str = include_str!("../asapv1_golden/ddsketch_positive_a001.hex"); const GOLDEN_DD_SIGNED: &str = include_str!("../asapv1_golden/ddsketch_signed_a001.hex"); const GOLDEN_DD_EMPTY: &str = include_str!("../asapv1_golden/ddsketch_empty_a001.hex"); const GOLDEN_DD_DELTA: &str = include_str!("../asapv1_golden/ddsketch_signed_a001_delta.hex"); +// The current DDSketch kinds `05 01` / `05 81`. +const GOLDEN_DD_BUCKETS_POSITIVE: &str = + include_str!("../asapv1_golden/ddsketch_buckets_positive_a001.hex"); +const GOLDEN_DD_BUCKETS_SIGNED: &str = + include_str!("../asapv1_golden/ddsketch_buckets_signed_a001.hex"); +const GOLDEN_DD_BUCKETS_EMPTY: &str = + include_str!("../asapv1_golden/ddsketch_buckets_empty_a001.hex"); +const GOLDEN_DD_BUCKETS_DELTA: &str = + include_str!("../asapv1_golden/ddsketch_buckets_signed_a001_delta.hex"); +const GOLDEN_DD_BUCKETS_ADMISSION: &str = + include_str!("../asapv1_golden/ddsketch_buckets_signed_a001_admission.hex"); const GOLDEN_KLL_F64: &str = include_str!("../asapv1_golden/kll_f64_k200.hex"); const GOLDEN_KLL_I64: &str = include_str!("../asapv1_golden/kll_i64_k200.hex"); const GOLDEN_KLL_DYN_F64: &str = include_str!("../asapv1_golden/kll_dynamic_f64_k200.hex"); @@ -128,6 +140,8 @@ const GOLDEN_PYRAMID: &str = const GOLDEN_UNIVMON_Q: &str = include_str!("../asapv1_golden/univmon_q_i32_l3_c8_r3.hex"); const GOLDEN_EHSL_CM: &str = include_str!("../asapv1_golden/ehsketchlist_cm_i32_fast_2x4.hex"); const GOLDEN_EHSL_DD: &str = include_str!("../asapv1_golden/ehsketchlist_ddsketch_a001.hex"); +const GOLDEN_EHSL_DD_BUCKETS: &str = + include_str!("../asapv1_golden/ehsketchlist_ddsketch_buckets_a001.hex"); const GOLDEN_EH: &str = include_str!("../asapv1_golden/eh_w100_k2_cm_2x4.hex"); /// The known P12 register pattern shared by all three HLL fixtures. @@ -1030,22 +1044,23 @@ fn kll_dynamic_i64_k200_matches_golden() { } // --------------------------------------------------------------------------- -// DDSketch and its delta: golden decodes to the known stores and scalars and -// round-trips, and the delta merged into the signed state gives the expected -// stores and scalars. Encoding the known state is checked in -// `src/sketches/ddsketch/wire.rs`, which can build it. +// DDSketch and its delta: golden decodes to the known stores and round-trips, +// and the delta merged into the signed state gives the expected stores. +// Encoding the known state is checked in `src/sketches/ddsketch/wire.rs`, +// which can build it. Each retired `05 00` / `05 80` fixture decodes, its +// `sum` / `min` / `max` dropped, and re-encodes as its `ddsketch_buckets_*` +// twin. // --------------------------------------------------------------------------- const DD_ALPHA: f64 = 0.01; -/// Decodes `golden`, checks the decoded stores, zero count and scalars, and +/// Decodes `golden`, checks the decoded stores and zero count, and /// re-serializes it. fn assert_dd_golden( golden: &str, positive: (&[u64], i32), negative: (&[u64], i32), zero_count: u64, - (sum, min, max): (f64, f64, f64), ) { let want = decode_hex(golden); let decoded = DdSketch::from_asapv1(&want).expect("decode"); @@ -1064,9 +1079,6 @@ fn assert_dd_golden( negative ); assert_eq!(decoded.zero_count(), zero_count); - assert_eq!(decoded.sum().to_bits(), sum.to_bits()); - assert_eq!(decoded.min().map(f64::to_bits), Some(min.to_bits())); - assert_eq!(decoded.max().map(f64::to_bits), Some(max.to_bits())); assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); } @@ -1079,27 +1091,24 @@ fn dd_buckets(counts: &[u64], offset: i32) -> Vec<(i32, u64)> { } #[test] -fn ddsketch_positive_a001_matches_golden() { +fn ddsketch_buckets_positive_a001_matches_golden() { let counts = [1, 0, 127, 128, 300, 65536, 4294967296]; - let scalars = (2181071000.0, 0.453125, 0.5078125); - assert_dd_golden(GOLDEN_DD_POSITIVE, (&counts, -40), (&[], 0), 0, scalars); + assert_dd_golden(GOLDEN_DD_BUCKETS_POSITIVE, (&counts, -40), (&[], 0), 0); } #[test] -fn ddsketch_signed_a001_matches_golden() { - let scalars = (2523.90625, -0.016, 515.0); +fn ddsketch_buckets_signed_a001_matches_golden() { assert_dd_golden( - GOLDEN_DD_SIGNED, + GOLDEN_DD_BUCKETS_SIGNED, (&[3, 0, 2], 310), (&[5, 1], -208), 7, - scalars, ); } #[test] -fn ddsketch_empty_a001_matches_golden() { - let want = decode_hex(GOLDEN_DD_EMPTY); +fn ddsketch_buckets_empty_a001_matches_golden() { + let want = decode_hex(GOLDEN_DD_BUCKETS_EMPTY); assert_eq!( DdSketch::new(DD_ALPHA).to_asapv1().expect("serialize"), want @@ -1108,19 +1117,36 @@ fn ddsketch_empty_a001_matches_golden() { assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); } +/// Each retired `05 00` state fixture decodes to its twin's stores and +/// re-encodes as the twin. #[test] -fn ddsketch_signed_a001_delta_matches_golden() { - let want = decode_hex(GOLDEN_DD_DELTA); - let indices = vec![-300, -100, -5, 7, 200, 400]; - let deltas = vec![1, 127, 128, 256, 65536, 4294967296]; - let negative_indices = vec![-208, -33, -1, 0, 130, 1000]; - let negative_deltas = vec![2, 3, 3, 1, 3, 1]; +fn ddsketch_retired_fixtures_decode_to_their_bucket_twins() { + let pairs = [ + (GOLDEN_DD_POSITIVE, GOLDEN_DD_BUCKETS_POSITIVE), + (GOLDEN_DD_SIGNED, GOLDEN_DD_BUCKETS_SIGNED), + (GOLDEN_DD_EMPTY, GOLDEN_DD_BUCKETS_EMPTY), + (GOLDEN_DD_ADMISSION, GOLDEN_DD_BUCKETS_ADMISSION), + ]; + for (retired, twin) in pairs { + let decoded = DdSketch::from_asapv1(&decode_hex(retired)).expect("decode retired"); + assert_eq!(decoded.to_asapv1().expect("re-encode"), decode_hex(twin)); + } + let delta = DdSketchDelta::from_asapv1(&decode_hex(GOLDEN_DD_DELTA)).expect("retired delta"); + assert_eq!( + delta.to_asapv1().expect("re-encode"), + decode_hex(GOLDEN_DD_BUCKETS_DELTA) + ); +} + +#[test] +fn ddsketch_buckets_signed_a001_delta_matches_golden() { + let want = decode_hex(GOLDEN_DD_BUCKETS_DELTA); let delta = DdSketchDelta::new( DD_ALPHA, - indices.clone(), - deltas.clone(), - negative_indices.clone(), - negative_deltas.clone(), + vec![-300, -100, -5, 7, 200, 400], + vec![1, 127, 128, 256, 65536, 4294967296], + vec![-208, -33, -1, 0, 130, 1000], + vec![2, 3, 3, 1, 3, 1], 9, ) .expect("valid"); @@ -1129,18 +1155,8 @@ fn ddsketch_signed_a001_delta_matches_golden() { let decoded = DdSketchDelta::from_asapv1(&want).expect("decode"); assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); - let mut state = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_SIGNED)).expect("state"); + let mut state = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_BUCKETS_SIGNED)).expect("state"); state.merge_delta(&decoded).expect("merge"); - - let gamma = (1.0 + DD_ALPHA) / (1.0 - DD_ALPHA); - let r = |k: i32| (1.0 + DD_ALPHA) * gamma.powf(f64::from(k)); - let mut sum = 2523.90625; - for (k, c) in indices.iter().zip(&deltas) { - sum += r(*k) * *c as f64; - } - for (k, c) in negative_indices.iter().zip(&negative_deltas) { - sum += -r(*k) * *c as f64; - } assert_eq!( dd_buckets(state.positive_store_counts(), state.positive_store_offset()), [ @@ -1167,9 +1183,6 @@ fn ddsketch_signed_a001_delta_matches_golden() { ] ); assert_eq!((state.zero_count(), state.get_count()), (16, 4295033384)); - assert_eq!(state.sum(), sum); - assert_eq!(state.min(), Some(-r(1000))); - assert_eq!(state.max(), Some(r(400))); } // --------------------------------------------------------------------------- @@ -1724,26 +1737,24 @@ fn cs_i64_regular_2x4_admission_matches_golden() { } #[test] -fn ddsketch_signed_a001_admission_matches_golden() { - let scalars = (2523.90625, -0.016, 515.0); - let signed = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_SIGNED)).expect("decode"); +fn ddsketch_buckets_signed_a001_admission_matches_golden() { + let signed = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_BUCKETS_SIGNED)).expect("decode"); assert_eq!( signed .with_admission_rate(0.125) .to_asapv1() .expect("serialize"), - decode_hex(GOLDEN_DD_ADMISSION) + decode_hex(GOLDEN_DD_BUCKETS_ADMISSION) ); assert_dd_golden( - GOLDEN_DD_ADMISSION, + GOLDEN_DD_BUCKETS_ADMISSION, (&[3, 0, 2], 310), (&[5, 1], -208), 7, - scalars, ); - let decoded = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_ADMISSION)).expect("decode"); + let decoded = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_BUCKETS_ADMISSION)).expect("decode"); assert_eq!(decoded.admission_rate(), 0.125); - assert_eq!((decoded.get_count(), decoded.sum()), (18, 2523.90625)); + assert_eq!(decoded.get_count(), 18); } // --------------------------------------------------------------------------- @@ -2028,11 +2039,13 @@ fn ehsketchlist_cm_i32_fast_2x4_matches_golden() { assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); } -/// The nested blocks are `ddsketch_signed_a001`'s metadata and payload. +/// The nested blocks are `ddsketch_buckets_signed_a001`'s metadata and payload, +/// under kind_id `05 01`. The retired `ehsketchlist_ddsketch_a001`, nesting +/// `05 00`, decodes to the same sketch and re-encodes as this fixture. #[test] -fn ehsketchlist_ddsketch_a001_matches_golden() { - let want = decode_hex(GOLDEN_EHSL_DD); - let dd = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_SIGNED)).expect("dd"); +fn ehsketchlist_ddsketch_buckets_a001_matches_golden() { + let want = decode_hex(GOLDEN_EHSL_DD_BUCKETS); + let dd = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_BUCKETS_SIGNED)).expect("dd"); let sketch = EhBucketSketch::DdSketch(dd); assert_eq!(sketch.to_asapv1().expect("serialize"), want); @@ -2042,9 +2055,12 @@ fn ehsketchlist_ddsketch_a001_matches_golden() { }; assert_eq!( inner.to_asapv1().expect("inner"), - decode_hex(GOLDEN_DD_SIGNED) + decode_hex(GOLDEN_DD_BUCKETS_SIGNED) ); assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); + + let retired = EhBucketSketch::from_asapv1(&decode_hex(GOLDEN_EHSL_DD)).expect("retired"); + assert_eq!(retired.to_asapv1().expect("re-encode"), want); } const EH_BUCKETS: [([[i32; 4]; 2], usize, u64, u64); 2] = [ diff --git a/tests/ddsketch_signed.rs b/tests/ddsketch_signed.rs index 22855f57..c8dffc11 100644 --- a/tests/ddsketch_signed.rs +++ b/tests/ddsketch_signed.rs @@ -87,7 +87,11 @@ fn tiny_values_are_counted_in_zero_bucket_and_survive_asapv1() { core.try_add(&-3.).unwrap(); let decoded = DdSketch::from_asapv1(&core.to_asapv1().unwrap()).unwrap(); assert_eq!(decoded.get_count(), 4); - assert_eq!(decoded.sum(), core.sum()); - assert_eq!(decoded.min(), core.min()); - assert_eq!(decoded.max(), core.max()); + assert_eq!(decoded.zero_count(), 3); + for q in [0.0, 0.5, 1.0] { + assert_eq!( + decoded.get_value_at_quantile(q), + core.get_value_at_quantile(q) + ); + } } diff --git a/tests/e2e/quantiles.rs b/tests/e2e/quantiles.rs index 787b3c33..0d7c892f 100644 --- a/tests/e2e/quantiles.rs +++ b/tests/e2e/quantiles.rs @@ -678,11 +678,11 @@ fn ddsketch_answers_the_ceil_nearest_rank_order_statistic() { } } -/// The sketch tracks the exact minimum and maximum beside the bucket store, and -/// `get_value_at_quantile` short-circuits `q <= 0` and `q >= 1` to them, so its -/// endpoints are exact. +/// `get_value_at_quantile` reads `q <= 0` and `q >= 1` from the lowest and +/// highest occupied buckets, so its endpoints are within alpha of the exact +/// minimum and maximum. #[test] -fn ddsketch_endpoints_are_exact() { +fn ddsketch_endpoints_are_within_alpha() { for &alpha in &DDS_ALPHAS { let values = log_uniform_f64(5_000, (1.0 + alpha) / (1.0 - alpha), 3..30, 4_242); let truth = NumericTruth::new(values.clone()); @@ -690,26 +690,13 @@ fn ddsketch_endpoints_are_exact() { for v in &values { core.add(v); } - assert_eq!( - core.get_value_at_quantile(0.0), - Some(truth.min()), - "core alpha={alpha}: q=0 must be the exact minimum" - ); - assert_eq!( - core.get_value_at_quantile(1.0), - Some(truth.max()), - "core alpha={alpha}: q=1 must be the exact maximum" - ); - assert_eq!( - core.min(), - Some(truth.min()), - "core alpha={alpha}: min() must be exact" - ); - assert_eq!( - core.max(), - Some(truth.max()), - "core alpha={alpha}: max() must be exact" - ); + for (q, want) in [(0.0, truth.min()), (1.0, truth.max())] { + let got = core.get_value_at_quantile(q).unwrap(); + assert!( + (got - want).abs() <= want.abs() * (alpha + 1e-9), + "core alpha={alpha}: q={q} answered {got}, exact {want}" + ); + } } } diff --git a/tests/e2e/wire.rs b/tests/e2e/wire.rs index 45a1b764..e68d4771 100644 --- a/tests/e2e/wire.rs +++ b/tests/e2e/wire.rs @@ -34,7 +34,7 @@ fn a_ddsketch_envelope_round_trips_and_names_its_kind() { sketch.add(&(1.0 + v as f64)); } let bytes = sketch.to_asapv1().expect("encode"); - assert_kind(&bytes, 0x05, 0x00, "DDSketch"); + assert_kind(&bytes, 0x05, 0x01, "DDSketch"); let decoded = DdSketch::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.get_count(), sketch.get_count(), "count"); assert_eq!(decoded.alpha(), sketch.alpha(), "alpha"); diff --git a/tests/pbt/ddsketch.rs b/tests/pbt/ddsketch.rs index f99cad3a..ab11e71c 100644 --- a/tests/pbt/ddsketch.rs +++ b/tests/pbt/ddsketch.rs @@ -105,23 +105,6 @@ proptest! { prop_assert_eq!(sketch.get_count(), values.len() as u64, "total count"); } - /// `sum` is carried on the wire and read back through `sum()`, so it is - /// the running total of the stream, not an estimate off the buckets. - #[test] - fn ddsketch_sum_tracks_the_stream( - alpha in alpha(), - values in signed_values(120), - ) { - let sketch = dd_of(alpha, &values); - let want = values.iter().fold(0.0f64, |acc, v| acc + v); - - prop_assert_eq!( - sketch.sum().to_bits(), want.to_bits(), - "alpha={}: sum {} over {} values, running total {}", - alpha, sketch.sum(), values.len(), want - ); - } - /// The first dense allocation is one `GROW_CHUNK` centred on the value's /// bucket, in both signed stores. The layout /// reaches the wire as `positive_store_counts` / `positive_store_offset`, where the Go @@ -148,11 +131,10 @@ proptest! { // ===== The paper's guarantee ===== - /// Sharper than the paper's bound: the sketch tracks the extremes outside - /// the buckets and clamps a bucket answer to them, so a lone value comes - /// back unrounded at every quantile. + /// A lone value comes back as its bucket's representative at every + /// quantile, within alpha of the value. #[test] - fn ddsketch_returns_a_lone_value_exactly( + fn ddsketch_returns_a_lone_value_within_alpha( alpha in alpha(), value in 1e-6f64..1e9, q in 0.0f64..=1.0, @@ -160,24 +142,28 @@ proptest! { let sketch = dd_of(alpha, &[value]); let answer = sketch.get_value_at_quantile(q).expect("a non-empty sketch answers"); - prop_assert_eq!(answer.to_bits(), value.to_bits()); + prop_assert!( + (answer - value).abs() <= value * (alpha + 1e-9), + "alpha={} q={}: got {}, value {}", alpha, q, answer, value + ); + prop_assert_eq!(Some(answer), sketch.get_value_at_quantile(0.0)); } - /// Also sharper than the bound: q=0 and q=1 read the tracked extremes - /// rather than a bucket, so they are exact and say nothing about alpha. + /// q=0 and q=1 read the lowest and highest occupied buckets, within alpha + /// of the smallest and largest values of the stream. #[test] - fn ddsketch_returns_the_signed_extremes_exactly( + fn ddsketch_returns_the_signed_extremes_within_alpha( alpha in alpha(), values in signed_values(120), ) { let sketch = dd_of(alpha, &values); let ordered = sorted(&values); - let last = ordered.len() - 1; + let (min, max) = (ordered[0], ordered[ordered.len() - 1]); + let low = sketch.get_value_at_quantile(0.0).expect("q=0"); + let high = sketch.get_value_at_quantile(1.0).expect("q=1"); - prop_assert_eq!(sketch.min(), Some(ordered[0]), "min over {} values", ordered.len()); - prop_assert_eq!(sketch.max(), Some(ordered[last]), "max over {} values", ordered.len()); - prop_assert_eq!(sketch.get_value_at_quantile(0.0), Some(ordered[0]), "q=0"); - prop_assert_eq!(sketch.get_value_at_quantile(1.0), Some(ordered[last]), "q=1"); + prop_assert!((low - min).abs() <= min.abs() * (alpha + 1e-9), "q=0: {} vs {}", low, min); + prop_assert!((high - max).abs() <= max.abs() * (alpha + 1e-9), "q=1: {} vs {}", high, max); } /// The paper's guarantee across the sign boundary, against the exact @@ -292,7 +278,6 @@ proptest! { sketch, |s: &DdSketch| s.get_count(), |s: &DdSketch| s.alpha().to_bits(), - |s: &DdSketch| s.sum().to_bits(), |s: &DdSketch| QUANTILES.iter() .map(|q| s.get_value_at_quantile(*q).map(f64::to_bits)) .collect::>(), From f961b90c537ae81a34aef80c863703271f98362a Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 28 Oct 2026 22:43:20 +0000 Subject: [PATCH 2/3] ddsketch: review fixes (octo-runtime test, eleven nested ids, docs, rank fallback) - octo.rs: the octo-runtime test read the removed min()/max(); it now checks quantiles 0 and 1 within alpha of the stream's extremes. - EhBucketSketch: spec rules 1-2, wire comments, API doc and test doc count eleven nested ids (ten encoded plus the retired 0x05 0x00). - docs/tests.md: rows for the delta kind checks, the e2e envelope kind and the octo-runtime test match the tests. - The within-alpha claim is for indexable magnitudes; smaller nonzero magnitudes count as zero and read 0.0. - value_at_rank returns the last occupied bucket instead of panicking if count ever exceeded the buckets. - New tests: the retired decoder refuses an int or float32 scalar and drops NaN ones; a rank past the buckets reads the last occupied one. Co-Authored-By: Claude Opus 5.5 --- docs/api/api_ddsketch.md | 7 +-- docs/api/api_eh_bucket_sketch.md | 3 +- docs/asapv1_wire_format.md | 8 ++-- docs/tests.md | 14 +++--- src/sketch_framework/eh_bucket_sketch/wire.rs | 11 +++-- src/sketch_framework/octo.rs | 9 +++- src/sketches/ddsketch.rs | 47 ++++++++++++++++--- src/sketches/ddsketch/wire.rs | 37 +++++++++++++++ 8 files changed, 109 insertions(+), 27 deletions(-) diff --git a/docs/api/api_ddsketch.md b/docs/api/api_ddsketch.md index bc2d2a48..85278492 100644 --- a/docs/api/api_ddsketch.md +++ b/docs/api/api_ddsketch.md @@ -36,9 +36,10 @@ fn quantile_interpolated(&self, q: f64) -> Result, String> fn get_count(&self) -> u64 ``` -Every answer is a bucket's representative: within `alpha` of a sample, the -extremes included (`q <= 0` reads the lowest occupied bucket, `q >= 1` the -highest). The sketch keeps no sum, minimum or maximum: those are not DDSketch +Every answer is a bucket's representative: within `alpha` of a sample of +indexable magnitude, the extremes included (`q <= 0` reads the lowest occupied +bucket, `q >= 1` the highest). A nonzero magnitude below the indexable range is +counted as zero and reads `0.0`. The sketch keeps no sum, minimum or maximum: those are not DDSketch readouts, and a caller that needs them keeps a separate exact aggregate beside the sketch. diff --git a/docs/api/api_eh_bucket_sketch.md b/docs/api/api_eh_bucket_sketch.md index 191d465d..c8ac6007 100644 --- a/docs/api/api_eh_bucket_sketch.md +++ b/docs/api/api_eh_bucket_sketch.md @@ -65,7 +65,8 @@ stripped, so each variant's own validation applies unchanged. An `ExponentialHistogram` bucket inlines the same triple, so there is one EhBucketSketch encoding. -The ten nested kind_ids: +The ten encoded nested kind_ids, plus the retired `0x05 0x00`, which decodes +into `DdSketch`: | Variant | Nested kind_id | Registry name | Feature | | ------- | -------------- | ------------- | ------- | diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 77c1020b..aa090227 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -747,7 +747,7 @@ Wire counter types are the base Count-Min's: **`"i32"`, `"i64"` and `"f64"`**. ` **DDSketch carries no hash-spec group.** It never hashes: it maps a raw `f64` through a logarithm and carries no hasher type parameter at all. The hash spec answers "how were keys hashed", and a sketch that does not hash has no truthful answer, so the group is omitted entirely. This is the Q-KLL precedent, and the reason DDSketch's metadata schema is not `HashProfile`-derived the way HLL's and Count-Min's are. -**No `sum`, `min` or `max`.** Every DDSketch readout comes from the buckets: a quantile, `q = 0` and `q = 1` included, is its bucket's representative and within `alpha` of a sample. A sum, minimum or maximum of the values is not a DDSketch readout but a separate exact aggregate kept beside the sketch. Carrying them in the state would also break deltas: a delta names buckets, not values, so a receiver could only fold in bucket representatives and would end with different scalars than the producer. +**No `sum`, `min` or `max`.** Every DDSketch readout comes from the buckets: a quantile, `q = 0` and `q = 1` included, is its bucket's representative and within `alpha` of a sample of indexable magnitude (a nonzero magnitude below the indexable range is counted as zero and reads `0.0`). A sum, minimum or maximum of the values is not a DDSketch readout but a separate exact aggregate kept beside the sketch. Carrying them in the state would also break deltas: a delta names buckets, not values, so a receiver could only fold in bucket representatives and would end with different scalars than the producer. **The retired `0x05 0x00`.** It carried `sum` / `min` / `max` as three f64 fields between `offset` and `negative_counts` (eight fields in all). A decoder still accepts it, reads the three scalars as float64 (Section 4) and drops them, and the result re-encodes as `0x05 0x01`. Nothing encodes `0x05 0x00`; per Q-VER the id is never reused. @@ -1177,16 +1177,16 @@ The `kind_id` is carried rather than a name, because Section 1's registry is alr **Feature-gating contract (cross-language; Go must honour it).** -1. The nested-id namespace is **fixed and identical in every build**. All ten ids dispatch whether or not the `experimental` feature is on. An id is registry bytes, never an enum ordinal and never a discriminant, so no id's meaning can shift because a variant is compiled out. +1. The nested-id namespace is **fixed and identical in every build**. All eleven ids (the ten encoded ones plus the retired `0x05 0x00`, which decodes into the DDSketch arm) dispatch whether or not the `experimental` feature is on. An id is registry bytes, never an enum ordinal and never a discriminant, so no id's meaning can shift because a variant is compiled out. 2. A decoder built **without** `experimental` that meets `0x0d 0x00` **fails closed**, before the blocks are assembled into anything, with an error naming the variant (`UniformSampling`) **and** the feature. It never misparses, skips, substitutes another variant, or falls through the unknown-id path. 3. An encoder built **without** `experimental` can never emit that id: the enum arm does not exist. -4. An id outside the ten is rejected as unknown. +4. An id outside the eleven is rejected as unknown. 5. The id-to-name lookup exists for **error messages only**. Encoding and decoding both dispatch on the bytes; the name influences neither. **Decode rules.** Fail **closed** on each, with an error and never a panic: 1. `kind_id` is `0x14 0x00`; any other id is rejected. The metadata is `metadata_version = 1` and nothing else; an unknown key or a missing key is rejected. -2. The nested `kind_id` is one of the ten; anything else, **including a sibling algorithm's id within the same family**, is rejected as unknown — and rejected **before** the triple is assembled into an envelope. Nothing is ever built from an id the table does not carry, so an id too long for the envelope's one-byte `kind_id_len` field never reaches the framing that would have to hold it: it draws the same unknown-id error as any other stranger. +2. The nested `kind_id` is one of the eleven in the table (ten encoded, plus the retired `0x05 0x00`); anything else, **including a sibling algorithm's id within the same family**, is rejected as unknown — and rejected **before** the triple is assembled into an envelope. Nothing is ever built from an id the table does not carry, so an id too long for the envelope's one-byte `kind_id_len` field never reaches the framing that would have to hold it: it draws the same unknown-id error as any other stranger. 3. A nested id this build does not carry is rejected before `descriptor` and `state` are assembled into anything, with an error naming the variant and the feature. The availability check runs **first**, ahead of rule 2's: a feature-gated id is a known id, so `0x0d 0x00` in a build without `experimental` always draws the feature error and never the unknown-id one. 4. `descriptor` and `state` are handed to that variant's own decoder, wrapped in the variant's own `kind_id`. Every check that decoder makes applies here: geometry bounds, length agreement, allocation guards, and the hash-profile pin. 5. A `descriptor` / `state` pair that does not belong to the declared id fails inside that variant's decoder; no partial state is produced. diff --git a/docs/tests.md b/docs/tests.md index 40ef2e46..cf5f65b0 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -225,6 +225,7 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | `quantile_interpolated_refuses_q_outside_the_unit_interval` | `q` outside `[0, 1]` is an error. | `-0.1`, `1.1`, NaN and `±inf` give `Err`, on a filled and on an empty sketch. | | `quantile_interpolated_signed_values` | Signed values interpolate across both stores and the zero bucket. | On `[-40, -20, -5, 0, 10, 30, 70]` for 101 `q`, the error is within `alpha ((1 - f) abs(lower) + f abs(upper))` of the exact interpolated value, and within `alpha` relative where both neighbours share a sign. | | `quantile_interpolated_matches_the_exact_value_on_a_large_stream` | A large signed stream meets the bound at every `q`. | 20,000 seeded values `±exp(U(-3, 6))` (20% negative) at `alpha = 0.01`, 201 `q` in `[0, 1]`: each answer is within `alpha ((1 - f) abs(lower) + f abs(upper))` of the exact interpolated quantile. | +| `a_rank_past_the_buckets_reads_the_last_occupied_bucket` | If `count` ever exceeded the buckets, a quantile reads the last occupied bucket rather than panicking. | Adds `-4.0`, raises `count` by `5` past the buckets, and verifies `q = 1` reads the representative of `-4.0`'s bucket; after adding `8.0`, verifies `get_value_at_quantile(1.0)` and `quantile_interpolated(1.0)` read `8.0`'s bucket. | | `insert_and_query_basic` | Basic insert/query preserves count semantics and quantile monotonicity. | Inserts mixed values `[0.0, -5.0, 1.0, 2.0, 3.0, 10.0, 50.0, 100.0, 1000.0]`, verifies the count is `9`, and that quantiles at `{0, 0.5, 0.9, 0.99, 1}` are non-decreasing and inside the stream's range widened by `alpha`. | | `empty_quantile_returns_none` | Empty sketch returns no quantiles and zero count. | For a new `DdSketch(alpha=0.01)`, asserts `get_value_at_quantile` returns `None` for `p in {0.0, 0.5, 1.0}` and `get_count() == 0`. | | `merge_two_sketches_combines_counts_and_bounds` | Merge combines counts, and the extreme quantiles read the extreme buckets. | Merges sketches built from `[1,2,3,4]` and `[5,10,20]`, verifies the count is `7`, that `q = 0` and `q = 1` are within `alpha` of `1` and `20`, and that the median lies between them. | @@ -232,9 +233,10 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | `ddsketch_envelope_structure_and_round_trip` | The envelope frames a sketch under kind_id `0x05 0x01`, and the bytes are stable across a round trip. | For a populated `alpha = 0.01` sketch, verifies the `ASAPv1` magic, kind_id `0x05 0x01`, metadata version `1` and `alpha`, the decoded positive store, identical re-serialized bytes, and unchanged quantiles. | | `ddsketch_payload_is_five_fields` | The payload is the two stores and the zero count, with no `count` and no scalars. | Verifies the payload reads back as a five-element array whose bucket counts sum to `get_count()`, with an empty negative store at offset `0` and a zero `zero_count`, and that the decoded sketch reports the same count. | | `ddsketch_decodes_the_retired_kind_with_scalars` | Bytes of the retired kind `0x05 0x00` still decode, their `sum` / `min` / `max` dropped. | Builds an eight-field `0x05 0x00` envelope around a populated sketch's stores, verifies it decodes to the same count and re-encodes to the sketch's own `0x05 0x01` bytes, and that a five-field payload under `0x05 0x00` is refused. | +| `ddsketch_retired_scalars_must_be_float64` | The retired scalars are read as float64 and then dropped. | Verifies a `0x05 0x00` payload with an int `sum` or a float32 `min` is refused, that NaN in all three scalars decodes to a count of `3`, and that the same payload with float64 scalars decodes. | | `ddsketch_empty_round_trip` | A sketch that saw nothing round-trips. | Verifies a fresh `DdSketch(alpha=0.01)` decodes back with a `count` of `0`, no quantile, an empty store, and identical re-serialized bytes. | | `ddsketch_merged_round_trip` | A store grown on both sides of a merge round-trips. | Merges 200 values against the same 200 scaled by `0.001`, then verifies the decoded sketch matches the source on `count` and on the quantiles at `{0, 0.5, 1}` and re-serializes to the same bytes. | -| `ddsketch_rejects_foreign_kind_id` | Another sketch's kind_id is refused even when the rest parses cleanly. | Verifies a `3x8` Count-Min envelope fails to decode as a `DdSketch`, and that an empty DDSketch envelope decodes but fails with its kind_id relabelled `0x02 0x00`. | +| `ddsketch_rejects_foreign_kind_id` | Another sketch's kind_id is refused even when the rest parses cleanly. | Verifies a `3x8` Count-Min envelope fails to decode as a `DdSketch`, and that an empty DDSketch envelope decodes but fails with its kind_id relabelled `0x02 0x00` or as the delta kind `0x05 0x81`. | | `dd_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the two `DdMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `DdMetadata`. | | `dd_metadata_rejects_a_missing_alpha_key` | `alpha` is required and can never be silently defaulted. | Encodes a named map holding only `metadata_version` and verifies it does not decode as `DdMetadata`. | | `ddsketch_rejects_another_metadata_version` | `metadata_version` is `1`. | Verifies a valid payload under `metadata_version` `0` and `2` each fails with a metadata-mismatch complaint. | @@ -249,8 +251,8 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | `representative_within_alpha_at_bucket_edges` | Every bucket's representative is within `alpha` of both of its edges. | For `alpha` in `{0.001, 0.01, 0.05, 0.1}` and bucket indices `{-100, -1, 0, 1, 7, 500}`, verifies `bin_representative(k)` lies in `[lower_bound(k), lower_bound(k + 1)]` and its relative error against each edge is at most `alpha + 1e-9`. | | `merge_alpha_mismatch_is_a_real_runtime_error` | Merging across index mappings is an error, and a matched merge is not. | Verifies merging an `alpha = 0.02` sketch into an `alpha = 0.01` sketch holding `5.0` returns `Err`, while merging two `alpha = 0.01` sketches holding `3.0` and `7.0` returns `Ok` and leaves `get_count()` at `2`. | | `untrackable_extreme_is_dropped` | A finite value outside the indexable range is not recorded, so the dense store never spans the gap. | For an `alpha = 0.01` sketch over `1..=2000`, adds `max_indexable * 10.0` and `min_indexable / 10.0` from `ddsketch_indexable_bounds(0.01)` and verifies both `get_count()` and the store's bucket span are unchanged, then that `max_indexable / 2.0` is still recorded and raises the count by `1`. | -| `decode_rejects_each_payload_rule` | A `0x05 0x80` payload breaking any delta rule fails to decode. | At `alpha = 0.01`, verifies indices `[-35418, 35448]` (`key(min_v)`, `key(max_v)`) in both stores decode, and that index `-35419`, index `35449`, and negative-store index `35449` each fail. | -| `decode_rejects_another_kind_and_new_another_alpha` | A delta decodes only under its own kind_id, and `new` checks `alpha`. | Verifies a delta envelope decodes and fails with its kind_id relabelled `0x05 0x00`, and that `new` refuses `alpha` `1.0`. | +| `decode_rejects_each_payload_rule` | A `0x05 0x81` payload breaking any delta rule fails to decode. | At `alpha = 0.01`, verifies indices `[-35418, 35448]` (`key(min_v)`, `key(max_v)`) in both stores decode, and that index `-35419`, index `35449`, and negative-store index `35449` each fail. | +| `decode_rejects_another_kind_and_new_another_alpha` | A delta decodes only under its own kind_id, and `new` checks `alpha`. | Verifies a `0x05 0x81` delta envelope decodes, that relabelled as the retired `0x05 0x80` it still decodes, that relabelled as the state kind `0x05 0x01` it fails, and that `new` refuses `alpha` `1.0`. | | `deltas_rebuild_the_producer_bit_for_bit` | Deltas of a producer's changed buckets rebuild its state exactly at a receiver. | Feeds three batches of signed values, zeros and tiny magnitudes to a producer, sends each batch's changed buckets as a `0x05 0x81` delta through the wire, and verifies the receiver's count, quantiles and non-zero cells equal the producer's after every batch. | | `merge_delta_adds_the_zero_count` | A zero-count delta adds to the zero count and the total. | Merges a count of `2` at bucket `10`, then a zero-count delta of `3`, and verifies the zero count is `3`, the total `5`, and `q = 0` reads `0.0`. | | `merge_delta_rejects_without_changing_the_state` | A rejected delta leaves the sketch unchanged. | Verifies a delta whose `alpha` is one ulp above `0.01` fails on an `alpha = 0.01` sketch, and that on a sketch holding `u64::MAX - 1` samples a delta whose negative count overflows the total and a delta whose `zero_count` overflows it each fail, leaving the count and the serialized bytes as they were. | @@ -1147,7 +1149,7 @@ Wire tests: [`src/sketch_framework/eh_bucket_sketch/wire.rs`](../src/sketch_fram | `supports_norm_whitelist_is_enforced` | Supports norm whitelist is enforced. | Validates norm capability matrix: `CountMinSketch`/`CountSketch`/`DdSketch` support `L1` only, while `CountSketchWithL2Norm`/`UnivMon` support `L2` only. | | `eh_bucket_sketch_round_trip_serialization` | The envelope frames a union under kind_id `0x14 0x00`, and the state survives a round trip. | For a `3x8` fast-path Count-Min variant holding one key, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x14 0x00`, and that the decode keeps a `sketch_type` of `CountMinSketch` and answers that key as the source does. | | `eh_bucket_sketch_every_variant_round_trips` | Every variant this build carries round-trips and keeps its type. | For each populated variant, verifies the decode reports the same `sketch_type` and re-serializes to the bytes it came from. | -| `eh_bucket_sketch_kind_ids_are_build_independent` | The ten nested kind_ids and their registry names are the same in every build. | Verifies `variant_name` maps each of the ten ids to its name and `0xff 0xff` to `None`, and that every populated variant emits the id the table gives it. | +| `eh_bucket_sketch_kind_ids_are_build_independent` | The ten encoded nested kind_ids, plus the retired `0x05 0x00`, and their registry names are the same in every build. | Verifies `variant_name` maps each of the ten ids to its name, the retired `0x05 0x00` to `DDSketch` and `0xff 0xff` to `None`, and that every populated variant emits the id the table gives it. | | `eh_bucket_sketch_rejects_an_experimental_kind_id` | An experimental variant's kind_id is refused without the feature. | Verifies crafted triples carrying the `CocoSketch`, `ElasticSketch`, and `UniformSampling` kind_ids each fail to decode, with a message naming the variant and "experimental" in builds without the feature. | | `eh_bucket_sketch_rejects_an_unknown_kind_id` | An unknown kind_id is refused. | Verifies a Count-Min triple relabelled `0xff 0xff` fails with a "not a wire variant" complaint. | | `eh_bucket_sketch_rejects_sibling_algorithm_kind_ids` | The nested ids are pinned to one algorithm each. | Verifies an HLL triple relabelled `0x01 0x01` or `0x01 0x03` fails to decode, and that a KLL triple relabelled `0x06 0x01` fails too. | @@ -1244,7 +1246,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `run_octo_univmon_reaches_the_deepest_layer` | The scaled per-layer threshold lets the deepest layer through. | Runs 60,000 inputs `U64(i % 4_096)` through a 12-layer `5x1024` `UnivMonOctoPlan` at a heap of `64` on 1 worker, and verifies layer `11` holds at least one non-zero counter. | | `run_octo_univmon_tracks_heavy_hitters_and_total_weight` | The reported total weight trails the stream without overshooting it, and layer 0 holds the hot keys. | Runs `U64(0..5)` at 4,000 occurrences each plus a tail of `U64(10_000..15_000)`, 25,000 inputs in all, through a 6-layer `3x256` `UnivMonOctoPlan` at a heap of `32` and a shared tau of `8` on 4 workers, and verifies `bucket_size` is at most 25,000 and above `0.9 * 25_000`, that layer 0's heap finds each of the five hot keys, and that no layer claims a complete candidate set. | | `octo_runtime_streaming_matches_the_batch_helper` | `OctoRuntime` and `run_octo` end in the same parent state. | Runs 30,000 inputs `U64(i % 1024)` through a `3x4096` `CountMinSketchOctoPlan` on 4 workers once through `run_octo` and once insert-by-insert through `OctoRuntime`, and verifies keys `0..128` estimate identically on both parents. | -| `finishing_a_ddsketch_run_answers_against_the_whole_stream` | `finish` flushes, so every sample has reached the parent. | Runs 20,000 near-distinct values `1.0 + i * 0.37` through `DdOctoPlan` at alpha `0.01` on 4 workers and verifies the parent's count equals `20_000`, that quantiles `0.0`, `0.1`, `0.5`, `0.9` and `1.0` are each within `2 * alpha` relative error of the exact value, and that `min` and `max` bracket the stream's extremes to within alpha. | +| `finishing_a_ddsketch_run_answers_against_the_whole_stream` | `finish` flushes, so every sample has reached the parent. | Runs 20,000 near-distinct values `1.0 + i * 0.37` through `DdOctoPlan` at alpha `0.01` on 4 workers and verifies the parent's count equals `20_000`, that quantiles `0.0`, `0.1`, `0.5`, `0.9` and `1.0` are each within `2 * alpha` relative error of the exact value, and that quantiles `0.0` and `1.0` are each within `alpha` relative error of the stream's exact minimum and maximum. | | `a_mid_stream_flush_makes_the_live_parent_answerable` | A mid-stream flush lands everything accepted so far and does not seal the runtime. | Inserts the first 10,000 of 20,000 values `1.0 + i * 0.37` into a `DdOctoPlan` runtime at alpha `0.01` on 4 workers, flushes, and verifies the read handle sees a count of exactly `10_000` and answers a median, then inserts the remaining 10,000 and verifies `finish` reports `20_000`. | | `a_live_read_right_after_flush_sees_every_input` | `flush` returns only once the aggregator has applied every flushed delta. | Over 30,000 rounds of one to seven inserts into a one-worker runtime whose worker emits only on flush, verifies a live read right after each `flush` reports every input so far, and that `finish` reports the same total. | | `a_worker_that_dies_mid_flush_fails_the_flush_rather_than_hanging_it` | A worker that panics during a flush fails the flush instead of hanging it. | With 2 workers whose `flush` panics, verifies that one insert followed by `flush` panics with "worker dropped during flush". | @@ -2057,7 +2059,7 @@ Test file: [`tests/e2e/wire.rs`](../tests/e2e/wire.rs) | --- | --- | --- | | `a_bloom_envelope_round_trips_on_both_paths_and_names_its_kind` | A Bloom envelope names kind_id `0x17 0x00` and preserves every answer on both hash paths. | Fills a `Bloom` and a `Bloom`, each `with_capacity(2_000, 0.01)`, with `DataInput::I64(0..2_000)`, verifies both envelopes carry a `kind_id_len` of `2` and the bytes `0x17 0x00`, that each decode reports all 2,000 members present, that the regular decode answers exactly as its source on the 2,000 non-members `10_000..12_000`, and that `inserted()` is unchanged. | | `a_coco_envelope_round_trips_and_names_its_kind` | A Coco envelope names kind_id `0x0c 0x00` and preserves every per-key estimate. | Fills `CocoSketch::::with_dimensions_and_seed(4, 512, 0xC0C0)` with the keys `flow-{k}` of a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the envelope's `kind_id_len` of `2` and bytes `0x0c 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | -| `a_ddsketch_envelope_round_trips_and_names_its_kind` | A DDSketch envelope names kind_id `0x05 0x01` and preserves its quantile answers exactly. | Fills `DdSketch::new(0.01)` with `1.0 + v` over `uniform_u64(20_000, 1_000_000, 0x1DE7_0001)`, verifies the envelope's `kind_id_len` of `2` and bytes `0x05 0x00`, and that the decode matches the source on `get_count()`, `alpha()`, and `get_value_at_quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`. | +| `a_ddsketch_envelope_round_trips_and_names_its_kind` | A DDSketch envelope names kind_id `0x05 0x01` and preserves its quantile answers exactly. | Fills `DdSketch::new(0.01)` with `1.0 + v` over `uniform_u64(20_000, 1_000_000, 0x1DE7_0001)`, verifies the envelope's `kind_id_len` of `2` and bytes `0x05 0x01`, and that the decode matches the source on `get_count()`, `alpha()`, and `get_value_at_quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`. | | `a_hydra_envelope_round_trips_and_names_the_counter_it_carries` | A Hydra envelope's kind_id names the counter it carries, and the schema and cells survive. | Builds `Hydra::with_schema(4, 512, ["region", "user"], HydraCounter::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))`, updates each of the nine region/user pairs 40 times with `DataInput::Str("event")`, verifies the `kind_id_len` of `2` and bytes `0x07 0x01`, an unchanged `schema()`, and equal `query_key` answers under `HydraQuery::Frequency` for all nine fully specified keys. | | `a_kll_dynamic_envelope_round_trips_and_names_its_kind` | A KllDynamic envelope names kind_id `0x06 0x01` and re-encodes byte-identically. | Fills `KllDynamic::::with_k_and_seed(200, 0x5EED_9001)` from `uniform_u64(20_000, 1_000_000, 0xC0DE_0001)`, verifies the `kind_id_len` of `2` and bytes `0x06 0x01`, an unchanged `count()`, bit-identical `quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`, and that the decode re-serializes to the same bytes. | | `a_space_saving_envelope_round_trips_and_names_its_kind` | A Space-Saving envelope names kind_id `0x18 0x00` and preserves every estimate. | Fills `SpaceSaving::::with_capacity(256)` from a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the `kind_id_len` of `2` and bytes `0x18 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | diff --git a/src/sketch_framework/eh_bucket_sketch/wire.rs b/src/sketch_framework/eh_bucket_sketch/wire.rs index bec8599c..1a78597f 100644 --- a/src/sketch_framework/eh_bucket_sketch/wire.rs +++ b/src/sketch_framework/eh_bucket_sketch/wire.rs @@ -23,7 +23,8 @@ //! //! ## The kind_id namespace is fixed in every build //! -//! All ten ids dispatch whatever features are on. A decoder built without +//! All eleven ids (ten encoded, plus the retired `0x05 0x00`) dispatch +//! whatever features are on. A decoder built without //! `experimental` rejects `0x0d 0x00` by name with the feature named, and its //! encoder can never emit it. A DDSketch bucket encodes as `0x05 0x01`; the //! retired `0x05 0x00` still decodes into the same variant, its `sum` / `min` / @@ -86,7 +87,8 @@ pub(crate) struct SketchState { pub(crate) state: Vec, } -/// Rejects a kind_id outside the ten. +/// Rejects a kind_id outside the eleven (ten encoded, plus the retired +/// `0x05 0x00`). pub(crate) fn unknown_variant(kind_id: &[u8]) -> RmpDecodeError { RmpDecodeError::Uncategorized(format!( "ASAPv1 EhBucketSketch: kind_id {kind_id:02x?} is not a wire variant" @@ -414,8 +416,9 @@ pub(crate) mod tests { } } - /// The ten nested kind_ids and their registry names are the same in every - /// build, and every variant emits the id the table gives it. + /// The ten encoded nested kind_ids, plus the retired `0x05 0x00`, and their + /// registry names are the same in every build, and every variant emits the + /// id the table gives it. #[test] fn eh_bucket_sketch_kind_ids_are_build_independent() { let table: [(&[u8], &str); 10] = [ diff --git a/src/sketch_framework/octo.rs b/src/sketch_framework/octo.rs index 80f90048..90f52cd6 100644 --- a/src/sketch_framework/octo.rs +++ b/src/sketch_framework/octo.rs @@ -4198,8 +4198,13 @@ mod runtime_tests { "q={q}: got {got}, truth {truth}, relative {relative:.4}" ); } - let (min, max) = (parent.min().unwrap(), parent.max().unwrap()); - assert!(min <= exact[0] * (1.0 + alpha) && max >= exact[exact.len() - 1] * (1.0 - alpha)); + // The extremes are the lowest and highest occupied buckets, each + // within alpha of the stream's true minimum and maximum. + let (lo, hi) = (exact[0], exact[exact.len() - 1]); + let min = parent.get_value_at_quantile(0.0).unwrap(); + let max = parent.get_value_at_quantile(1.0).unwrap(); + assert!((min - lo).abs() <= alpha * lo, "q=0: {min} vs min {lo}"); + assert!((max - hi).abs() <= alpha * hi, "q=1: {max} vs max {hi}"); } #[test] diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 328187e1..d63be802 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -333,7 +333,9 @@ impl DdSketch { /// empty or `q` is NaN: the representative of the bucket holding the /// order statistic at nearest rank `ceil(q * n)` (1-based). `q <= 0` reads /// the lowest occupied bucket and `q >= 1` the highest, so every answer, - /// the extremes included, is within `alpha` relative of a sample. + /// the extremes included, is within `alpha` relative of a sample when that + /// sample's magnitude is indexable. A nonzero magnitude below the + /// indexable range is counted as zero and reads `0.0`. pub fn get_value_at_quantile(&self, q: f64) -> Option { if self.count == 0 || q.is_nan() { return None; @@ -386,26 +388,40 @@ impl DdSketch { /// representative of the bucket holding it, `0.0` in the zero bucket. fn value_at_rank(&self, rank: u64) -> f64 { debug_assert!(rank < self.count); + // The last occupied bucket walked, returned if the buckets ever held + // fewer samples than `count` says (no code path allows it). + let mut last = 0.0; let mut seen = 0u64; let negative = self.negative_store.counts.as_slice(); for (i, &count) in negative.iter().enumerate().rev() { + if count == 0 { + continue; + } seen += count; + last = -self.bin_representative(self.negative_store.offset + i as i32); if seen > rank { - return -self.bin_representative(self.negative_store.offset + i as i32); + return last; } } - seen += self.zero_count; - if seen > rank { - return 0.0; + if self.zero_count > 0 { + seen += self.zero_count; + last = 0.0; + if seen > rank { + return last; + } } let positive = self.positive_store.counts.as_slice(); for (i, &count) in positive.iter().enumerate() { + if count == 0 { + continue; + } seen += count; + last = self.bin_representative(self.positive_store.offset + i as i32); if seen > rank { - return self.bin_representative(self.positive_store.offset + i as i32); + return last; } } - unreachable!("rank {rank} is below the bucket total {}", self.count) + last } /// Returns the total number of samples inserted so far. @@ -647,6 +663,23 @@ mod tests { sketch } + /// If `count` ever exceeded the buckets, a quantile reads the last occupied + /// bucket instead of panicking. + #[test] + fn a_rank_past_the_buckets_reads_the_last_occupied_bucket() { + let mut sketch = DdSketch::new(0.01); + sketch.add(&-4.0); + sketch.count += 5; + assert_eq!( + sketch.get_value_at_quantile(1.0), + Some(-sketch.bin_representative(sketch.key_for(4.0))) + ); + sketch.add(&8.0); + let top = sketch.bin_representative(sketch.key_for(8.0)); + assert_eq!(sketch.get_value_at_quantile(1.0), Some(top)); + assert_eq!(sketch.quantile_interpolated(1.0), Ok(Some(top))); + } + #[test] fn quantile_interpolated_two_values() { let sketch = sketch_of(0.01, &[20.0, 40.0]); diff --git a/src/sketches/ddsketch/wire.rs b/src/sketches/ddsketch/wire.rs index ec73c8a7..54f544d2 100644 --- a/src/sketches/ddsketch/wire.rs +++ b/src/sketches/ddsketch/wire.rs @@ -397,6 +397,43 @@ mod tests { assert!(DdSketch::from_asapv1(&short).is_err()); } + /// The retired scalars are read as float64 (Section 4) and then dropped: + /// an int or a float32 in their place is refused, and any float64 value, + /// NaN included, decodes. + #[test] + fn ddsketch_retired_scalars_must_be_float64() { + let metadata = encode_metadata(ALPHA, 1.0).unwrap(); + let retired = |payload: Vec| { + DdSketch::from_asapv1(&envelope::encode( + kind::DDSKETCH_WITH_SCALARS, + &metadata, + &payload, + )) + }; + let empty: Vec = Vec::new(); + let int_sum = + rmp_serde::to_vec(&(vec![3u64], 7i32, 6i64, 1.5f64, 2.5f64, &empty, 0i32, 0u64)); + assert!(retired(int_sum.unwrap()).is_err(), "int sum"); + let f32_min = + rmp_serde::to_vec(&(vec![3u64], 7i32, 6.0f64, 1.5f32, 2.5f64, &empty, 0i32, 0u64)); + assert!(retired(f32_min.unwrap()).is_err(), "float32 min"); + let nan = rmp_serde::to_vec(&( + vec![3u64], + 7i32, + f64::NAN, + f64::NAN, + f64::NAN, + &empty, + 0i32, + 0u64, + )); + let decoded = retired(nan.unwrap()).expect("NaN scalars are dropped"); + assert_eq!(decoded.get_count(), 3); + let floats = + rmp_serde::to_vec(&(vec![3u64], 7i32, 6.0f64, 1.5f64, 2.5f64, &empty, 0i32, 0u64)); + assert!(retired(floats.unwrap()).is_ok(), "control"); + } + #[test] fn ddsketch_empty_round_trip() { let sketch = DdSketch::new(ALPHA); From d2c187ea71feb07d0bc17a425a08c1ee5a681fea Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 29 Oct 2026 00:00:44 +0000 Subject: [PATCH 3/3] chore: point asapv1_golden at golden-bytes main (#4 merged) Co-Authored-By: Claude Opus 5.5 --- asapv1_golden | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asapv1_golden b/asapv1_golden index 08440a11..f92a8cea 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit 08440a1196027e5b75fe779462f0262fa1e34544 +Subproject commit f92a8ceacdaf0f5018182a24a253c9e035e31ae7