From c0be4a4f6c130eb0cd14f87f17a12f715eddf328 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:46:24 +0200 Subject: [PATCH 1/4] fix(lore-0135): make the carry bound conditional on a fresh venue surviving The unconditional form blanked sources on 52% of the table to prevent zero evictions. The defect it guards - a stale venue outvoting a live one in the unweighted 5.5 median - needs a live venue to victimise, so the guard now fires only when one survives. Measured before applying this time, both variants over the same live data (4152 assets): zero_but_vwap_ok 30->0, zero_price_usd 868->368, and empty_sources 840->368. Every metric improves. empty_sources, zero_price_usd and zero_vwap all land on 368 exactly - an asset has a price, sources and a vwap, or none of the three. Both arms pinned by fixtures and proven non-vacuous: disabling the filter fails MIX, making it unconditional fails STA with sources={}. --- packages/prices-clickhouse/schema/current.sql | 72 +++++++++++++----- packages/prices-clickhouse/src/lib.rs | 9 ++- .../prices-clickhouse/tests/current_mv_it.rs | 75 ++++++++++++------- 3 files changed, 107 insertions(+), 49 deletions(-) diff --git a/packages/prices-clickhouse/schema/current.sql b/packages/prices-clickhouse/schema/current.sql index b942a2bf..8ca8d42e 100644 --- a/packages/prices-clickhouse/schema/current.sql +++ b/packages/prices-clickhouse/schema/current.sql @@ -31,7 +31,8 @@ -- -- Columns (task 0072 completes the set; all ten are now written): -- price_usd — latest PRICED close in the 24h window (argMaxIf, 0135); --- NOT age-bounded — see the unfiltered CTE for why +-- NOT age-bounded — see the unfiltered CTE for why. The +-- per-venue pipeline IS bounded, but conditionally: see 1b -- price_xlm — price_usd re-expressed in XLM (÷ the XLM/USD close) -- change_24h_pct — vs the oldest close inside the 24h window -- change_7d_pct — vs the oldest priced close in the [7d, 5d] band of @@ -122,21 +123,16 @@ WITH -- reading 0 here and being dropped by `src_price > 0` below as a side -- effect of enrichment timing. -- - -- The carry is BOUNDED, and the bound lives HERE and nowhere else. The - -- reason is scope, not a number: the defect 0135 introduces is a dead - -- venue voting in the §5.5 median, which is a per-venue problem, so the - -- guard belongs in the per-venue pipeline. A carried close feeding - -- `sources` and `vwap_24h` asserts "this venue quotes X right now"; once - -- it is hours old that assertion is false, and because the median is - -- UNWEIGHTED, two stale venues can outvote and evict the one live venue - -- (the 3-source case in PR #228's review). + -- Freshness is measured but NOT yet applied — level 1b decides that, per + -- asset. Both columns take the latest priced close; they differ only in + -- whether a candle older than CARRY_BOUND may supply it, so for a source + -- that IS fresh the two are identical. -- -- CARRY_BOUND = 2h, derived from the enrichment SCHEDULE — `rate(1 hour)` -- in infra/envs/production.json, so two cycles, and a venue quoting -- normally survives one missed pass. Deliberately NOT fitted to an -- observed lag: as of 2026-08-20 enrichment was failing on every run -- (task 0215), so a measured figure would encode a broken pipeline. - -- Revisit once 0111 and 0215 land. -- -- Reference is `now()`, NOT the asset's own newest candle. An earlier -- revision compared against `max(timestamp)`, which spans quote legs @@ -148,14 +144,54 @@ WITH SELECT asset_id AS asset_id, source AS source, + argMaxIf(close_usd, timestamp, close_usd > 0) AS src_price, argMaxIf(close_usd, timestamp, - close_usd > 0 AND timestamp >= now() - INTERVAL 2 HOUR) AS src_price, + close_usd > 0 AND timestamp >= now() - INTERVAL 2 HOUR) AS src_price_fresh, sum(volume_quote_usd) AS src_volume FROM prices.price_ohlcv_1m FINAL WHERE timestamp >= now() - INTERVAL 24 HOUR GROUP BY asset_id, source ), + -- Level 1b — the bound is CONDITIONAL, and that is the whole design. + -- + -- What it guards: a venue whose last quote is hours old still votes in the + -- §5.5 median, which is UNWEIGHTED, so two stale venues can outvote and + -- evict the one live venue (the 3-source case in PR #228's review). That + -- defect needs a live venue to victimise. + -- + -- So: drop stale venues ONLY when a fresh one survives. If every venue on + -- an asset is stale there is nothing to defend, and dropping them all is + -- pure loss — it empties `sources` and zeroes `vwap_24h` for an asset we + -- have perfectly usable, if unfresh, prices for. + -- + -- ⚠️ This is not a hypothetical refinement. The unconditional form shipped + -- to prod on 2026-08-21 and was rolled back within the hour: measured + -- against the same data, it blanked `sources` on **2,284 of 4,365 assets + -- (52%)** while preventing **zero** evictions — 74% of the table had every + -- venue stale, because enrichment had been down for two days (0215). The + -- population where the defect can actually occur (>= 3 sources, mixed + -- fresh/stale) was **7 assets**, and on all 7 the fresh-only median sat + -- within 1% of the all-source median, far inside the 20% threshold. + -- + -- Kept anyway, deliberately: that zero is not evidence of future safety. + -- The mixed population is small precisely BECAUSE almost nothing is fresh + -- right now; when 0215 and 0111 land, fresh venues multiply and the mixed + -- population grows with them. The risk rises as the pipeline recovers. + -- Conditional costs nothing by construction, so it is the cheap side of + -- that bet. + per_source_kept AS ( + SELECT + asset_id, + source, + src_price, + src_price_fresh, + src_volume, + max(src_price_fresh > 0) OVER (PARTITION BY asset_id) AS asset_has_fresh + FROM per_source + WHERE src_price > 0 + ), + -- Level 2 — collapse sources into per-asset arrays so the median filter can -- run as an array pipeline. Decimal arrays are kept for the JSON (exact), -- Float64 arrays for the arithmetic (see numeric strategy above). @@ -167,12 +203,14 @@ WITH groupArray(src_volume) AS vols_dec, groupArray(toFloat64(src_price)) AS prices_f, groupArray(toFloat64(src_volume)) AS vols_f - FROM per_source - WHERE src_price > 0 -- explicit rule (0135 C2): a source with - -- NO priced candle in the whole window - -- carries no signal; with the argMaxIf - -- above this can no longer drop a source - -- that merely has an un-enriched tip + FROM per_source_kept + WHERE NOT asset_has_fresh OR src_price_fresh > 0 + -- explicit rule (0135 C2 + the + -- conditional bound): a source with NO + -- priced candle in the window is + -- already gone at level 1b; here a + -- STALE source is dropped only when the + -- asset still has a fresh one left GROUP BY asset_id ), diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index 0c233e85..83ca6ad4 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -469,11 +469,12 @@ mod tests { guarded += stmt.matches("argMinIf(close_usd, timestamp,").count(); } - // Non-vacuity: 3 argMaxIf (xlm_usd scalar, per_source, unfiltered) + - // 2 argMinIf (ref_7d, open_24h). A drop means a projection lost its - // guard or the expression was reworded and this test has gone blind. + // Non-vacuity: 4 argMaxIf (xlm_usd scalar, per_source's src_price and + // src_price_fresh, unfiltered) + 2 argMinIf (ref_7d, open_24h). A drop + // means a projection lost its guard or the expression was reworded and + // this test has gone blind. assert_eq!( - guarded, 5, + guarded, 6, "current.sql guarded close_usd aggregate count changed — verify \ every site still skips un-enriched rows, then update this count" ); diff --git a/packages/prices-clickhouse/tests/current_mv_it.rs b/packages/prices-clickhouse/tests/current_mv_it.rs index 3071efc7..7063964b 100644 --- a/packages/prices-clickhouse/tests/current_mv_it.rs +++ b/packages/prices-clickhouse/tests/current_mv_it.rs @@ -173,8 +173,9 @@ async fn current_prices_mv_computes_price_volume_and_market_cap() { /// fixture exercising the load-bearing change_7d numerator guard /// 7 ZER — priced HISTORY, un-enriched TIP → 0135: latest priced close wins /// 8 DIP — a genuine ~-100% crash → must NOT be swallowed by 0138's guard -/// 10 STA — priced history OLDER than the 2h carry bound → the ASYMMETRY: -/// price_usd still publishes it, the venue drops out of sources/vwap +/// 10 STA — EVERY venue stale → the conditional bound must NOT fire; the +/// venue is kept, because there is no live venue to protect +/// 14 MIX — one fresh venue, one stale → the bound FIRES: stale is dropped /// 11 LAG — SINGLE source, priced history, un-enriched tip → carried (the /// shape the XLM acceptance criterion names) /// 12 TRI — three sources, one carried past an un-enriched tip → the §5.5 @@ -199,7 +200,7 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { (6,'EXO','classic','GEXO','',1), (7,'ZER','classic','GZER','',1), \ (8,'DIP','classic','GDIP','',1), (10,'STA','classic','GSTA','',1), \ (11,'LAG','classic','GLAG','',1), (12,'TRI','classic','GTRI','',1), \ - (13,'NRB','classic','GNRB','',1)" + (13,'NRB','classic','GNRB','',1), (14,'MIX','classic','GMIX','',1)" )) .execute() .await @@ -265,6 +266,11 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { // 13 NRB — ordinary priced asset; the interesting part is its 1h // reference below, which is too RECENT for the [7d, 5d] band. (13, 5, "2.00", "1000", "sdex"), + // 14 MIX — the conditional bound FIRING. sdex last quoted 190 min ago + // (stale), soroswap 5 min ago (fresh). Because a fresh venue survives, + // the stale one is dropped: it must not vote in the §5.5 median. + (14, 190, "2.00", "800", "sdex"), + (14, 5, "1.00", "400", "soroswap"), ]; for (i, (asset, mins, cu, vol, src)) in rows.iter().enumerate() { admin @@ -350,7 +356,7 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { .fetch_one() .await .expect("count"); - if n >= 11 { + if n >= 12 { ready = true; break; } @@ -555,46 +561,59 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { "market_cap_usd must be 1.90 * 1000 = 1900, got {zer_mc}" ); - // ── 0135 ASYMMETRY: the bound guards the venues, not the headline ────── - // Asset 10's only priced close is 190 min old. The per-venue guard drops - // sdex (a venue that last quoted 3h ago is not quoting "now"), so - // `sources` empties and vwap goes to its sentinel — while `price_usd` - // still publishes the close, because blanking a price we hold is the - // outcome 0135 exists to remove. Measured on prod 2026-08-20: 1,091 of - // 4,444 assets (24.5%) already publish a hard zero without any bound. + // ── conditional bound, ARM A: every venue stale → do NOT fire ───────── + // Asset 10's only venue last quoted 190 min ago, past the 2h bound. There + // is no live venue to protect, so dropping it would be pure loss — it + // would empty `sources` and zero `vwap_24h` on an asset whose price we + // hold. Measured on prod 2026-08-21, the unconditional form did exactly + // that to 2,284 of 4,365 assets (52%) while preventing zero evictions. let sta_p = scalar_f64(&admin, &f("price_usd", 10)).await; assert!( (sta_p - 2.0).abs() < 1e-9, - "price_usd is NOT age-bounded — it must still publish the 190-min-old \ - priced close, got {sta_p}" + "price_usd is not age-bounded, so the 190-min-old close still publishes, got {sta_p}" ); let sta_srcs: String = admin .query(&s("sources", 10)) .fetch_one() .await .expect("sta sources"); - assert_eq!( - sta_srcs, "{}", - "a venue whose last quote is beyond the carry bound must drop out of \ - sources, so it cannot vote in the §5.5 median" + assert!( + sta_srcs.contains("sdex"), + "with NO fresh venue on the asset the stale one must be KEPT — dropping \ + it defends nothing and blanks the row, got {sta_srcs}" ); let sta_vwap = scalar_f64(&admin, &f("vwap_24h", 10)).await; assert!( - sta_vwap.abs() < 1e-9, - "vwap must be the sentinel when no venue is currently quoting, got {sta_vwap}" + (sta_vwap - 2.0).abs() < 1e-6, + "vwap must still come from the kept venue, got {sta_vwap}" + ); + + // ── conditional bound, ARM B: a fresh venue survives → DO fire ───────── + // Asset 14 has sdex stale at 2.00 (190 min) and soroswap fresh at 1.00 + // (5 min). Here the defect is live: unweighted, the stale venue votes in + // the median and its price is the one an outlier check would anchor on. + // So the stale venue is dropped and only the fresh one survives. + let mix_srcs: String = admin + .query(&s("sources", 14)) + .fetch_one() + .await + .expect("mix sources"); + assert!( + mix_srcs.contains("soroswap") && !mix_srcs.contains("sdex"), + "a stale venue must be dropped when a fresh one survives, got {mix_srcs}" ); - // The row stays internally coherent: change_* are computed FROM the - // published price, never fabricated against it. - let sta_24 = scalar_f64(&admin, &f("change_24h_pct", 10)).await; + let mix_vwap = scalar_f64(&admin, &f("vwap_24h", 14)).await; assert!( - sta_24.abs() < 1e-9, - "one priced candle means open_24h == price_usd, so 0%, got {sta_24}" + (mix_vwap - 1.0).abs() < 1e-6, + "vwap must weight the FRESH venue only (1.00), not the blend with the \ + stale 2.00 — a blend means the bound did not fire, got {mix_vwap}" ); - let sta_7d = scalar_f64(&admin, &f("change_7d_pct", 10)).await; + // price_usd is unbounded and venue-blind: it is simply the newest priced + // close on the asset, which here is the fresh one. + let mix_p = scalar_f64(&admin, &f("price_usd", 14)).await; assert!( - (sta_7d - 100.0).abs() < 1e-2, - "change_7d_pct must come from the published 2.00 against the 1.00 \ - baseline = +100%, got {sta_7d}" + (mix_p - 1.0).abs() < 1e-9, + "price_usd is the newest priced close regardless of venue, got {mix_p}" ); // ── the change_7d numerator guard, on the only shape that reaches it ─── From 1f8d83f8aefda8864e1d53fe953cc590fe616d14 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:15:33 +0200 Subject: [PATCH 2/4] docs(lore-0135): re-measure the conditional bound on a healthy pipeline; retract the growth argument 0215 was fixed 2026-08-24 14:46 UTC. Re-took the counterfactual ~16h in. The change holds on independent data: zero_but_vwap_ok 27 -> 0, zero_price_usd 707 -> 259, empty_sources 680 -> 259, zero_vwap 680 -> 259. The three-way coherence reproduces at a different value than the 08-21 run, so it follows from the construction rather than from that day's data. Retracts the PR #241 claim that the guard's value would grow as enrichment recovered. It shrank: mixed 35 -> 15, at-risk (mixed AND >=3 sources) 7 -> 1. Assets move wholesale from all-stale to all-fresh, not through a mixed state. Guard kept on a corrected argument: its value is anti-correlated with pipeline health, it costs nothing by construction, and removing it is a two-line revert that changes none of the metrics above. --- ...0135_FEATURE_price-usd-outlier-exposure.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lore/1-tasks/active/0135_FEATURE_price-usd-outlier-exposure.md b/lore/1-tasks/active/0135_FEATURE_price-usd-outlier-exposure.md index 23493462..59db7ff6 100644 --- a/lore/1-tasks/active/0135_FEATURE_price-usd-outlier-exposure.md +++ b/lore/1-tasks/active/0135_FEATURE_price-usd-outlier-exposure.md @@ -243,6 +243,44 @@ history: documented "grep the definition to confirm the apply landed" check reports a successful deploy as failed unless it greps for function names. Cost me one false alarm mid-deploy. + - date: 2026-08-25 + status: active + who: stkrolikiewicz + note: > + **Re-measured on a healthy pipeline; one of my own arguments is + retracted.** BE fixed [[0215]] on 2026-08-24 14:46 UTC (zero errors + since, against 3/hour and no successful run for the preceding 26 days), + so the numbers behind the conditional bound were re-taken ~16 h in. + **The change itself holds, on data 4 days and one pipeline-state apart.** + Old vs conditional over 3,166 assets: `zero_but_vwap_ok` 27 → **0**, + `zero_price_usd` 707 → **259**, `empty_sources` 680 → **259** (421 + assets regain a `sources` object), `zero_vwap` 680 → **259**. The + three-way coherence reproduces exactly — an asset has a price, sources + and a VWAP, or none of the three — which the 08-21 run also showed at a + different value (368). Reproducing on independent data means it follows + from the construction, not from that day's data. + **Retracted:** PR #241 argued the guard would grow in value as + enrichment recovered, because fresh venues would multiply and with them + the mixed fresh/stale population. Measured, the opposite happened. + Freshness did rise (all-fresh assets 656/15.5% → 849/27%), but assets + moved wholesale from all-stale to all-fresh rather than through a mixed + state: **mixed 35 → 15, and the at-risk group (mixed AND ≥3 sources, + the only shape where the §5.5 median can evict anything) 7 → 1.** + A venue that trades gets a fresh price and one that does not, does not; + the mix is a transient, and a consistent pipeline yields fewer of them. + **The guard is kept on a corrected argument:** its value is + anti-correlated with pipeline health. It prevents nothing measurable + today (1 at-risk asset), was worth 7 during an outage that ran 26 days + before [[0144]] noticed it, and the defect it blocks — a stale venue + outvoting a live one in an unweighted median — is silent and + indistinguishable from a real price downstream. It costs nothing by + construction and is already pinned by non-vacuous fixtures. Deleting it + is a clean two-line revert that changes none of the numbers above; they + come from C2's carry, not from the guard. Recorded as a judgement call, + not a measurement. + Re-measure again after [[0111]], and after BE's partition-limited scan + (their next step, now that their own 300 s Lambda cap is the binding + constraint) changes enrichment throughput a second time. --- # price_usd is not outlier-protected From 64dad35684b219c766387dc029688734db120c88 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:05:12 +0200 Subject: [PATCH 3/4] fix(0135): gate liveness on the venue's candle, not its priced close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses all six findings from the PR #241 review. Finding 1 (the substantive one). src_price_fresh tested the newest PRICED close, so a venue that is actively quoting was evicted whenever enrichment had not reached it -- conflating a dead venue with a lagging one, and reproducing one level down the exact defect scope correction C2 exists to prevent. Replaced with src_is_live = max(timestamp) >= now() - 2h, which has no close_usd term at all: liveness is a property of quoting, so it is measured on candles. Verified read-only on prod ClickHouse 26.3.10.60 against the review's own probe (asset 99 in the check): a venue quoting 1 min ago with $1,000,000 of volume but last enriched 3 h ago is now KEPT and dominates the VWAP at 1.00, where the old predicate evicted it in favour of a $10 venue and published 1.20. ARM A / ARM B / the mask fixtures reproduce identically under both predicates, so the change is narrow. Finding 2. The "price, sources and VWAP, or none of the three, BY CONSTRUCTION" claim was too strong: median is quantile(0.5) and interpolates, so an even-count set straddling the midpoint by > OUTLIER_PCT clears entirely. [1,1,3,3] -> median 2.00 -> all four deviate 50% -> sources '{}', vwap 0, beside a real price_usd of 1.00. Verified on prod. Documented at the mask, and pinned by new fixture asset 15. It matters more after C2's carry, which newly routes the whole all-dead population through the mask. Whether interpolation is the right median for an even count is 0217's call. Findings 3-6, doc drift: the asset 10 fixture comment described the old behaviour and contradicted its own assertion; current.sql still carried the growth argument this PR retracts; ARM B ran on two sources, where the §5.5 mask is a documented no-op, so it asserted the bound without exercising the defect the bound exists for; the change_*_pct rationale described a bound the unfiltered CTE never applied. Also: ARM A needed a fixture change the review did not flag. Asset 10 had an un-enriched tip at 1 min, which under candle-based liveness makes the venue LIVE -- the fixture would have silently stopped testing the all-dead arm. Tip row removed and the reason recorded in the fixture. Lint count 6 -> 5: liveness is no longer a close_usd aggregate, and must not be, since counting close_usd there is the finding-1 defect. --- packages/prices-clickhouse/schema/current.sql | 116 +++++++++++++----- packages/prices-clickhouse/src/lib.rs | 16 ++- .../prices-clickhouse/tests/current_mv_it.rs | 106 +++++++++++++--- 3 files changed, 181 insertions(+), 57 deletions(-) diff --git a/packages/prices-clickhouse/schema/current.sql b/packages/prices-clickhouse/schema/current.sql index 8ca8d42e..d2fb0e53 100644 --- a/packages/prices-clickhouse/schema/current.sql +++ b/packages/prices-clickhouse/schema/current.sql @@ -78,6 +78,27 @@ -- is that source (deviation 0, always kept) — harmless but pointless. So the -- filter is a no-op below 3 sources by construction, not by luck. -- +-- ⚠️ At >= 3 it can, however, clear EVERY source. `median` is `quantile(0.5)` +-- and INTERPOLATES on an even count, so a set straddling the midpoint by more +-- than OUTLIER_PCT leaves nothing behind. Verified on prod 26.3.10.60: +-- +-- [1.00, 1.00, 3.00, 3.00] -> median 2.00 -> every element deviates 50% +-- -> sources = '{}', vwap_24h = 0 +-- +-- while `price_usd` — venue-blind and taken from `unfiltered` — still +-- publishes 1.00. So an asset CAN carry a price with no sources beside it, and +-- the three-way coherence 0135 measures on prod (empty_sources = zero_price_usd +-- = zero_vwap, 259 each on 2026-08-25) is an observed property of the current +-- data, NOT an invariant of this SQL. Raised in the PR #241 review, finding 2, +-- against a PR-body claim of "by construction" that was too strong. +-- +-- It matters more after 0135's C2 carry than before it: assets whose every +-- venue has an un-enriched tip used to arrive here with zero kept sources and +-- never armed the mask. They now arrive with their carried prices, so the +-- whole all-dead population — 65% of prod assets on 2026-08-25 — is newly +-- routed through it. Pinned by fixture asset 15; whether interpolation is the +-- right median for an even count is 0217's call, not this file's. +-- -- OUTLIER_PCT = 0.20 (20%). A starting value, deliberately loose. The asymmetry -- matters: wrongly EXCLUDING a legitimate venue silently removes it from the -- `sources` JSON and from the VWAP weighting (a visible, wrong answer), while @@ -123,10 +144,10 @@ WITH -- reading 0 here and being dropped by `src_price > 0` below as a side -- effect of enrichment timing. -- - -- Freshness is measured but NOT yet applied — level 1b decides that, per - -- asset. Both columns take the latest priced close; they differ only in - -- whether a candle older than CARRY_BOUND may supply it, so for a source - -- that IS fresh the two are identical. + -- Liveness is measured but NOT yet applied — level 1b decides that, per + -- asset. Note the two columns read DIFFERENT things on purpose: src_price + -- is a price and skips un-enriched candles; src_is_live is a quoting + -- signal and counts them. -- -- CARRY_BOUND = 2h, derived from the enrichment SCHEDULE — `rate(1 hour)` -- in infra/envs/production.json, so two cycles, and a venue quoting @@ -135,18 +156,31 @@ WITH -- (task 0215), so a measured figure would encode a broken pipeline. -- -- Reference is `now()`, NOT the asset's own newest candle. An earlier - -- revision compared against `max(timestamp)`, which spans quote legs - -- enrichment can never price — for a venue trading continuously on such a - -- leg that reference advances forever, the bound becomes unsatisfiable and - -- the venue drops out with nothing wrong. `now()` asks the question this - -- column actually needs answered: is the venue's last quote still current? + -- revision compared the venue against `max(timestamp)` ACROSS the asset, + -- which spans quote legs enrichment can never price — for a venue trading + -- continuously on such a leg that reference advances forever, the bound + -- becomes unsatisfiable and the venue drops out with nothing wrong. Below, + -- `max(timestamp)` is the venue's OWN newest candle against the wall + -- clock, which is a different question and has no such failure mode. + -- + -- ⚠️ src_is_live tests the venue's newest CANDLE, and there is deliberately + -- no `close_usd > 0` in that predicate. It asks "is this venue still + -- QUOTING?", which is not the same question as "has enrichment reached + -- this venue lately?" (PR #241 review, finding 1). An earlier revision + -- tested the newest *priced* close and so conflated a dead venue with one + -- enrichment simply had not caught up to — reproducing, one level down, + -- the exact defect C2 above exists to prevent. Measured on the review's + -- probe: a venue quoting 1 min ago carrying $1,000,000 of volume, last + -- enriched 3 h ago, was evicted in favour of one carrying $10 that + -- happened to be enriched 5 min ago — publishing a `vwap_24h` drawn from + -- 0.001% of the `volume_24h_usd` the same row reports. Liveness is a + -- property of quoting, so it is measured on candles. per_source AS ( SELECT asset_id AS asset_id, source AS source, argMaxIf(close_usd, timestamp, close_usd > 0) AS src_price, - argMaxIf(close_usd, timestamp, - close_usd > 0 AND timestamp >= now() - INTERVAL 2 HOUR) AS src_price_fresh, + max(timestamp) >= now() - INTERVAL 2 HOUR AS src_is_live, sum(volume_quote_usd) AS src_volume FROM prices.price_ohlcv_1m FINAL WHERE timestamp >= now() - INTERVAL 24 HOUR @@ -155,39 +189,51 @@ WITH -- Level 1b — the bound is CONDITIONAL, and that is the whole design. -- - -- What it guards: a venue whose last quote is hours old still votes in the - -- §5.5 median, which is UNWEIGHTED, so two stale venues can outvote and + -- What it guards: a venue that stopped quoting hours ago still votes in + -- the §5.5 median, which is UNWEIGHTED, so two dead venues can outvote and -- evict the one live venue (the 3-source case in PR #228's review). That -- defect needs a live venue to victimise. -- - -- So: drop stale venues ONLY when a fresh one survives. If every venue on - -- an asset is stale there is nothing to defend, and dropping them all is - -- pure loss — it empties `sources` and zeroes `vwap_24h` for an asset we - -- have perfectly usable, if unfresh, prices for. + -- So: drop dead venues ONLY when a live one survives. If every venue on an + -- asset is dead there is nothing to defend, and dropping them all is pure + -- loss — it empties `sources` and zeroes `vwap_24h` for an asset we have + -- perfectly usable, if stale, prices for. -- -- ⚠️ This is not a hypothetical refinement. The unconditional form shipped -- to prod on 2026-08-21 and was rolled back within the hour: measured -- against the same data, it blanked `sources` on **2,284 of 4,365 assets -- (52%)** while preventing **zero** evictions — 74% of the table had every - -- venue stale, because enrichment had been down for two days (0215). The - -- population where the defect can actually occur (>= 3 sources, mixed - -- fresh/stale) was **7 assets**, and on all 7 the fresh-only median sat - -- within 1% of the all-source median, far inside the 20% threshold. + -- venue stale, because enrichment had been down for two days (0215). + -- + -- Sizing, measured twice, and NOT trending the way an earlier revision of + -- this comment predicted: + -- + -- pipeline down (08-21) healthy (08-25) + -- mixed live/dead 35 15 + -- ...and >= 3 sources 7 1 <- at-risk + -- + -- That revision argued the mixed population would GROW as enrichment + -- recovered, so the guard would be worth more over time. Retracted: it + -- shrank by more than half. Assets move wholesale from all-dead to + -- all-live rather than through a mixed state — a venue that trades gets a + -- fresh candle, one that does not, does not. The mix is a transient, and a + -- consistent pipeline produces fewer of them. -- - -- Kept anyway, deliberately: that zero is not evidence of future safety. - -- The mixed population is small precisely BECAUSE almost nothing is fresh - -- right now; when 0215 and 0111 land, fresh venues multiply and the mixed - -- population grows with them. The risk rises as the pipeline recovers. - -- Conditional costs nothing by construction, so it is the cheap side of - -- that bet. + -- Kept on the corrected argument: the guard's value is ANTI-correlated + -- with pipeline health. At 1 at-risk asset it prevents nothing measurable + -- today; it was worth 7 during an outage that ran 26 days before 0144 + -- noticed it. The defect it blocks is silent and indistinguishable from a + -- real price downstream, and the conditional form costs nothing by + -- construction. Removing it is a two-line revert that changes none of the + -- measured gains, which come from C2's carry rather than from the guard. per_source_kept AS ( SELECT asset_id, source, src_price, - src_price_fresh, src_volume, - max(src_price_fresh > 0) OVER (PARTITION BY asset_id) AS asset_has_fresh + src_is_live, + max(src_is_live) OVER (PARTITION BY asset_id) AS asset_has_live FROM per_source WHERE src_price > 0 ), @@ -204,7 +250,7 @@ WITH groupArray(toFloat64(src_price)) AS prices_f, groupArray(toFloat64(src_volume)) AS vols_f FROM per_source_kept - WHERE NOT asset_has_fresh OR src_price_fresh > 0 + WHERE NOT asset_has_live OR src_is_live -- explicit rule (0135 C2 + the -- conditional bound): a source with NO -- priced candle in the window is @@ -353,8 +399,14 @@ SELECT -- while their `vwap_24h` and `sources` carried the real price. -100 is NOT -- a sentinel — it passes every consumer-side "0 means unavailable" guard -- the views.sql interop contract documents, because it looks like data. - -- With 0135's bounded argMaxIf the numerator is 0 when the window is - -- unpriced OR the carry bound is exceeded. For change_24h_pct that case + -- With 0135's guarded argMaxIf the numerator is 0 when the window holds no + -- priced candle at all — and ONLY then. It is not subject to the liveness + -- bound: the numerator reads `unfiltered.price_usd`, which is deliberately + -- venue-blind and unbounded, while the bound lives in `per_source` and has + -- never touched this projection. An earlier revision of this comment said + -- "or the carry bound is exceeded", which was never true and became more + -- misleading once the bound stopped being an argMaxIf predicate at all + -- (PR #241 review, finding 6). For change_24h_pct the unpriced case -- usually zeroes open_24h too (same table, same window), but NOT always — -- an over-bound asset has priced history and a real open_24h — and for -- change_7d_pct the denominator comes from a DIFFERENT table diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index 08358568..1b6cc14f 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -470,12 +470,18 @@ mod tests { guarded += stmt.matches("argMinIf(close_usd, timestamp,").count(); } - // Non-vacuity: 4 argMaxIf (xlm_usd scalar, per_source's src_price and - // src_price_fresh, unfiltered) + 2 argMinIf (ref_7d, open_24h). A drop - // means a projection lost its guard or the expression was reworded and - // this test has gone blind. + // Non-vacuity: 3 argMaxIf (xlm_usd scalar, per_source's src_price, + // unfiltered) + 2 argMinIf (ref_7d, open_24h). A drop means a + // projection lost its guard or the expression was reworded and this + // test has gone blind. + // + // Was 6 until the PR #241 review: per_source's src_price_fresh was a + // second argMaxIf carrying the bound in its predicate. Liveness is now + // `max(timestamp) >= now() - INTERVAL 2 HOUR`, which is not a + // close_usd aggregate at all — it must NOT be, since counting + // close_usd there is exactly the finding-1 defect. assert_eq!( - guarded, 6, + guarded, 5, "current.sql guarded close_usd aggregate count changed — verify \ every site still skips un-enriched rows, then update this count" ); diff --git a/packages/prices-clickhouse/tests/current_mv_it.rs b/packages/prices-clickhouse/tests/current_mv_it.rs index 7063964b..cbe0922e 100644 --- a/packages/prices-clickhouse/tests/current_mv_it.rs +++ b/packages/prices-clickhouse/tests/current_mv_it.rs @@ -173,9 +173,12 @@ async fn current_prices_mv_computes_price_volume_and_market_cap() { /// fixture exercising the load-bearing change_7d numerator guard /// 7 ZER — priced HISTORY, un-enriched TIP → 0135: latest priced close wins /// 8 DIP — a genuine ~-100% crash → must NOT be swallowed by 0138's guard -/// 10 STA — EVERY venue stale → the conditional bound must NOT fire; the +/// 10 STA — EVERY venue dead → the conditional bound must NOT fire; the /// venue is kept, because there is no live venue to protect -/// 14 MIX — one fresh venue, one stale → the bound FIRES: stale is dropped +/// 14 MIX — one live venue, TWO dead → the bound FIRES: the dead pair is +/// dropped before it can outvote the live one in the §5.5 median +/// 15 EVN — four dead venues straddling the interpolated median → the mask +/// clears ALL of them, leaving a price beside empty `sources` /// 11 LAG — SINGLE source, priced history, un-enriched tip → carried (the /// shape the XLM acceptance criterion names) /// 12 TRI — three sources, one carried past an un-enriched tip → the §5.5 @@ -245,13 +248,19 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { // 0138 control: a REAL near-total crash. The guard must leave this alone. (8, 30, "2.00", "500", "sdex"), (8, 1, "0.0001", "10", "sdex"), - // 10 STA — the carry BOUND, and the ASYMMETRY it creates. Its only - // priced close is 190 min old (> 2h), so the per-venue guard drops - // sdex from `sources` and the vwap — while `price_usd`, deliberately - // unbounded, still publishes that close. Price without venues is the - // intended shape: we hold a price, no venue is quoting right now. + // 10 STA — ARM A of the conditional bound. EVERY venue here is dead: + // sdex's newest candle is 190 min old, past the 2h bound. With no live + // venue to protect there is nothing to defend, so the bound must NOT + // fire — sdex is KEPT, `sources` names it and `vwap_24h` comes from + // it. Dropping it would blank the row for an asset we hold a perfectly + // good, merely stale, price for: that is the 2026-08-21 prod rollback, + // reproduced as a test. + // + // There is deliberately NO un-enriched tip row here. Liveness is + // measured on the newest CANDLE, not the newest priced close (PR #241 + // review, finding 1), so a tip at 1 min would make sdex LIVE and this + // fixture would silently stop testing ARM A at all. (10, 190, "2.00", "500", "sdex"), - (10, 1, "0", "0", "sdex"), // 11 LAG — single source, priced history INSIDE the bound: the carry // must keep the venue alive on every column. (11, 30, "2.00", "300", "sdex"), @@ -269,8 +278,31 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { // 14 MIX — the conditional bound FIRING. sdex last quoted 190 min ago // (stale), soroswap 5 min ago (fresh). Because a fresh venue survives, // the stale one is dropped: it must not vote in the §5.5 median. - (14, 190, "2.00", "800", "sdex"), + // 14 MIX — ARM B: a live venue survives, so the bound FIRES. Two dead + // venues at 5.00 and one live at 1.00. THREE sources, because that is + // the whole point: the §5.5 mask is a documented no-op below 3 + // (current.sql), so the two-source version of this fixture asserted + // the bound while never exercising the defect the bound exists for + // (PR #241 review, finding 5). At three, WITHOUT the bound the median + // is 5.00, the live venue deviates 80% and IS the one evicted — + // publishing a vwap of 5.00 from two venues that stopped quoting hours + // ago. With the bound the dead pair is dropped before it can vote. + (14, 190, "5.00", "800", "sdex"), + (14, 200, "5.00", "700", "aquarius"), (14, 5, "1.00", "400", "soroswap"), + // 15 EVN — the §5.5 mask clearing EVERY source. Four dead venues at + // 1.00, 1.00, 3.00, 3.00: `median` is `quantile(0.5)` and interpolates + // to 2.00, so all four deviate 50% and none survives. The row still + // publishes `price_usd` 1.00, which is venue-blind. That pairing — + // a real price beside `sources = {}` and `vwap_24h = 0` — disproves + // the "price, sources and VWAP, or none of the three, BY CONSTRUCTION" + // claim (PR #241 review, finding 2). Reachable only because C2's carry + // brings all-dead assets into the mask at all; before it they arrived + // with zero kept sources and it never armed. + (15, 200, "1.00", "100", "sdex"), + (15, 210, "1.00", "100", "soroswap"), + (15, 220, "3.00", "100", "aquarius"), + (15, 230, "3.00", "100", "phoenix"), ]; for (i, (asset, mins, cu, vol, src)) in rows.iter().enumerate() { admin @@ -356,7 +388,7 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { .fetch_one() .await .expect("count"); - if n >= 12 { + if n >= 13 { ready = true; break; } @@ -588,34 +620,68 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { "vwap must still come from the kept venue, got {sta_vwap}" ); - // ── conditional bound, ARM B: a fresh venue survives → DO fire ───────── - // Asset 14 has sdex stale at 2.00 (190 min) and soroswap fresh at 1.00 - // (5 min). Here the defect is live: unweighted, the stale venue votes in - // the median and its price is the one an outlier check would anchor on. - // So the stale venue is dropped and only the fresh one survives. + // ── conditional bound, ARM B: a live venue survives → DO fire ────────── + // Asset 14 has sdex and aquarius DEAD at 5.00 (190/200 min) and soroswap + // LIVE at 1.00 (5 min). Three sources, so the §5.5 mask genuinely arms and + // the defect is real: unweighted, the dead pair's 5.00 is the median, the + // live venue deviates 80% and is the one the mask would evict. The bound + // drops the dead pair before it can vote. let mix_srcs: String = admin .query(&s("sources", 14)) .fetch_one() .await .expect("mix sources"); assert!( - mix_srcs.contains("soroswap") && !mix_srcs.contains("sdex"), - "a stale venue must be dropped when a fresh one survives, got {mix_srcs}" + mix_srcs.contains("soroswap") + && !mix_srcs.contains("sdex") + && !mix_srcs.contains("aquarius"), + "both dead venues must be dropped when a live one survives, got {mix_srcs}" ); let mix_vwap = scalar_f64(&admin, &f("vwap_24h", 14)).await; assert!( (mix_vwap - 1.0).abs() < 1e-6, - "vwap must weight the FRESH venue only (1.00), not the blend with the \ - stale 2.00 — a blend means the bound did not fire, got {mix_vwap}" + "vwap must come from the LIVE venue (1.00). 5.00 means the bound did \ + not fire and the dead pair outvoted it — the exact defect this arm \ + exists to pin; got {mix_vwap}" ); // price_usd is unbounded and venue-blind: it is simply the newest priced - // close on the asset, which here is the fresh one. + // close on the asset, which here is the live one. let mix_p = scalar_f64(&admin, &f("price_usd", 14)).await; assert!( (mix_p - 1.0).abs() < 1e-9, "price_usd is the newest priced close regardless of venue, got {mix_p}" ); + // ── the §5.5 mask can clear EVERY source (review finding 2) ──────────── + // Asset 15's four dead venues straddle the interpolated median 2.00 by + // 50%, so nothing survives — while price_usd still publishes. This is the + // counterexample to "by construction": an asset CAN hold a price with no + // sources and no VWAP beside it. Whether interpolation is the right median + // for an even count is task 0217's decision, not this test's. + let evn_srcs: String = admin + .query(&s("sources", 15)) + .fetch_one() + .await + .expect("evn sources"); + assert!( + !evn_srcs.contains("sdex") && !evn_srcs.contains("phoenix"), + "an even-count set straddling the median by > OUTLIER_PCT must clear \ + entirely — if this keeps sources, the mask's arming rule changed and \ + current.sql's note about it is stale; got {evn_srcs}" + ); + let evn_vwap = scalar_f64(&admin, &f("vwap_24h", 15)).await; + assert!( + evn_vwap.abs() < 1e-9, + "vwap must be 0 when every source is filtered out, got {evn_vwap}" + ); + let evn_p = scalar_f64(&admin, &f("price_usd", 15)).await; + assert!( + (evn_p - 1.0).abs() < 1e-9, + "price_usd is venue-blind and must still publish the newest priced \ + close — this pairing is what disproves the three-way invariant; \ + got {evn_p}" + ); + // ── the change_7d numerator guard, on the only shape that reaches it ─── // Asset 6 has no priced candle at all (price_usd = 0) but DOES have a real // 1h baseline of 1.00 — ref_7d reads a different table, so the pairing is From 3aec619b7dd6f30d9acee370a9f48255193d4c43 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:20:10 +0200 Subject: [PATCH 4/4] =?UTF-8?q?test(0135):=20pin=20the=20finding-1=20fix?= =?UTF-8?q?=20=E2=80=94=20no=20existing=20fixture=20discriminated=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assets 10, 14 and 15 behave identically under candle-liveness and priced-close-liveness (verified on prod 26.3.10.60), so a revert of the review's finding-1 fix would have left the whole suite green. The fix was unpinned. Adds asset 16 LIV, the review's own probe as a fixture: sdex quoting at 1 min with $1,000,000 of volume but last enriched 3 h ago, against soroswap at $10 enriched 5 min ago. candle liveness -> both kept, vwap ~= 1.00 (sdex carries the weight) priced liveness -> sdex evicted, vwap 1.20 from $10 of a $1,000,020 asset Proven non-vacuous: reverting the predicate fails the new assertion with sources = {"soroswap":{"price":"1.2","volume_24h":"10"}}, which is the defect the review reported, reproduced by the test. Full IT suite run locally against ClickHouse 26.3.10.60, the pinned prod version: 3 passed. --- .../prices-clickhouse/tests/current_mv_it.rs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/prices-clickhouse/tests/current_mv_it.rs b/packages/prices-clickhouse/tests/current_mv_it.rs index cbe0922e..af42c855 100644 --- a/packages/prices-clickhouse/tests/current_mv_it.rs +++ b/packages/prices-clickhouse/tests/current_mv_it.rs @@ -179,6 +179,8 @@ async fn current_prices_mv_computes_price_volume_and_market_cap() { /// dropped before it can outvote the live one in the §5.5 median /// 15 EVN — four dead venues straddling the interpolated median → the mask /// clears ALL of them, leaving a price beside empty `sources` +/// 16 LIV — a LIVE venue whose newest priced close is past the bound → it +/// must be KEPT: liveness is measured on candles, not enrichment /// 11 LAG — SINGLE source, priced history, un-enriched tip → carried (the /// shape the XLM acceptance criterion names) /// 12 TRI — three sources, one carried past an un-enriched tip → the §5.5 @@ -303,6 +305,24 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { (15, 210, "1.00", "100", "soroswap"), (15, 220, "3.00", "100", "aquarius"), (15, 230, "3.00", "100", "phoenix"), + // 16 LIV — the ONLY fixture that discriminates candle-liveness from + // enrichment-freshness, and the shape PR #241's review found (finding + // 1). sdex is quoting RIGHT NOW and carries essentially all the + // volume, but its newest candle is un-enriched and its newest PRICED + // close is 3 h old — older than the bound. soroswap is tiny and + // happens to have been enriched 5 min ago. + // + // Liveness on CANDLES (correct): sdex is live, both venues are kept, + // and vwap ~= 1.00 because sdex carries the weight. + // Liveness on PRICED CLOSES (the defect): sdex reads as dead, is + // evicted in favour of soroswap, and the row publishes vwap 1.20 — + // drawn from $10 of the $1,000,020 the same row reports as volume. + // + // Assets 10/14/15 behave IDENTICALLY under both predicates, so + // without this fixture a revert to the old form stays green. + (16, 1, "0", "1000000", "sdex"), + (16, 180, "1.00", "10", "sdex"), + (16, 5, "1.20", "10", "soroswap"), ]; for (i, (asset, mins, cu, vol, src)) in rows.iter().enumerate() { admin @@ -388,7 +408,7 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { .fetch_one() .await .expect("count"); - if n >= 13 { + if n >= 14 { ready = true; break; } @@ -682,6 +702,30 @@ async fn current_prices_mv_writes_0072_columns_and_filters_outliers() { got {evn_p}" ); + // ── liveness is a property of QUOTING, not of enrichment (finding 1) ── + // Asset 16's sdex quotes at 1 min with $1,000,000 of volume but was last + // enriched 3 h ago; soroswap is $10 enriched 5 min ago. Gating on the + // newest PRICED close evicts sdex and publishes soroswap's 1.20 as the + // VWAP of a $1,000,020 asset. Gating on the newest CANDLE keeps it. + let liv_srcs: String = admin + .query(&s("sources", 16)) + .fetch_one() + .await + .expect("liv sources"); + assert!( + liv_srcs.contains("sdex") && liv_srcs.contains("soroswap"), + "a venue quoting 1 min ago must be LIVE even though its newest priced \ + close is past the bound — dropping it is the defect PR #241's review \ + found; got {liv_srcs}" + ); + let liv_vwap = scalar_f64(&admin, &f("vwap_24h", 16)).await; + assert!( + (liv_vwap - 1.0).abs() < 1e-3, + "vwap must be dominated by the venue holding the volume (~1.00). 1.20 \ + means liveness was gated on enrichment and the $1,000,000 venue was \ + evicted in favour of a $10 one; got {liv_vwap}" + ); + // ── the change_7d numerator guard, on the only shape that reaches it ─── // Asset 6 has no priced candle at all (price_usd = 0) but DOES have a real // 1h baseline of 1.00 — ref_7d reads a different table, so the pairing is