From 1adedc46606e16dffabfb1d1bb9dded075fbde9e Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Mon, 21 Sep 2026 18:27:00 +0530 Subject: [PATCH] evidence: live / replay / synthetic latency basis fixture (mixed-aggregate failure) --- .../README.md | 44 +++++++ .../fixture.json | 111 ++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 conformance/evidence/live-replay-synthetic-latency-basis/README.md create mode 100644 conformance/evidence/live-replay-synthetic-latency-basis/fixture.json diff --git a/conformance/evidence/live-replay-synthetic-latency-basis/README.md b/conformance/evidence/live-replay-synthetic-latency-basis/README.md new file mode 100644 index 0000000..8404685 --- /dev/null +++ b/conformance/evidence/live-replay-synthetic-latency-basis/README.md @@ -0,0 +1,44 @@ +# Live / replay / synthetic latency basis (mixed-aggregate failure) + +> Origin: contributed by Chandan Kumar (@modelpath-dev), 2026-09-21, via +> direct reply after the counting-grain / provenance discussion with Roy Tong. +> Evidence-directory entry for the live-only vs mixed-basis aggregate rule. + +## What it pins + +One `operation_id` carries three invocation rows with different latency: + +| Row | provenance | latency_ms | role | +| --- | --- | ---: | --- | +| live | live | 100 | observed production attempt | +| replay | replay | 20 | derived from a stored prior result | +| synthetic | synthetic | 300 | declared synthetic; not production | + +A schema that tags provenance can enforce: + +- `operation_id` = one user intent +- each try stays its own row +- production averages draw from **live** rows only +- **replay** rows sit in their own view +- **synthetic** rows are excluded from production aggregates entirely + +## The number pair that matters + +| View | Mean latency_ms | How | +| --- | ---: | --- | +| **live-only (correct)** | **100** | mean of live rows only | +| **all-rows (wrong)** | **140** | mean of live + replay + synthetic = (100+20+300)/3 | + +That pair is the mixed-basis failure in one look: the wrong mean looks +decisive, but it mixes measurement bases. + +## Expected behavior + +- Live view mean latency = 100 +- Replay view mean latency = 20 +- Synthetic excluded from production aggregates +- Do not emit 140 as a measured production latency + +## Claim boundary + +Synthetic conformance fixture; no real provider traffic is described. diff --git a/conformance/evidence/live-replay-synthetic-latency-basis/fixture.json b/conformance/evidence/live-replay-synthetic-latency-basis/fixture.json new file mode 100644 index 0000000..f2f4018 --- /dev/null +++ b/conformance/evidence/live-replay-synthetic-latency-basis/fixture.json @@ -0,0 +1,111 @@ +{ + "fixture_id": "live-replay-synthetic-latency-basis-v1", + "fixture_status": "synthetic_conformance_fixture", + "attribution": { + "contributor": "Chandan Kumar", + "github": "@modelpath-dev", + "email": "cml.codes@gmail.com", + "intended_use": "AgentMeasure evidence directory", + "date": "2026-09-21" + }, + "scenario": { + "description": "Same operation_id has one live row and one replay row with different latency, plus one synthetic row. Correct aggregates use live-only; the wrong all-rows mean mixes bases.", + "operation_id": "op-eval-dashboard-001", + "invocations": [ + { + "row_id": "row-live-1", + "operation_id": "op-eval-dashboard-001", + "attempt_index": 1, + "provenance": "live", + "outcome": "success", + "latency_ms": 100, + "validity": "normal", + "note": "observed production attempt" + }, + { + "row_id": "row-replay-1", + "operation_id": "op-eval-dashboard-001", + "attempt_index": 1, + "provenance": "replay", + "outcome": "success", + "latency_ms": 20, + "validity": "normal", + "note": "derived from a stored prior result; not a second live observation" + }, + { + "row_id": "row-synthetic-1", + "operation_id": "op-eval-dashboard-001", + "attempt_index": 1, + "provenance": "synthetic", + "outcome": "success", + "latency_ms": 300, + "validity": "normal", + "note": "declared synthetic; excluded from production aggregates" + } + ] + }, + "evidence": [ + { + "field": "invocations[].provenance", + "value": ["live", "replay", "synthetic"], + "status": "observed", + "basis": "Each row declares its measurement basis explicitly." + }, + { + "field": "operation_id", + "value": "op-eval-dashboard-001", + "status": "observed", + "basis": "All three rows share one user intent." + }, + { + "field": "latency_ms_by_provenance", + "value": {"live": 100, "replay": 20, "synthetic": 300}, + "status": "observed", + "basis": "Latencies differ; mixing them changes the mean." + } + ], + "expected_aggregates": { + "live_only_view": { + "rows": ["row-live-1"], + "mean_latency_ms": 100, + "status": "correct_production_aggregate" + }, + "replay_only_view": { + "rows": ["row-replay-1"], + "mean_latency_ms": 20, + "status": "side_view_not_production" + }, + "synthetic_excluded_from_production": { + "rows_excluded": ["row-synthetic-1"], + "status": "required" + }, + "wrong_all_rows_mean": { + "rows": ["row-live-1", "row-replay-1", "row-synthetic-1"], + "mean_latency_ms": 140, + "formula": "(100 + 20 + 300) / 3", + "status": "mixed_basis_failure" + } + }, + "expected_behavior": { + "decision": "separate_views_by_provenance", + "rule": "operation_id groups intent; each try stays its own row; production averages default to provenance=live only; replay has its own view; synthetic is excluded from production aggregates.", + "forbid": [ + "Do not report 140 as measured production mean latency.", + "Do not average live with replay in the same production mean.", + "Do not include synthetic rows in production aggregates." + ] + }, + "acceptance_test": { + "input": "three invocation rows under one operation_id with provenance tags", + "pass_if": [ + "live-only mean_latency_ms equals 100", + "replay-only mean_latency_ms equals 20", + "synthetic row is excluded from production aggregates", + "wrong all-rows mean of 140 is not emitted as measured" + ], + "fail_if": [ + "an implementation emits 140 (or any mixed live+replay+synthetic mean) as the production latency" + ] + }, + "claim_boundary": "Synthetic conformance fixture. It is not a real provider observation, an endorsement of any dashboard, or a claim about a specific product's current behavior." +}