diff --git a/evidence/indexer-255-20261029/patch/README.md b/evidence/indexer-255-20261029/patch/README.md new file mode 100644 index 000000000..aa14457cd --- /dev/null +++ b/evidence/indexer-255-20261029/patch/README.md @@ -0,0 +1,24 @@ +# Candidate + +Runtime candidate: commit `6da28494` on baseline `ab03f3ffb8f2e5bc02c304828c5c41890eef673e`. + +`Bundle._manifest` / `_view` may reuse validated resource path/content hashes only within one Collect invocation. Keys contain the exact path and immutable bytes; metadata, required resources and dependency revisions are still validated for each bundle. Public manifest/view calls remain uncached. Default selection, ranking, limits, explicit identities and context resources are unchanged. + +The same packaged library bytes were repeatedly validated and hashed for each generated skill owner. This removes repeated resource work, not corpus contents or provenance checks. No cross-request state is introduced. Tests exercise changed bytes at the same path, multiple owners, mutable metadata/dependencies, unsafe paths and request lifetime. + +Tradeoff: the request-local memo retains distinct resource bytes/keys until that Collect call completes. It is not a cross-request cache, but memory overhead is nonzero. Peak memory is not measured in this campaign and no memory-improvement claim is made. + +## Withheld from production + +The final feature branch reverts the runtime change in `23e769ac`; production runtime matches the baseline. The candidate-specific regression tests are retained with the candidate, not active against baseline private APIs. This is evidence-only delivery, not promotion of an unaccepted optimization. + +`candidate.patch` preserves the source/test delta. The 2.2KB `candidate.bundle` additionally preserves the **exact** original candidate commit so replication does not depend on a temporary branch surviving a squash merge. It requires the baseline commit already in this repository: + +```sh +git bundle verify evidence/indexer-255-20261029/patch/candidate.bundle +git fetch evidence/indexer-255-20261029/patch/candidate.bundle \ + refs/heads/issue255-indexer-evidence:refs/heads/reproduce-indexer-255-candidate +git worktree add /tmp/indexer-255-candidate 6da28494891b21cdc89f632d84d763bc997b7d15 +``` + +Do not use the final evidence-only branch as the optimized arm. No more performance studies were run after the single prospective confirmation. diff --git a/evidence/indexer-255-20261029/patch/candidate.bundle b/evidence/indexer-255-20261029/patch/candidate.bundle new file mode 100644 index 000000000..d6fd83fd8 Binary files /dev/null and b/evidence/indexer-255-20261029/patch/candidate.bundle differ diff --git a/evidence/indexer-255-20261029/patch/candidate.patch b/evidence/indexer-255-20261029/patch/candidate.patch new file mode 100644 index 000000000..c0343a68a --- /dev/null +++ b/evidence/indexer-255-20261029/patch/candidate.patch @@ -0,0 +1,165 @@ +From 6da28494891b21cdc89f632d84d763bc997b7d15 Mon Sep 17 00:00:00 2001 +From: Codex +Date: Thu, 29 Oct 2026 05:40:54 -0700 +Subject: [PATCH] perf(indexer): reuse exact resource validation within collect + requests + +Signed-off-by: Codex +--- + src/inferrsi/knowledge_operations.py | 29 +++++++++++-- + tests/test_collect_resource_memo.py | 65 ++++++++++++++++++++++++++++ + 2 files changed, 90 insertions(+), 4 deletions(-) + create mode 100644 tests/test_collect_resource_memo.py + +diff --git a/src/inferrsi/knowledge_operations.py b/src/inferrsi/knowledge_operations.py +index 63920c05..c582ea0a 100644 +--- a/src/inferrsi/knowledge_operations.py ++++ b/src/inferrsi/knowledge_operations.py +@@ -126,6 +126,11 @@ class Bundle: + object.__setattr__(self, "metadata", _normalized(self.metadata, "metadata")) + + def manifest(self) -> dict: ++ return self._manifest() ++ ++ def _manifest(self, resources_seen: dict | None = None) -> dict: ++ # Only Collect shares this memo within one invocation. Keys include exact ++ # immutable bytes, never a mutable Bundle or a resource path alone. + record = self.metadata + if not isinstance(record.get("id"), str) or not re.fullmatch(IDENTITY, record["id"]): + raise ValueError("Invalid bundle identity") +@@ -149,10 +154,19 @@ class Bundle: + if not isinstance(record["sources"], list) or not record["sources"]: + raise ValueError("Bundle requires sources, including explicit unknown provenance") + _sources(record["sources"]) ++ resources = {} + for path, data in self.content.items(): +- safe_path(path) + if not isinstance(data, bytes): ++ safe_path(path) + raise ValueError("Bundle content must be bytes") ++ key = (path, data) ++ cached = resources_seen.get(key) if resources_seen is not None else None ++ if cached is None: ++ safe_path(path) ++ cached = _hash(data) ++ if resources_seen is not None: ++ resources_seen[key] = cached ++ resources[path] = cached + if record["entrypoint"] not in self.content: + raise ValueError("Missing entrypoint in complete bundle") + for required in record.get("required_resources", []): +@@ -161,7 +175,7 @@ class Bundle: + continue + if safe_path(required) not in self.content: + raise ValueError(f"Missing required resource: {required}") +- resources = {path: _hash(data) for path, data in sorted(self.content.items())} ++ resources = dict(sorted(resources.items())) + for dependency, revision in self.dependencies.items(): + safe_path(dependency) + if not re.fullmatch(r"sha256:[0-9a-f]{64}", revision): +@@ -190,7 +204,10 @@ class Bundle: + return "sha256:" + _hash(_json(self.manifest())) + + def view(self, *, material: bool = False) -> dict: +- manifest = self.manifest() ++ return self._view(material=material) ++ ++ def _view(self, *, material: bool = False, resources_seen: dict | None = None) -> dict: ++ manifest = self._manifest(resources_seen) + result = { + **self.metadata, + "revision": "sha256:" + _hash(_json(manifest)), +@@ -557,6 +574,7 @@ class KnowledgeReader: + + cases = self._cases() if kind else {} + selected, omitted, seen = [], [], set() ++ resources_seen: dict[tuple[str, bytes], str] = {} + used = 0 + ranked, contexts, context_omissions = [], [], [] + context_used = 0 +@@ -614,7 +632,10 @@ class KnowledgeReader: + omitted.append({"id": reference, "reason": "filters"}) + continue + bundle = self._read(qualified, pinned.get(qualified)) +- view = {**bundle.view(material=material), "authority": authority, "ref": qualified} ++ view = { ++ **bundle._view(material=material, resources_seen=resources_seen), ++ "authority": authority, "ref": qualified, ++ } + entry = bundle.content[bundle.metadata["entrypoint"]] + haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() + except IdentityError as exc: +diff --git a/tests/test_collect_resource_memo.py b/tests/test_collect_resource_memo.py +new file mode 100644 +index 00000000..0c468e5d +--- /dev/null ++++ b/tests/test_collect_resource_memo.py +@@ -0,0 +1,65 @@ ++"""Request-local resource reuse must not cache mutable Bundle validation.""" ++import pytest ++ ++from inferrsi.knowledge_operations import Bundle, KnowledgeReader, RevisionConflict, _hash ++ ++CGROUP = "inferrsi.common.skills.cgroup_resource_controls" ++ ++ ++def small_bundle(data=b"original"): ++ return Bundle({"id": "example", "kind": "skill", "sources": [{"type": "unknown"}], ++ "applicability": {}, "state": "active", "limitations": [], ++ "attribution": {}, "entrypoint": "SKILL.md"}, {"SKILL.md": data}, {}) ++ ++ ++def test_memo_keys_exact_bytes_not_path_or_mutable_bundle(): ++ memo = {} ++ bundle = small_bundle() ++ original = bundle._view(resources_seen=memo) ++ assert bundle._view(resources_seen=memo) == original ++ other = small_bundle(b"other owner") ++ assert other._view(resources_seen=memo)["resources"]["SKILL.md"] == _hash(b"other owner") ++ bundle.content["SKILL.md"] = b"changed" ++ changed = bundle._view(resources_seen=memo) ++ assert changed["revision"] != original["revision"] ++ assert changed["resources"]["SKILL.md"] == _hash(b"changed") ++ assert bundle.view() == changed ++ bundle.metadata["limitations"].append("new limitation") ++ assert bundle._view(resources_seen=memo)["revision"] != changed["revision"] ++ bundle.metadata["id"] = "INVALID" ++ with pytest.raises(ValueError, match="identity"): ++ bundle._view(resources_seen=memo) ++ ++ ++def test_memo_does_not_skip_security_or_dependency_validation(): ++ bundle, memo = small_bundle(), {} ++ bundle._view(resources_seen=memo) ++ bundle.dependencies["SKILL.md"] = "sha256:" + "0" * 64 ++ with pytest.raises(RevisionConflict): ++ bundle._view(resources_seen=memo) ++ bundle.dependencies.clear() ++ bundle.content["../SKILL.md"] = b"original" ++ with pytest.raises(ValueError, match="Unsafe"): ++ bundle._view(resources_seen=memo) ++ del bundle.content["../SKILL.md"] ++ bundle.content["SKILL.md"] = bytearray(b"original") ++ with pytest.raises(ValueError, match="bytes"): ++ bundle._view(resources_seen=memo) ++ ++ ++def test_collect_resource_memo_is_not_retained_between_requests(monkeypatch): ++ reader = KnowledgeReader() ++ seen = [] ++ original = Bundle._view ++ ++ def capture(self, *, material=False, resources_seen=None): ++ seen.append(resources_seen) ++ return original(self, material=material, resources_seen=resources_seen) ++ ++ monkeypatch.setattr(Bundle, "_view", capture) ++ one = reader.collect(ids=[CGROUP]) ++ first = seen[-1] ++ two = reader.collect(ids=[CGROUP]) ++ assert one == two ++ assert first is not seen[-1] ++ assert first and seen[-1] +-- +2.52.0 + diff --git a/evidence/indexer-255-20261029/plan/README.md b/evidence/indexer-255-20261029/plan/README.md new file mode 100644 index 000000000..c9d229134 --- /dev/null +++ b/evidence/indexer-255-20261029/plan/README.md @@ -0,0 +1,21 @@ +# Indexer #255: bounded CPU comparison and accuracy assessment + +Owner: Dong1017. Scope and closure acceptance: [Issue #255](https://github.com/InferMatrixLab/InferRSI/issues/255). This is a harness-plane CPU retrieval campaign, not model or hardware efficacy evidence. + +`protocol.json` is the frozen comparison contract. The exact baseline already includes the historical knowledge trim; this campaign does not repeat or claim that work. Candidate cap: one request-local resource-validation optimization. No paid services, new models, GPUs, synthetic corpus or ranking rewrite. Stop at one complete comparison; preserve any gate failures rather than revise gates to fit outcomes. + +The source-only profile identified repeated Bundle resource validation and SHA-256 as the dominant controllable cost. A profiled reused-reader cgroup query took 1.767 seconds, with manifest 1.327 seconds, including approximately 47,278 SHA/path operations. Profile timings are diagnostic only, not the campaign result. + +## Reproduce + +First recover the preserved candidate commit using `../patch/README.md`. The final feature branch contains evidence only, not the optimized runtime. Use two clean worktrees at the baseline and candidate commits, the same Python executable, and dependencies recorded in `python-environment.txt`. Run sequentially without competing benchmark jobs: + +```sh +python tools/evaluate_indexer.py \ + --baseline /path/to/baseline --candidate /path/to/candidate \ + --protocol evidence/indexer-255-20261029/plan/protocol.json \ + --cases tests/fixtures/indexer/development.json tests/fixtures/indexer/heldout.json \ + --output /path/to/new-results +``` + +A fresh reader is not a cold filesystem cache. Full CLI latency includes interpreter startup/output capture; JSON decoding is separately recorded. All 20 paired AB/BA groups and slow samples are retained. Bootstrap resamples the group statistic, not individual queries. Tail estimates from 20 samples are descriptive and noisy, not production SLOs. diff --git a/evidence/indexer-255-20261029/plan/confirmation-v2.json b/evidence/indexer-255-20261029/plan/confirmation-v2.json new file mode 100644 index 000000000..8892beeb1 --- /dev/null +++ b/evidence/indexer-255-20261029/plan/confirmation-v2.json @@ -0,0 +1,33 @@ +{ + "version": 2, + "purpose": "One prospective focused replication of unexplained full-CLI catalog-search tail; never replace or pool v1 FAIL", + "baseline_revision": "ab03f3ffb8f2e5bc02c304828c5c41890eef673e", + "candidate_revision": "6da28494891b21cdc89f632d84d763bc997b7d15", + "candidate_source_sha256": "16bbb0e523e4026dc4f189e81b96e79988c795b5e54b5abb66305d295a62bf31", + "frozen_v1_runner_sha256": "ea5b91ff17e80a422fd373621528dfe74ff144976bb55d6432501df452757ca5", + "command": "python -m inferrsi knowledge search cgroupv2 --limit 5 --json", + "timing": "Exactly v1 tools.evaluate_indexer.cli(mode=search): full subprocess including interpreter startup and stdout capture; JSON decode separately outside clock", + "warmup": "No explicit warmup; repeated process calls under existing OS cache, not cold cache", + "samples": 120, + "order": "60 AB and60 BA shuffled with random.Random(25502); paired successive subprocess calls; sequential only", + "negative_control": { + "pairs": 40, + "order": "20 AB and20 BA shuffled with random.Random(25503), both labels use baseline tree; after primary confirmation; diagnostic only, never pooled or acceptance substitute" + }, + "gates": { + "p50_max_regression_percent": 10, + "p95_max_regression_percent": 20, + "output_parity": "Exact canonical output SHA-256 per pair; constant within both arms", + "identity": "Same exact baseline/candidate runtime/corpus as v1, source/corpus before/after unchanged", + "failure": "Any completed guard miss FAIL; missing/changed evidence INCONCLUSIVE. No trimming or pooling. Exactly one confirmation study regardless of outcome; do not repeat to obtain PASS." + }, + "statistics": { + "p50": "median", + "p95": "nearest rank ceil(.95*n)", + "paired_bootstrap": "10000 paired group resamples random.Random(25504) for p50 and p95 relative change; percentile2.5/97.5 intervals descriptive, no extra posthoc acceptance rule" + }, + "raw": "Every sample latency/decode_ns/outputhash/bytecount, pair index/order, UTC start and monotonic offsets; preserve failures/timeouts separately", + "budget": "One fixed study including40baseline-baselinepairs; CPU only, estimated3-5minutes, no install/model/GPU/service calls", + "coordination": "Start only after parent freeze receipt and otherworkers acknowledge no heavy CPU work; never edit source/runner/protocol mid-run", + "decision": "This study alone cannot turn v1 into PASS or authorize runtime merge; assess all evidence and uncertainty explicitly" +} diff --git a/evidence/indexer-255-20261029/plan/integrity-manifest.json b/evidence/indexer-255-20261029/plan/integrity-manifest.json new file mode 100644 index 000000000..c4a6a5e4a --- /dev/null +++ b/evidence/indexer-255-20261029/plan/integrity-manifest.json @@ -0,0 +1,35 @@ +{ + "patch/README.md": "56b5d4a5e996d5d094ffdde2edbe27285be1dad98cd3ef19c939d62722376463", + "patch/candidate.bundle": "a3ab0986e7c5d39867ef8f2137f4a1c273c5f52daa95e571373ffa01435ad5c7", + "patch/candidate.patch": "a0c3e5230c489008525244af0bfe7c8ea483034c11ce5a835ef24a4b8a06669a", + "plan/README.md": "05a545d7e729c206df72aec1cf7715a7505c3ba6c6a4562f27f5899a318654cf", + "plan/confirmation-v2.json": "40de4eefba8712d3d07f2e48049b0899c831f94bfcfa252c1f8841525a337fa4", + "plan/protocol.json": "e4964329a20e6752b8149ab270733c0bbb258d0f948bdca9e86da6fe8f028956", + "plan/python-environment.txt": "e61e77e6b46a835af8b82e4fbc997d36b8ac9fbb415ab33cccc3b6f434874192", + "results/baseline-lint.txt": "f23163c00e3cd478afeb6fe5126d1758f9e94349e65067ffcec30aee311ecb58", + "results/baseline-smoke.json": "dd3f8876a759efbd733142838a797730be76cf956d6c3f39a40caaff900a4c00", + "results/confirmation-v2.json": "34d5b863c9f8dc75369d8bca4050019b3b33acf8e5bd26889ff0595850a65e26", + "results/final-evidence-tree-tests.txt": "d798e533b9e00ddacbdd77be63b15a1936f4853c3502c3010711093403006845", + "results/final-lint.txt": "f23163c00e3cd478afeb6fe5126d1758f9e94349e65067ffcec30aee311ecb58", + "results/focused-tests.txt": "c8f559b45ab7a92c5c94970cb03b7d481e9bede52082ced61b1f80ba5e09ef9d", + "results/full-suite-initial.txt": "e6332f13837daa4b5aa3389750fa2342472daec28dac3d465850d9c9e1870439", + "results/import-proof-baseline.json": "1e2e9d0e2b318c06582548995dc79614ce087cd946a5874da57b83d41b331736", + "results/import-proof-indexer-evidence.json": "1e2e9d0e2b318c06582548995dc79614ce087cd946a5874da57b83d41b331736", + "results/paired-run/A-mandatory-audit.json": "7fd24578d05007a916389e0080ad36ca24572a3f4a5598e7a99594cb52e6d869", + "results/paired-run/B-mandatory-audit.json": "cddaaa095a8b6645c75c3504cad70c4d30bd54bc9069f8c114006d784b892642", + "results/paired-run/timing.json": "588d37dfc6464c9894fd19a69d2db37f0635f8a9991c051e15131ca1f317a934", + "results/raw/3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f.json.gz": "7af84694f52243cf0433a7212af5d038c6dbca8a08e125ef8c0ef7fe2dae90a6", + "results/raw/517e374c3f13d9d22dafad37d51368d1f4e5669842f9bc0dadc1bb88cba3d968.json.gz": "0c070b69dcc6ea010a24c637a160b19ea8a7384eb40bb17636db0eaa42906a0a", + "results/raw/a35be36b859e6c4fd5bd7b7ec38920794f77adb322e9ae26498f31a9b4a91b6c.json.gz": "22a4ada0c2dab381ee9f8e3b342d0418d9e0668a16379fde069bae4b1eb0e9ad", + "results/raw/fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0.json.gz": "0de101743d3cdb7a1eef38a7a398d71ce372bb19f9380b85000b45d052f3341c", + "results/raw-archives.json": "0f0ce2ed6da87046936b811b02930f1a9d0f6cc8d27c8c2077e6c08d65cd19d0", + "results/report.md": "38186030ae6cf8a94d202c4c1d6ae41bad8b19d9060f63dd350ae8c66896cb2a", + "results/targeted-env-final.txt": "dd2c815dfa55647cde1bace7510b74c4de3cca07ed800f2687acd150abfdabd8", + "results/targeted-env-initial.txt": "fedbef13168bf9473c9abd869cbf1830872e634601122b1f69b1b015d1828063", + "validator/README.md": "2217992077ac3fd5aee72cee23c2d90b8598bbde3cca634bccfa17cd263d0845", + "validator/confirm_cli_search.py": "b11b4cf08b6dceb43be447516d5abb740bb2de8c2fdfd7a9ee3f820a1174b55d", + "validator/frozen_evaluate_indexer.py": "ea5b91ff17e80a422fd373621528dfe74ff144976bb55d6432501df452757ca5", + "validator/label-review.md": "edf4946fef7485d968845407e3c4b8035e43fe634e56ecb0b32aa64f022f4262", + "validator/supplemental_required_audit.py": "ce1cf9b796ce8e029191220bf55e6924a7cb2b7a364f062db4631433a36568ff", + "validator/verify_raw_archives.py": "dc730ed4536278cf925e0663c99d6c703ab3fa0cfcbbb828b5bf1065e6aded44" +} diff --git a/evidence/indexer-255-20261029/plan/protocol.json b/evidence/indexer-255-20261029/plan/protocol.json new file mode 100644 index 000000000..3ccc5084f --- /dev/null +++ b/evidence/indexer-255-20261029/plan/protocol.json @@ -0,0 +1,139 @@ +{ + "version": 1, + "issue": "https://github.com/InferMatrixLab/InferRSI/issues/255", + "baseline_revision": "ab03f3ffb8f2e5bc02c304828c5c41890eef673e", + "plane": "harness", + "scope": "CPU-only packaged retrieval; request-local resource-validation reuse; no ranking/content changes or downstream agent efficacy claim", + "queries": [ + { + "id": "dev01", + "query": "", + "relevance": "cgroupv2 cpu quota", + "limit": 5 + }, + { + "id": "dev02", + "query": "", + "relevance": "P2pNccl KV transfer completion", + "limit": 5 + }, + { + "id": "dev03", + "query": "", + "relevance": "DLRover checkpoint replica recovery", + "limit": 5 + }, + { + "id": "dev04", + "query": "", + "relevance": "weight loading mmap checkpoint peak memory", + "limit": 5 + } + ], + "primary": { + "mode": "reused-reader relevance Collect API", + "query_ids": [ + "dev01", + "dev02", + "dev03", + "dev04" + ], + "gain_min_percent": 15, + "bootstrap_lower_bound_must_exceed": 0, + "each_query_p50_max_regression_percent": 5 + }, + "secondary": [ + { + "mode": "fresh-reader relevance API", + "query_id": "dev01" + }, + { + "mode": "reused-reader default Collect API", + "query_id": "dev01", + "override": { + "query": "cgroupv2", + "relevance": null + } + }, + { + "mode": "reused-reader relevance context API", + "query_id": "dev01", + "override": { + "context_bytes": 262144 + } + }, + { + "mode": "catalog search API", + "query_id": "dev01", + "override": { + "query": "cgroupv2", + "relevance": null + } + }, + { + "mode": "full CLI relevance Collect", + "query_id": "dev01" + }, + { + "mode": "full CLI catalog search", + "query_id": "dev01", + "override": { + "query": "cgroupv2", + "relevance": null + } + } + ], + "secondary_limits": { + "each_path_p50_max_regression_percent": 10, + "each_path_p95_max_regression_percent": 20, + "p95_note": "Twenty samples is a noisy sampled tail, not a production SLO estimate; failures remain failures, no post-hoc gate relaxation" + }, + "measurement": { + "groups": 20, + "order": "even zero-based groups AB, odd groups BA; A baseline, B candidate", + "isolation": "sequential child process per arm/group, same Python executable and host; do not run other benchmark workers concurrently", + "reused_reader": "new KnowledgeReader per mode/query, one unmeasured identical invocation, then one measured invocation", + "fresh_reader": "KnowledgeReader construction plus first collect measured, no explicit warmup; fresh reader is not cold OS filesystem cache", + "cli": "one complete subprocess per mode/group including interpreter start and stdout capture; no explicit warmup; same query/mode/limit with no filters as paired API (Collect CLI has no repo filter); decoding timed separately", + "clock": "time.perf_counter_ns", + "summary": "p50 median; p95 nearest-rank ceil(0.95*n); retain every raw observation including slow samples", + "primary_group_statistic": "median of four paired query relative reductions 100*(baseline-candidate)/baseline; final statistic median across20 groups", + "confidence_interval": "percentile bootstrap of20 group statistics,10000 resamples,random.Random(255),2.5th/97.5th percentiles; group is resampling unit", + "failure": "No trimming, cherry picking or retry pooling. Incomplete run is INCONCLUSIVE. Complete gate miss is FAIL. Preserve failed attempts separately. Stop measurement if source or corpus changes.", + "integrity": "Record exact source/corpus hashes before and after; baseline/candidate corpus hashes must match. Record serialized output SHA256/bytes for every sample; full outputs retained for correctness run. Require paired output equality. No cache across requests or readers." + }, + "accuracy": { + "semantic_cases": 12, + "development": 8, + "heldout": 4, + "heldout_policy": "Independent reviewer creates4 distinct intent-family cases without running search. Labels unavailable to implementer until candidate code freeze; no tuning on heldout outcomes.", + "contract_cases": 6, + "metrics": [ + "partial-label Hit@5", + "partial-label MRR@5", + "known forbidden hit count", + "complete required-resource delivery", + "intentional budget omission", + "six contract checks" + ], + "interpretation": "Partial labels are not exhaustive relevance ground truth or complete Recall. Same result before/after is no accuracy gain. Report each failure and coverage boundary. No model judge or downstream task-effect measurement.", + "candidate_gate": "Efficiency-only candidate preserves every semantic/contract output and passes all contract checks; heldout labels assess baseline capability, not tuned to claim an improvement. Any later accuracy change needs a separate prospective protocol/variant.", + "heldout_sha256": "10e13080cfb4d27de40dc92166c261cf6ccf47776d08111f2e85d7c8edc2d19f", + "delivery_denominator": "Report complete delivery conditional on retrieved relevant items, separately from missed relevant items and intentional budget omission." + }, + "required_regressions": [ + "mutable Bundle content/metadata/dependencies", + "same path changed bytes", + "unsafe resource paths", + "dependency revision mismatch", + "stale generated source projection", + "checkout updates between calls", + "explicit IDs/order/revision pins", + "context budget completeness" + ], + "artifact_layout": { + "cases": "tests/fixtures/indexer/", + "runner": "tools/evaluate_indexer.py", + "frozen_protocol_and_raw_results": "evidence/indexer-255-20261029/" + } +} diff --git a/evidence/indexer-255-20261029/plan/python-environment.txt b/evidence/indexer-255-20261029/plan/python-environment.txt new file mode 100644 index 000000000..b59b29ffe --- /dev/null +++ b/evidence/indexer-255-20261029/plan/python-environment.txt @@ -0,0 +1,7 @@ +iniconfig==2.3.1 +packaging==26.3 +pluggy==1.7.0 +Pygments==2.21.0 +pytest==9.2.0 +PyYAML==6.0.3 +ruff==0.17.3 diff --git a/evidence/indexer-255-20261029/results/baseline-lint.txt b/evidence/indexer-255-20261029/results/baseline-lint.txt new file mode 100644 index 000000000..31331459d --- /dev/null +++ b/evidence/indexer-255-20261029/results/baseline-lint.txt @@ -0,0 +1,374 @@ +evidence/modular-refactor-20261013/validator/compare.py:1:1: I001 [*] Import block is un-sorted or un-formatted +evidence/modular-refactor-20261013/validator/compare.py:16:101: E501 Line too long (102 > 100) +evidence/modular-refactor-20261013/validator/compare.py:17:101: E501 Line too long (109 > 100) +evidence/modular-refactor-20261013/validator/compare.py:27:101: E501 Line too long (147 > 100) +evidence/modular-refactor-20261013/validator/compare.py:30:101: E501 Line too long (118 > 100) +evidence/modular-refactor-20261013/validator/compare.py:35:101: E501 Line too long (167 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:10:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:31:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:36:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:40:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:41:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:10:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:31:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:36:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:40:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:41:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/test_reference_completion.py:32:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/test_reference_completion.py:33:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/test_reference_completion.py:34:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/test_reference_completion.py:32:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/test_reference_completion.py:33:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/test_reference_completion.py:34:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bwrap-build-v2/extract_source.py:11:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v1/diagnose.py:29:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v1/review-rejected-v1/diagnose.py:27:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v1/review-rejected-v1/diagnose.py:82:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v2/diagnose.py:29:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-dependency-v1/diagnose.py:37:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/synthetic_probe.py:24:1: E402 Module level import not at top of file +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/synthetic_probe.py:94:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/test_reference_completion.py:32:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/test_reference_completion.py:33:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/test_reference_completion.py:34:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/check_wheels.py:13:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/check_wheels.py:41:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/prefetch.py:105:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/verify_wheels.py:55:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:36:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:40:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:41:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:42:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:52:1: E731 Do not assign a `lambda` expression, use a `def` +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:57:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:58:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:124:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/prefetch.py:105:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:87:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:88:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:89:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:92:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:95:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:96:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:97:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:98:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:101:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:102:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:105:43: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:107:42: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:108:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:125:54: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:125:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:132:58: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:137:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:142:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:146:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:148:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:152:101: E501 Line too long (116 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:154:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:157:101: E501 Line too long (125 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:164:41: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:166:89: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:166:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:167:90: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:167:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:25:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:33:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:38:30: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:53:30: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:56:31: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:57:25: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:66:1: E402 Module level import not at top of file +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:66:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:69:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:91:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:99:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:102:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:103:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:112:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:135:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:137:58: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:142:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:145:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:151:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:154:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:162:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:172:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:176:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:56:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:57:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:58:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:60:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:69:26: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:72:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:69:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:75:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:87:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:105:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:110:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:121:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:122:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:123:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:127:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:130:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:131:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:132:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:134:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:140:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:143:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:145:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:63:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:69:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:75:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:87:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:105:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:110:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:121:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:122:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:123:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:127:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:130:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:131:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:132:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:134:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:140:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:143:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:145:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_probe.py:6:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_probe.py:87:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/http_readiness.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/http_readiness.py:114:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/official_client_probe_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:21:45: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:23:27: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:23:49: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:24:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:26:5: E731 Do not assign a `lambda` expression, use a `def` +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:27:13: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:33:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:34:101: E501 Line too long (186 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:38:101: E501 Line too long (206 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:39:38: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:39:59: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:39:73: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:44:70: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:49:42: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:51:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:57:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:58:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:61:30: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:63:25: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:65:73: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:65:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:66:101: E501 Line too long (812 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:69:101: E501 Line too long (158 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:72:24: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:21:45: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:23:27: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:23:49: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:24:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:26:5: E731 Do not assign a `lambda` expression, use a `def` +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:27:13: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:33:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:34:101: E501 Line too long (186 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:38:101: E501 Line too long (170 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:38: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:70: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:91: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:105: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:44:63: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:49:42: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:51:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:57:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:58:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:61:30: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:63:25: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:65:73: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:65:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:66:101: E501 Line too long (1033 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:69:101: E501 Line too long (158 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:72:24: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:86:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:94:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:96:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:110:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:117:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:118:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:121:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:122:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:131:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:154:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:51: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:80: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:111: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:161:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:164:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:170:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:173:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:175:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:178:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:181:101: E501 Line too long (147 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:182:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:188:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:192:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:196:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:86:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:94:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:96:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:97:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:98:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:110:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:117:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:118:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:121:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:122:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:129:101: E501 Line too long (146 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:131:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:154:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:51: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:80: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:111: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:161:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:164:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:165:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:166:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:172:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:175:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:177:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:185:101: E501 Line too long (147 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:186:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:188:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:193:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:197:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:201:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:86:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:94:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:96:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:97:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:98:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:111:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:118:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:119:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:122:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:123:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:130:101: E501 Line too long (175 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:133:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:138:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:156:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:51: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:80: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:111: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:163:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:166:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:167:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:168:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:174:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:177:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:179:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:187:101: E501 Line too long (147 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:188:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:190:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:195:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:199:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:203:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:8:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:16:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:18:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:19:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:20:101: E501 Line too long (105 > 100) +examples/qwen21_turbo_b300.py:6:1: I001 [*] Import block is un-sorted or un-formatted +examples/qwen21_turbo_b300.py:14:8: F401 [*] `subprocess` imported but unused +examples/qwen21_turbo_b300.py:28:101: E501 Line too long (126 > 100) +examples/qwen21_turbo_b300.py:29:101: E501 Line too long (121 > 100) +examples/qwen21_turbo_b300.py:30:101: E501 Line too long (126 > 100) +examples/qwen21_turbo_b300.py:33:101: E501 Line too long (105 > 100) +examples/qwen21_turbo_b300.py:34:101: E501 Line too long (106 > 100) +examples/qwen21_turbo_b300.py:35:101: E501 Line too long (103 > 100) +examples/qwen21_turbo_b300.py:44:101: E501 Line too long (108 > 100) +examples/qwen21_turbo_b300.py:50:101: E501 Line too long (113 > 100) +examples/qwen21_turbo_b300.py:58:101: E501 Line too long (127 > 100) +examples/qwen21_turbo_b300.py:60:101: E501 Line too long (106 > 100) +examples/qwen21_turbo_b300.py:61:101: E501 Line too long (139 > 100) +examples/qwen21_turbo_b300.py:69:101: E501 Line too long (140 > 100) +examples/qwen21_turbo_b300.py:70:101: E501 Line too long (107 > 100) +examples/qwen21_turbo_b300.py:75:101: E501 Line too long (116 > 100) +examples/qwen21_turbo_b300.py:76:101: E501 Line too long (119 > 100) +examples/qwen21_turbo_b300.py:88:101: E501 Line too long (101 > 100) +examples/qwen21_turbo_b300.py:128:101: E501 Line too long (122 > 100) +examples/qwen21_turbo_b300.py:131:101: E501 Line too long (128 > 100) +examples/qwen21_turbo_b300.py:134:101: E501 Line too long (111 > 100) +examples/qwen21_turbo_b300.py:145:101: E501 Line too long (103 > 100) +examples/qwen21_turbo_b300.py:146:101: E501 Line too long (104 > 100) +examples/qwen21_turbo_b300.py:149:101: E501 Line too long (107 > 100) +examples/qwen21_turbo_b300.py:152:101: E501 Line too long (125 > 100) +examples/qwen21_turbo_b300.py:173:101: E501 Line too long (152 > 100) +examples/qwen21_turbo_b300.py:184:101: E501 Line too long (110 > 100) +examples/qwen21_turbo_b300.py:185:50: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:186:46: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:187:45: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:202:17: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:241:101: E501 Line too long (101 > 100) +examples/qwen21_turbo_b300.py:268:26: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:2:1: I001 [*] Import block is un-sorted or un-formatted +examples/qwen21_turbo_quality.py:7:8: F401 [*] `sys` imported but unused +examples/qwen21_turbo_quality.py:14:101: E501 Line too long (123 > 100) +examples/qwen21_turbo_quality.py:39:5: I001 [*] Import block is un-sorted or un-formatted +examples/qwen21_turbo_quality.py:44:60: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:58:101: E501 Line too long (116 > 100) +examples/qwen21_turbo_quality.py:59:101: E501 Line too long (119 > 100) +examples/qwen21_turbo_quality.py:70:101: E501 Line too long (102 > 100) +examples/qwen21_turbo_quality.py:74:101: E501 Line too long (126 > 100) +examples/qwen21_turbo_quality.py:75:33: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:76:101: E501 Line too long (101 > 100) +examples/qwen21_turbo_quality.py:85:54: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:86:57: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:95:47: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:98:101: E501 Line too long (109 > 100) +examples/qwen21_turbo_quality.py:111:28: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:115:78: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:115:101: E501 Line too long (114 > 100) +examples/qwen21_turbo_quality.py:129:101: E501 Line too long (102 > 100) +examples/qwen21_turbo_quality.py:132:37: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:133:13: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:135:101: E501 Line too long (130 > 100) +examples/qwen21_turbo_quality.py:136:101: E501 Line too long (142 > 100) +examples/qwen21_turbo_quality.py:137:30: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:142:24: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_validation_adapter.py:8:1: I001 [*] Import block is un-sorted or un-formatted +Found 372 errors. +[*] 27 fixable with `--fix` (3 unsafe fixes available with `--fix --unsafe-fixes`). diff --git a/evidence/indexer-255-20261029/results/baseline-smoke.json b/evidence/indexer-255-20261029/results/baseline-smoke.json new file mode 100644 index 000000000..bee42df0d --- /dev/null +++ b/evidence/indexer-255-20261029/results/baseline-smoke.json @@ -0,0 +1 @@ +{"hot_relevance/dev01": {"ns": 1808219064, "bytes": 760539, "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641"}, "hot_relevance/dev02": {"ns": 1868916449, "bytes": 760449, "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe"}, "hot_relevance/dev03": {"ns": 3367375495, "bytes": 757742, "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5"}, "hot_relevance/dev04": {"ns": 2122132588, "bytes": 760602, "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc"}, "fresh_relevance": {"ns": 1371296071, "bytes": 760539, "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641"}, "default": {"ns": 829143426, "bytes": 150213, "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491"}, "context": {"ns": 1198960097, "bytes": 1007130, "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f"}, "search": {"ns": 32348407, "bytes": 4317, "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947"}, "cli_relevance": {"ns": 2139961023, "decode_ns": 7214941, "bytes": 823797, "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641"}, "cli_search": {"ns": 516039455, "decode_ns": 97933, "bytes": 4991, "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947"}} diff --git a/evidence/indexer-255-20261029/results/confirmation-v2.json b/evidence/indexer-255-20261029/results/confirmation-v2.json new file mode 100644 index 000000000..e11be4cee --- /dev/null +++ b/evidence/indexer-255-20261029/results/confirmation-v2.json @@ -0,0 +1,3182 @@ +{ + "study": "confirmation-v2", + "protocol_sha256": "40de4eefba8712d3d07f2e48049b0899c831f94bfcfa252c1f8841525a337fa4", + "wrapper_sha256": "b11b4cf08b6dceb43be447516d5abb740bb2de8c2fdfd7a9ee3f820a1174b55d", + "before": { + "A": { + "python": { + "sha256": "054a6a2941e6b134876e55c7fb71a2052ab76dbf33017f8d4ba235c03277cd9b", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "ab03f3ffb8f2e5bc02c304828c5c41890eef673e", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + }, + "B": { + "python": { + "sha256": "094006cbb640aaee0ebcc2835dc3549525637cecf9517ef6ff656fb8e4800106", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "6da28494891b21cdc89f632d84d763bc997b7d15", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + } + }, + "started_utc": "2026-10-29T13:15:45.703818+00:00", + "verdict": "PASS", + "confirmation": [ + { + "index": 0, + "order": "AB", + "start_offset_ns": 56364, + "A": { + "ns": 309400056, + "decode_ns": 111417, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 59626 + }, + "B": { + "ns": 241962937, + "decode_ns": 85484, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 309691056 + } + }, + { + "index": 1, + "order": "BA", + "start_offset_ns": 552289815, + "B": { + "ns": 156260279, + "decode_ns": 78255, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 552291797 + }, + "A": { + "ns": 212195867, + "decode_ns": 82908, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 708729707 + } + }, + { + "index": 2, + "order": "AB", + "start_offset_ns": 923004983, + "A": { + "ns": 214995585, + "decode_ns": 78547, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 923007497 + }, + "B": { + "ns": 252152332, + "decode_ns": 120721, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 1138622600 + } + }, + { + "index": 3, + "order": "BA", + "start_offset_ns": 1391351588, + "B": { + "ns": 247352018, + "decode_ns": 80606, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 1391352824 + }, + "A": { + "ns": 209152535, + "decode_ns": 76801, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 1639379710 + } + }, + { + "index": 4, + "order": "BA", + "start_offset_ns": 1849103303, + "B": { + "ns": 243505446, + "decode_ns": 85365, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 1849104582 + }, + "A": { + "ns": 272843979, + "decode_ns": 79083, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 2092802736 + } + }, + { + "index": 5, + "order": "BA", + "start_offset_ns": 2366729252, + "B": { + "ns": 471068114, + "decode_ns": 155497, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 2366731010 + }, + "A": { + "ns": 265026903, + "decode_ns": 86349, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 2838061334 + } + }, + { + "index": 6, + "order": "AB", + "start_offset_ns": 3103982193, + "A": { + "ns": 513293454, + "decode_ns": 87462, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 3103984487 + }, + "B": { + "ns": 279744916, + "decode_ns": 117483, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 3617544662 + } + }, + { + "index": 7, + "order": "BA", + "start_offset_ns": 3898195996, + "B": { + "ns": 204313577, + "decode_ns": 77492, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 3898197409 + }, + "A": { + "ns": 239370120, + "decode_ns": 151672, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 4102686072 + } + }, + { + "index": 8, + "order": "AB", + "start_offset_ns": 4342733525, + "A": { + "ns": 208670669, + "decode_ns": 100987, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 4342734945 + }, + "B": { + "ns": 258450506, + "decode_ns": 80956, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 4551647034 + } + }, + { + "index": 9, + "order": "AB", + "start_offset_ns": 4810731584, + "A": { + "ns": 202132893, + "decode_ns": 74942, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 4810733277 + }, + "B": { + "ns": 447616502, + "decode_ns": 87736, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 5013040706 + } + }, + { + "index": 10, + "order": "BA", + "start_offset_ns": 5461331864, + "B": { + "ns": 180949325, + "decode_ns": 79316, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 5461333280 + }, + "A": { + "ns": 204979177, + "decode_ns": 357435, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 5642460415 + } + }, + { + "index": 11, + "order": "AB", + "start_offset_ns": 5848662299, + "A": { + "ns": 226238771, + "decode_ns": 84651, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 5848663992 + }, + "B": { + "ns": 188393700, + "decode_ns": 77622, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 6075092178 + } + }, + { + "index": 12, + "order": "AB", + "start_offset_ns": 6264287363, + "A": { + "ns": 178747499, + "decode_ns": 415556, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 6264289136 + }, + "B": { + "ns": 204168759, + "decode_ns": 102325, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 6443606178 + } + }, + { + "index": 13, + "order": "BA", + "start_offset_ns": 6648563179, + "B": { + "ns": 207891028, + "decode_ns": 78958, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 6648565108 + }, + "A": { + "ns": 226090853, + "decode_ns": 124418, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 6856635334 + } + }, + { + "index": 14, + "order": "BA", + "start_offset_ns": 7084036712, + "B": { + "ns": 230756041, + "decode_ns": 101578, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 7084038890 + }, + "A": { + "ns": 184656030, + "decode_ns": 77646, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 7315017560 + } + }, + { + "index": 15, + "order": "BA", + "start_offset_ns": 7500771701, + "B": { + "ns": 182812047, + "decode_ns": 356917, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 7500773927 + }, + "A": { + "ns": 196540894, + "decode_ns": 78612, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 7684048311 + } + }, + { + "index": 16, + "order": "AB", + "start_offset_ns": 7881477356, + "A": { + "ns": 204774439, + "decode_ns": 77554, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 7881478896 + }, + "B": { + "ns": 236490760, + "decode_ns": 84094, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 8086427712 + } + }, + { + "index": 17, + "order": "BA", + "start_offset_ns": 8323945474, + "B": { + "ns": 179092053, + "decode_ns": 80105, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 8323947260 + }, + "A": { + "ns": 210465284, + "decode_ns": 85725, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 8503229456 + } + }, + { + "index": 18, + "order": "BA", + "start_offset_ns": 8714743639, + "B": { + "ns": 182190024, + "decode_ns": 140958, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 8714745303 + }, + "A": { + "ns": 206184107, + "decode_ns": 78312, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 8897177180 + } + }, + { + "index": 19, + "order": "BA", + "start_offset_ns": 9104104374, + "B": { + "ns": 199980767, + "decode_ns": 149796, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 9104105929 + }, + "A": { + "ns": 289763878, + "decode_ns": 72524, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 9304341857 + } + }, + { + "index": 20, + "order": "AB", + "start_offset_ns": 9594890940, + "A": { + "ns": 269364602, + "decode_ns": 80187, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 9594893075 + }, + "B": { + "ns": 213634271, + "decode_ns": 80686, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 9864440968 + } + }, + { + "index": 21, + "order": "BA", + "start_offset_ns": 10078930454, + "B": { + "ns": 189746550, + "decode_ns": 78187, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 10078932126 + }, + "A": { + "ns": 227430818, + "decode_ns": 180750, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 10268886417 + } + }, + { + "index": 22, + "order": "AB", + "start_offset_ns": 10497744922, + "A": { + "ns": 279119098, + "decode_ns": 82153, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 10497747404 + }, + "B": { + "ns": 276949143, + "decode_ns": 81089, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 10777054970 + } + }, + { + "index": 23, + "order": "AB", + "start_offset_ns": 11055219420, + "A": { + "ns": 338198977, + "decode_ns": 83189, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 11055221805 + }, + "B": { + "ns": 337580421, + "decode_ns": 360174, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 11393608463 + } + }, + { + "index": 24, + "order": "AB", + "start_offset_ns": 11732336740, + "A": { + "ns": 274728652, + "decode_ns": 83913, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 11732338303 + }, + "B": { + "ns": 540608746, + "decode_ns": 89388, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 12007284326 + } + }, + { + "index": 25, + "order": "AB", + "start_offset_ns": 12549138476, + "A": { + "ns": 588228264, + "decode_ns": 80420, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 12549140628 + }, + "B": { + "ns": 338037640, + "decode_ns": 114574, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 13137762076 + } + }, + { + "index": 26, + "order": "BA", + "start_offset_ns": 13477520099, + "B": { + "ns": 299675646, + "decode_ns": 81431, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 13477523321 + }, + "A": { + "ns": 291370979, + "decode_ns": 111503, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 13777382884 + } + }, + { + "index": 27, + "order": "BA", + "start_offset_ns": 14070020390, + "B": { + "ns": 329650219, + "decode_ns": 87507, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 14070023148 + }, + "A": { + "ns": 379548302, + "decode_ns": 80759, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 14399893785 + } + }, + { + "index": 28, + "order": "AB", + "start_offset_ns": 14780925864, + "A": { + "ns": 302413046, + "decode_ns": 82680, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 14780927707 + }, + "B": { + "ns": 260413591, + "decode_ns": 82311, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 15083528044 + } + }, + { + "index": 29, + "order": "AB", + "start_offset_ns": 15345248614, + "A": { + "ns": 274782020, + "decode_ns": 80914, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 15345250772 + }, + "B": { + "ns": 382505850, + "decode_ns": 85578, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 15620215318 + } + }, + { + "index": 30, + "order": "BA", + "start_offset_ns": 16003729305, + "B": { + "ns": 310989995, + "decode_ns": 451002, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 16003730696 + }, + "A": { + "ns": 264391528, + "decode_ns": 444883, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 16315277802 + } + }, + { + "index": 31, + "order": "AB", + "start_offset_ns": 16581383330, + "A": { + "ns": 282585985, + "decode_ns": 81588, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 16581385380 + }, + "B": { + "ns": 276709592, + "decode_ns": 80234, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 16864804203 + } + }, + { + "index": 32, + "order": "AB", + "start_offset_ns": 17142478003, + "A": { + "ns": 278862932, + "decode_ns": 75778, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 17142479753 + }, + "B": { + "ns": 249458501, + "decode_ns": 79797, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 17421516493 + } + }, + { + "index": 33, + "order": "BA", + "start_offset_ns": 17673158087, + "B": { + "ns": 311353068, + "decode_ns": 87318, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 17673160477 + }, + "A": { + "ns": 288979788, + "decode_ns": 81770, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 17984707075 + } + }, + { + "index": 34, + "order": "BA", + "start_offset_ns": 18274948706, + "B": { + "ns": 256090044, + "decode_ns": 79305, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 18274950742 + }, + "A": { + "ns": 255200287, + "decode_ns": 187213, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 18531221408 + } + }, + { + "index": 35, + "order": "BA", + "start_offset_ns": 18788020482, + "B": { + "ns": 858910778, + "decode_ns": 104593, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 18788023265 + }, + "A": { + "ns": 268303191, + "decode_ns": 79568, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 19647165579 + } + }, + { + "index": 36, + "order": "AB", + "start_offset_ns": 19916718466, + "A": { + "ns": 293971460, + "decode_ns": 77065, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 19916720654 + }, + "B": { + "ns": 202971790, + "decode_ns": 116714, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 20210913384 + } + }, + { + "index": 37, + "order": "BA", + "start_offset_ns": 20414964641, + "B": { + "ns": 257508724, + "decode_ns": 568853, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 20414966217 + }, + "A": { + "ns": 241481072, + "decode_ns": 250411, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 20673149774 + } + }, + { + "index": 38, + "order": "AB", + "start_offset_ns": 20916302071, + "A": { + "ns": 223767188, + "decode_ns": 81914, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 20916304068 + }, + "B": { + "ns": 290381357, + "decode_ns": 78059, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 21140257840 + } + }, + { + "index": 39, + "order": "AB", + "start_offset_ns": 21431772084, + "A": { + "ns": 239759145, + "decode_ns": 150977, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 21431774969 + }, + "B": { + "ns": 253813332, + "decode_ns": 81523, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 21671791829 + } + }, + { + "index": 40, + "order": "BA", + "start_offset_ns": 21926874774, + "B": { + "ns": 286006243, + "decode_ns": 180580, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 21926876940 + }, + "A": { + "ns": 288392938, + "decode_ns": 79180, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 22213555341 + } + }, + { + "index": 41, + "order": "BA", + "start_offset_ns": 22505742676, + "B": { + "ns": 233920391, + "decode_ns": 85210, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 22505745658 + }, + "A": { + "ns": 261104006, + "decode_ns": 83476, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 22740185930 + } + }, + { + "index": 42, + "order": "BA", + "start_offset_ns": 23002551638, + "B": { + "ns": 267136768, + "decode_ns": 350833, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 23002553493 + }, + "A": { + "ns": 220215613, + "decode_ns": 79470, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 23270796793 + } + }, + { + "index": 43, + "order": "BA", + "start_offset_ns": 23492030136, + "B": { + "ns": 267769175, + "decode_ns": 111189, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 23492031830 + }, + "A": { + "ns": 230193546, + "decode_ns": 82529, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 23760012475 + } + }, + { + "index": 44, + "order": "BA", + "start_offset_ns": 23991307511, + "B": { + "ns": 221325007, + "decode_ns": 77307, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 23991309218 + }, + "A": { + "ns": 226434232, + "decode_ns": 106152, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 24212813896 + } + }, + { + "index": 45, + "order": "BA", + "start_offset_ns": 24440971084, + "B": { + "ns": 292731056, + "decode_ns": 79605, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 24440974002 + }, + "A": { + "ns": 291999952, + "decode_ns": 81256, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 24733959329 + } + }, + { + "index": 46, + "order": "AB", + "start_offset_ns": 25028598277, + "A": { + "ns": 282487269, + "decode_ns": 87077, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 25028601279 + }, + "B": { + "ns": 458351254, + "decode_ns": 91996, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 25311289187 + } + }, + { + "index": 47, + "order": "AB", + "start_offset_ns": 25774025524, + "A": { + "ns": 237351373, + "decode_ns": 81371, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 25774028776 + }, + "B": { + "ns": 270430525, + "decode_ns": 87256, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 26011631271 + } + }, + { + "index": 48, + "order": "AB", + "start_offset_ns": 26290276980, + "A": { + "ns": 241115129, + "decode_ns": 87391, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 26290280097 + }, + "B": { + "ns": 304853827, + "decode_ns": 87178, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 26531610985 + } + }, + { + "index": 49, + "order": "BA", + "start_offset_ns": 26838024144, + "B": { + "ns": 219593368, + "decode_ns": 83041, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 26838026395 + }, + "A": { + "ns": 239739047, + "decode_ns": 99236, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 27057806668 + } + }, + { + "index": 50, + "order": "BA", + "start_offset_ns": 27299241948, + "B": { + "ns": 177900371, + "decode_ns": 81026, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 27299243866 + }, + "A": { + "ns": 223986629, + "decode_ns": 76335, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 27477325635 + } + }, + { + "index": 51, + "order": "BA", + "start_offset_ns": 27703876631, + "B": { + "ns": 261697053, + "decode_ns": 95623, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 27703878953 + }, + "A": { + "ns": 217347106, + "decode_ns": 150637, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 27965769408 + } + }, + { + "index": 52, + "order": "BA", + "start_offset_ns": 28185641354, + "B": { + "ns": 216704215, + "decode_ns": 93380, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 28185644440 + }, + "A": { + "ns": 307199370, + "decode_ns": 121476, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 28402551845 + } + }, + { + "index": 53, + "order": "BA", + "start_offset_ns": 28711095755, + "B": { + "ns": 255981060, + "decode_ns": 79563, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 28711097453 + }, + "A": { + "ns": 206048071, + "decode_ns": 84387, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 28967257832 + } + }, + { + "index": 54, + "order": "AB", + "start_offset_ns": 29174708004, + "A": { + "ns": 195356669, + "decode_ns": 497697, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 29174709987 + }, + "B": { + "ns": 320359154, + "decode_ns": 205706, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 29371013822 + } + }, + { + "index": 55, + "order": "BA", + "start_offset_ns": 29693433771, + "B": { + "ns": 615203427, + "decode_ns": 234506, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 29693436449 + }, + "A": { + "ns": 558643871, + "decode_ns": 88762, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 30309286841 + } + }, + { + "index": 56, + "order": "BA", + "start_offset_ns": 30872693789, + "B": { + "ns": 484417306, + "decode_ns": 86522, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 30872696738 + }, + "A": { + "ns": 314484998, + "decode_ns": 2163714, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 31357310511 + } + }, + { + "index": 57, + "order": "BA", + "start_offset_ns": 31682094572, + "B": { + "ns": 393550960, + "decode_ns": 286001, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 31682097620 + }, + "A": { + "ns": 410282896, + "decode_ns": 89765, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 32076229504 + } + }, + { + "index": 58, + "order": "BA", + "start_offset_ns": 32497732071, + "B": { + "ns": 397924270, + "decode_ns": 87456, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 32497744518 + }, + "A": { + "ns": 316378858, + "decode_ns": 85082, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 32895902304 + } + }, + { + "index": 59, + "order": "BA", + "start_offset_ns": 33218926496, + "B": { + "ns": 281056500, + "decode_ns": 78514, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 33218930141 + }, + "A": { + "ns": 251133353, + "decode_ns": 189046, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 33500170008 + } + }, + { + "index": 60, + "order": "AB", + "start_offset_ns": 33753558239, + "A": { + "ns": 248092028, + "decode_ns": 83025, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 33753560804 + }, + "B": { + "ns": 253771176, + "decode_ns": 116133, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 34002636577 + } + }, + { + "index": 61, + "order": "BA", + "start_offset_ns": 34257751824, + "B": { + "ns": 399664418, + "decode_ns": 85409, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 34257753452 + }, + "A": { + "ns": 256733335, + "decode_ns": 78960, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 34657609391 + } + }, + { + "index": 62, + "order": "BA", + "start_offset_ns": 34917137170, + "B": { + "ns": 231771837, + "decode_ns": 82790, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 34917139512 + }, + "A": { + "ns": 170104824, + "decode_ns": 80214, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 35149097965 + } + }, + { + "index": 63, + "order": "AB", + "start_offset_ns": 35320912004, + "A": { + "ns": 193701736, + "decode_ns": 75100, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 35320915111 + }, + "B": { + "ns": 214866553, + "decode_ns": 95025, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 35514788666 + } + }, + { + "index": 64, + "order": "BA", + "start_offset_ns": 35731701752, + "B": { + "ns": 213377544, + "decode_ns": 77673, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 35731704866 + }, + "A": { + "ns": 197817879, + "decode_ns": 106597, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 35945260825 + } + }, + { + "index": 65, + "order": "BA", + "start_offset_ns": 36146548502, + "B": { + "ns": 171889547, + "decode_ns": 81960, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 36146551328 + }, + "A": { + "ns": 251512866, + "decode_ns": 286180, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 36318626190 + } + }, + { + "index": 66, + "order": "BA", + "start_offset_ns": 36572090619, + "B": { + "ns": 240112365, + "decode_ns": 88104, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 36572093334 + }, + "A": { + "ns": 480225531, + "decode_ns": 162244, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 36812424922 + } + }, + { + "index": 67, + "order": "AB", + "start_offset_ns": 37295345942, + "A": { + "ns": 282005492, + "decode_ns": 140445, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 37295348852 + }, + "B": { + "ns": 533403677, + "decode_ns": 130960, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 37578370321 + } + }, + { + "index": 68, + "order": "AB", + "start_offset_ns": 38113476612, + "A": { + "ns": 599972041, + "decode_ns": 88697, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 38113479170 + }, + "B": { + "ns": 423554321, + "decode_ns": 92605, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 38713651942 + } + }, + { + "index": 69, + "order": "AB", + "start_offset_ns": 39138817368, + "A": { + "ns": 536408504, + "decode_ns": 234282, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 39138819937 + }, + "B": { + "ns": 662093490, + "decode_ns": 120376, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 39675664138 + } + }, + { + "index": 70, + "order": "BA", + "start_offset_ns": 40339797016, + "B": { + "ns": 502305186, + "decode_ns": 97674, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 40339799434 + }, + "A": { + "ns": 408624169, + "decode_ns": 105132, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 40842318970 + } + }, + { + "index": 71, + "order": "AB", + "start_offset_ns": 41252333416, + "A": { + "ns": 437788261, + "decode_ns": 242746, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 41252334985 + }, + "B": { + "ns": 509561440, + "decode_ns": 304572, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 41690552189 + } + }, + { + "index": 72, + "order": "AB", + "start_offset_ns": 42203874557, + "A": { + "ns": 385510777, + "decode_ns": 89484, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 42203884497 + }, + "B": { + "ns": 443734911, + "decode_ns": 86313, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 42589663250 + } + }, + { + "index": 73, + "order": "AB", + "start_offset_ns": 43035428544, + "A": { + "ns": 403726203, + "decode_ns": 143499, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 43035431313 + }, + "B": { + "ns": 398249172, + "decode_ns": 89108, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 43439416170 + } + }, + { + "index": 74, + "order": "BA", + "start_offset_ns": 43840586869, + "B": { + "ns": 380200670, + "decode_ns": 116140, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 43840589905 + }, + "A": { + "ns": 515120335, + "decode_ns": 89909, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 44221018828 + } + }, + { + "index": 75, + "order": "BA", + "start_offset_ns": 44738318421, + "B": { + "ns": 311913607, + "decode_ns": 89556, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 44738321788 + }, + "A": { + "ns": 207520716, + "decode_ns": 86429, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 45050790411 + } + }, + { + "index": 76, + "order": "AB", + "start_offset_ns": 45262118541, + "A": { + "ns": 221061849, + "decode_ns": 87077, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 45262121176 + }, + "B": { + "ns": 463586107, + "decode_ns": 81289, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 45483484145 + } + }, + { + "index": 77, + "order": "BA", + "start_offset_ns": 45949494753, + "B": { + "ns": 441128834, + "decode_ns": 88758, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 45949497325 + }, + "A": { + "ns": 182895621, + "decode_ns": 121130, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 46390856066 + } + }, + { + "index": 78, + "order": "BA", + "start_offset_ns": 46576632961, + "B": { + "ns": 412510344, + "decode_ns": 295621, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 46576635684 + }, + "A": { + "ns": 221695500, + "decode_ns": 88264, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 46989559840 + } + }, + { + "index": 79, + "order": "AB", + "start_offset_ns": 47213975289, + "A": { + "ns": 227045870, + "decode_ns": 85907, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 47213978636 + }, + "B": { + "ns": 168257399, + "decode_ns": 79202, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 47441246777 + } + }, + { + "index": 80, + "order": "AB", + "start_offset_ns": 47611667830, + "A": { + "ns": 195855981, + "decode_ns": 227694, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 47611670060 + }, + "B": { + "ns": 218597583, + "decode_ns": 84900, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 47807940891 + } + }, + { + "index": 81, + "order": "AB", + "start_offset_ns": 48029330407, + "A": { + "ns": 205346378, + "decode_ns": 84098, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 48029333844 + }, + "B": { + "ns": 173017043, + "decode_ns": 87273, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 48234896404 + } + }, + { + "index": 82, + "order": "BA", + "start_offset_ns": 48409981116, + "B": { + "ns": 249345466, + "decode_ns": 990781, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 48409983610 + }, + "A": { + "ns": 346042018, + "decode_ns": 106996, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 48660935423 + } + }, + { + "index": 83, + "order": "BA", + "start_offset_ns": 49010074575, + "B": { + "ns": 193108426, + "decode_ns": 83005, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 49010078146 + }, + "A": { + "ns": 181918213, + "decode_ns": 123183, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 49203375080 + } + }, + { + "index": 84, + "order": "BA", + "start_offset_ns": 49387774954, + "B": { + "ns": 219424598, + "decode_ns": 86845, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 49387777297 + }, + "A": { + "ns": 214681110, + "decode_ns": 169332, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 49607398745 + } + }, + { + "index": 85, + "order": "BA", + "start_offset_ns": 49824341455, + "B": { + "ns": 208807310, + "decode_ns": 86733, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 49824344115 + }, + "A": { + "ns": 183200897, + "decode_ns": 83159, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 50033372627 + } + }, + { + "index": 86, + "order": "AB", + "start_offset_ns": 50219083991, + "A": { + "ns": 224812534, + "decode_ns": 89408, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 50219087011 + }, + "B": { + "ns": 215508449, + "decode_ns": 82997, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 50444098137 + } + }, + { + "index": 87, + "order": "BA", + "start_offset_ns": 50662877037, + "B": { + "ns": 211644642, + "decode_ns": 90975, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 50662880496 + }, + "A": { + "ns": 296834132, + "decode_ns": 92894, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 50874745241 + } + }, + { + "index": 88, + "order": "AB", + "start_offset_ns": 51174168899, + "A": { + "ns": 452559542, + "decode_ns": 122420, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 51174171844 + }, + "B": { + "ns": 401889491, + "decode_ns": 215765, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 51626971152 + } + }, + { + "index": 89, + "order": "AB", + "start_offset_ns": 52032695000, + "A": { + "ns": 170717326, + "decode_ns": 112501, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 52032698360 + }, + "B": { + "ns": 198378205, + "decode_ns": 84416, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 52203642528 + } + }, + { + "index": 90, + "order": "AB", + "start_offset_ns": 52404881041, + "A": { + "ns": 199478893, + "decode_ns": 89722, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 52404883787 + }, + "B": { + "ns": 306550643, + "decode_ns": 140068, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 52604558483 + } + }, + { + "index": 91, + "order": "AB", + "start_offset_ns": 52919642807, + "A": { + "ns": 182937341, + "decode_ns": 78152, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 52919653134 + }, + "B": { + "ns": 182545876, + "decode_ns": 99064, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 53102773631 + } + }, + { + "index": 92, + "order": "BA", + "start_offset_ns": 53288038235, + "B": { + "ns": 189957667, + "decode_ns": 88526, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 53288041321 + }, + "A": { + "ns": 209952243, + "decode_ns": 88319, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 53478197728 + } + }, + { + "index": 93, + "order": "AB", + "start_offset_ns": 53692475102, + "A": { + "ns": 191193166, + "decode_ns": 81987, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 53692477861 + }, + "B": { + "ns": 483213825, + "decode_ns": 91787, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 53883892286 + } + }, + { + "index": 94, + "order": "BA", + "start_offset_ns": 54369296100, + "B": { + "ns": 349457581, + "decode_ns": 209536, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 54369298949 + }, + "A": { + "ns": 286023250, + "decode_ns": 89450, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 54719594629 + } + }, + { + "index": 95, + "order": "AB", + "start_offset_ns": 55010575635, + "A": { + "ns": 253105702, + "decode_ns": 85942, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 55010579154 + }, + "B": { + "ns": 575444743, + "decode_ns": 120976, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 55263909826 + } + }, + { + "index": 96, + "order": "AB", + "start_offset_ns": 55842973560, + "A": { + "ns": 212755877, + "decode_ns": 222050, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 55842977051 + }, + "B": { + "ns": 305607013, + "decode_ns": 141284, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 56056205615 + } + }, + { + "index": 97, + "order": "BA", + "start_offset_ns": 56363869442, + "B": { + "ns": 308870431, + "decode_ns": 80536, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 56363871818 + }, + "A": { + "ns": 215683173, + "decode_ns": 598323, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 56672960464 + } + }, + { + "index": 98, + "order": "AB", + "start_offset_ns": 56893988011, + "A": { + "ns": 219190076, + "decode_ns": 87839, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 56893990402 + }, + "B": { + "ns": 188239422, + "decode_ns": 269457, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 57113378460 + } + }, + { + "index": 99, + "order": "AB", + "start_offset_ns": 57307147125, + "A": { + "ns": 162897143, + "decode_ns": 84774, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 57307150780 + }, + "B": { + "ns": 176848326, + "decode_ns": 82496, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 57470238007 + } + }, + { + "index": 100, + "order": "AB", + "start_offset_ns": 57649236822, + "A": { + "ns": 234883525, + "decode_ns": 90131, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 57649238625 + }, + "B": { + "ns": 219834432, + "decode_ns": 89891, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 57884321542 + } + }, + { + "index": 101, + "order": "AB", + "start_offset_ns": 58107003678, + "A": { + "ns": 201090398, + "decode_ns": 88588, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 58107006333 + }, + "B": { + "ns": 184313524, + "decode_ns": 250015, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 58308301285 + } + }, + { + "index": 102, + "order": "BA", + "start_offset_ns": 58495453934, + "B": { + "ns": 187867950, + "decode_ns": 178536, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 58495456557 + }, + "A": { + "ns": 193899514, + "decode_ns": 83967, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 58683613203 + } + }, + { + "index": 103, + "order": "BA", + "start_offset_ns": 58879749137, + "B": { + "ns": 209313892, + "decode_ns": 81142, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 58879752167 + }, + "A": { + "ns": 189869461, + "decode_ns": 85504, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 59089257816 + } + }, + { + "index": 104, + "order": "AB", + "start_offset_ns": 59282134881, + "A": { + "ns": 178226535, + "decode_ns": 78684, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 59282180801 + }, + "B": { + "ns": 203742356, + "decode_ns": 81543, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 59460617929 + } + }, + { + "index": 105, + "order": "AB", + "start_offset_ns": 59666634422, + "A": { + "ns": 257755803, + "decode_ns": 87716, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 59666636644 + }, + "B": { + "ns": 207211873, + "decode_ns": 186562, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 59924816738 + } + }, + { + "index": 106, + "order": "AB", + "start_offset_ns": 60134064114, + "A": { + "ns": 192257939, + "decode_ns": 83191, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 60134066913 + }, + "B": { + "ns": 228501000, + "decode_ns": 170638, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 60326522480 + } + }, + { + "index": 107, + "order": "AB", + "start_offset_ns": 60561180476, + "A": { + "ns": 175948241, + "decode_ns": 81222, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 60561190708 + }, + "B": { + "ns": 321369433, + "decode_ns": 2937408, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 60737336967 + } + }, + { + "index": 108, + "order": "BA", + "start_offset_ns": 61072222429, + "B": { + "ns": 357754848, + "decode_ns": 35476522, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 61072233586 + }, + "A": { + "ns": 865458484, + "decode_ns": 462061, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 61467610550 + } + }, + { + "index": 109, + "order": "BA", + "start_offset_ns": 62339829051, + "B": { + "ns": 327633119, + "decode_ns": 123524, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 62339864949 + }, + "A": { + "ns": 2060393543, + "decode_ns": 398772, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 62667749760 + } + }, + { + "index": 110, + "order": "AB", + "start_offset_ns": 64731992493, + "A": { + "ns": 606812072, + "decode_ns": 131234, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 64732001866 + }, + "B": { + "ns": 441038450, + "decode_ns": 94462, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 65339071087 + } + }, + { + "index": 111, + "order": "AB", + "start_offset_ns": 65785796414, + "A": { + "ns": 462299181, + "decode_ns": 285557, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 65785799404 + }, + "B": { + "ns": 931687125, + "decode_ns": 90962, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 66248589240 + } + }, + { + "index": 112, + "order": "AB", + "start_offset_ns": 67183487290, + "A": { + "ns": 204097369, + "decode_ns": 198163, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 67183496820 + }, + "B": { + "ns": 1604318210, + "decode_ns": 227162, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 67387933768 + } + }, + { + "index": 113, + "order": "AB", + "start_offset_ns": 68995425898, + "A": { + "ns": 685833903, + "decode_ns": 88960, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 68995436378 + }, + "B": { + "ns": 402455002, + "decode_ns": 83614, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 69681471274 + } + }, + { + "index": 114, + "order": "AB", + "start_offset_ns": 70086310929, + "A": { + "ns": 233282355, + "decode_ns": 83089, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 70086313219 + }, + "B": { + "ns": 569495160, + "decode_ns": 194511, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 70320538720 + } + }, + { + "index": 115, + "order": "AB", + "start_offset_ns": 70903095588, + "A": { + "ns": 699348166, + "decode_ns": 1852825, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 70903106215 + }, + "B": { + "ns": 371834046, + "decode_ns": 79588, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 71606177402 + } + }, + { + "index": 116, + "order": "AB", + "start_offset_ns": 71980042492, + "A": { + "ns": 288222421, + "decode_ns": 1127484, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 71980044736 + }, + "B": { + "ns": 275121820, + "decode_ns": 285998, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 72269504275 + } + }, + { + "index": 117, + "order": "BA", + "start_offset_ns": 72606125540, + "B": { + "ns": 809235640, + "decode_ns": 97689, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 72606136574 + }, + "A": { + "ns": 215072192, + "decode_ns": 84896, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 73415625964 + } + }, + { + "index": 118, + "order": "BA", + "start_offset_ns": 73632654001, + "B": { + "ns": 580710886, + "decode_ns": 155324, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 73632656580 + }, + "A": { + "ns": 660498523, + "decode_ns": 174333, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 74213665000 + } + }, + { + "index": 119, + "order": "AB", + "start_offset_ns": 74877606526, + "A": { + "ns": 252061112, + "decode_ns": 85368, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 74877623464 + }, + "B": { + "ns": 247706894, + "decode_ns": 83606, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 75129950230 + } + } + ], + "negative_control": [ + { + "index": 0, + "order": "BA", + "start_offset_ns": 75383668489, + "B": { + "ns": 259963173, + "decode_ns": 102216, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 75383672129 + }, + "A": { + "ns": 246662967, + "decode_ns": 80578, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 75643909919 + } + }, + { + "index": 1, + "order": "AB", + "start_offset_ns": 75893224639, + "A": { + "ns": 286259158, + "decode_ns": 78799, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 75893228646 + }, + "B": { + "ns": 316005002, + "decode_ns": 76460, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 76179669513 + } + }, + { + "index": 2, + "order": "BA", + "start_offset_ns": 76497622567, + "B": { + "ns": 321386348, + "decode_ns": 88984, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 76497625197 + }, + "A": { + "ns": 205235002, + "decode_ns": 79302, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 76819212451 + } + }, + { + "index": 3, + "order": "AB", + "start_offset_ns": 77026971412, + "A": { + "ns": 344838712, + "decode_ns": 122415, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 77026974429 + }, + "B": { + "ns": 295040924, + "decode_ns": 113995, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 77372746594 + } + }, + { + "index": 4, + "order": "BA", + "start_offset_ns": 77670136981, + "B": { + "ns": 374573675, + "decode_ns": 84797, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 77670139310 + }, + "A": { + "ns": 333138636, + "decode_ns": 82059, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 78044927838 + } + }, + { + "index": 5, + "order": "BA", + "start_offset_ns": 78380552443, + "B": { + "ns": 472310333, + "decode_ns": 86715, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 78380554865 + }, + "A": { + "ns": 313806087, + "decode_ns": 80316, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 78853065480 + } + }, + { + "index": 6, + "order": "BA", + "start_offset_ns": 79171945235, + "B": { + "ns": 295715893, + "decode_ns": 91519, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 79171947937 + }, + "A": { + "ns": 284553159, + "decode_ns": 75155, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 79467887594 + } + }, + { + "index": 7, + "order": "BA", + "start_offset_ns": 79754651119, + "B": { + "ns": 240765275, + "decode_ns": 120231, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 79754653660 + }, + "A": { + "ns": 253563908, + "decode_ns": 214071, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 79995661906 + } + }, + { + "index": 8, + "order": "AB", + "start_offset_ns": 80251877856, + "A": { + "ns": 238179579, + "decode_ns": 74139, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 80251880066 + }, + "B": { + "ns": 240369993, + "decode_ns": 79790, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 80490232742 + } + }, + { + "index": 9, + "order": "AB", + "start_offset_ns": 80732999148, + "A": { + "ns": 248999702, + "decode_ns": 75650, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 80733001582 + }, + "B": { + "ns": 265467780, + "decode_ns": 84696, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 80982481075 + } + }, + { + "index": 10, + "order": "BA", + "start_offset_ns": 81250059343, + "B": { + "ns": 226023641, + "decode_ns": 73255, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 81250062588 + }, + "A": { + "ns": 251501500, + "decode_ns": 104581, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 81476259370 + } + }, + { + "index": 11, + "order": "BA", + "start_offset_ns": 81730400696, + "B": { + "ns": 246890325, + "decode_ns": 83128, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 81730404744 + }, + "A": { + "ns": 306619744, + "decode_ns": 98070, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 81977486669 + } + }, + { + "index": 12, + "order": "AB", + "start_offset_ns": 82287058132, + "A": { + "ns": 336390808, + "decode_ns": 85540, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 82287060916 + }, + "B": { + "ns": 360099356, + "decode_ns": 113561, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 82623651132 + } + }, + { + "index": 13, + "order": "AB", + "start_offset_ns": 82987661791, + "A": { + "ns": 303442272, + "decode_ns": 156895, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 82987664359 + }, + "B": { + "ns": 327640110, + "decode_ns": 91784, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 83291558290 + } + }, + { + "index": 14, + "order": "AB", + "start_offset_ns": 83632315248, + "A": { + "ns": 191812292, + "decode_ns": 85869, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 83632319237 + }, + "B": { + "ns": 226574503, + "decode_ns": 82477, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 83824330682 + } + }, + { + "index": 15, + "order": "BA", + "start_offset_ns": 84053073150, + "B": { + "ns": 227606908, + "decode_ns": 79706, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 84053075504 + }, + "A": { + "ns": 235751785, + "decode_ns": 251526, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 84280894910 + } + }, + { + "index": 16, + "order": "BA", + "start_offset_ns": 84519941639, + "B": { + "ns": 200601603, + "decode_ns": 80796, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 84519944158 + }, + "A": { + "ns": 218250268, + "decode_ns": 728998, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 84720786948 + } + }, + { + "index": 17, + "order": "AB", + "start_offset_ns": 84944542923, + "A": { + "ns": 211198440, + "decode_ns": 111363, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 84944545623 + }, + "B": { + "ns": 195544967, + "decode_ns": 98328, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 85155962930 + } + }, + { + "index": 18, + "order": "AB", + "start_offset_ns": 85354187097, + "A": { + "ns": 194717545, + "decode_ns": 79130, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 85354189445 + }, + "B": { + "ns": 213102400, + "decode_ns": 81669, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 85549090645 + } + }, + { + "index": 19, + "order": "BA", + "start_offset_ns": 85764769400, + "B": { + "ns": 202319108, + "decode_ns": 76679, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 85764771640 + }, + "A": { + "ns": 274283812, + "decode_ns": 84593, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 85967338507 + } + }, + { + "index": 20, + "order": "BA", + "start_offset_ns": 86245056856, + "B": { + "ns": 196823196, + "decode_ns": 394606, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 86245059548 + }, + "A": { + "ns": 329283202, + "decode_ns": 85073, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 86442384049 + } + }, + { + "index": 21, + "order": "AB", + "start_offset_ns": 86775614416, + "A": { + "ns": 343365386, + "decode_ns": 107569, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 86775617437 + }, + "B": { + "ns": 364262730, + "decode_ns": 88834, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 87119252512 + } + }, + { + "index": 22, + "order": "AB", + "start_offset_ns": 87487030211, + "A": { + "ns": 212236234, + "decode_ns": 72850, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 87487033184 + }, + "B": { + "ns": 189801678, + "decode_ns": 76855, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 87699441739 + } + }, + { + "index": 23, + "order": "AB", + "start_offset_ns": 87893228516, + "A": { + "ns": 188204223, + "decode_ns": 78836, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 87893230796 + }, + "B": { + "ns": 219919461, + "decode_ns": 79948, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 88081616968 + } + }, + { + "index": 24, + "order": "AB", + "start_offset_ns": 88304190828, + "A": { + "ns": 222921225, + "decode_ns": 75664, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 88304193217 + }, + "B": { + "ns": 201069506, + "decode_ns": 79354, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 88527291844 + } + }, + { + "index": 25, + "order": "BA", + "start_offset_ns": 88731364830, + "B": { + "ns": 184948328, + "decode_ns": 147924, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 88731366927 + }, + "A": { + "ns": 193919070, + "decode_ns": 79055, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 88916580547 + } + }, + { + "index": 26, + "order": "BA", + "start_offset_ns": 89114658419, + "B": { + "ns": 243709567, + "decode_ns": 154937, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 89114661293 + }, + "A": { + "ns": 234825385, + "decode_ns": 75924, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 89358628789 + } + }, + { + "index": 27, + "order": "BA", + "start_offset_ns": 89596242108, + "B": { + "ns": 205534235, + "decode_ns": 156078, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 89596244703 + }, + "A": { + "ns": 198630526, + "decode_ns": 77519, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 89802045685 + } + }, + { + "index": 28, + "order": "BA", + "start_offset_ns": 90004993182, + "B": { + "ns": 211984181, + "decode_ns": 75695, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 90004995996 + }, + "A": { + "ns": 202776160, + "decode_ns": 77457, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 90217158298 + } + }, + { + "index": 29, + "order": "BA", + "start_offset_ns": 90422539578, + "B": { + "ns": 188536312, + "decode_ns": 138524, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 90422542056 + }, + "A": { + "ns": 202301951, + "decode_ns": 80092, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 90611322877 + } + }, + { + "index": 30, + "order": "AB", + "start_offset_ns": 90816043367, + "A": { + "ns": 187187518, + "decode_ns": 78676, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 90816045707 + }, + "B": { + "ns": 229359635, + "decode_ns": 70946, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 91003534533 + } + }, + { + "index": 31, + "order": "AB", + "start_offset_ns": 91236219321, + "A": { + "ns": 233542520, + "decode_ns": 99513, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 91236223027 + }, + "B": { + "ns": 195078835, + "decode_ns": 77305, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 91470022718 + } + }, + { + "index": 32, + "order": "AB", + "start_offset_ns": 91667965102, + "A": { + "ns": 173471072, + "decode_ns": 77384, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 91667967702 + }, + "B": { + "ns": 192058137, + "decode_ns": 122223, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 91841689596 + } + }, + { + "index": 33, + "order": "AB", + "start_offset_ns": 92037467484, + "A": { + "ns": 214848366, + "decode_ns": 81698, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 92037469838 + }, + "B": { + "ns": 307678995, + "decode_ns": 82072, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 92252504471 + } + }, + { + "index": 34, + "order": "AB", + "start_offset_ns": 92563640696, + "A": { + "ns": 323849630, + "decode_ns": 152779, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 92563643483 + }, + "B": { + "ns": 313209384, + "decode_ns": 127483, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 92887825868 + } + }, + { + "index": 35, + "order": "BA", + "start_offset_ns": 93207710719, + "B": { + "ns": 225616613, + "decode_ns": 81320, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 93207715056 + }, + "A": { + "ns": 212524231, + "decode_ns": 79309, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 93433525784 + } + }, + { + "index": 36, + "order": "AB", + "start_offset_ns": 93649386960, + "A": { + "ns": 196851547, + "decode_ns": 77060, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 93649390894 + }, + "B": { + "ns": 195029197, + "decode_ns": 82788, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 93846530065 + } + }, + { + "index": 37, + "order": "BA", + "start_offset_ns": 94044246745, + "B": { + "ns": 196092919, + "decode_ns": 76709, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 94044249756 + }, + "A": { + "ns": 213576795, + "decode_ns": 79820, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 94240541446 + } + }, + { + "index": 38, + "order": "BA", + "start_offset_ns": 94457474814, + "B": { + "ns": 200732312, + "decode_ns": 81829, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 94457477981 + }, + "A": { + "ns": 184956894, + "decode_ns": 76254, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 94658395778 + } + }, + { + "index": 39, + "order": "AB", + "start_offset_ns": 94846351052, + "A": { + "ns": 201421585, + "decode_ns": 102493, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 94846354211 + }, + "B": { + "ns": 197419516, + "decode_ns": 77498, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947", + "start_offset_ns": 95048066810 + } + } + ], + "after": { + "A": { + "python": { + "sha256": "054a6a2941e6b134876e55c7fb71a2052ab76dbf33017f8d4ba235c03277cd9b", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "ab03f3ffb8f2e5bc02c304828c5c41890eef673e", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + }, + "B": { + "python": { + "sha256": "094006cbb640aaee0ebcc2835dc3549525637cecf9517ef6ff656fb8e4800106", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "6da28494891b21cdc89f632d84d763bc997b7d15", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + } + }, + "summary": { + "confirmation": { + "A": { + "n": 120, + "p50_ms": 240.437137, + "p95_ms": 599.972041 + }, + "B": { + "n": 120, + "p50_ms": 264.4169105, + "p95_ms": 580.710886 + }, + "p50_ms_change_percent": 9.973406687170772, + "p95_ms_change_percent": -3.2103420965911345, + "paired_bootstrap_95ci": { + "p50_ms": [ + 1.201447633216124, + 20.918694001959558 + ], + "p95_ms": [ + -22.225530895051126, + 50.861821534432636 + ] + }, + "all_output_parity": true + }, + "negative_control": { + "A": { + "n": 40, + "p50_ms": 228.2318725, + "p95_ms": 336.390808 + }, + "B": { + "n": 40, + "p50_ms": 226.299072, + "p95_ms": 364.26273 + }, + "p50_ms_change_percent": -0.84685827567752, + "p95_ms_change_percent": 8.285577767630325, + "paired_bootstrap_95ci": { + "p50_ms": [ + -9.181766013546655, + 7.593644823179013 + ], + "p95_ms": [ + -5.98226979592964, + 41.775910074867454 + ] + }, + "all_output_parity": true + } + }, + "failed_gates": [], + "verdict_scope": "Focused confirmation only; v1 remains FAIL", + "ended_utc": "2026-10-29T13:17:22.170911+00:00" +} diff --git a/evidence/indexer-255-20261029/results/final-evidence-tree-tests.txt b/evidence/indexer-255-20261029/results/final-evidence-tree-tests.txt new file mode 100644 index 000000000..a07b2bfb0 --- /dev/null +++ b/evidence/indexer-255-20261029/results/final-evidence-tree-tests.txt @@ -0,0 +1,5 @@ +.............................................ss......................... [ 79%] +................... [100%] +=========================== short test summary info ============================ +SKIPPED [2] tests/test_knowledge_operations.py:277: NTFS junctions are Windows-only +89 passed, 2 skipped in 24.23s diff --git a/evidence/indexer-255-20261029/results/final-lint.txt b/evidence/indexer-255-20261029/results/final-lint.txt new file mode 100644 index 000000000..31331459d --- /dev/null +++ b/evidence/indexer-255-20261029/results/final-lint.txt @@ -0,0 +1,374 @@ +evidence/modular-refactor-20261013/validator/compare.py:1:1: I001 [*] Import block is un-sorted or un-formatted +evidence/modular-refactor-20261013/validator/compare.py:16:101: E501 Line too long (102 > 100) +evidence/modular-refactor-20261013/validator/compare.py:17:101: E501 Line too long (109 > 100) +evidence/modular-refactor-20261013/validator/compare.py:27:101: E501 Line too long (147 > 100) +evidence/modular-refactor-20261013/validator/compare.py:30:101: E501 Line too long (118 > 100) +evidence/modular-refactor-20261013/validator/compare.py:35:101: E501 Line too long (167 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:10:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:31:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:36:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:40:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.initial.py:41:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:10:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:31:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:36:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:40:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-benign-retry-v2/check_loader.py:41:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/test_reference_completion.py:32:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/test_reference_completion.py:33:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v1/test_reference_completion.py:34:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/test_reference_completion.py:32:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/test_reference_completion.py:33:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bounds-v2/test_reference_completion.py:34:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-bwrap-build-v2/extract_source.py:11:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v1/diagnose.py:29:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v1/review-rejected-v1/diagnose.py:27:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v1/review-rejected-v1/diagnose.py:82:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-context-v2/diagnose.py:29:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-elf-dependency-v1/diagnose.py:37:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/synthetic_probe.py:24:1: E402 Module level import not at top of file +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/synthetic_probe.py:94:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/test_reference_completion.py:32:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/test_reference_completion.py:33:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-evaluator-synthetic-v3/test_reference_completion.py:34:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-executor-v1/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:3:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:428:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:443:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-full164-v1/runtime.py:444:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/check_wheels.py:13:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/check_wheels.py:41:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/prefetch.py:105:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-install-v3/verify_wheels.py:55:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:36:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:40:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:41:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:42:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:52:1: E731 Do not assign a `lambda` expression, use a `def` +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:57:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:58:101: E501 Line too long (110 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/check_prefetch.py:124:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/diagnostics/sandbox-offline-prefetch-v1/prefetch.py:105:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:87:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:88:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:89:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:92:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:95:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:96:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:97:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:98:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:101:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:102:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:105:43: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:107:42: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:108:101: E501 Line too long (104 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:125:54: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:125:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:132:58: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:137:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:142:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:146:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:148:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:152:101: E501 Line too long (116 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:154:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:157:101: E501 Line too long (125 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:164:41: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:166:89: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:166:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:167:90: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/http_probe.py:167:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:25:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:33:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:38:30: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:53:30: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:56:31: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:57:25: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:66:1: E402 Module level import not at top of file +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:66:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/patch/flashinfer-small-m/operator_probe.py:69:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:91:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:99:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:102:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:103:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:112:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:135:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:137:58: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:142:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:145:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:151:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:154:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:162:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:172:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/baseline_long_readiness.py:176:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:56:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:57:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:58:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:60:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:69:26: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_forced96.py:72:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:69:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:75:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:87:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:105:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:110:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:121:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:122:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:123:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:127:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:130:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:131:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:132:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:134:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:140:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:143:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http.py:145:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:63:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:69:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:75:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:87:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:105:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:110:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:121:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:122:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:123:101: E501 Line too long (107 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:127:101: E501 Line too long (117 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:130:101: E501 Line too long (103 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:131:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:132:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:134:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:140:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:143:101: E501 Line too long (109 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_long_http_v2.py:145:101: E501 Line too long (112 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_probe.py:6:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/graph_probe.py:87:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/http_readiness.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/http_readiness.py:114:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/official_client_probe_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:21:45: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:23:27: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:23:49: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:24:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:26:5: E731 Do not assign a `lambda` expression, use a `def` +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:27:13: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:33:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:34:101: E501 Line too long (186 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:38:101: E501 Line too long (206 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:39:38: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:39:59: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:39:73: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:44:70: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:49:42: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:51:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:57:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:58:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:61:30: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:63:25: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:65:73: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:65:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:66:101: E501 Line too long (812 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:69:101: E501 Line too long (158 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval.py:72:24: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:21:45: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:23:27: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:23:49: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:24:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:26:5: E731 Do not assign a `lambda` expression, use a `def` +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:27:13: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:33:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:34:101: E501 Line too long (186 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:38:101: E501 Line too long (170 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:38: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:70: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:91: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:101: E501 Line too long (113 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:39:105: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:44:63: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:49:42: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:51:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:57:48: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:58:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:61:30: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:63:25: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:65:73: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:65:101: E501 Line too long (115 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:66:101: E501 Line too long (1033 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:69:101: E501 Line too long (158 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/prepare_retrieval_v2.py:72:24: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:86:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:94:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:96:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:110:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:117:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:118:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:121:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:122:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:131:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:154:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:51: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:80: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:156:111: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:161:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:164:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:170:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:173:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:175:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:178:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:181:101: E501 Line too long (147 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:182:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:188:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:192:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline.py:196:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:86:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:94:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:96:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:97:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:98:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:110:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:117:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:118:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:121:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:122:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:129:101: E501 Line too long (146 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:131:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:154:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:51: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:80: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:156:111: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:161:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:164:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:165:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:166:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:172:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:175:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:177:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:185:101: E501 Line too long (147 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:186:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:188:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:193:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:197:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v2.py:201:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:86:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:94:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:96:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:97:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:98:101: E501 Line too long (105 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:111:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:118:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:119:28: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:122:47: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:123:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:130:101: E501 Line too long (175 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:133:40: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:138:101: E501 Line too long (101 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:156:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:51: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:80: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:101: E501 Line too long (120 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:158:111: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:163:101: E501 Line too long (122 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:166:101: E501 Line too long (118 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:167:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:168:101: E501 Line too long (128 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:174:101: E501 Line too long (114 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:177:101: E501 Line too long (106 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:179:101: E501 Line too long (119 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:187:101: E501 Line too long (147 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:188:101: E501 Line too long (102 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:190:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:195:101: E501 Line too long (108 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:199:101: E501 Line too long (154 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/diagnostics/retrieval_baseline_v3.py:203:101: E501 Line too long (135 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:2:1: I001 [*] Import block is un-sorted or un-formatted +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:8:29: E701 Multiple statements on one line (colon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:16:101: E501 Line too long (126 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:18:18: E702 Multiple statements on one line (semicolon) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:19:101: E501 Line too long (134 > 100) +evidence/qwen38-h20-fp8-single-request-vllm032/validator/quality-execution-8k-v1/retrieval_score.py:20:101: E501 Line too long (105 > 100) +examples/qwen21_turbo_b300.py:6:1: I001 [*] Import block is un-sorted or un-formatted +examples/qwen21_turbo_b300.py:14:8: F401 [*] `subprocess` imported but unused +examples/qwen21_turbo_b300.py:28:101: E501 Line too long (126 > 100) +examples/qwen21_turbo_b300.py:29:101: E501 Line too long (121 > 100) +examples/qwen21_turbo_b300.py:30:101: E501 Line too long (126 > 100) +examples/qwen21_turbo_b300.py:33:101: E501 Line too long (105 > 100) +examples/qwen21_turbo_b300.py:34:101: E501 Line too long (106 > 100) +examples/qwen21_turbo_b300.py:35:101: E501 Line too long (103 > 100) +examples/qwen21_turbo_b300.py:44:101: E501 Line too long (108 > 100) +examples/qwen21_turbo_b300.py:50:101: E501 Line too long (113 > 100) +examples/qwen21_turbo_b300.py:58:101: E501 Line too long (127 > 100) +examples/qwen21_turbo_b300.py:60:101: E501 Line too long (106 > 100) +examples/qwen21_turbo_b300.py:61:101: E501 Line too long (139 > 100) +examples/qwen21_turbo_b300.py:69:101: E501 Line too long (140 > 100) +examples/qwen21_turbo_b300.py:70:101: E501 Line too long (107 > 100) +examples/qwen21_turbo_b300.py:75:101: E501 Line too long (116 > 100) +examples/qwen21_turbo_b300.py:76:101: E501 Line too long (119 > 100) +examples/qwen21_turbo_b300.py:88:101: E501 Line too long (101 > 100) +examples/qwen21_turbo_b300.py:128:101: E501 Line too long (122 > 100) +examples/qwen21_turbo_b300.py:131:101: E501 Line too long (128 > 100) +examples/qwen21_turbo_b300.py:134:101: E501 Line too long (111 > 100) +examples/qwen21_turbo_b300.py:145:101: E501 Line too long (103 > 100) +examples/qwen21_turbo_b300.py:146:101: E501 Line too long (104 > 100) +examples/qwen21_turbo_b300.py:149:101: E501 Line too long (107 > 100) +examples/qwen21_turbo_b300.py:152:101: E501 Line too long (125 > 100) +examples/qwen21_turbo_b300.py:173:101: E501 Line too long (152 > 100) +examples/qwen21_turbo_b300.py:184:101: E501 Line too long (110 > 100) +examples/qwen21_turbo_b300.py:185:50: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:186:46: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:187:45: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:202:17: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_b300.py:241:101: E501 Line too long (101 > 100) +examples/qwen21_turbo_b300.py:268:26: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:2:1: I001 [*] Import block is un-sorted or un-formatted +examples/qwen21_turbo_quality.py:7:8: F401 [*] `sys` imported but unused +examples/qwen21_turbo_quality.py:14:101: E501 Line too long (123 > 100) +examples/qwen21_turbo_quality.py:39:5: I001 [*] Import block is un-sorted or un-formatted +examples/qwen21_turbo_quality.py:44:60: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:58:101: E501 Line too long (116 > 100) +examples/qwen21_turbo_quality.py:59:101: E501 Line too long (119 > 100) +examples/qwen21_turbo_quality.py:70:101: E501 Line too long (102 > 100) +examples/qwen21_turbo_quality.py:74:101: E501 Line too long (126 > 100) +examples/qwen21_turbo_quality.py:75:33: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:76:101: E501 Line too long (101 > 100) +examples/qwen21_turbo_quality.py:85:54: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:86:57: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:95:47: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:98:101: E501 Line too long (109 > 100) +examples/qwen21_turbo_quality.py:111:28: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:115:78: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:115:101: E501 Line too long (114 > 100) +examples/qwen21_turbo_quality.py:129:101: E501 Line too long (102 > 100) +examples/qwen21_turbo_quality.py:132:37: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:133:13: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:135:101: E501 Line too long (130 > 100) +examples/qwen21_turbo_quality.py:136:101: E501 Line too long (142 > 100) +examples/qwen21_turbo_quality.py:137:30: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_quality.py:142:24: E701 Multiple statements on one line (colon) +examples/qwen21_turbo_validation_adapter.py:8:1: I001 [*] Import block is un-sorted or un-formatted +Found 372 errors. +[*] 27 fixable with `--fix` (3 unsafe fixes available with `--fix --unsafe-fixes`). diff --git a/evidence/indexer-255-20261029/results/focused-tests.txt b/evidence/indexer-255-20261029/results/focused-tests.txt new file mode 100644 index 000000000..7f9e8857d --- /dev/null +++ b/evidence/indexer-255-20261029/results/focused-tests.txt @@ -0,0 +1,2 @@ +................................................ss.............. [100%] +62 passed, 2 skipped in 17.88s diff --git a/evidence/indexer-255-20261029/results/full-suite-initial.txt b/evidence/indexer-255-20261029/results/full-suite-initial.txt new file mode 100644 index 000000000..df3f10536 --- /dev/null +++ b/evidence/indexer-255-20261029/results/full-suite-initial.txt @@ -0,0 +1,270 @@ +..................................................................... [ 2%] +................................................................ [ 5%] +........................................................................ [ 8%] +.........F.............................................................. [ 10%] +........................................................................ [ 13%] +..........................................ssss............... [ 16%] +...........................sFFFFF....................................... [ 19%] +........................................................................ [ 21%] +................................................ss.............................................................................................................................. [ 28%] +......................................................................................................... [ 33%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 44%] +........................................................................ [ 47%] +........................................................................ [ 50%] +........................................................................ [ 53%] +......................................................................F. [ 55%] +........................................................................ [ 58%] +.............sss............................................................................FF................... [ 63%] +........................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +............................................. [ 72%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 81%] +........................................................................ [ 84%] +............................................... [ 86%] +............................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 94%] +........................................................................ [ 97%] +....................................................................... [100%] +=================================== FAILURES =================================== +__________________________ test_checked_in_inventory ___________________________ + + def test_checked_in_inventory(): + repo = Path(__file__).resolve().parents[1] + result = validate_inventory(repo) +> assert result["errors"] == [] +E AssertionError: assert ['inventory d...255-20261029'] == [] +E +E Left contains one more item: 'inventory directories differ: indexer-255-20261029' +E Use -v to get more diff + +tests/test_evidence_inventory.py:147: AssertionError +________________ test_package_only_commands_work_without_pyyaml ________________ + +without_yaml = PosixPath('/tmp/pytest-of-agent/pytest-1/test_package_only_commands_wor0/blocked') + + def test_package_only_commands_work_without_pyyaml(without_yaml): + env = _cli_env(without_yaml) + listed = _run(["collect", "--limit", "1", "--json"], env) +> assert listed.returncode == 0, listed.stderr +E AssertionError: /workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python: No module named inferrsi +E +E assert 1 == 0 +E + where 1 = CompletedProcess(args=['/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python', '-m', 'inferrsi', 'collect', '--lim...returncode=1, stdout='', stderr='/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python: No module named inferrsi\n').returncode + +tests/test_knowledge_install.py:54: AssertionError +____________ test_skill_root_without_pyyaml_fails_with_install_hint ____________ + +without_yaml = PosixPath('/tmp/pytest-of-agent/pytest-1/test_skill_root_without_pyyaml0/blocked') + + def test_skill_root_without_pyyaml_fails_with_install_hint(without_yaml): + result = _run( + ["collect", "--id", f"checkout:{SOURCE}", "--skill-root", str(SKILLS)], + _cli_env(without_yaml), + ) +> assert result.returncode == 2 +E AssertionError: assert 1 == 2 +E + where 1 = CompletedProcess(args=['/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python', '-m', 'inferrsi', 'collect', '--id'...returncode=1, stdout='', stderr='/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python: No module named inferrsi\n').returncode + +tests/test_knowledge_install.py:67: AssertionError +____________ test_install_hint_is_actionable_outside_the_repository ____________ + +without_yaml = PosixPath('/tmp/pytest-of-agent/pytest-1/test_install_hint_is_actionabl0/blocked') +tmp_path = PosixPath('/tmp/pytest-of-agent/pytest-1/test_install_hint_is_actionabl0') + + def test_install_hint_is_actionable_outside_the_repository(without_yaml, tmp_path): + # The hint must not depend on the caller's working directory. + result = _run( + ["collect", "--id", f"checkout:{SOURCE}", "--skill-root", str(SKILLS)], + _cli_env(without_yaml), + cwd=tmp_path, + ) +> assert result.returncode == 2 +E AssertionError: assert 1 == 2 +E + where 1 = CompletedProcess(args=['/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python', '-m', 'inferrsi', 'collect', '--id'...returncode=1, stdout='', stderr='/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python: No module named inferrsi\n').returncode + +tests/test_knowledge_install.py:80: AssertionError +__________ test_checkout_chain_collect_read_resource_through_real_cli __________ + +tmp_path = PosixPath('/tmp/pytest-of-agent/pytest-1/test_checkout_chain_collect_re0') + + def test_checkout_chain_collect_read_resource_through_real_cli(tmp_path): + env = _cli_env() + collected = _run( + ["collect", "--id", f"checkout:{SOURCE}", "--skill-root", str(SKILLS), "--json"], env + ) +> assert collected.returncode == 0, collected.stderr +E AssertionError: /workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python: No module named inferrsi +E +E assert 1 == 0 +E + where 1 = CompletedProcess(args=['/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python', '-m', 'inferrsi', 'collect', '--id'...returncode=1, stdout='', stderr='/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python: No module named inferrsi\n').returncode + +tests/test_knowledge_install.py:90: AssertionError +____________ test_corrupted_store_database_fails_without_traceback _____________ + +tmp_path = PosixPath('/tmp/pytest-of-agent/pytest-1/test_corrupted_store_database_0') + + def test_corrupted_store_database_fails_without_traceback(tmp_path): + workspace = tmp_path / "broken-store" + workspace.mkdir() + (workspace / "knowledge.sqlite3").write_bytes(b"this is not a database") + result = _run(["read", "some-id", "--store", str(workspace)], _cli_env()) +> assert result.returncode == 2 +E AssertionError: assert 1 == 2 +E + where 1 = CompletedProcess(args=['/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python', '-m', 'inferrsi', 'read', 'some-id'...returncode=1, stdout='', stderr='/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python: No module named inferrsi\n').returncode + +tests/test_knowledge_install.py:125: AssertionError +_________ test_control_imports_need_no_inference_or_image_dependencies _________ + + def test_control_imports_need_no_inference_or_image_dependencies(): + result = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import inferrsi.cli; " + "import inferrsi.model.validators.image; " + "assert not {'torch', 'vllm', 'numpy', 'PIL', 'skimage'}.intersection(sys.modules)", + ], + check=False, + capture_output=True, + text=True, + ) +> assert result.returncode == 0, result.stderr +E AssertionError: Traceback (most recent call last): +E File "", line 1, in +E ModuleNotFoundError: No module named 'inferrsi' +E +E assert 1 == 0 +E + where 1 = CompletedProcess(args=['/workspace/scratch/6ddc7a7a8db4/issue255-venv/bin/python', '-c', "import sys; import inferrsi....(most recent call last):\n File "", line 1, in \nModuleNotFoundError: No module named \'inferrsi\'\n').returncode + +tests/test_module_boundaries.py:145: AssertionError +______ test_cleanup_after_leader_exit_preserves_unrelated_process[False] _______ + +tmp_path = PosixPath('/tmp/pytest-of-agent/pytest-1/test_cleanup_after_leader_exit0') +supervised = False + + @pytest.mark.skipif(sys.platform != "linux", reason="Linux process ownership contract") + @pytest.mark.parametrize("supervised", [False, True]) + def test_cleanup_after_leader_exit_preserves_unrelated_process(tmp_path, supervised): + server = tmp_path / "server.py" + server.write_text( + "import pathlib, subprocess, sys, time\n" + "child = subprocess.Popen([sys.executable, '-c', " + "'import os, pathlib, signal, sys, time; '" + "'signal.signal(signal.SIGTERM, signal.SIG_IGN); '" + "'pathlib.Path(sys.argv[1]).write_text(str(os.getpid())); '" + "'time.sleep(60)', sys.argv[1]])\n" + "while not pathlib.Path(sys.argv[1]).exists(): time.sleep(.01)\n", + encoding="utf-8", + ) + # The helper runs in its own group, as it would inside the trial executor. + helper = """ + import json, os, sys + from inferrsi.infra.processes import launch_server, stop_server + with open(sys.argv[3], 'wb') as log: + process, owner = launch_server([sys.executable, sys.argv[1], sys.argv[2]], + '.', os.environ.copy(), log) + print(json.dumps(owner), flush=True) + process.wait(timeout=10) + cleanup = stop_server(process, owner, grace_seconds=.1) + print(json.dumps(cleanup), flush=True) + """ + environment = os.environ.copy() + environment.pop("INFERRSI_RESULT_PATH", None) + if supervised: + environment["INFERRSI_RESULT_PATH"] = "synthetic-result" + unrelated = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + worker_file = tmp_path / "worker.pid" + runner = subprocess.Popen( + [sys.executable, "-c", helper, str(server), str(worker_file), str(tmp_path / "log")], + env=environment, start_new_session=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + owner = None + try: + stdout, stderr = runner.communicate(timeout=15) + if stdout: + owner = json.loads(stdout.splitlines()[0]) +> assert runner.returncode == 0, stderr +E AssertionError: Traceback (most recent call last): +E File "", line 3, in +E ModuleNotFoundError: No module named 'inferrsi' +E +E assert 1 == 0 +E + where 1 = .returncode + +tests/test_server_processes.py:95: AssertionError +_______ test_cleanup_after_leader_exit_preserves_unrelated_process[True] _______ + +tmp_path = PosixPath('/tmp/pytest-of-agent/pytest-1/test_cleanup_after_leader_exit1') +supervised = True + + @pytest.mark.skipif(sys.platform != "linux", reason="Linux process ownership contract") + @pytest.mark.parametrize("supervised", [False, True]) + def test_cleanup_after_leader_exit_preserves_unrelated_process(tmp_path, supervised): + server = tmp_path / "server.py" + server.write_text( + "import pathlib, subprocess, sys, time\n" + "child = subprocess.Popen([sys.executable, '-c', " + "'import os, pathlib, signal, sys, time; '" + "'signal.signal(signal.SIGTERM, signal.SIG_IGN); '" + "'pathlib.Path(sys.argv[1]).write_text(str(os.getpid())); '" + "'time.sleep(60)', sys.argv[1]])\n" + "while not pathlib.Path(sys.argv[1]).exists(): time.sleep(.01)\n", + encoding="utf-8", + ) + # The helper runs in its own group, as it would inside the trial executor. + helper = """ + import json, os, sys + from inferrsi.infra.processes import launch_server, stop_server + with open(sys.argv[3], 'wb') as log: + process, owner = launch_server([sys.executable, sys.argv[1], sys.argv[2]], + '.', os.environ.copy(), log) + print(json.dumps(owner), flush=True) + process.wait(timeout=10) + cleanup = stop_server(process, owner, grace_seconds=.1) + print(json.dumps(cleanup), flush=True) + """ + environment = os.environ.copy() + environment.pop("INFERRSI_RESULT_PATH", None) + if supervised: + environment["INFERRSI_RESULT_PATH"] = "synthetic-result" + unrelated = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + worker_file = tmp_path / "worker.pid" + runner = subprocess.Popen( + [sys.executable, "-c", helper, str(server), str(worker_file), str(tmp_path / "log")], + env=environment, start_new_session=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + owner = None + try: + stdout, stderr = runner.communicate(timeout=15) + if stdout: + owner = json.loads(stdout.splitlines()[0]) +> assert runner.returncode == 0, stderr +E AssertionError: Traceback (most recent call last): +E File "", line 3, in +E ModuleNotFoundError: No module named 'inferrsi' +E +E assert 1 == 0 +E + where 1 = .returncode + +tests/test_server_processes.py:95: AssertionError +=============================== warnings summary =============================== +tests/test_skill_eval_retained.py::RetainedArchiveTests::test_duplicate_and_oversize + /opt/codex/runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/zipfile/__init__.py:1625: UserWarning: Duplicate name: 'one' + return self._open_to_write(zinfo, force_zip64=force_zip64) + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +=========================== short test summary info ============================ +SKIPPED [4] tests/test_image_modules.py:102: could not import 'numpy': No module named 'numpy' +SKIPPED [1] tests/test_knowledge_fresh_install.py:69: opt-in fresh-venv smoke; set RUN_FRESH_VENV=1 to build a venv and install this tree +SKIPPED [2] tests/test_knowledge_operations.py:277: NTFS junctions are Windows-only +SKIPPED [3] tests/test_qwen_quality_compatibility.py:22: could not import 'numpy': No module named 'numpy' +9 failed, 2504 passed, 10 skipped, 1 warning, 284 subtests passed in 583.02s (0:09:43) diff --git a/evidence/indexer-255-20261029/results/import-proof-baseline.json b/evidence/indexer-255-20261029/results/import-proof-baseline.json new file mode 100644 index 000000000..e20604690 --- /dev/null +++ b/evidence/indexer-255-20261029/results/import-proof-baseline.json @@ -0,0 +1,40 @@ +{ + "all_modules_in_declared_arm": true, + "modules": { + "inferrsi": "src/inferrsi/__init__.py", + "inferrsi._skill_selection": "src/inferrsi/_skill_selection.py", + "inferrsi.arguments": "src/inferrsi/arguments.py", + "inferrsi.catalog": "src/inferrsi/catalog.py", + "inferrsi.claim": "src/inferrsi/claim.py", + "inferrsi.cli": "src/inferrsi/cli.py", + "inferrsi.execution": "src/inferrsi/execution.py", + "inferrsi.harness": "src/inferrsi/harness/__init__.py", + "inferrsi.harness.evaluation": "src/inferrsi/harness/evaluation.py", + "inferrsi.harness.reporting": "src/inferrsi/harness/reporting.py", + "inferrsi.harness.runner": "src/inferrsi/harness/runner.py", + "inferrsi.harness.scenarios": "src/inferrsi/harness/scenarios/__init__.py", + "inferrsi.harness.scenarios.external": "src/inferrsi/harness/scenarios/external.py", + "inferrsi.infra": "src/inferrsi/infra/__init__.py", + "inferrsi.infra.artifacts": "src/inferrsi/infra/artifacts.py", + "inferrsi.infra.commands": "src/inferrsi/infra/commands.py", + "inferrsi.infra.executors": "src/inferrsi/infra/executors/__init__.py", + "inferrsi.infra.executors.local": "src/inferrsi/infra/executors/local.py", + "inferrsi.infra.inspection": "src/inferrsi/infra/inspection.py", + "inferrsi.infra.platforms": "src/inferrsi/infra/platforms/__init__.py", + "inferrsi.infra.platforms.cuda": "src/inferrsi/infra/platforms/cuda.py", + "inferrsi.infra.platforms.mlx": "src/inferrsi/infra/platforms/mlx.py", + "inferrsi.infra.processes": "src/inferrsi/infra/processes.py", + "inferrsi.infra.targets": "src/inferrsi/infra/targets/__init__.py", + "inferrsi.infra.targets.git": "src/inferrsi/infra/targets/git.py", + "inferrsi.inspection": "src/inferrsi/inspection.py", + "inferrsi.knowledge_operations": "src/inferrsi/knowledge_operations.py", + "inferrsi.knowledge_store": "src/inferrsi/knowledge_store.py", + "inferrsi.model": "src/inferrsi/model/__init__.py", + "inferrsi.model.protocols": "src/inferrsi/model/protocols/__init__.py", + "inferrsi.model.protocols.metrics": "src/inferrsi/model/protocols/metrics.py", + "inferrsi.reporting": "src/inferrsi/reporting.py", + "inferrsi.research": "src/inferrsi/research.py", + "inferrsi.runspec": "src/inferrsi/runspec.py", + "inferrsi.summary_chart": "src/inferrsi/summary_chart.py" + } +} diff --git a/evidence/indexer-255-20261029/results/import-proof-indexer-evidence.json b/evidence/indexer-255-20261029/results/import-proof-indexer-evidence.json new file mode 100644 index 000000000..e20604690 --- /dev/null +++ b/evidence/indexer-255-20261029/results/import-proof-indexer-evidence.json @@ -0,0 +1,40 @@ +{ + "all_modules_in_declared_arm": true, + "modules": { + "inferrsi": "src/inferrsi/__init__.py", + "inferrsi._skill_selection": "src/inferrsi/_skill_selection.py", + "inferrsi.arguments": "src/inferrsi/arguments.py", + "inferrsi.catalog": "src/inferrsi/catalog.py", + "inferrsi.claim": "src/inferrsi/claim.py", + "inferrsi.cli": "src/inferrsi/cli.py", + "inferrsi.execution": "src/inferrsi/execution.py", + "inferrsi.harness": "src/inferrsi/harness/__init__.py", + "inferrsi.harness.evaluation": "src/inferrsi/harness/evaluation.py", + "inferrsi.harness.reporting": "src/inferrsi/harness/reporting.py", + "inferrsi.harness.runner": "src/inferrsi/harness/runner.py", + "inferrsi.harness.scenarios": "src/inferrsi/harness/scenarios/__init__.py", + "inferrsi.harness.scenarios.external": "src/inferrsi/harness/scenarios/external.py", + "inferrsi.infra": "src/inferrsi/infra/__init__.py", + "inferrsi.infra.artifacts": "src/inferrsi/infra/artifacts.py", + "inferrsi.infra.commands": "src/inferrsi/infra/commands.py", + "inferrsi.infra.executors": "src/inferrsi/infra/executors/__init__.py", + "inferrsi.infra.executors.local": "src/inferrsi/infra/executors/local.py", + "inferrsi.infra.inspection": "src/inferrsi/infra/inspection.py", + "inferrsi.infra.platforms": "src/inferrsi/infra/platforms/__init__.py", + "inferrsi.infra.platforms.cuda": "src/inferrsi/infra/platforms/cuda.py", + "inferrsi.infra.platforms.mlx": "src/inferrsi/infra/platforms/mlx.py", + "inferrsi.infra.processes": "src/inferrsi/infra/processes.py", + "inferrsi.infra.targets": "src/inferrsi/infra/targets/__init__.py", + "inferrsi.infra.targets.git": "src/inferrsi/infra/targets/git.py", + "inferrsi.inspection": "src/inferrsi/inspection.py", + "inferrsi.knowledge_operations": "src/inferrsi/knowledge_operations.py", + "inferrsi.knowledge_store": "src/inferrsi/knowledge_store.py", + "inferrsi.model": "src/inferrsi/model/__init__.py", + "inferrsi.model.protocols": "src/inferrsi/model/protocols/__init__.py", + "inferrsi.model.protocols.metrics": "src/inferrsi/model/protocols/metrics.py", + "inferrsi.reporting": "src/inferrsi/reporting.py", + "inferrsi.research": "src/inferrsi/research.py", + "inferrsi.runspec": "src/inferrsi/runspec.py", + "inferrsi.summary_chart": "src/inferrsi/summary_chart.py" + } +} diff --git a/evidence/indexer-255-20261029/results/paired-run/A-mandatory-audit.json b/evidence/indexer-255-20261029/results/paired-run/A-mandatory-audit.json new file mode 100644 index 000000000..b46aa190b --- /dev/null +++ b/evidence/indexer-255-20261029/results/paired-run/A-mandatory-audit.json @@ -0,0 +1,781 @@ +{ + "arm": "A", + "scope": "all emitted contexts for 8 dev and 4 frozen heldout queries; absent relevant retrieval is not counted as delivery success", + "checks": [ + { + "case": "heldout01", + "ref": "package:inferrsi.ascend.skills.torchnpu_pinned_host_memory_lifecycle", + "required": [ + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/SKILL.md", + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/references/source-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.common.skills.training_resource_budget", + "required": [ + "data/skill_library/methods/training-resource-budget/SKILL.md", + "data/skill_library/methods/training-resource-budget/references/mindspeed-historical.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/cases/paper-grounded-cases.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/knowledge/offload-delta.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/notes/training-memory-scheduling.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/references/source-readset.md", + "data/skill_library/methods/training-resource-budget/references/sources.md", + "data/skill_library/methods/training-resource-budget/references/stas-decision-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.cuda.skills.audit_flare_session_lifecycle", + "required": [ + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/SKILL.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-notes.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-ledger.json", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.common.skills.audit_vllm_kv_lifetimes", + "required": [ + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/SKILL.md", + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/references/source-notes.md", + "data/skill_library/repositories/vllm/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.common.skills.dcp_async_save_performance", + "required": [ + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md", + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout02", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout02", + "ref": "package:inferrsi.common.skills.repository_evolution_analysis", + "required": [ + "data/skill_library/methods/repository-evolution-analysis/SKILL.md", + "data/skill_library/methods/repository-evolution-analysis/references/rtk-consumer-path-transition.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout02", + "ref": "package:inferrsi.common.skills.performance_ceiling_analysis", + "required": [ + "data/skill_library/methods/performance-ceiling-analysis/SKILL.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/cases/decision-cases.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/govern-optimization-experiments/procedure.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/govern-optimization-experiments/references/source-notes.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/knowledge/bounds-predictions-and-decisions.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/MECHANISMS.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/README.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/READSET.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/redirect-cross-layer-optimization/procedure.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/redirect-cross-layer-optimization/references/source-notes.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/agentic-kernel.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/flashvector.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/inferencebench.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/prior-coverage.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/vidur.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/STATIC_CHECKS.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.dcp_async_save_performance", + "required": [ + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md", + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.checkpoint_recovery_consistency", + "required": [ + "data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md", + "data/skill_library/methods/checkpoint-recovery-consistency/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.training_resource_budget", + "required": [ + "data/skill_library/methods/training-resource-budget/SKILL.md", + "data/skill_library/methods/training-resource-budget/references/mindspeed-historical.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/cases/paper-grounded-cases.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/knowledge/offload-delta.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/notes/training-memory-scheduling.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/references/source-readset.md", + "data/skill_library/methods/training-resource-budget/references/sources.md", + "data/skill_library/methods/training-resource-budget/references/stas-decision-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.kv_tier_management", + "required": [ + "data/skill_library/repositories/lmcache/kv-tier-management/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.audit_sglang_diffusion_lifetimes", + "required": [ + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/SKILL.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/references/source-notes.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/LICENSE", + "data/skill_library/repositories/sglang/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.video_diffusion_optimization", + "required": [ + "data/skill_library/methods/video-diffusion-optimization/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.ascend.skills.ascend_video_diffusion_optimization", + "required": [ + "data/skill_library/hardware/ascend/ascend-video-diffusion-optimization/SKILL.md", + "data/skill_library/methods/video-diffusion-optimization/SKILL.md", + "data/skill_library/methods/video-diffusion-optimization/scripts/evaluate_target.py", + "data/skill_library/methods/video-diffusion-optimization/assets/campaign.template.json", + "data/skill_library/hardware/ascend/ascend-video-diffusion-optimization/references/field-notes-video-diffusion.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.generation_cache_lifecycle", + "required": [ + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/SKILL.md", + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/references/source-map.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.cgroup_resource_controls", + "required": [ + "data/skill_library/hardware/host/cgroup-resource-controls/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.host_overhead_optimization", + "required": [ + "data/skill_library/hardware/host/host-overhead-optimization/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.os_process_placement", + "required": [ + "data/skill_library/hardware/host/os-process-placement/SKILL.md", + "data/skill_library/hardware/host/os-process-placement/references/llama-numa-thread-readset.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.project_maya", + "required": [ + "data/skill_library/repositories/project-maya/SKILL.md", + "data/skill_library/repositories/project-maya/references/repo-provenance.md", + "data/skill_library/repositories/project-maya/references/source-identity.json", + "data/skill_library/repositories/project-maya/LICENSE", + "data/skill_library/repositories/project-maya/sub-skills/model-runtime/references/notices.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.training_resource_budget", + "required": [ + "data/skill_library/methods/training-resource-budget/SKILL.md", + "data/skill_library/methods/training-resource-budget/references/mindspeed-historical.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/cases/paper-grounded-cases.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/knowledge/offload-delta.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/notes/training-memory-scheduling.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/references/source-readset.md", + "data/skill_library/methods/training-resource-budget/references/sources.md", + "data/skill_library/methods/training-resource-budget/references/stas-decision-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.audit_vllm_kv_lifetimes", + "required": [ + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/SKILL.md", + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/references/source-notes.md", + "data/skill_library/repositories/vllm/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.audit_recurrent_kv_finalization", + "required": [ + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/SKILL.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-notes.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-identity.json", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/NOTICE.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.cuda.skills.audit_flare_session_lifecycle", + "required": [ + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/SKILL.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-notes.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-ledger.json", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.kv_tier_management", + "required": [ + "data/skill_library/repositories/lmcache/kv-tier-management/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.bound_windowed_attention_replay", + "required": [ + "data/skill_library/repositories/deepinfer/bound-windowed-attention-replay/SKILL.md", + "data/skill_library/repositories/deepinfer/bound-windowed-attention-replay/references/source-notes.md", + "data/skill_library/repositories/deepinfer/bound-windowed-attention-replay/references/validation.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.checkpoint_recovery_consistency", + "required": [ + "data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md", + "data/skill_library/methods/checkpoint-recovery-consistency/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.dcp_async_save_performance", + "required": [ + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md", + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.ascend.skills.ascend_model_adaptation", + "required": [ + "data/skill_library/hardware/ascend/ascend-model-adaptation/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.audit_h3_metal_fasth3_semantics", + "required": [ + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/SKILL.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-notes.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-identity.json", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/repo-provenance.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/THIRD_PARTY_NOTICES.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.checkpoint_recovery_consistency", + "required": [ + "data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md", + "data/skill_library/methods/checkpoint-recovery-consistency/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.audit_sd_cpp_residency", + "required": [ + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/SKILL.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/source-notes.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/loader-storage-boundaries.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.kv_tier_management", + "required": [ + "data/skill_library/repositories/lmcache/kv-tier-management/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.ascend.skills.torchnpu_pinned_host_memory_lifecycle", + "required": [ + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/SKILL.md", + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/references/source-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_device_health", + "required": [ + "data/skill_library/hardware/ascend/ascend-device-health/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.torchnpu_device_memory_lifecycle", + "required": [ + "data/skill_library/hardware/ascend/torchnpu-device-memory-lifecycle/SKILL.md", + "data/skill_library/hardware/ascend/torchnpu-device-memory-lifecycle/references/source-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_operator_profiling", + "required": [ + "data/skill_library/hardware/ascend/ascend-operator-profiling/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_operator_tuning", + "required": [ + "data/skill_library/hardware/ascend/ascend-operator-tuning/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_distributed_parallel", + "required": [ + "data/skill_library/hardware/ascend/ascend-distributed-parallel/SKILL.md", + "data/skill_library/hardware/ascend/ascend-distributed-parallel/references/awesome-ascend-communication-tests.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_h3_runtime_residency", + "required": [ + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/SKILL.md", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/source-notes.md", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/source-identity.json", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/minimax-h3-weight-inventory.json", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/THIRD_PARTY_NOTICES.md", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_h3_metal_fasth3_semantics", + "required": [ + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/SKILL.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-notes.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-identity.json", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/repo-provenance.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/THIRD_PARTY_NOTICES.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_sd_cpp_residency", + "required": [ + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/SKILL.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/source-notes.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/loader-storage-boundaries.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_sglang_diffusion_lifetimes", + "required": [ + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/SKILL.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/references/source-notes.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/LICENSE", + "data/skill_library/repositories/sglang/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.comfy_ui", + "required": [ + "data/skill_library/repositories/comfy-ui/SKILL.md", + "data/skill_library/repositories/comfy-ui/references/repo-provenance.md", + "data/skill_library/repositories/comfy-ui/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.audit_fixed_workload_claim", + "required": [ + "data/skill_library/methods/audit-fixed-workload-claim/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.storage_io_measurement", + "required": [ + "data/skill_library/hardware/host/storage-io-measurement/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.dense_performance_evidence", + "required": [ + "data/skill_library/methods/dense-performance-evidence/SKILL.md", + "data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.md", + "data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.husky", + "required": [ + "data/skill_library/repositories/husky/SKILL.md", + "data/skill_library/repositories/husky/references/repo-provenance.md", + "data/skill_library/repositories/husky/references/source-availability.json" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.claim_contract", + "required": [ + "data/skill_library/workflows/claim-contract/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.storage_io_measurement", + "required": [ + "data/skill_library/hardware/host/storage-io-measurement/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.generation_cache_lifecycle", + "required": [ + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/SKILL.md", + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/references/source-map.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.cuda.skills.audit_flare_session_lifecycle", + "required": [ + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/SKILL.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-notes.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-ledger.json", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.audit_recurrent_kv_finalization", + "required": [ + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/SKILL.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-notes.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-identity.json", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/NOTICE.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + } + ], + "passed": true +} diff --git a/evidence/indexer-255-20261029/results/paired-run/B-mandatory-audit.json b/evidence/indexer-255-20261029/results/paired-run/B-mandatory-audit.json new file mode 100644 index 000000000..24ac6691f --- /dev/null +++ b/evidence/indexer-255-20261029/results/paired-run/B-mandatory-audit.json @@ -0,0 +1,781 @@ +{ + "arm": "B", + "scope": "all emitted contexts for 8 dev and 4 frozen heldout queries; absent relevant retrieval is not counted as delivery success", + "checks": [ + { + "case": "heldout01", + "ref": "package:inferrsi.ascend.skills.torchnpu_pinned_host_memory_lifecycle", + "required": [ + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/SKILL.md", + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/references/source-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.common.skills.training_resource_budget", + "required": [ + "data/skill_library/methods/training-resource-budget/SKILL.md", + "data/skill_library/methods/training-resource-budget/references/mindspeed-historical.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/cases/paper-grounded-cases.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/knowledge/offload-delta.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/notes/training-memory-scheduling.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/references/source-readset.md", + "data/skill_library/methods/training-resource-budget/references/sources.md", + "data/skill_library/methods/training-resource-budget/references/stas-decision-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.cuda.skills.audit_flare_session_lifecycle", + "required": [ + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/SKILL.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-notes.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-ledger.json", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.common.skills.audit_vllm_kv_lifetimes", + "required": [ + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/SKILL.md", + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/references/source-notes.md", + "data/skill_library/repositories/vllm/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout01", + "ref": "package:inferrsi.common.skills.dcp_async_save_performance", + "required": [ + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md", + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout02", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout02", + "ref": "package:inferrsi.common.skills.repository_evolution_analysis", + "required": [ + "data/skill_library/methods/repository-evolution-analysis/SKILL.md", + "data/skill_library/methods/repository-evolution-analysis/references/rtk-consumer-path-transition.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout02", + "ref": "package:inferrsi.common.skills.performance_ceiling_analysis", + "required": [ + "data/skill_library/methods/performance-ceiling-analysis/SKILL.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/cases/decision-cases.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/govern-optimization-experiments/procedure.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/govern-optimization-experiments/references/source-notes.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/knowledge/bounds-predictions-and-decisions.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/MECHANISMS.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/README.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/READSET.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/redirect-cross-layer-optimization/procedure.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/redirect-cross-layer-optimization/references/source-notes.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/agentic-kernel.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/flashvector.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/inferencebench.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/prior-coverage.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/references/vidur.md", + "data/skill_library/methods/performance-ceiling-analysis/references/papers-20261015/STATIC_CHECKS.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.dcp_async_save_performance", + "required": [ + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md", + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.checkpoint_recovery_consistency", + "required": [ + "data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md", + "data/skill_library/methods/checkpoint-recovery-consistency/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.training_resource_budget", + "required": [ + "data/skill_library/methods/training-resource-budget/SKILL.md", + "data/skill_library/methods/training-resource-budget/references/mindspeed-historical.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/cases/paper-grounded-cases.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/knowledge/offload-delta.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/notes/training-memory-scheduling.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/references/source-readset.md", + "data/skill_library/methods/training-resource-budget/references/sources.md", + "data/skill_library/methods/training-resource-budget/references/stas-decision-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout03", + "ref": "package:inferrsi.common.skills.kv_tier_management", + "required": [ + "data/skill_library/repositories/lmcache/kv-tier-management/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.audit_sglang_diffusion_lifetimes", + "required": [ + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/SKILL.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/references/source-notes.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/LICENSE", + "data/skill_library/repositories/sglang/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.video_diffusion_optimization", + "required": [ + "data/skill_library/methods/video-diffusion-optimization/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.ascend.skills.ascend_video_diffusion_optimization", + "required": [ + "data/skill_library/hardware/ascend/ascend-video-diffusion-optimization/SKILL.md", + "data/skill_library/methods/video-diffusion-optimization/SKILL.md", + "data/skill_library/methods/video-diffusion-optimization/scripts/evaluate_target.py", + "data/skill_library/methods/video-diffusion-optimization/assets/campaign.template.json", + "data/skill_library/hardware/ascend/ascend-video-diffusion-optimization/references/field-notes-video-diffusion.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.generation_cache_lifecycle", + "required": [ + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/SKILL.md", + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/references/source-map.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "heldout04", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.cgroup_resource_controls", + "required": [ + "data/skill_library/hardware/host/cgroup-resource-controls/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.host_overhead_optimization", + "required": [ + "data/skill_library/hardware/host/host-overhead-optimization/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.os_process_placement", + "required": [ + "data/skill_library/hardware/host/os-process-placement/SKILL.md", + "data/skill_library/hardware/host/os-process-placement/references/llama-numa-thread-readset.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.project_maya", + "required": [ + "data/skill_library/repositories/project-maya/SKILL.md", + "data/skill_library/repositories/project-maya/references/repo-provenance.md", + "data/skill_library/repositories/project-maya/references/source-identity.json", + "data/skill_library/repositories/project-maya/LICENSE", + "data/skill_library/repositories/project-maya/sub-skills/model-runtime/references/notices.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev01", + "ref": "package:inferrsi.common.skills.training_resource_budget", + "required": [ + "data/skill_library/methods/training-resource-budget/SKILL.md", + "data/skill_library/methods/training-resource-budget/references/mindspeed-historical.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/cases/paper-grounded-cases.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/INTEGRATION_NOTES.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/knowledge/offload-delta.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/notes/training-memory-scheduling.md", + "data/skill_library/methods/training-resource-budget/references/papers-20261015/references/source-readset.md", + "data/skill_library/methods/training-resource-budget/references/sources.md", + "data/skill_library/methods/training-resource-budget/references/stas-decision-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.audit_vllm_kv_lifetimes", + "required": [ + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/SKILL.md", + "data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/references/source-notes.md", + "data/skill_library/repositories/vllm/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.audit_recurrent_kv_finalization", + "required": [ + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/SKILL.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-notes.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-identity.json", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/NOTICE.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.cuda.skills.audit_flare_session_lifecycle", + "required": [ + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/SKILL.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-notes.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-ledger.json", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.kv_tier_management", + "required": [ + "data/skill_library/repositories/lmcache/kv-tier-management/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev02", + "ref": "package:inferrsi.common.skills.bound_windowed_attention_replay", + "required": [ + "data/skill_library/repositories/deepinfer/bound-windowed-attention-replay/SKILL.md", + "data/skill_library/repositories/deepinfer/bound-windowed-attention-replay/references/source-notes.md", + "data/skill_library/repositories/deepinfer/bound-windowed-attention-replay/references/validation.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.checkpoint_recovery_consistency", + "required": [ + "data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md", + "data/skill_library/methods/checkpoint-recovery-consistency/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.dcp_async_save_performance", + "required": [ + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md", + "data/skill_library/frameworks/pytorch/dcp-async-save-performance/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.ascend.skills.ascend_model_adaptation", + "required": [ + "data/skill_library/hardware/ascend/ascend-model-adaptation/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev03", + "ref": "package:inferrsi.common.skills.audit_h3_metal_fasth3_semantics", + "required": [ + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/SKILL.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-notes.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-identity.json", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/repo-provenance.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/THIRD_PARTY_NOTICES.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.checkpoint_recovery_consistency", + "required": [ + "data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md", + "data/skill_library/methods/checkpoint-recovery-consistency/references/sources.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.audit_sd_cpp_residency", + "required": [ + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/SKILL.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/source-notes.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/loader-storage-boundaries.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.common.skills.kv_tier_management", + "required": [ + "data/skill_library/repositories/lmcache/kv-tier-management/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev04", + "ref": "package:inferrsi.ascend.skills.torchnpu_pinned_host_memory_lifecycle", + "required": [ + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/SKILL.md", + "data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/references/source-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_device_health", + "required": [ + "data/skill_library/hardware/ascend/ascend-device-health/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.torchnpu_device_memory_lifecycle", + "required": [ + "data/skill_library/hardware/ascend/torchnpu-device-memory-lifecycle/SKILL.md", + "data/skill_library/hardware/ascend/torchnpu-device-memory-lifecycle/references/source-notes.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_operator_profiling", + "required": [ + "data/skill_library/hardware/ascend/ascend-operator-profiling/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_operator_tuning", + "required": [ + "data/skill_library/hardware/ascend/ascend-operator-tuning/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev05", + "ref": "package:inferrsi.ascend.skills.ascend_distributed_parallel", + "required": [ + "data/skill_library/hardware/ascend/ascend-distributed-parallel/SKILL.md", + "data/skill_library/hardware/ascend/ascend-distributed-parallel/references/awesome-ascend-communication-tests.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_h3_runtime_residency", + "required": [ + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/SKILL.md", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/source-notes.md", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/source-identity.json", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/minimax-h3-weight-inventory.json", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/references/THIRD_PARTY_NOTICES.md", + "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_h3_metal_fasth3_semantics", + "required": [ + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/SKILL.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-notes.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/source-identity.json", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/repo-provenance.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/references/THIRD_PARTY_NOTICES.md", + "data/skill_library/repositories/h3-metal/audit-h3-metal-fasth3-semantics/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_sd_cpp_residency", + "required": [ + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/SKILL.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/source-notes.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/references/loader-storage-boundaries.md", + "data/skill_library/repositories/stable-diffusion-cpp/audit-sd-cpp-residency/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.audit_sglang_diffusion_lifetimes", + "required": [ + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/SKILL.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/references/source-notes.md", + "data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/LICENSE", + "data/skill_library/repositories/sglang/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev06", + "ref": "package:inferrsi.common.skills.comfy_ui", + "required": [ + "data/skill_library/repositories/comfy-ui/SKILL.md", + "data/skill_library/repositories/comfy-ui/references/repo-provenance.md", + "data/skill_library/repositories/comfy-ui/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.audit_fixed_workload_claim", + "required": [ + "data/skill_library/methods/audit-fixed-workload-claim/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.storage_io_measurement", + "required": [ + "data/skill_library/hardware/host/storage-io-measurement/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.dense_performance_evidence", + "required": [ + "data/skill_library/methods/dense-performance-evidence/SKILL.md", + "data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.md", + "data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.husky", + "required": [ + "data/skill_library/repositories/husky/SKILL.md", + "data/skill_library/repositories/husky/references/repo-provenance.md", + "data/skill_library/repositories/husky/references/source-availability.json" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev07", + "ref": "package:inferrsi.common.skills.claim_contract", + "required": [ + "data/skill_library/workflows/claim-contract/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.storage_io_measurement", + "required": [ + "data/skill_library/hardware/host/storage-io-measurement/SKILL.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.generation_cache_lifecycle", + "required": [ + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/SKILL.md", + "data/skill_library/repositories/mlx-lm/generation-cache-lifecycle/references/source-map.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.weight_loading_optimization", + "required": [ + "data/skill_library/methods/weight-loading-optimization/SKILL.md", + "data/skill_library/methods/weight-loading-optimization/references/llama-loading-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/fastsafetensors-transfer-readset.md", + "data/skill_library/methods/weight-loading-optimization/references/flare-preparation-boundary.md" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.cuda.skills.audit_flare_session_lifecycle", + "required": [ + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/SKILL.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-notes.md", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/references/source-ledger.json", + "data/skill_library/repositories/flare-local-inference/audit-flare-session-lifecycle/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + }, + { + "case": "dev08", + "ref": "package:inferrsi.common.skills.audit_recurrent_kv_finalization", + "required": [ + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/SKILL.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-notes.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/source-identity.json", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/references/NOTICE.md", + "data/skill_library/repositories/vllm-rlt/audit-recurrent-kv-finalization/LICENSE" + ], + "missing": [], + "bad_bytes_or_hash": [], + "revision_match": true, + "passed": true + } + ], + "passed": true +} diff --git a/evidence/indexer-255-20261029/results/paired-run/timing.json b/evidence/indexer-255-20261029/results/paired-run/timing.json new file mode 100644 index 000000000..f9191bfbe --- /dev/null +++ b/evidence/indexer-255-20261029/results/paired-run/timing.json @@ -0,0 +1,2490 @@ +{ + "protocol_sha256": "e4964329a20e6752b8149ab270733c0bbb258d0f948bdca9e86da6fe8f028956", + "before": { + "A": { + "python": { + "sha256": "054a6a2941e6b134876e55c7fb71a2052ab76dbf33017f8d4ba235c03277cd9b", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "ab03f3ffb8f2e5bc02c304828c5c41890eef673e", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + }, + "B": { + "python": { + "sha256": "094006cbb640aaee0ebcc2835dc3549525637cecf9517ef6ff656fb8e4800106", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "6da28494891b21cdc89f632d84d763bc997b7d15", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + } + }, + "groups": [ + { + "index": 0, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 2192417853, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1468063041, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1151153499, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 739927954, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 969180958, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1123800902, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 763944166, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 26641882, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 2077478298, + "decode_ns": 3689144, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 289246891, + "decode_ns": 87519, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 439597032, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 523390992, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 534354256, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 588859963, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 701117729, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 491698701, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 536659236, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 29046385, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 628926630, + "decode_ns": 3877078, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 185635393, + "decode_ns": 102293, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 1, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 388750639, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 382002845, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 322760370, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 449472874, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 582241314, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 286907634, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 312582707, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 19704962, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 945846843, + "decode_ns": 6707026, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 303365353, + "decode_ns": 85337, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 748298517, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 850781058, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 867649323, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1342090825, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1012027667, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 963292882, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 928474399, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 18021982, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1279066458, + "decode_ns": 2392276, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 175593830, + "decode_ns": 137845, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 2, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 821361837, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 895674453, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 922983760, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 979535880, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1196176432, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1103940384, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 788518420, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 28271979, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1509367135, + "decode_ns": 7130116, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 304085435, + "decode_ns": 86240, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 409746885, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 462106434, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 435344549, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 291453428, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 451228468, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 517480226, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 247885615, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 14739956, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 853452707, + "decode_ns": 3045870, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 392055378, + "decode_ns": 87525, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 3, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 471985797, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 433568940, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 559064789, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 564392411, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 684514951, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 335471543, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 321546914, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 33652156, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 863999928, + "decode_ns": 2780769, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 289022807, + "decode_ns": 163728, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 1182463302, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1478221971, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 919835741, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1067442404, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1272839457, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 14974432311, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1997781318, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 146252837, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 3585047088, + "decode_ns": 2687278, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 230775084, + "decode_ns": 83115, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 4, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 1679977201, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1710639522, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1160995515, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 996980994, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1655787512, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1590916284, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1392895753, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 45310406, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1287715910, + "decode_ns": 8036734, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 240125247, + "decode_ns": 86915, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 478543336, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 550976003, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 595003996, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 621241438, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 850055099, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 544849923, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 576877234, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 48972492, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 705561099, + "decode_ns": 7047539, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 188462317, + "decode_ns": 83119, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 5, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 640506682, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 551109995, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 743191535, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 656427828, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 926149971, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 577084828, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 684311817, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 51893539, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 907319847, + "decode_ns": 6992417, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 222205876, + "decode_ns": 88065, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 1533416729, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1156883138, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1808626236, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1542325212, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1140502669, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1102348270, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1021266291, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 49062380, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1860112917, + "decode_ns": 13964510, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 307849565, + "decode_ns": 103277, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 6, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 1369914022, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1431003053, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1272604156, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1145326446, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1163979578, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1466459660, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 819441233, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 15098680, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1134315306, + "decode_ns": 5547826, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 189350513, + "decode_ns": 78229, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 523996129, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 568859103, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 483405391, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 716338541, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 769245384, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 526419955, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 578280703, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 31145770, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1087490973, + "decode_ns": 3984863, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 364829948, + "decode_ns": 441310, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 7, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 546500018, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 697313723, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 602837881, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 822682515, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 865195102, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 498052427, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 497891169, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 41009417, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1236139264, + "decode_ns": 13169167, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 387124374, + "decode_ns": 172273, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 1237123071, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1163255094, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1241733479, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1272011990, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1315448142, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1215546019, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1409262595, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 40646228, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1319600762, + "decode_ns": 3290115, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 245057924, + "decode_ns": 728888, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 8, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 1176517407, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1204971055, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 2729518770, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1726842507, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1663928970, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1481279649, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1227175510, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 44257073, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1857946181, + "decode_ns": 3933984, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 452442446, + "decode_ns": 89424, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 525372731, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 590541191, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 599090309, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 551886221, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 661908366, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 570437151, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 571610123, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 34702953, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1158436198, + "decode_ns": 4877331, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 224884784, + "decode_ns": 85588, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 9, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 422886190, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 445550630, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 531025871, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 436583507, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 643934774, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 500973677, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 767317485, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 26470031, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1087110252, + "decode_ns": 9743521, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 379331852, + "decode_ns": 89841, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 1467162316, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 2398593566, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1353046615, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1768296812, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1104500276, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1191180994, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1867403833, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 68667544, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 2198695721, + "decode_ns": 14218703, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 441270733, + "decode_ns": 95102, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 10, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 1870655050, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 2093382892, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 2782157453, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 2948174943, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 2662439832, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 2378695621, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 2211802143, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 79961876, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1449726223, + "decode_ns": 8709056, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 301365887, + "decode_ns": 82391, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 1267837426, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 900120825, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 823415833, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 835429410, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1010095086, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 780453350, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 685492131, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 71678475, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 2634816330, + "decode_ns": 13924917, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 1757819000, + "decode_ns": 108670, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 11, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 795251399, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 478645470, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 376483631, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 390388051, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 527214133, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 434408686, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 531486392, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 46328734, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 865661052, + "decode_ns": 5683540, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 170352593, + "decode_ns": 80848, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 1370247385, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1368990479, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1206625076, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 860656321, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 992198628, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1028099878, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 826283107, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 23421590, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1655936385, + "decode_ns": 6757343, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 450213823, + "decode_ns": 89291, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 12, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 805735896, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 817264095, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 909236860, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 760945355, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1176050878, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1096185573, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 940065805, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 45227062, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1692721104, + "decode_ns": 4480498, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 332884651, + "decode_ns": 151371, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 315729425, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 465246935, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 765713844, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 569681597, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 590418023, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1045363687, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 516081625, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 31337960, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1220975998, + "decode_ns": 6708652, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 335322363, + "decode_ns": 90566, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 13, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 476756128, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 493259976, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 464923356, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 505140506, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 664407385, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 441037994, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 481151445, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 51119512, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1104708449, + "decode_ns": 6888724, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 331125104, + "decode_ns": 153322, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 2006328124, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 2198783178, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1467044131, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 2321435239, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 2015165906, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1902433827, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1838318176, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 53133167, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1394415009, + "decode_ns": 10113776, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 183201932, + "decode_ns": 1584400, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 14, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 2146299652, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1455925556, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 979427033, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1149377360, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1496113779, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1470672541, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1137388953, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 31520633, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1202675171, + "decode_ns": 5823060, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 433263401, + "decode_ns": 92486, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 300817020, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 315442397, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 264939265, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 475040589, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1229131219, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 512093196, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 667173231, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 44995023, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 831905195, + "decode_ns": 14361857, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 251436945, + "decode_ns": 86255, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 15, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 451027555, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 551270062, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 568391056, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 306733609, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 417888397, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 289850027, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 319144075, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 17515873, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 778852261, + "decode_ns": 10799779, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 152152218, + "decode_ns": 86920, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 2085766683, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 2552013458, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1938502350, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1515073595, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1120990169, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 866589319, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1113078708, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 44898243, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1680160614, + "decode_ns": 7419006, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 289781553, + "decode_ns": 85153, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 16, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 899843228, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1326899438, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 860982360, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1021646101, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1190671767, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1736705825, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1023115559, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 27327551, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1300236714, + "decode_ns": 2541207, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 145561722, + "decode_ns": 177307, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 649420849, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 749527169, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1018328634, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 812103832, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1263894106, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1732235805, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 2606079582, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 170544781, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 701612092, + "decode_ns": 9892347, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 687563553, + "decode_ns": 125685, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 17, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 831760450, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 986717195, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 814528625, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 995251174, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1256382055, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 763969374, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 792784530, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 58492453, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1435456524, + "decode_ns": 14205948, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 182736790, + "decode_ns": 81317, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 1032387266, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1186279019, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 903008816, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1151972171, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1569874020, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1073648063, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 918375295, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 17670736, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1396316742, + "decode_ns": 7590483, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 180594824, + "decode_ns": 81043, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 18, + "order": "AB", + "A": { + "hot_relevance/dev01": { + "ns": 900672675, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 3024011018, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1551744954, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 942121233, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 961034902, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1744193190, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 1999429609, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 32882777, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1687905949, + "decode_ns": 4675907, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 211737028, + "decode_ns": 80159, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "B": { + "hot_relevance/dev01": { + "ns": 490342028, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 986411643, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 519572714, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 536070542, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 703530148, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 442872046, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 951958721, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 64291613, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 3350624703, + "decode_ns": 21551676, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 534668283, + "decode_ns": 121930, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + }, + { + "index": 19, + "order": "BA", + "B": { + "hot_relevance/dev01": { + "ns": 580687095, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 922926029, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 845779897, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 2133971589, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 2153177067, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1057960642, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 594995502, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 20218101, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1942229994, + "decode_ns": 4058395, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 389900225, + "decode_ns": 91219, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + }, + "A": { + "hot_relevance/dev01": { + "ns": 2009073893, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "hot_relevance/dev02": { + "ns": 1291460335, + "bytes": 760449, + "output_sha256": "4e04ab986bb8fce8155ec3fbf41ce5f4678ae88466438b878285122c7cd0affe" + }, + "hot_relevance/dev03": { + "ns": 1093157525, + "bytes": 757742, + "output_sha256": "43ae08f1bd292a09130e9ca9c2f09243a6f6e473908ef9f43a9d72dd9d9c0ee5" + }, + "hot_relevance/dev04": { + "ns": 1660667526, + "bytes": 760602, + "output_sha256": "16532c667bae53db3c1b076e65385f4d3e7b303dda9c997e1a9d402781ca8dcc" + }, + "fresh_relevance": { + "ns": 1739606229, + "bytes": 760539, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "default": { + "ns": 1433822984, + "bytes": 150213, + "output_sha256": "de4efdd504352a398d0223d5f498d3910650acde68699b2fb33edb2378ae1491" + }, + "context": { + "ns": 787725039, + "bytes": 1007130, + "output_sha256": "c8593bd1fc29832fd5125b8068adc5dd85086d4455b5792231acc5c828b1a17f" + }, + "search": { + "ns": 30863524, + "bytes": 4317, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + }, + "cli_relevance": { + "ns": 1001441479, + "decode_ns": 5431019, + "bytes": 823797, + "output_sha256": "74ebfe24500a396883fbacf1e31b72366198d26b4a89bf7f053a30ce09c78641" + }, + "cli_search": { + "ns": 401501183, + "decode_ns": 476005, + "bytes": 4991, + "output_sha256": "7bc555327cc8408e47507bcd2dafcc279c1eb9a91f5e89262b9b2f0ca6f5a947" + } + } + } + ], + "verdict": "FAIL", + "after": { + "A": { + "python": { + "sha256": "054a6a2941e6b134876e55c7fb71a2052ab76dbf33017f8d4ba235c03277cd9b", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "ab03f3ffb8f2e5bc02c304828c5c41890eef673e", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + }, + "B": { + "python": { + "sha256": "094006cbb640aaee0ebcc2835dc3549525637cecf9517ef6ff656fb8e4800106", + "files": 81 + }, + "corpus": { + "sha256": "411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0", + "files": 751 + }, + "revision": "6da28494891b21cdc89f632d84d763bc997b7d15", + "environment": { + "python": "3.12.14", + "system": "Linux", + "release": "6.18.55", + "machine": "x86_64", + "cpu_count": 9 + } + } + }, + "summary": { + "paths": { + "hot_relevance/dev01": { + "A": { + "n": 20, + "p50_ms": 1370.0807034999998, + "p95_ms": 2146.299652 + }, + "B": { + "n": 20, + "p50_ms": 484.442682, + "p95_ms": 831.76045 + }, + "p50_ms_change_percent": -64.64130319021021, + "p95_ms_change_percent": -61.24676956337689, + "output_parity": true + }, + "hot_relevance/dev02": { + "A": { + "n": 20, + "p50_ms": 1399.996766, + "p95_ms": 2552.013458 + }, + "B": { + "n": 20, + "p50_ms": 551.042999, + "p95_ms": 986.411643 + }, + "p50_ms_change_percent": -60.639694863409424, + "p95_ms_change_percent": -61.347709985313095, + "output_parity": true + }, + "hot_relevance/dev03": { + "A": { + "n": 20, + "p50_ms": 1183.8102955, + "p95_ms": 2729.51877 + }, + "B": { + "n": 20, + "p50_ms": 563.7279225, + "p95_ms": 845.779897 + }, + "p50_ms_change_percent": -52.380214579743864, + "p95_ms_change_percent": -69.01358927090287, + "output_parity": true + }, + "hot_relevance/dev04": { + "A": { + "n": 20, + "p50_ms": 1150.6747655, + "p95_ms": 2321.435239 + }, + "B": { + "n": 20, + "p50_ms": 567.037004, + "p95_ms": 995.251174 + }, + "p50_ms_change_percent": -50.72134881192021, + "p95_ms_change_percent": -57.127764872358775, + "output_parity": true + }, + "fresh_relevance": { + "A": { + "n": 20, + "p50_ms": 1193.4240995, + "p95_ms": 2015.165906 + }, + "B": { + "n": 20, + "p50_ms": 702.3239385, + "p95_ms": 1263.894106 + }, + "p50_ms_change_percent": -41.150514825848795, + "p95_ms_change_percent": -37.28089075758708, + "output_parity": true + }, + "default": { + "A": { + "n": 20, + "p50_ms": 1324.6845015, + "p95_ms": 2378.695621 + }, + "B": { + "n": 20, + "p50_ms": 514.786711, + "p95_ms": 1057.960642 + }, + "p50_ms_change_percent": -61.138919462175046, + "p95_ms_change_percent": -55.523496463358555, + "output_parity": true + }, + "context": { + "A": { + "n": 20, + "p50_ms": 1068.0971335, + "p95_ms": 1999.429609 + }, + "B": { + "n": 20, + "p50_ms": 574.2436785, + "p95_ms": 951.958721 + }, + "p50_ms_change_percent": -46.2367550207455, + "p95_ms_change_percent": -52.38848536027657, + "output_parity": true + }, + "search": { + "A": { + "n": 20, + "p50_ms": 36.7645025, + "p95_ms": 79.961876 + }, + "B": { + "n": 20, + "p50_ms": 37.856184999999996, + "p95_ms": 71.678475 + }, + "p50_ms_change_percent": 2.9693928266811165, + "p95_ms_change_percent": -10.359187921003754, + "output_parity": true + }, + "cli_relevance": { + "A": { + "n": 20, + "p50_ms": 1479.546679, + "p95_ms": 2198.695721 + }, + "B": { + "n": 20, + "p50_ms": 1016.4785475, + "p95_ms": 2634.81633 + }, + "p50_ms_change_percent": -31.29797376943726, + "p95_ms_change_percent": 19.83542355745549, + "output_parity": true + }, + "cli_search": { + "A": { + "n": 20, + "p50_ms": 289.514222, + "p95_ms": 450.213823 + }, + "B": { + "n": 20, + "p50_ms": 317.24522850000005, + "p95_ms": 687.563553 + }, + "p50_ms_change_percent": 9.57846088127583, + "p95_ms_change_percent": 52.719334208447876, + "output_parity": true + } + }, + "primary_reduction_percent": 58.610526361217275, + "primary_95ci": [ + 52.53945596397199, + 61.345765247234304 + ] + }, + "failed_gates": [ + "cli_search p95 regression" + ], + "verdict_scope": "Timing only; accuracy and contracts reported separately", + "contracts_pass": true, + "accuracy_output_parity": true, + "cases_sha256": { + "development.json": "cdaa0f020281a689b38015cf33932c5681b46b84053c3f9666effb32914c96b2" + } +} diff --git a/evidence/indexer-255-20261029/results/post-manifest-verification.txt b/evidence/indexer-255-20261029/results/post-manifest-verification.txt new file mode 100644 index 000000000..8c63500f7 --- /dev/null +++ b/evidence/indexer-255-20261029/results/post-manifest-verification.txt @@ -0,0 +1,19 @@ +Post-manifest software verification — 2026-10-29 UTC + +Scope: affected evidence inventory test after final33-entry manifest regeneration. +This is not a full-suite receipt or a benchmark/efficacy result. +Original pre-manifest89-pass log is retained unchanged and is not final-tree proof. +This receipt is Git-versioned and intentionally outside the campaign manifest +to avoid a circular dependency between test output and its input hash manifest. + +Manifest SHA256: 2cbe16d6f7709a71bb8362949d0675cb66c9bd83c4fd3e5c84d42d84af16ef86 +Inventory test SHA256: 51965b9ff80f1c8b2c6fa123346c0bac993685c6e27685ba23929dec82d2bb64 + +Command: PYTHONPATH=src -m pytest tests/test_evidence_inventory.py -rs +Environment: sanitized child environment; no installation, model or hardware calls. + +................. [100%] +17 passed in 1.10s + +Additional check: tools/validate_evidence.py returned errors=[] and checked_hashes=199. +Lint: ruff check tests/test_evidence_inventory.py passed. diff --git a/evidence/indexer-255-20261029/results/raw-archives.json b/evidence/indexer-255-20261029/results/raw-archives.json new file mode 100644 index 000000000..974ba950a --- /dev/null +++ b/evidence/indexer-255-20261029/results/raw-archives.json @@ -0,0 +1,53 @@ +{ + "format": "gzip, mtime=0; byte-identical inputs share one archive", + "files": [ + { + "original_path": "results/baseline-contract-smoke.json", + "original_sha256": "a35be36b859e6c4fd5bd7b7ec38920794f77adb322e9ae26498f31a9b4a91b6c", + "original_bytes": 540743, + "archive": "results/raw/a35be36b859e6c4fd5bd7b7ec38920794f77adb322e9ae26498f31a9b4a91b6c.json.gz", + "archive_sha256": "22a4ada0c2dab381ee9f8e3b342d0418d9e0668a16379fde069bae4b1eb0e9ad", + "archive_bytes": 136920 + }, + { + "original_path": "results/paired-run/A-cases.json", + "original_sha256": "fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0", + "original_bytes": 8248857, + "archive": "results/raw/fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0.json.gz", + "archive_sha256": "0de101743d3cdb7a1eef38a7a398d71ce372bb19f9380b85000b45d052f3341c", + "archive_bytes": 1960685 + }, + { + "original_path": "results/paired-run/A-heldout.json", + "original_sha256": "3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f", + "original_bytes": 3967752, + "archive": "results/raw/3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f.json.gz", + "archive_sha256": "7af84694f52243cf0433a7212af5d038c6dbca8a08e125ef8c0ef7fe2dae90a6", + "archive_bytes": 1022977 + }, + { + "original_path": "results/paired-run/B-cases.json", + "original_sha256": "fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0", + "original_bytes": 8248857, + "archive": "results/raw/fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0.json.gz", + "archive_sha256": "0de101743d3cdb7a1eef38a7a398d71ce372bb19f9380b85000b45d052f3341c", + "archive_bytes": 1960685 + }, + { + "original_path": "results/paired-run/B-heldout.json", + "original_sha256": "3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f", + "original_bytes": 3967752, + "archive": "results/raw/3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f.json.gz", + "archive_sha256": "7af84694f52243cf0433a7212af5d038c6dbca8a08e125ef8c0ef7fe2dae90a6", + "archive_bytes": 1022977 + }, + { + "original_path": "results/paired-run/contracts.json", + "original_sha256": "517e374c3f13d9d22dafad37d51368d1f4e5669842f9bc0dadc1bb88cba3d968", + "original_bytes": 1195539, + "archive": "results/raw/517e374c3f13d9d22dafad37d51368d1f4e5669842f9bc0dadc1bb88cba3d968.json.gz", + "archive_sha256": "0c070b69dcc6ea010a24c637a160b19ea8a7384eb40bb17636db0eaa42906a0a", + "archive_bytes": 275870 + } + ] +} diff --git a/evidence/indexer-255-20261029/results/raw/3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f.json.gz b/evidence/indexer-255-20261029/results/raw/3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f.json.gz new file mode 100644 index 000000000..bb451ae6f Binary files /dev/null and b/evidence/indexer-255-20261029/results/raw/3f3d873aaa65657ef4e3213e15847dbd90c9e30207621238fc2a35d2c9d5419f.json.gz differ diff --git a/evidence/indexer-255-20261029/results/raw/517e374c3f13d9d22dafad37d51368d1f4e5669842f9bc0dadc1bb88cba3d968.json.gz b/evidence/indexer-255-20261029/results/raw/517e374c3f13d9d22dafad37d51368d1f4e5669842f9bc0dadc1bb88cba3d968.json.gz new file mode 100644 index 000000000..8be3a5666 Binary files /dev/null and b/evidence/indexer-255-20261029/results/raw/517e374c3f13d9d22dafad37d51368d1f4e5669842f9bc0dadc1bb88cba3d968.json.gz differ diff --git a/evidence/indexer-255-20261029/results/raw/a35be36b859e6c4fd5bd7b7ec38920794f77adb322e9ae26498f31a9b4a91b6c.json.gz b/evidence/indexer-255-20261029/results/raw/a35be36b859e6c4fd5bd7b7ec38920794f77adb322e9ae26498f31a9b4a91b6c.json.gz new file mode 100644 index 000000000..855b5eb94 Binary files /dev/null and b/evidence/indexer-255-20261029/results/raw/a35be36b859e6c4fd5bd7b7ec38920794f77adb322e9ae26498f31a9b4a91b6c.json.gz differ diff --git a/evidence/indexer-255-20261029/results/raw/fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0.json.gz b/evidence/indexer-255-20261029/results/raw/fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0.json.gz new file mode 100644 index 000000000..869c7d5ed Binary files /dev/null and b/evidence/indexer-255-20261029/results/raw/fbe219d8332596850e32d3d64ed4e4d566644d630c6e007338fd834af83fd2c0.json.gz differ diff --git a/evidence/indexer-255-20261029/results/report.md b/evidence/indexer-255-20261029/results/report.md new file mode 100644 index 000000000..bb37d2100 --- /dev/null +++ b/evidence/indexer-255-20261029/results/report.md @@ -0,0 +1,97 @@ +# Indexer efficiency and retrieval accuracy — Issue #255 + +**Frozen comparison verdict: FAIL. Do not accept the runtime optimization on this evidence.** + +The final feature branch is **evidence-only**: production runtime is restored to baseline; the rejected candidate and its tests are retained as a small exact-commit bundle/patch under `patch/`. + +The primary request-local optimization is promising, but the unchanged acceptance contract includes a CLI-search tail guard that failed. This report preserves that failure; it is not an unconditional performance PASS or issue-closure decision. Retrieval results are identical before/after: this is an accuracy assessment, not an accuracy improvement. + +## Headline table + +| Dimension | Baseline → candidate | Disposition | +| --- | --- | --- | +| Primary paired reused-reader relevance latency | 58.61% reduction; group-bootstrap 95% CI 52.54–61.35% | Primary gate passes, **overall timing FAIL** | +| CLI catalog search p95, v1 | 450.21 → 687.56 ms (+52.72%; allowed ≤20%) | Guard fails; no sample removed | +| Separate CLI confirmation v2 | p50 +9.9734%; p95 −3.2103%, 120 pairs | Point gates pass; broad uncertainty, does not overwrite v1 FAIL | +| Frozen development partial-label Hit@5 / MRR@5 | 7/8 (87.5%) / 0.875 → identical | dev02 label ambiguous; not ground-truth full-query accuracy | +| Independently held-out partial-label Hit@5 / MRR@5 | 3/4 (75%) / 0.750 → identical | Unseen queries, not fully family-disjoint | +| Relevant-item emitted-byte fidelity/core delivery | dev 7/7, held-out 3/3 → identical | Conditional on retrieval; misses excluded and shown below | +| Supplemental all-required-resource audit | 58/58 emitted contexts per arm, exact bytes/hash/revision | Post-run independent audit, not frozen validation | +| Contract checks | 6/6 per arm, identical | No-match, incompatible filter, explicit ID, exact/undersized budget, stale projection | +| Output parity | Every one of 20 paired groups ×10 paths; all12 semantic cases | Exact serialized output equality | + +## Reproducibility and interpretation + +- Baseline: `ab03f3ffb8f2e5bc02c304828c5c41890eef673e`; candidate runtime: `6da28494891b21cdc89f632d84d763bc997b7d15`. +- Corpus: 751 packaged files, SHA-256 `411bd47a168f1549164f96fb3e4db4ba27e73edbb8d65b19d3bbc961897679c0`, identical before/after and across arms. Historical trim was already in baseline. +- Linux x86_64, Python3.12.14, reported CPU count9. Same interpreter and explicit source-root imports verified for both arms. Sanitized child environments; no model calls, hardware experiments or paid services. +- Frozen protocol: `plan/protocol.json`; commands and dependency versions: `plan/README.md` and `plan/python-environment.txt`. 20 alternating AB/BA groups, one warmup per reused-reader query/mode. Fresh reader ≠ cold filesystem cache. Full CLI includes interpreter startup and output capture. +- One runtime candidate; no tuning or rerun using held-out results. Small partial-positive labels do not establish complete Recall/Precision, downstream agent success or hardware efficacy. Peak memory was not measured; the per-request memo has nonzero retention overhead. + +## All latency paths (milliseconds) + +| Path | A p50 | B p50 | p50 change | A p95 | B p95 | p95 change | n/arm | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| hot_relevance/dev01 | 1370.08 | 484.44 | -64.64% | 2146.30 | 831.76 | -61.25% | 20 | +| hot_relevance/dev02 | 1400.00 | 551.04 | -60.64% | 2552.01 | 986.41 | -61.35% | 20 | +| hot_relevance/dev03 | 1183.81 | 563.73 | -52.38% | 2729.52 | 845.78 | -69.01% | 20 | +| hot_relevance/dev04 | 1150.67 | 567.04 | -50.72% | 2321.44 | 995.25 | -57.13% | 20 | +| fresh_relevance | 1193.42 | 702.32 | -41.15% | 2015.17 | 1263.89 | -37.28% | 20 | +| default | 1324.68 | 514.79 | -61.14% | 2378.70 | 1057.96 | -55.52% | 20 | +| context | 1068.10 | 574.24 | -46.24% | 1999.43 | 951.96 | -52.39% | 20 | +| search | 36.76 | 37.86 | +2.97% | 79.96 | 71.68 | -10.36% | 20 | +| cli_relevance | 1479.55 | 1016.48 | -31.30% | 2198.70 | 2634.82 | +19.84% | 20 | +| cli_search | 289.51 | 317.25 | +9.58% | 450.21 | 687.56 | +52.72% | 20 | + +Negative change means faster. p95 is nearest rank from20 samples, a noisy descriptive tail—not a production SLO. Gates were not changed after measurement. The primary statistic is the median of within-group median four-query reductions, not a ratio of the table medians. + +## Per-case retrieval results + +| Case | Query intent | Known positive rank A/B | Required/core delivery among retrieved positives | Note | +| --- | --- | --- | --- | --- | +| dev01 | cgroupv2 cpu quota | 1 / 1 | 1/1 | partial positive label | +| dev02 | P2pNccl KV transfer completion | miss / miss | 0/0 | ambiguous mixed-intent label | +| dev03 | DLRover checkpoint replica recovery | 1 / 1 | 1/1 | partial positive label | +| dev04 | weight loading mmap checkpoint peak memory | 1 / 1 | 1/1 | partial positive label | +| dev05 | Ascend npu-smi device health before launch | 1 / 1 | 1/1 | partial positive label | +| dev06 | H3 MiniMax AdaLN SSD residency | 1 / 1 | 1/1 | partial positive label | +| dev07 | fixed workload throughput denominator timing claim | 1 / 1 | 1/1 | partial positive label | +| dev08 | storage fio direct IO page cache measurement | 1 / 1 | 1/1 | partial positive label | +| heldout01 | TorchNPU nonblocking transfer finished submitting but pinned RAM remains occupied; when may a CPU buffer be reused? | 1 / 1 | 1/1 | partial positive label | +| heldout02 | Checkpoint loading got faster on the second run. Does high disk utilization prove the SSD is the bottleneck, or could cached reads explain it? | miss / miss | 0/0 | natural-language positive missed; other returned items may still be relevant | +| heldout03 | After asynchronous training save staging completes, may the next checkpoint overwrite its cached CPU snapshot while the previous writer is still uploading? | 1 / 1 | 1/1 | partial positive label | +| heldout04 | SGLang video generation request timed out and its job record was deleted. How can I tell whether the worker stopped computing and released its resident components? | 1 / 1 | 1/1 | partial positive label | + +The seven source-valid development cases have a separately labeled sensitivity subtotal Hit@5=7/7 and MRR@5=1.0; this does **not** replace the frozen8-case diagnostic. dev02 combines KV-transfer intent with an AFD hidden-state skill that explicitly excludes KV-block release. Heldout02 asks about repeated checkpoint-load speed/cache versus SSD bottlenecks and misses the labeled storage I/O skill despite the explicit `fio`/`page cache` development query succeeding. This is a useful lexical-versus-natural-query coverage gap, not evidence that every returned alternative was wrong. See `validator/label-review.md`. + +## CLI tail diagnosis (observation, not a waiver) + +- `catalog.search_knowledge` and its CLI search handler are unchanged and do not invoke Collect/Bundle manifests. The CLI imports the knowledge-operations module, but the optimized function is not executed on this search path. This narrows the plausible mechanism; it does not prove the observed regression is noise. +- Candidate CLI-search largest observations were group10 (AB) 1757.82ms, group16 (AB) 687.56ms, group18 (AB) 534.67ms. Baseline maximum452.44ms and p95 450.21ms. The p95 failure remains even though the maximum is not used as p95. +- CLI JSON decoding is measured separately and is sub-2ms; it does not explain the end-to-end tail. Catalog API p50 changes only+2.97%. Startup/import/scheduling/output capture were not separately instrumented, and host contention telemetry was not collected: attribution is unresolved. +- All raw samples and AB/BA order remain in `results/paired-run/timing.json`. A new confirmation study, if approved and prospectively frozen, must be a separate result and cannot replace this FAIL. + +## Separate prospective confirmation v2 + +One focused replication was frozen in [Issue #255](https://github.com/InferMatrixLab/InferRSI/issues/255#issuecomment-6497690434) before execution. Same runtime candidate and exact v1 CLI invocation; 120 paired samples in balanced randomized AB/BA order (seed25502), then40 separate baseline/baseline diagnostic pairs (seed25503). All outputs and before/after source identities match. No samples or studies were pooled, dropped or repeated to obtain a pass. + +| Study | n/arm | A p50 | B p50 | p50 change (paired95%CI) | A p95 | B p95 | p95 change (paired95%CI) | +| --- | ---: | ---: | ---: | --- | ---: | ---: | --- | +| Confirmation v2 | 120 | 240.44 | 264.42 | +9.9734% (+1.20…+20.92%) | 599.97 | 580.71 | −3.21% (−22.23…+50.86%) | +| Baseline/baseline negative control | 40 | 228.23 | 226.30 | −0.85% (−9.18…+7.59%) | 336.39 | 364.26 | +8.29% (−5.98…+41.78%) | + +Times are milliseconds. **v2's prespecified point gates pass**, narrowly at the10% median guard. Its uncertainty does not exclude a meaningful median or tail regression. The negative control illustrates variability but does not prove all candidate regression is environmental. v1 remains **FAIL** and runtime promotion is not justified by treating v2 as a replacement. These are separate studies, not an unconditional combined PASS. Raw chronology and bootstrap results: `results/confirmation-v2.json`; prospective protocol/wrapper hashes are recorded there. + +## Software checks and remaining acceptance blocker + +- Focused runtime/selection/knowledge tests:62 passed,2 Windows-only skips. New tests cover exact-byte/path reuse, changed owners/bytes, mutable metadata/dependencies, unsafe paths, non-byte content and no cross-request memo. +- Initial complete suite:2504 passed,9 failed,10 skipped; retained unchanged in `results/full-suite-initial.txt`. Eight failures were subprocess import errors from an uninstalled isolated environment; one was new evidence inventory registration/count. No full-suite PASS is claimed. +- After explicit source-path setup and inventory registration/count correction, unaffected inventory/module-boundary/server-process/summary tests:30 passed. This does not substitute for five outside-repository installed-package smoke tests. +- Repository CI lint command `ruff check src tests tools` and this campaign validator directory both pass. Broader `ruff check .` reports372 unchanged historical-evidence findings; baseline/final concise outputs are byte-identical and retained. +- The earlier focused source/harness/inventory/summary/module/server run recorded **89 passed,2 Windows-only skips** (`results/final-evidence-tree-tests.txt`), but it preceded inclusion of the final33-entry campaign integrity manifest. Despite that log filename, it did **not** prove the final published tree passed. Hosted CI then correctly exposed the stale inventory assertion (166 expected versus199 checked). The assertion is corrected to199; `results/post-manifest-verification.txt` records the affected test rerun after final manifest regeneration. This later software receipt is Git-versioned but deliberately excluded from the campaign integrity manifest to avoid a self-referential test-log/hash cycle. Original logs and benchmark data are retained unchanged. Production runtime equals baseline; candidate-only tests remain in the preserved patch. +- Editable package/build-dependency installation was denied by the execution approval gate; no retry or bypass. Installed-package smoke remains blocked pending specific authorization. Optional image dependencies and opt-in fresh-venv/Windows tests were not silently counted as passes. +- Runtime candidate is **not accepted** because the frozen CLI p95 guard failed. Evidence delivery is distinct from optimization acceptance and from closing the tracker. + +## Raw evidence packaging + +Large complete JSON outputs are losslessly gzip-compressed (`mtime=0`) and byte-identical A/B outputs share one object. `results/raw-archives.json` records every original path, original hash/byte count and compressed hash/byte count. This preserves full outputs without26MB of expanded duplicate text. `validator/verify_raw_archives.py` verifies and optionally restores them. Supplemental mandatory-resource audit is post-run and explicitly distinguished from the frozen runner. diff --git a/evidence/indexer-255-20261029/results/targeted-env-final.txt b/evidence/indexer-255-20261029/results/targeted-env-final.txt new file mode 100644 index 000000000..19f40bc19 --- /dev/null +++ b/evidence/indexer-255-20261029/results/targeted-env-final.txt @@ -0,0 +1,2 @@ +.............................. [100%] +30 passed in 2.57s diff --git a/evidence/indexer-255-20261029/results/targeted-env-initial.txt b/evidence/indexer-255-20261029/results/targeted-env-initial.txt new file mode 100644 index 000000000..4a0e0f05d --- /dev/null +++ b/evidence/indexer-255-20261029/results/targeted-env-initial.txt @@ -0,0 +1,14 @@ +................F............. [100%] +=================================== FAILURES =================================== +__________________________ test_checked_in_inventory ___________________________ + + def test_checked_in_inventory(): + repo = Path(__file__).resolve().parents[1] + result = validate_inventory(repo) + assert result["errors"] == [] +> assert len(result["bundles"]) == 9 +E AssertionError: assert 10 == 9 +E + where 10 = len([{'name': 'indexer-255-20261029', 'kind': 'campaign'}, {'name': '_template', 'kind': 'template'}, {'name': 'qwen38-h20...ame': 'modular-refactor-20261013', 'kind': 'campaign'}, {'name': 'qwen21-turbo-b300-20261010', 'kind': 'archive'}, ...]) + +tests/test_evidence_inventory.py:148: AssertionError +1 failed, 29 passed in 2.52s diff --git a/evidence/indexer-255-20261029/validator/README.md b/evidence/indexer-255-20261029/validator/README.md new file mode 100644 index 000000000..cb6133891 --- /dev/null +++ b/evidence/indexer-255-20261029/validator/README.md @@ -0,0 +1,20 @@ +# Frozen evaluator + +- Runner: `tools/evaluate_indexer.py`, SHA-256 `ea5b91ff17e80a422fd373621528dfe74ff144976bb55d6432501df452757ca5`. +- Development cases: `tests/fixtures/indexer/development.json`, SHA-256 `cdaa0f020281a689b38015cf33932c5681b46b84053c3f9666effb32914c96b2`. +- Independent held-out commitment: `10e13080cfb4d27de40dc92166c261cf6ccf47776d08111f2e85d7c8edc2d19f`; four cases created by a separate source reviewer without retrieval observations and withheld until candidate code freeze. +- Protocol: `../plan/protocol.json`, SHA-256 `e4964329a20e6752b8149ab270733c0bbb258d0f948bdca9e86da6fe8f028956`. + +Protocol accepted prospectively in [Issue #255](https://github.com/InferMatrixLab/InferRSI/issues/255#issuecomment-6496822362). Runner/development hashes communicated to the coordinating reviewer before candidate edits. Baseline smoke checks all six contracts and exact API/CLI output equality. Smoke timings are diagnostic, **not** the controlled comparison. + +One fixed efficiency candidate, no ranking changes and no held-out tuning. Contract/accuracy results remain separate from timing verdict. Partial labels cannot establish corpus Recall or exhaustive Precision. A miss is reported, not relabeled. + +## Separate prospective confirmation + +`confirm_cli_search.py` and `../plan/confirmation-v2.json` define one focused 120-pair CLI-search replication plus 40 baseline/baseline diagnostic pairs. It neither replaces nor pools with the original FAIL. Its hashes and approval receipt must precede execution; all results and uncertainty are reported separately. + +Confirmation v2 protocol SHA-256 `40de4eefba8712d3d07f2e48049b0899c831f94bfcfa252c1f8841525a337fa4`; wrapper SHA-256 `b11b4cf08b6dceb43be447516d5abb740bb2de8c2fdfd7a9ee3f820a1174b55d`. Prospective receipt: https://github.com/InferMatrixLab/InferRSI/issues/255#issuecomment-6497690434. Exactly one run completed; point PASS is distinct from v1 FAIL and does not remove uncertainty. + +`frozen_evaluate_indexer.py` is the byte-identical v1 evaluator snapshot (same `ea5b...` hash). It remains replayable if the generic runner later evolves; invoke it with the same arguments. The confirmation wrapper intentionally checks the exact v1 generic-runner hash and refuses a changed evaluator rather than silently replaying different logic. Restore the snapshot in an isolated reproduction checkout when replaying that wrapper. + +`verify_raw_archives.py` validates the original and compressed hashes and can reconstruct every original JSON path under an explicit destination. `supplemental_required_audit.py` accepts plain/gzip inputs and is a portable post-run adaptation of the independent audit, not a prospectively frozen gate. diff --git a/evidence/indexer-255-20261029/validator/confirm_cli_search.py b/evidence/indexer-255-20261029/validator/confirm_cli_search.py new file mode 100644 index 000000000..6de8dc5ce --- /dev/null +++ b/evidence/indexer-255-20261029/validator/confirm_cli_search.py @@ -0,0 +1,110 @@ +"""One prospective CLI-search replication; independent of v1, no pooling/retries.""" +import argparse +import json +import random +import statistics +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO)) +from tools.evaluate_indexer import cli, digest, identity, percentile, write # noqa: E402 + + +def summary(groups): + rows = {} + for arm in "AB": + values = [g[arm]["ns"] / 1e6 for g in groups] + rows[arm] = {"n": len(values), "p50_ms": statistics.median(values), + "p95_ms": percentile(values, .95)} + for metric in ("p50_ms", "p95_ms"): + rows[metric + "_change_percent"] = 100 * (rows["B"][metric] / rows["A"][metric] - 1) + rng = random.Random(25504) + distributions = {"p50_ms": [], "p95_ms": []} + for _ in range(10000): + sample = rng.choices(groups, k=len(groups)) + values = {arm: [g[arm]["ns"] for g in sample] for arm in "AB"} + for metric, estimator in (("p50_ms", statistics.median), + ("p95_ms", lambda v: percentile(v, .95))): + distributions[metric].append(100 * (estimator(values["B"]) / + estimator(values["A"]) - 1)) + rows["paired_bootstrap_95ci"] = { + metric: [percentile(v, .025), percentile(v, .975)] + for metric, v in distributions.items() + } + hashes = {g[arm]["output_sha256"] for g in groups for arm in "AB"} + rows["all_output_parity"] = len(hashes) == 1 + return rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--protocol", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + raise ValueError("Refusing to overwrite a prior study") + protocol = json.loads(args.protocol.read_text()) + roots = {"A": args.baseline.resolve(), "B": args.candidate.resolve()} + assert digest((REPO / "tools/evaluate_indexer.py").read_bytes()) == protocol[ + "frozen_v1_runner_sha256"] + assert digest((roots["B"] / "src/inferrsi/knowledge_operations.py").read_bytes()) == protocol[ + "candidate_source_sha256"] + before = {a: identity(root) for a, root in roots.items()} + assert before["A"]["revision"] == protocol["baseline_revision"] + assert before["B"]["revision"] == protocol["candidate_revision"] + assert before["A"]["corpus"] == before["B"]["corpus"] + v1 = json.loads((REPO / "evidence/indexer-255-20261029/results/paired-run/timing.json") + .read_text()) + assert before == v1["before"] + result = {"study": "confirmation-v2", "protocol_sha256": digest(args.protocol.read_bytes()), + "wrapper_sha256": digest(Path(__file__).read_bytes()), "before": before, + "started_utc": datetime.now(timezone.utc).isoformat(), + "verdict": "INCONCLUSIVE", "confirmation": [], "negative_control": []} + start = time.monotonic_ns() + try: + phases = (("confirmation", protocol["samples"], 25502), + ("negative_control", protocol["negative_control"]["pairs"], 25503)) + for name, count, seed in phases: + order = ["AB"] * (count // 2) + ["BA"] * (count // 2) + random.Random(seed).shuffle(order) + for index, sequence in enumerate(order): + row = {"index": index, "order": sequence, + "start_offset_ns": time.monotonic_ns() - start} + for arm in sequence: + root = roots[arm] if name == "confirmation" else roots["A"] + offset = time.monotonic_ns() - start + row[arm] = {**cli(root, "search", {}), "start_offset_ns": offset} + result[name].append(row) + write(args.output, result) + if (index + 1) % 20 == 0: + print(name, index + 1, flush=True) + result["after"] = {a: identity(root) for a, root in roots.items()} + assert result["after"] == before + result["summary"] = {name: summary(result[name]) + for name in ("confirmation", "negative_control")} + actual = result["summary"]["confirmation"] + failed = [] + if not actual["all_output_parity"]: + failed.append("output mismatch") + for metric in ("p50", "p95"): + if actual[metric + "_ms_change_percent"] > protocol["gates"][ + metric + "_max_regression_percent"]: + failed.append(metric + " regression") + result["failed_gates"] = failed + result["verdict"] = "FAIL" if failed else "PASS" + result["verdict_scope"] = "Focused confirmation only; v1 remains FAIL" + except Exception as exc: + result["error"] = type(exc).__name__ + ": " + str(exc) + raise + finally: + result["ended_utc"] = datetime.now(timezone.utc).isoformat() + write(args.output, result) + + +if __name__ == "__main__": + main() diff --git a/evidence/indexer-255-20261029/validator/frozen_evaluate_indexer.py b/evidence/indexer-255-20261029/validator/frozen_evaluate_indexer.py new file mode 100644 index 000000000..222d54112 --- /dev/null +++ b/evidence/indexer-255-20261029/validator/frozen_evaluate_indexer.py @@ -0,0 +1,311 @@ +"""Read-only, CPU-only retrieval evidence runner; no model or hardware invocation. + +Use --help. Baseline/candidate paths and frozen protocol are explicit inputs. +This tool measures real packaged retrieval, not a synthetic workload or CI gate. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import random +import statistics +import subprocess +import sys +import time +from pathlib import Path + + +def encode(value): + return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() + + +def digest(value): + return hashlib.sha256(value).hexdigest() + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def identity(root): + groups = {} + for name, paths in ( + ("python", sorted((root / "src").rglob("*.py"))), + ("corpus", sorted((root / "src/inferrsi/data").rglob("*"))), + ): + members = { + p.relative_to(root).as_posix(): digest(p.read_bytes()) + for p in paths if p.is_file() and "__pycache__" not in p.parts + } + groups[name] = {"sha256": digest(encode(members)), "files": len(members)} + groups["revision"] = subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip() + groups["environment"] = { + "python": platform.python_version(), "system": platform.system(), + "release": platform.release(), "machine": platform.machine(), "cpu_count": os.cpu_count(), + } + return groups + + +def options(case, context=False): + value = {"query": case.get("query", ""), "limit": case.get("limit", 5)} + value.update(case.get("filters", {})) + if case.get("relevance") is not None: + value["relevance"] = case["relevance"] + if context: + value["context_bytes"] = case.get("context_bytes", 262144) + return value + + +def child(root, action, payload): + env = {"PATH": os.defpath, "PYTHONPATH": str(root / "src"), "PYTHONDONTWRITEBYTECODE": "1"} + run = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--worker", str(root), action], + input=json.dumps(payload), text=True, capture_output=True, env=env, check=True, timeout=180, + ) + return json.loads(run.stdout) + + +def cli(root, mode, case): + args = [sys.executable, "-m", "inferrsi"] + if mode == "search": + args += ["knowledge", "search", "cgroupv2", "--limit", "5", "--json"] + else: + args += ["collect", case["query"], "--relevance", case["relevance"], + "--limit", str(case["limit"]), "--json"] + start = time.perf_counter_ns() + run = subprocess.run(args, capture_output=True, check=True, timeout=180, env={ + "PATH": os.defpath, "PYTHONPATH": str(root / "src"), "PYTHONDONTWRITEBYTECODE": "1", + }) + elapsed = time.perf_counter_ns() - start + start = time.perf_counter_ns() + result = json.loads(run.stdout) + decode = time.perf_counter_ns() - start + return {"ns": elapsed, "decode_ns": decode, "bytes": len(run.stdout), + "output_sha256": digest(encode(result))} + + +def worker(root, action, payload): + sys.path.insert(0, str(root / "src")) + from inferrsi.catalog import search_knowledge + from inferrsi.knowledge_operations import KnowledgeReader + + if action == "identity": + return identity(root) + if action == "bench": + rows = {} + for case in payload["queries"]: + reader = KnowledgeReader() + args = options(case) + reader.collect(**args) + start = time.perf_counter_ns() + value = reader.collect(**args) + elapsed = time.perf_counter_ns() - start + rows["hot_relevance/" + case["id"]] = { + "ns": elapsed, "bytes": len(encode(value)), "output_sha256": digest(encode(value))} + case = payload["queries"][0] + for mode in ("fresh_relevance", "default", "context", "search"): + args = options(case, context=mode == "context") + if mode in ("default", "search"): + args.pop("relevance", None) + args["query"] = "cgroupv2" + reader = KnowledgeReader() + call = search_knowledge if mode == "search" else reader.collect + if mode != "fresh_relevance": + call(**args) + start = time.perf_counter_ns() + if mode == "fresh_relevance": + value = KnowledgeReader().collect(**args) + else: + value = call(**args) + elapsed = time.perf_counter_ns() - start + rows[mode] = {"ns": elapsed, "bytes": len(encode(value)), + "output_sha256": digest(encode(value))} + for mode in ("relevance", "search"): + rows["cli_" + mode] = cli(root, mode, case) + for api, cli_name in (("hot_relevance/dev01", "cli_relevance"), + ("search", "cli_search")): + if rows[api]["output_sha256"] != rows[cli_name]["output_sha256"]: + raise ValueError("API/CLI payload mismatch: " + api) + return rows + if action == "contracts": + from inferrsi._skill_selection import SkillIndexError, indexed_skill + reader = KnowledgeReader() + cgroup = "inferrsi.common.skills.cgroup_resource_controls" + afd = "inferrsi.cuda.skills.audit_afd_p2p_exchanges" + rows = [] + def check(name, passed, output): + rows.append({"id": name, "passed": bool(passed), "output": output}) + result = reader.collect(relevance="floristry orchid") + check("no_match", result["no_match"], result) + result = reader.collect(relevance="P2pNccl", backend="ascend") + check("incompatible_filter", result["no_match"], result) + result = reader.collect(ids=[cgroup], relevance="unrelated orchid") + check("explicit_id_unrelated", [x["id"] for x in result["items"]] == [cgroup], result) + small = reader.collect(ids=[afd], context_bytes=1) + required = small["context_omissions"][0]["required_bytes"] + exact = reader.collect(ids=[afd], context_bytes=required) + check("exact_required_budget", bool(exact["contexts"]) and + exact["context_bytes"] == required and not exact["context_omissions"], exact) + short = reader.collect(ids=[afd], context_bytes=required - 1) + check("undersized_required_budget", not short["contexts"] and + short["context_omissions"][0]["reason"] == "context_limit", short) + bundle = reader.read(cgroup) + record = reader.records()[cgroup] + core = record["skill_documents"]["original"] + bundle.content[core] += b"\nchanged source" + bundle.dependencies.clear() # isolate projection validation from dependency validation + view = {**bundle.view(), "authority": "package"} + rejected = False + try: + indexed_skill(record, view, bundle.content) + except SkillIndexError: + rejected = True + check("stale_source_projection", rejected, {"rejected": rejected}) + return rows + if action == "cases": + rows = [] + for case in payload: + reader = KnowledgeReader() + result = reader.collect(**options(case, context=True)) + ids = [item["id"] for item in result["items"]] + ranks = [ids.index(i) + 1 for i in case["relevant_ids"] if i in ids] + selected = [i for i in case["relevant_ids"] if i in ids] + complete = [] + for target in selected: + context = next((c for c in result["contexts"] + if c["ref"] == "package:" + target), None) + checks = [] + if context: + bundle = reader.read(target, revision=context["revision"]) + # Read the exact revision once; verify every complete emitted resource. + for resource in context["resources"]: + data = bundle.content[resource["path"]] + checks.append(data == resource["content"].encode() + and digest(data) == resource["sha256"]) + paths = [r["path"] for r in context["resources"]] + checks.extend(any(p.endswith(s) for p in paths) + for s in case.get("required_resource_suffixes", [])) + checks.append(bundle.revision == context["revision"]) + complete.append(bool(context) and all(checks)) + rows.append({"id": case["id"], "split": case["split"], "hit_at_5": bool(ranks), + "rr_at_5": 1 / min(ranks) if ranks else 0, + "retrieved_relevant": len(selected), "complete_relevant": sum(complete), + "forbidden_hits": sorted(set(ids) & set(case.get("forbidden_ids", []))), + "output_sha256": digest(encode(result)), "result": result}) + return rows + raise ValueError(action) + + +def percentile(values, q): + return sorted(values)[max(0, math.ceil(q * len(values)) - 1)] + + +def summarize(groups): + names = list(groups[0]["A"]) + rows = {} + for name in names: + arms = {arm: [g[arm][name]["ns"] / 1e6 for g in groups] for arm in ("A", "B")} + rows[name] = { + arm: {"n": len(v), "p50_ms": statistics.median(v), "p95_ms": percentile(v, .95)} + for arm, v in arms.items() + } + for stat in ("p50_ms", "p95_ms"): + rows[name][stat + "_change_percent"] = ( + 100 * (rows[name]["B"][stat] / rows[name]["A"][stat] - 1)) + rows[name]["output_parity"] = all( + g["A"][name]["output_sha256"] == g["B"][name]["output_sha256"] for g in groups) + gains = [statistics.median([ + 100 * (g["A"][name]["ns"] - g["B"][name]["ns"]) / g["A"][name]["ns"] + for name in names if name.startswith("hot_relevance/")]) for g in groups] + rng = random.Random(255) + bootstrap = [statistics.median(rng.choices(gains, k=len(gains))) for _ in range(10000)] + return {"paths": rows, "primary_reduction_percent": statistics.median(gains), + "primary_95ci": [percentile(bootstrap, .025), percentile(bootstrap, .975)]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--worker", type=Path) + parser.add_argument("action", nargs="?") + parser.add_argument("--baseline", type=Path) + parser.add_argument("--candidate", type=Path) + parser.add_argument("--protocol", type=Path) + parser.add_argument("--cases", type=Path, nargs="*") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.worker: + print(json.dumps(worker(args.worker.resolve(), args.action, json.load(sys.stdin)))) + return + if not all((args.baseline, args.candidate, args.protocol, args.output)): + parser.error("baseline, candidate, protocol and output are required") + protocol = json.loads(args.protocol.read_text()) + roots = {"A": args.baseline.resolve(), "B": args.candidate.resolve()} + before = {a: child(p, "identity", {}) for a, p in roots.items()} + if before["A"]["revision"] != protocol["baseline_revision"]: + raise ValueError("baseline revision mismatch") + if before["A"]["corpus"] != before["B"]["corpus"]: + raise ValueError("corpus mismatch") + result = {"protocol_sha256": digest(args.protocol.read_bytes()), "before": before, + "groups": [], "verdict": "INCONCLUSIVE"} + try: + for index in range(protocol["measurement"]["groups"]): + order = "AB" if index % 2 == 0 else "BA" + group = {"index": index, "order": order} + for arm in order: + group[arm] = child(roots[arm], "bench", protocol) + result["groups"].append(group) + write(args.output / "timing.json", result) + print(f"Completed group {index + 1}", flush=True) + result["after"] = {a: child(p, "identity", {}) for a, p in roots.items()} + if result["after"] != before: + raise ValueError("source/corpus/environment changed during measurement") + result["summary"] = summarize(result["groups"]) + summary = result["summary"] + failed = [] + if summary["primary_reduction_percent"] < protocol["primary"]["gain_min_percent"]: + failed.append("primary practical gain") + if summary["primary_95ci"][0] <= 0: + failed.append("primary confidence interval") + for name, row in summary["paths"].items(): + primary = name.startswith("hot_relevance/") + p50_limit = (protocol["primary"]["each_query_p50_max_regression_percent"] + if primary else protocol["secondary_limits"][ + "each_path_p50_max_regression_percent"]) + if row["p50_ms_change_percent"] > p50_limit: + failed.append(name + " p50 regression") + if not primary and row["p95_ms_change_percent"] > protocol["secondary_limits"][ + "each_path_p95_max_regression_percent"]: + failed.append(name + " p95 regression") + if not row["output_parity"]: + failed.append(name + " output mismatch") + result["failed_gates"] = failed + result["verdict"] = "FAIL" if failed else "PASS" + result["verdict_scope"] = "Timing only; accuracy and contracts reported separately" + contracts = {arm: child(root, "contracts", {}) for arm, root in roots.items()} + write(args.output / "contracts.json", contracts) + result["contracts_pass"] = (contracts["A"] == contracts["B"] and + all(r["passed"] for r in contracts["A"])) + if args.cases: + cases = [c for path in args.cases for c in json.loads(path.read_text())["cases"]] + outcomes = {arm: child(root, "cases", cases) for arm, root in roots.items()} + for arm, rows in outcomes.items(): + write(args.output / (arm + "-cases.json"), rows) + result["accuracy_output_parity"] = outcomes["A"] == outcomes["B"] + result["cases_sha256"] = {path.name: digest(path.read_bytes()) for path in args.cases} + except Exception as exc: + result["error"] = type(exc).__name__ + ": " + str(exc) + result["verdict"] = "INCONCLUSIVE" + raise + finally: + write(args.output / "timing.json", result) + + +if __name__ == "__main__": + main() diff --git a/evidence/indexer-255-20261029/validator/label-review.md b/evidence/indexer-255-20261029/validator/label-review.md new file mode 100644 index 000000000..167a54769 --- /dev/null +++ b/evidence/indexer-255-20261029/validator/label-review.md @@ -0,0 +1,9 @@ +# Independent source-label audit (before retrieval scoring) + +These qualifications do not modify frozen inputs, numerical gates, candidate code or raw scores. + +1. **dev02 is ambiguous.** The query says “P2pNccl KV transfer completion”, while the positive `audit-afd-p2p-exchanges` concerns hidden-state AFD exchanges and explicitly excludes KV-block release. P2pNccl/completion is related, but a hit cannot prove the full query was answered correctly. Keep the original 8-development-case label-hit score as a frozen diagnostic; any 7-source-valid-case subtotal must be separately named, never silently substituted. +2. **Held-out intent families are not fully disjoint.** Independently selected heldout02 and dev08 both concern `storage_io_measurement`. The four held-out queries/labels were unavailable during implementation and were not tuned, but this is unseen-query assessment, not family-disjoint generalization. +3. Positives are partial labels, not exhaustive corpus relevance judgments. No negative-ID labels were justified. No agent answer, actual hardware execution, or downstream task success is measured. + +The timing query remains a valid fixed workload and output-parity check regardless of its label ambiguity. Efficiency-only optimization cannot establish improved relevance. diff --git a/evidence/indexer-255-20261029/validator/supplemental_required_audit.py b/evidence/indexer-255-20261029/validator/supplemental_required_audit.py new file mode 100644 index 000000000..b724cb2af --- /dev/null +++ b/evidence/indexer-255-20261029/validator/supplemental_required_audit.py @@ -0,0 +1,85 @@ +"""Supplemental post-run audit of saved retrieval results, not a frozen validator. + +Portable adaptation of the independent audit used after the first timing run. +Inputs are saved case-result JSON arrays, optionally gzip compressed. This checks +all emitted contexts, not whether missing relevant items should have been found. +""" +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import sys +from pathlib import Path + + +def read_cases(path: Path) -> list[dict]: + """Load a raw case-result array without changing its semantics.""" + if path.suffix == ".gz": + with gzip.open(path, "rt", encoding="utf-8") as stream: + return json.load(stream) + return json.loads(path.read_text(encoding="utf-8")) + + +def audit(root: Path, arm: str, inputs: list[Path]) -> dict: + sys.path.insert(0, str(root.resolve() / "src")) + from inferrsi._skill_selection import indexed_skill + from inferrsi.knowledge_operations import KnowledgeReader + + reader = KnowledgeReader() + audits = [] + for input_path in inputs: + for row in read_cases(input_path): + for context in row["result"]["contexts"]: + bundle = reader.read(context["ref"], revision=context["revision"]) + view = next( + item for item in row["result"]["items"] + if item["ref"] == context["ref"] + ) + _, required, _ = indexed_skill( + reader.records()[view["id"]], view, bundle.content, + ) + emitted = {resource["path"]: resource for resource in context["resources"]} + missing = sorted(set(required) - emitted.keys()) + bad = [ + path for path, resource in emitted.items() + if resource["content"].encode() != bundle.content[path] + or resource["sha256"] != hashlib.sha256(bundle.content[path]).hexdigest() + ] + revision_match = bundle.revision == context["revision"] + audits.append({ + "case": row["id"], + "ref": context["ref"], + "required": required, + "missing": missing, + "bad_bytes_or_hash": bad, + "revision_match": revision_match, + "passed": not missing and not bad and revision_match, + }) + return { + "arm": arm, + "scope": ( + "all emitted contexts in supplied saved case results; absent relevant " + "retrieval is not counted as delivery success" + ), + "checks": audits, + "passed": all(check["passed"] for check in audits), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", required=True, type=Path) + parser.add_argument("--arm", required=True) + parser.add_argument("--inputs", required=True, nargs="+", type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + result = audit(args.root, args.arm, args.inputs) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(args.arm, len(result["checks"]), result["passed"]) + + +if __name__ == "__main__": + main() diff --git a/evidence/indexer-255-20261029/validator/verify_raw_archives.py b/evidence/indexer-255-20261029/validator/verify_raw_archives.py new file mode 100644 index 000000000..a90d6715e --- /dev/null +++ b/evidence/indexer-255-20261029/validator/verify_raw_archives.py @@ -0,0 +1,44 @@ +"""Verify lossless compressed evidence; optionally restore outside the repository. + +python evidence/indexer-255-20261029/validator/verify_raw_archives.py +python .../verify_raw_archives.py --restore /tmp/indexer-255-raw +""" +import argparse +import gzip +import hashlib +import json +from pathlib import Path + + +def safe(root, relative): + path = (root / relative).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError("Evidence path escapes root") + return path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--restore", type=Path) + args = parser.parse_args() + bundle = Path(__file__).resolve().parents[1] + entries = json.loads((bundle / "results/raw-archives.json").read_text())["files"] + for entry in entries: + compressed = safe(bundle, entry["archive"]).read_bytes() + assert len(compressed) == entry["archive_bytes"] + assert hashlib.sha256(compressed).hexdigest() == entry["archive_sha256"] + raw = gzip.decompress(compressed) + assert len(raw) == entry["original_bytes"] + assert hashlib.sha256(raw).hexdigest() == entry["original_sha256"] + json.loads(raw) + if args.restore: + target = safe(args.restore, entry["original_path"]) + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != raw: + raise ValueError("Refusing to overwrite different evidence: " + str(target)) + target.write_bytes(raw) + print(f"Verified {len(entries)} original JSON artifacts") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/indexer/README.md b/tests/fixtures/indexer/README.md new file mode 100644 index 000000000..26229369e --- /dev/null +++ b/tests/fixtures/indexer/README.md @@ -0,0 +1,9 @@ +# Small frozen retrieval assessment + +- `development.json`: 8 source-reviewed partial-positive queries, visible before the candidate. +- `heldout.json`: 4 independently source-reviewed unseen queries/labels, released only after runtime candidate freeze. +- Reproduce with `tools/evaluate_indexer.py`; issue-scoped gates, exact revisions, raw data and the efficiency/accuracy table live under `evidence/indexer-255-20261029/`. + +These are not an exhaustive question bank or complete relevance ground truth. Partial-label Hit@5/MRR@5 measure whether a known positive appears, not full Recall/Precision or an agent's task success. Explicit-ID/resource-budget/no-match/stale-projection contracts are checked separately. + +Known label qualifications are retained rather than silently repaired after seeing results: dev02 mixes KV-transfer intent with an AFD hidden-state skill; heldout02 shares a target skill family with dev08. The latter tests unseen natural wording, not family-disjoint generalization. See the evidence's independent label review and per-case table. diff --git a/tests/fixtures/indexer/development.json b/tests/fixtures/indexer/development.json new file mode 100644 index 000000000..22e3b5485 --- /dev/null +++ b/tests/fixtures/indexer/development.json @@ -0,0 +1,190 @@ +{ + "schema_version": 1, + "label_scope": "Partial source-reviewed positives; not exhaustive relevance ground truth. No observed ranking used to select labels.", + "cases": [ + { + "id": "dev01", + "split": "development", + "query": "", + "relevance": "cgroupv2 cpu quota", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.cgroup_resource_controls" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/hardware/host/cgroup-resource-controls/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/hardware/host/cgroup-resource-controls/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/hardware/host/cgroup-resource-controls/SKILL.md": "fd1f5a7e6a55714730044155f5b38c1d700383d4e16c3e91963bbfbe1c773af5" + }, + "rationale": "CPU throttling and quota attribution, not GPU utilization." + }, + { + "id": "dev02", + "split": "development", + "query": "", + "relevance": "P2pNccl KV transfer completion", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.cuda.skills.audit_afd_p2p_exchanges" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/repositories/afd-plugin/audit-afd-p2p-exchanges/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/repositories/afd-plugin/audit-afd-p2p-exchanges/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/repositories/afd-plugin/audit-afd-p2p-exchanges/SKILL.md": "a46110891eca860915e314d6921a22856d631453b22b17837fe30262d6780535" + }, + "rationale": "Send/receive completion and ownership in AFD P2P exchanges." + }, + { + "id": "dev03", + "split": "development", + "query": "", + "relevance": "DLRover checkpoint replica recovery", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.checkpoint_recovery_consistency" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/methods/checkpoint-recovery-consistency/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/methods/checkpoint-recovery-consistency/SKILL.md": "6b7db1eb2780dec27858cb7aa0a3c093c5b5dfa4c30261aebfeaa255213ff688" + }, + "rationale": "Distributed checkpoint validity and replica recovery." + }, + { + "id": "dev04", + "split": "development", + "query": "", + "relevance": "weight loading mmap checkpoint peak memory", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.weight_loading_optimization" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/methods/weight-loading-optimization/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/methods/weight-loading-optimization/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/methods/weight-loading-optimization/SKILL.md": "a1bf0e2ec066d5f30225c6cc5fb603f830f48eb5e2aee3fc3d084ed9d8aebe2f" + }, + "rationale": "Storage-to-device startup and weight loading, distinct from per-request compute." + }, + { + "id": "dev05", + "split": "development", + "query": "", + "relevance": "Ascend npu-smi device health before launch", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.ascend.skills.ascend_device_health" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/hardware/ascend/ascend-device-health/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/hardware/ascend/ascend-device-health/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/hardware/ascend/ascend-device-health/SKILL.md": "03829039860ece35519f2e4c339a48b5014ec25e07a2c150a3efb6a3a4e35b4a" + }, + "rationale": "NPU device availability and health before launching a task." + }, + { + "id": "dev06", + "split": "development", + "query": "", + "relevance": "H3 MiniMax AdaLN SSD residency", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.audit_h3_runtime_residency" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/repositories/h3-c/audit-h3-runtime-residency/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/repositories/h3-c/audit-h3-runtime-residency/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/repositories/h3-c/audit-h3-runtime-residency/SKILL.md": "fdb4d9483c6efa836f9e59617da605fa9c2186e11c2041b05d5c81850ba0475d" + }, + "rationale": "MiniMax-H3 source-defined residency lifetime and storage path." + }, + { + "id": "dev07", + "split": "development", + "query": "", + "relevance": "fixed workload throughput denominator timing claim", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.audit_fixed_workload_claim" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/methods/audit-fixed-workload-claim/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/methods/audit-fixed-workload-claim/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/methods/audit-fixed-workload-claim/SKILL.md": "2be89b1340a3deb978c8e1792d18aeecbe8f5de81d4daf542389f53826b8ebee" + }, + "rationale": "Audit benchmark timing denominator and comparable fixed work." + }, + { + "id": "dev08", + "split": "development", + "query": "", + "relevance": "storage fio direct IO page cache measurement", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.storage_io_measurement" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "/hardware/host/storage-io-measurement/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/hardware/host/storage-io-measurement/SKILL.md" + ], + "evidence_sha256": { + "src/inferrsi/data/skill_library/hardware/host/storage-io-measurement/SKILL.md": "63b883c10a8d3e7d918abfc7175b72e2b401ff1e02697f94b8382ec1c8f369b7" + }, + "rationale": "Storage I/O measurement with cache and device/process attribution." + } + ] +} diff --git a/tests/fixtures/indexer/heldout.json b/tests/fixtures/indexer/heldout.json new file mode 100644 index 000000000..05d60c064 --- /dev/null +++ b/tests/fixtures/indexer/heldout.json @@ -0,0 +1,112 @@ +{ + "schema_version": 1, + "baseline_sha": "ab03f3ffb8f2e5bc02c304828c5c41890eef673e", + "frozen_at_utc": "2026-10-29T12:32:20Z", + "purpose": "Independent, source-reviewed partial relevance labels for four diagnostic retrieval queries. Not a comprehensive gold set or downstream agent efficacy test. Created without running retrieval or inspecting ranks. Held out from candidate optimization until code freeze.", + "metric_contract": { + "positive_queries": "Report relevant Hit@5 and reciprocal rank of first listed relevant ID; other returned IDs are unjudged, not necessarily irrelevant. Do not calculate complete Recall or false-positive precision from these partial labels.", + "delivery": "For each retrieved labelled relevant item, verify its context carries the matching revision and all specified mandatory resource suffixes; missing retrieval and budget omission must be distinct.", + "judgment_limits": "Single assessor/source-grounded labels, four queries, candidate-only corpus, no human validation or production representativeness claim. No forbidden-ID labels are imposed: broader companion skills may also be useful." + }, + "cases": [ + { + "id": "heldout01", + "split": "heldout", + "intent_family": "host_copy_lifetime", + "query": "", + "relevance": "TorchNPU nonblocking transfer finished submitting but pinned RAM remains occupied; when may a CPU buffer be reused?", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.ascend.skills.torchnpu_pinned_host_memory_lifecycle" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "hardware/ascend/torchnpu-pinned-host-memory-lifecycle/SKILL.md", + "hardware/ascend/torchnpu-pinned-host-memory-lifecycle/references/source-notes.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/SKILL.md" + ], + "source_sha256": { + "src/inferrsi/data/skill_library/hardware/ascend/torchnpu-pinned-host-memory-lifecycle/SKILL.md": "23e93ea7c98f99743509fdc04139d78ed2e7123c63e80b366c6beaba7eb70d0d" + }, + "rationale": "The pinned-host allocator procedure distinguishes CPU ownership, pending copy events, active/cached RAM, and completion before reuse; device-HBM tuning is a separate domain." + }, + { + "id": "heldout02", + "split": "heldout", + "intent_family": "storage_measurement", + "query": "", + "relevance": "Checkpoint loading got faster on the second run. Does high disk utilization prove the SSD is the bottleneck, or could cached reads explain it?", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.storage_io_measurement" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "hardware/host/storage-io-measurement/SKILL.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/hardware/host/storage-io-measurement/SKILL.md" + ], + "source_sha256": { + "src/inferrsi/data/skill_library/hardware/host/storage-io-measurement/SKILL.md": "63b883c10a8d3e7d918abfc7175b72e2b401ff1e02697f94b8382ec1c8f369b7" + }, + "rationale": "The procedure requires a warm/cold/unknown cache ledger and corroborating queue/latency/workload evidence; utilization alone does not establish storage saturation." + }, + { + "id": "heldout03", + "split": "heldout", + "intent_family": "snapshot_reuse", + "query": "", + "relevance": "After asynchronous training save staging completes, may the next checkpoint overwrite its cached CPU snapshot while the previous writer is still uploading?", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.dcp_async_save_performance" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "frameworks/pytorch/dcp-async-save-performance/SKILL.md", + "frameworks/pytorch/dcp-async-save-performance/references/sources.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md" + ], + "source_sha256": { + "src/inferrsi/data/skill_library/frameworks/pytorch/dcp-async-save-performance/SKILL.md": "645139ee0c88ec865cad952e24cda35f4648d4b777b732c9b7d274b6c7830845" + }, + "rationale": "The DCP completion table separates source-mutation readiness after staging from cached-snapshot reuse only after previous upload completion." + }, + { + "id": "heldout04", + "split": "heldout", + "intent_family": "diffusion_cancellation", + "query": "", + "relevance": "SGLang video generation request timed out and its job record was deleted. How can I tell whether the worker stopped computing and released its resident components?", + "filters": {}, + "limit": 5, + "context_bytes": 262144, + "relevant_ids": [ + "inferrsi.common.skills.audit_sglang_diffusion_lifetimes" + ], + "forbidden_ids": [], + "required_resource_suffixes": [ + "repositories/sglang/audit-sglang-diffusion-lifetimes/SKILL.md", + "repositories/sglang/audit-sglang-diffusion-lifetimes/references/source-notes.md" + ], + "evidence_paths": [ + "src/inferrsi/data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/SKILL.md" + ], + "source_sha256": { + "src/inferrsi/data/skill_library/repositories/sglang/audit-sglang-diffusion-lifetimes/SKILL.md": "b0a4ba4751087126143ef86983798deb0ed00e24b6ff06bf6ef76a9df6e274f2" + }, + "rationale": "The SGLang diffusion procedure distinguishes socket timeout/store deletion from worker/device completion and component-use residency; its source shows abort remains TODO." + } + ] +} diff --git a/tests/test_evidence_inventory.py b/tests/test_evidence_inventory.py index 745aabf80..13741c71c 100644 --- a/tests/test_evidence_inventory.py +++ b/tests/test_evidence_inventory.py @@ -145,6 +145,6 @@ def test_checked_in_inventory(): repo = Path(__file__).resolve().parents[1] result = validate_inventory(repo) assert result["errors"] == [] - assert len(result["bundles"]) == 9 - assert result["checked_hashes"] == 166 + assert len(result["bundles"]) == 10 + assert result["checked_hashes"] == 199 assert result["unavailable"] # Historical raw artifacts are not in this checkout. diff --git a/tests/test_indexer_evaluation_summary.py b/tests/test_indexer_evaluation_summary.py new file mode 100644 index 000000000..7427ed052 --- /dev/null +++ b/tests/test_indexer_evaluation_summary.py @@ -0,0 +1,23 @@ +"""The evidence summary resamples paired groups and retains every observation.""" +from tools.evaluate_indexer import percentile, summarize + + +def test_constant_paired_groups_have_exact_gain_and_interval(): + groups = [] + for _ in range(20): + groups.append({arm: { + name: {"ns": value, "output_sha256": "same"} + for name in ("hot_relevance/dev01", "hot_relevance/dev02", "search") + } for arm, value in (("A", 100), ("B", 80))}) + result = summarize(groups) + assert result["primary_reduction_percent"] == 20 + assert result["primary_95ci"] == [20, 20] + assert result["paths"]["search"]["A"]["n"] == 20 + assert result["paths"]["search"]["output_parity"] + groups[0]["B"]["search"]["output_sha256"] = "different" + assert not summarize(groups)["paths"]["search"]["output_parity"] + + +def test_tail_is_nearest_rank_not_interpolated_or_trimmed(): + assert percentile(list(range(1, 21)), .95) == 19 + assert percentile([1], .95) == 1 diff --git a/tools/evaluate_indexer.py b/tools/evaluate_indexer.py new file mode 100644 index 000000000..222d54112 --- /dev/null +++ b/tools/evaluate_indexer.py @@ -0,0 +1,311 @@ +"""Read-only, CPU-only retrieval evidence runner; no model or hardware invocation. + +Use --help. Baseline/candidate paths and frozen protocol are explicit inputs. +This tool measures real packaged retrieval, not a synthetic workload or CI gate. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import random +import statistics +import subprocess +import sys +import time +from pathlib import Path + + +def encode(value): + return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() + + +def digest(value): + return hashlib.sha256(value).hexdigest() + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def identity(root): + groups = {} + for name, paths in ( + ("python", sorted((root / "src").rglob("*.py"))), + ("corpus", sorted((root / "src/inferrsi/data").rglob("*"))), + ): + members = { + p.relative_to(root).as_posix(): digest(p.read_bytes()) + for p in paths if p.is_file() and "__pycache__" not in p.parts + } + groups[name] = {"sha256": digest(encode(members)), "files": len(members)} + groups["revision"] = subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip() + groups["environment"] = { + "python": platform.python_version(), "system": platform.system(), + "release": platform.release(), "machine": platform.machine(), "cpu_count": os.cpu_count(), + } + return groups + + +def options(case, context=False): + value = {"query": case.get("query", ""), "limit": case.get("limit", 5)} + value.update(case.get("filters", {})) + if case.get("relevance") is not None: + value["relevance"] = case["relevance"] + if context: + value["context_bytes"] = case.get("context_bytes", 262144) + return value + + +def child(root, action, payload): + env = {"PATH": os.defpath, "PYTHONPATH": str(root / "src"), "PYTHONDONTWRITEBYTECODE": "1"} + run = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--worker", str(root), action], + input=json.dumps(payload), text=True, capture_output=True, env=env, check=True, timeout=180, + ) + return json.loads(run.stdout) + + +def cli(root, mode, case): + args = [sys.executable, "-m", "inferrsi"] + if mode == "search": + args += ["knowledge", "search", "cgroupv2", "--limit", "5", "--json"] + else: + args += ["collect", case["query"], "--relevance", case["relevance"], + "--limit", str(case["limit"]), "--json"] + start = time.perf_counter_ns() + run = subprocess.run(args, capture_output=True, check=True, timeout=180, env={ + "PATH": os.defpath, "PYTHONPATH": str(root / "src"), "PYTHONDONTWRITEBYTECODE": "1", + }) + elapsed = time.perf_counter_ns() - start + start = time.perf_counter_ns() + result = json.loads(run.stdout) + decode = time.perf_counter_ns() - start + return {"ns": elapsed, "decode_ns": decode, "bytes": len(run.stdout), + "output_sha256": digest(encode(result))} + + +def worker(root, action, payload): + sys.path.insert(0, str(root / "src")) + from inferrsi.catalog import search_knowledge + from inferrsi.knowledge_operations import KnowledgeReader + + if action == "identity": + return identity(root) + if action == "bench": + rows = {} + for case in payload["queries"]: + reader = KnowledgeReader() + args = options(case) + reader.collect(**args) + start = time.perf_counter_ns() + value = reader.collect(**args) + elapsed = time.perf_counter_ns() - start + rows["hot_relevance/" + case["id"]] = { + "ns": elapsed, "bytes": len(encode(value)), "output_sha256": digest(encode(value))} + case = payload["queries"][0] + for mode in ("fresh_relevance", "default", "context", "search"): + args = options(case, context=mode == "context") + if mode in ("default", "search"): + args.pop("relevance", None) + args["query"] = "cgroupv2" + reader = KnowledgeReader() + call = search_knowledge if mode == "search" else reader.collect + if mode != "fresh_relevance": + call(**args) + start = time.perf_counter_ns() + if mode == "fresh_relevance": + value = KnowledgeReader().collect(**args) + else: + value = call(**args) + elapsed = time.perf_counter_ns() - start + rows[mode] = {"ns": elapsed, "bytes": len(encode(value)), + "output_sha256": digest(encode(value))} + for mode in ("relevance", "search"): + rows["cli_" + mode] = cli(root, mode, case) + for api, cli_name in (("hot_relevance/dev01", "cli_relevance"), + ("search", "cli_search")): + if rows[api]["output_sha256"] != rows[cli_name]["output_sha256"]: + raise ValueError("API/CLI payload mismatch: " + api) + return rows + if action == "contracts": + from inferrsi._skill_selection import SkillIndexError, indexed_skill + reader = KnowledgeReader() + cgroup = "inferrsi.common.skills.cgroup_resource_controls" + afd = "inferrsi.cuda.skills.audit_afd_p2p_exchanges" + rows = [] + def check(name, passed, output): + rows.append({"id": name, "passed": bool(passed), "output": output}) + result = reader.collect(relevance="floristry orchid") + check("no_match", result["no_match"], result) + result = reader.collect(relevance="P2pNccl", backend="ascend") + check("incompatible_filter", result["no_match"], result) + result = reader.collect(ids=[cgroup], relevance="unrelated orchid") + check("explicit_id_unrelated", [x["id"] for x in result["items"]] == [cgroup], result) + small = reader.collect(ids=[afd], context_bytes=1) + required = small["context_omissions"][0]["required_bytes"] + exact = reader.collect(ids=[afd], context_bytes=required) + check("exact_required_budget", bool(exact["contexts"]) and + exact["context_bytes"] == required and not exact["context_omissions"], exact) + short = reader.collect(ids=[afd], context_bytes=required - 1) + check("undersized_required_budget", not short["contexts"] and + short["context_omissions"][0]["reason"] == "context_limit", short) + bundle = reader.read(cgroup) + record = reader.records()[cgroup] + core = record["skill_documents"]["original"] + bundle.content[core] += b"\nchanged source" + bundle.dependencies.clear() # isolate projection validation from dependency validation + view = {**bundle.view(), "authority": "package"} + rejected = False + try: + indexed_skill(record, view, bundle.content) + except SkillIndexError: + rejected = True + check("stale_source_projection", rejected, {"rejected": rejected}) + return rows + if action == "cases": + rows = [] + for case in payload: + reader = KnowledgeReader() + result = reader.collect(**options(case, context=True)) + ids = [item["id"] for item in result["items"]] + ranks = [ids.index(i) + 1 for i in case["relevant_ids"] if i in ids] + selected = [i for i in case["relevant_ids"] if i in ids] + complete = [] + for target in selected: + context = next((c for c in result["contexts"] + if c["ref"] == "package:" + target), None) + checks = [] + if context: + bundle = reader.read(target, revision=context["revision"]) + # Read the exact revision once; verify every complete emitted resource. + for resource in context["resources"]: + data = bundle.content[resource["path"]] + checks.append(data == resource["content"].encode() + and digest(data) == resource["sha256"]) + paths = [r["path"] for r in context["resources"]] + checks.extend(any(p.endswith(s) for p in paths) + for s in case.get("required_resource_suffixes", [])) + checks.append(bundle.revision == context["revision"]) + complete.append(bool(context) and all(checks)) + rows.append({"id": case["id"], "split": case["split"], "hit_at_5": bool(ranks), + "rr_at_5": 1 / min(ranks) if ranks else 0, + "retrieved_relevant": len(selected), "complete_relevant": sum(complete), + "forbidden_hits": sorted(set(ids) & set(case.get("forbidden_ids", []))), + "output_sha256": digest(encode(result)), "result": result}) + return rows + raise ValueError(action) + + +def percentile(values, q): + return sorted(values)[max(0, math.ceil(q * len(values)) - 1)] + + +def summarize(groups): + names = list(groups[0]["A"]) + rows = {} + for name in names: + arms = {arm: [g[arm][name]["ns"] / 1e6 for g in groups] for arm in ("A", "B")} + rows[name] = { + arm: {"n": len(v), "p50_ms": statistics.median(v), "p95_ms": percentile(v, .95)} + for arm, v in arms.items() + } + for stat in ("p50_ms", "p95_ms"): + rows[name][stat + "_change_percent"] = ( + 100 * (rows[name]["B"][stat] / rows[name]["A"][stat] - 1)) + rows[name]["output_parity"] = all( + g["A"][name]["output_sha256"] == g["B"][name]["output_sha256"] for g in groups) + gains = [statistics.median([ + 100 * (g["A"][name]["ns"] - g["B"][name]["ns"]) / g["A"][name]["ns"] + for name in names if name.startswith("hot_relevance/")]) for g in groups] + rng = random.Random(255) + bootstrap = [statistics.median(rng.choices(gains, k=len(gains))) for _ in range(10000)] + return {"paths": rows, "primary_reduction_percent": statistics.median(gains), + "primary_95ci": [percentile(bootstrap, .025), percentile(bootstrap, .975)]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--worker", type=Path) + parser.add_argument("action", nargs="?") + parser.add_argument("--baseline", type=Path) + parser.add_argument("--candidate", type=Path) + parser.add_argument("--protocol", type=Path) + parser.add_argument("--cases", type=Path, nargs="*") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.worker: + print(json.dumps(worker(args.worker.resolve(), args.action, json.load(sys.stdin)))) + return + if not all((args.baseline, args.candidate, args.protocol, args.output)): + parser.error("baseline, candidate, protocol and output are required") + protocol = json.loads(args.protocol.read_text()) + roots = {"A": args.baseline.resolve(), "B": args.candidate.resolve()} + before = {a: child(p, "identity", {}) for a, p in roots.items()} + if before["A"]["revision"] != protocol["baseline_revision"]: + raise ValueError("baseline revision mismatch") + if before["A"]["corpus"] != before["B"]["corpus"]: + raise ValueError("corpus mismatch") + result = {"protocol_sha256": digest(args.protocol.read_bytes()), "before": before, + "groups": [], "verdict": "INCONCLUSIVE"} + try: + for index in range(protocol["measurement"]["groups"]): + order = "AB" if index % 2 == 0 else "BA" + group = {"index": index, "order": order} + for arm in order: + group[arm] = child(roots[arm], "bench", protocol) + result["groups"].append(group) + write(args.output / "timing.json", result) + print(f"Completed group {index + 1}", flush=True) + result["after"] = {a: child(p, "identity", {}) for a, p in roots.items()} + if result["after"] != before: + raise ValueError("source/corpus/environment changed during measurement") + result["summary"] = summarize(result["groups"]) + summary = result["summary"] + failed = [] + if summary["primary_reduction_percent"] < protocol["primary"]["gain_min_percent"]: + failed.append("primary practical gain") + if summary["primary_95ci"][0] <= 0: + failed.append("primary confidence interval") + for name, row in summary["paths"].items(): + primary = name.startswith("hot_relevance/") + p50_limit = (protocol["primary"]["each_query_p50_max_regression_percent"] + if primary else protocol["secondary_limits"][ + "each_path_p50_max_regression_percent"]) + if row["p50_ms_change_percent"] > p50_limit: + failed.append(name + " p50 regression") + if not primary and row["p95_ms_change_percent"] > protocol["secondary_limits"][ + "each_path_p95_max_regression_percent"]: + failed.append(name + " p95 regression") + if not row["output_parity"]: + failed.append(name + " output mismatch") + result["failed_gates"] = failed + result["verdict"] = "FAIL" if failed else "PASS" + result["verdict_scope"] = "Timing only; accuracy and contracts reported separately" + contracts = {arm: child(root, "contracts", {}) for arm, root in roots.items()} + write(args.output / "contracts.json", contracts) + result["contracts_pass"] = (contracts["A"] == contracts["B"] and + all(r["passed"] for r in contracts["A"])) + if args.cases: + cases = [c for path in args.cases for c in json.loads(path.read_text())["cases"]] + outcomes = {arm: child(root, "cases", cases) for arm, root in roots.items()} + for arm, rows in outcomes.items(): + write(args.output / (arm + "-cases.json"), rows) + result["accuracy_output_parity"] = outcomes["A"] == outcomes["B"] + result["cases_sha256"] = {path.name: digest(path.read_bytes()) for path in args.cases} + except Exception as exc: + result["error"] = type(exc).__name__ + ": " + str(exc) + result["verdict"] = "INCONCLUSIVE" + raise + finally: + write(args.output / "timing.json", result) + + +if __name__ == "__main__": + main() diff --git a/tools/evidence_inventory.json b/tools/evidence_inventory.json index 5ef5f2630..563753256 100644 --- a/tools/evidence_inventory.json +++ b/tools/evidence_inventory.json @@ -1,6 +1,10 @@ { "version": 1, "bundles": { + "indexer-255-20261029": { + "kind": "campaign", + "manifests": [{"path": "plan/integrity-manifest.json", "schema": "bundle-sha256"}] + }, "_template": {"kind": "template", "manifests": []}, "qwen38-h20-fp8-single-request-vllm032": { "kind": "campaign", "manifests": [