diff --git a/reviews/retrofit_20261006/CONTRACT.md b/reviews/retrofit_20261006/CONTRACT.md new file mode 100644 index 000000000..8ce445c7a --- /dev/null +++ b/reviews/retrofit_20261006/CONTRACT.md @@ -0,0 +1,135 @@ +# Retrofit evidence records — contract v0.1 + +**Proposed** shared format for the retrofit dispatch in +[main#275 6018658478](https://github.com/d6g8k5htny-coder/main/issues/275#issuecomment-6018658478) +(contract role accepted in [6018831775](https://github.com/d6g8k5htny-coder/main/issues/275#issuecomment-6018831775)). +It follows the closure-evidence protocol proposed in +[main#276](https://github.com/d6g8k5htny-coder/main/pull/276) and the decision record v1.1 in +[main#275](https://github.com/d6g8k5htny-coder/main/issues/275). + +**Scientific effect: NONE.** A record states that an exact source object does or does not have a record on one +evidence axis. It is not a review, an acceptance, a status, a register transition or an independence claim. No +shard edits proof, review or result bytes, `GRAPH.json`, `STATUS.md`, `PROOF_INDEX.md` or any index. + +## Files + +Each shard writes exactly one file, `reviews/retrofit_20261006//RECORDS.json`, plus an optional `NOTE.md` in +the same directory. One writer per shard directory. Shards never write here, in another shard's directory or in +shared code. `CONTRACT_EXAMPLE.json` beside this file shows the shape and is not an authoritative record. + +The inventory map of dispatch row I is out of scope for v0.1. Its membership classes are not evidence states. + +## Shape + +```json +{ + "schema": "retrofit-records/v0.1", + "shard": "B1_side24", + "baseline": {"Math-": "08f86862f859ac4804b4fd6c6477a7ed0421e37f", + "main": "cddbb7f6cf3f57f3b495277f7da148287e027b19"}, + "author": {"provider": "...", "model_or_agent": "...", "session": "... or UNKNOWN"}, + "scientific_effect": "NONE", + "status_authority": false, + "records": [{ + "id": "B1-001", + "subject": {"repository": "Math-", "commit": "<40 hex>", "path": "relative/path", "blob": "<40 hex>"}, + "delta": false, + "axis": "numerical_reproduction", + "state": "recorded", + "evidence": [{"kind": "file", "repository": "Math-", "commit": "<40 hex>", "path": "...", "blob": "<40 hex>"}, + {"kind": "github_comment", "repository": "main", "ref": "6018658478"}, + {"kind": "github_review", "repository": "Math-", "ref": "", "commit": "<40 hex>"}, + {"kind": "workflow_run", "repository": "Math-", "ref": "", "attempt": 1, "job": null, + "run_head_sha": "<40 hex>", "checked_commit": null, "purpose": "check", "conclusion": "failure", + "expected_conclusion": "success"}], + "performer": {"provider": "...", "model_or_agent": "...", "session": "... or UNKNOWN"}, + "exposure": "what the performer had read", + "independence_credit": 0, + "alias_of": null, + "notes": "" + }] +} +``` + +Every object has exactly the keys shown, and no others. In particular there is no `status`, `verdict` or +`classification` field. + +| Field | Rule | +|---|---| +| `shard` | Matches its directory name (`[A-Z][A-Za-z0-9_]*`) | +| `baseline` | Exactly the pinned v0.1 cut. A later cut is a new contract version | +| `subject` | An exact object: repository, commit, path and Git blob. With `delta: false` the commit is the baseline. A candidate object sets `delta: true` and pins its own commit. When verified against a clone, the commit must be a commit object and the path must name a blob in it (not a tree or a submodule) | +| `axis` | `source_review`, `provider_distinct_review`, `blind_reconstruction`, `adversarial_attack`, `formal_evidence`, `numerical_reproduction`, `novelty`, `human_reading`, `custody`, `observable_statement` | +| `state` | `recorded` (evidence linked), `not_recorded` (checked, none found), `unknown` (not checked), `not_applicable`. Availability only; see below | +| `evidence` | Required for `recorded`, empty for `not_recorded` and `not_applicable`. One item per native object. A `file` item carries commit, path and blob. `github_comment`, `github_review` and `workflow_run` items carry the numeric GitHub id as `ref`. A `github_review` also carries its native `commit_id` as `commit`. A `github_comment` has no commit: a SHA quoted in a comment body is never a native binding. A `workflow_run` also carries `attempt` (positive integer), `job` (numeric job id as a string, or `null` for the whole run), `run_head_sha` (the run's native `head_sha`), `checked_commit` (see below, or `null`), `purpose` (`check` or `negative_control`), `conclusion` (GitHub's native value, copied as served: `success`, `failure`, `cancelled`, `skipped`, `timed_out`, `neutral`, `action_required`, `stale` or `startup_failure`) and `expected_conclusion` (`success` or `failure`) | +| `performer` | Who produced the evidence, not who wrote the record. `UNKNOWN` where unknown; never `Human` or Dylan's name for an AI executor. This is the writer's obligation and a reviewer's check: the validator enforces the shape only and cannot tell whether an attribution is true | +| `exposure` | What that performer had read. Required | +| `independence_credit` | Always the integer `0`. Organizational independence is not established by these records | +| `alias_of` | `null`, or the `id` of a canonical record in the same file (one whose own `alias_of` is `null`) when this record restates exactly that record's evidence, for example for a byte alias of its subject. The alias has the same axis, the same state and the same evidence items (in any order). There are no chains, so there are no cycles. A copy of a verdict posted as a separate native object is its own evidence item in its own record, with the original named in `notes`. Aliases are counted separately and never as additional evidence. Aliasing is within one file; the aggregate does not deduplicate across shards | + +Quoted verdict words may appear inside `notes` as quotations. They never become a field value. + +### State is availability, not outcome + +`recorded` means linked evidence exists. It does not mean the referenced event succeeded or that it supports the +subject. A failed run, a review that asks for changes, an integration intention and a landing receipt are each +`recorded` as their own evidence items; none of them is collapsed into another or into a pass. A negative control +that fails as intended is recorded with `purpose: negative_control`, `conclusion: failure` and +`expected_conclusion: failure`, so it is not mistaken for a failed check. A later event never relabels an earlier +one: a later successful run is a new item and the earlier failure stays as served. + +### Run head is not the checked commit + +`run_head_sha` is the run's native `head_sha`: the run-associated source head only, never evidence of what was checked +out or executed. For a `pull_request` run it is the pull request's head, while the default checkout is the merge ref +(`refs/pull//merge`), a different commit; an explicit checkout can differ again. +`checked_commit` is the commit the job actually checked out, taken only from that run's own evidence (its checkout +step or a receipt it produced), and that evidence is recorded as its own item in the same record. When the checkout was +not inspected, `checked_commit` is `null`; it is never copied from `run_head_sha`. For example, Math- run +`37412915949` (`pull_request`) has `run_head_sha` `89170cf0…` and checked the merge commit `3e56e964…`, whose parents +are the base `532bc63f…` and that head. + +## Validation + +```sh +python3 -B -S tools/retrofit_record_check.py --repo [--main-repo
] reviews/retrofit_20261006//RECORDS.json +python3 -B -S tools/retrofit_record_check.py --aggregate --repo reviews/retrofit_20261006/*/RECORDS.json +``` + +The checker rejects: +- duplicate JSON keys and `NaN` or `Infinity`; +- unknown or missing keys, unknown axes or states, and status words used as a state; +- a non-baseline commit without `delta: true`; +- a `subject` or `file` blob that differs from Git (with `--repo` or `--main-repo`); +- a `recorded` state without evidence; +- a `github_review` without its native 40-hex `commit`, or a `github_comment` that carries a commit; +- a `workflow_run` without the full binding (attempt, job, run head, checked commit or `null`, purpose, native + conclusion, expected conclusion), or with a value outside those enums; +- non-zero or Boolean independence credit; +- an `alias_of` that dangles, names itself or another alias, or differs from its canonical record in axis, state or + evidence; +- a commit that is not a commit object, or a path that is not a blob, when verified against a clone; +- the same input file twice (under any spelling) and two inputs that declare the same shard. + +A malformed value of any JSON type is reported as a violation of its file; the remaining files are still read. + +`--aggregate` prints, per exact subject (full repository, commit, path and blob) and axis, the states recorded by +each shard, with aliases counted separately. `evidence_items_non_alias` counts evidence items as listed; +`distinct_evidence_items` counts identical items once across all inputs. Neither is a count of independent evidence +or of coverage of any inventory. It writes nothing. A later, separately reviewed change may join landed records onto the downstream-gate +graph view (Math-#381). That join is not part of v0.1. + +## Not in v0.1 + +- No scientific status, acceptance or promotion field. +- No shared manifest or index. +- No schema for the inventory map. +- No automatic import into GRAPH, STATUS or PROOF_INDEX. +- No selection of a run as a success. A later consumer may use a `workflow_run` item for a claim only when its + `checked_commit` is not `null` and, with its `job` and `purpose`, binds to that claim, and its `conclusion` equals + its `expected_conclusion`. These are necessary filters, never sufficient: the consumer must also read the checkout + evidence recorded beside the run, and a `checked_commit` without such an item is treated as unknown. Run success or + a matching pull-request head never establishes coverage of a base. + +Changes to this contract take a new version and their own nonauthor review. Records written under v0.1 stay valid +under v0.1. diff --git a/reviews/retrofit_20261006/CONTRACT_EXAMPLE.json b/reviews/retrofit_20261006/CONTRACT_EXAMPLE.json new file mode 100644 index 000000000..42e221bc5 --- /dev/null +++ b/reviews/retrofit_20261006/CONTRACT_EXAMPLE.json @@ -0,0 +1,76 @@ +{ + "schema": "retrofit-records/v0.1", + "shard": "EXAMPLE", + "baseline": { + "Math-": "08f86862f859ac4804b4fd6c6477a7ed0421e37f", + "main": "cddbb7f6cf3f57f3b495277f7da148287e027b19" + }, + "author": { + "provider": "Anthropic", + "model_or_agent": "Claude (Claude Code)", + "session": "session_017Mi3hxjaxV45x6zo6o1ee3" + }, + "scientific_effect": "NONE", + "status_authority": false, + "records": [ + { + "id": "EX-001", + "subject": { + "repository": "Math-", + "commit": "08f86862f859ac4804b4fd6c6477a7ed0421e37f", + "path": "coefficients/side24_v1/PROOF.md", + "blob": "44b66f04f89fcd87383b3603fa69f1feb64cdddd" + }, + "delta": false, + "axis": "numerical_reproduction", + "state": "recorded", + "evidence": [ + { + "kind": "file", + "repository": "Math-", + "commit": "08f86862f859ac4804b4fd6c6477a7ed0421e37f", + "path": "reviews/side24_v1_coefficient_claude_20260929/REVIEW.md", + "blob": "665d1683f33a7180957923c3cb2abce8ade235d1" + }, + { + "kind": "file", + "repository": "Math-", + "commit": "08f86862f859ac4804b4fd6c6477a7ed0421e37f", + "path": "reviews/side24_v1_coefficient_claude_20260929/coefficient_review_check.py", + "blob": "c47d337d6601b7fce5a2436836635837b46cfe35" + } + ], + "performer": { + "provider": "Anthropic", + "model_or_agent": "Claude (Claude Code)", + "session": "session_01NMeKEismAyeqgdB4sy2NJU" + }, + "exposure": "Per the review's provenance table: the reviewer had consumed the published side24_v1 endpoints in earlier work and redid the derivations from the note's text before reading the first review.", + "independence_credit": 0, + "alias_of": null, + "notes": "Illustrative example of the v0.1 shape only. The recomputation uses the same Stirling-series method family with different parameters (precision 90, shift 120, B2..B40, own pi); it is not a structurally different method. Shard B1 owns the authoritative SIDE24 records." + }, + { + "id": "EX-002", + "subject": { + "repository": "Math-", + "commit": "08f86862f859ac4804b4fd6c6477a7ed0421e37f", + "path": "coefficients/side24_v1/PROOF.md", + "blob": "44b66f04f89fcd87383b3603fa69f1feb64cdddd" + }, + "delta": false, + "axis": "adversarial_attack", + "state": "unknown", + "evidence": [], + "performer": { + "provider": "UNKNOWN", + "model_or_agent": "UNKNOWN", + "session": "UNKNOWN" + }, + "exposure": "Not searched for this example.", + "independence_credit": 0, + "alias_of": null, + "notes": "Illustrative: 'unknown' means not checked, which is different from 'not_recorded' (checked and none found)." + } + ] +} diff --git a/tests/test_retrofit_record_check.py b/tests/test_retrofit_record_check.py new file mode 100644 index 000000000..93d4d661a --- /dev/null +++ b/tests/test_retrofit_record_check.py @@ -0,0 +1,411 @@ +"""Behavior of tools/retrofit_record_check.py against reviews/retrofit_20261006/CONTRACT.md v0.1.""" +import copy +import importlib.util +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +TOOL = ROOT / 'tools' / 'retrofit_record_check.py' +EXAMPLE = ROOT / 'reviews' / 'retrofit_20261006' / 'CONTRACT_EXAMPLE.json' +SPEC = importlib.util.spec_from_file_location('retrofit_record_check', TOOL) +rc = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(rc) + + +def example(): + return rc.load_strict(EXAMPLE.read_text(encoding='utf-8')) + + +def git(*args): + return subprocess.run(['git', '-C', str(ROOT), *args], capture_output=True, text=True, check=True).stdout.strip() + + +class Shape(unittest.TestCase): + def test_example_is_valid(self): + self.assertEqual(rc.validate(example()), []) + + def mutated(self, change): + data = example() + change(data) + return rc.validate(data) + + def test_closed_vocabulary_and_keys(self): + cases = { + 'status word as state': lambda d: d['records'][0].update(state='PROVED_REVIEWED'), + 'CLOSED as state': lambda d: d['records'][0].update(state='CLOSED'), + 'unknown axis': lambda d: d['records'][0].update(axis='closure'), + 'extra status field': lambda d: d['records'][0].update(status='ACCEPT'), + 'extra top-level field': lambda d: d.update(classification='PROVED_REVIEWED'), + 'missing exposure': lambda d: d['records'][0].pop('exposure'), + 'status authority true': lambda d: d.update(status_authority=True), + 'status authority zero': lambda d: d.update(status_authority=0), + 'scientific effect': lambda d: d.update(scientific_effect='ACCEPT'), + 'other baseline': lambda d: d['baseline'].update(main='0' * 40), + 'independence one': lambda d: d['records'][0].update(independence_credit=1), + 'independence boolean': lambda d: d['records'][0].update(independence_credit=False), + 'recorded without evidence': lambda d: d['records'][0].update(evidence=[]), + 'not_recorded with evidence': lambda d: d['records'][0].update(state='not_recorded'), + 'non-baseline without delta': lambda d: d['records'][0]['subject'].update(commit='a' * 40), + 'delta not boolean': lambda d: d['records'][0].update(delta=0), + 'path traversal': lambda d: d['records'][0]['subject'].update(path='../STATUS.md'), + 'absolute path': lambda d: d['records'][0]['subject'].update(path='/etc/passwd'), + 'uppercase blob': lambda d: d['records'][0]['subject'].update(blob='A' * 40), + 'duplicate id': lambda d: d['records'][1].update(id='EX-001'), + 'dangling alias': lambda d: d['records'][1].update(alias_of='EX-999'), + 'self alias': lambda d: d['records'][1].update(alias_of='EX-002'), + 'performer missing keys': lambda d: d['records'][0].update(performer={'provider': 'UNKNOWN'}), + 'empty performer field': lambda d: d['records'][0]['performer'].update(session=''), + 'comment ref not numeric': lambda d: d['records'][0]['evidence'].append( + {'kind': 'github_comment', 'repository': 'main', 'ref': 'latest'}), + 'unknown evidence kind': lambda d: d['records'][0]['evidence'].append( + {'kind': 'chat', 'repository': 'main', 'ref': '1'}), + 'unknown repository': lambda d: d['records'][0]['subject'].update(repository='elsewhere'), + 'empty records': lambda d: d.update(records=[]), + } + for name, change in cases.items(): + with self.subTest(case=name): + self.assertNotEqual(self.mutated(change), []) + + def test_delta_is_accepted_when_well_formed(self): + def change(d): + d['records'][0].update(delta=True) + d['records'][0]['subject'].update(commit='b' * 40) + self.assertEqual(self.mutated(change), []) + + def test_validator_does_not_judge_attribution(self): + # Truthful attribution is the writer's obligation and a reviewer's check; the validator only enforces the + # shape. A complete human-shaped performer is well formed, as is UNKNOWN. + def change(d): + d['records'][0].update(performer={'provider': 'Human', 'model_or_agent': 'Human reader', + 'session': 'UNKNOWN'}) + self.assertEqual(self.mutated(change), []) + + def test_malformed_values_give_violations_not_exceptions(self): + wrong = ([], {}, None, 1, 1.5, True, 'x', [{'kind': 'file'}]) + repos = {'Math-': ROOT} + + def leaves(node, path=()): + yield path + if isinstance(node, dict): + for key, value in node.items(): + yield from leaves(value, path + (key,)) + elif isinstance(node, list): + for index, value in enumerate(node): + yield from leaves(value, path + (index,)) + + original = example() + for path in leaves(original): + if not path: + continue + here = original + for step in path: + here = here[step] + for value in wrong: + if type(value) is type(here) and value == here: + continue + data = example() + parent = data + for step in path[:-1]: + parent = parent[step] + parent[path[-1]] = copy.deepcopy(value) + with self.subTest(path=path, value=value): + # Git is consulted only where a repository name is the substituted value. + errors = rc.validate(data, shard_dir='EXAMPLE', + repos=repos if path[-1] == 'repository' else None) + self.assertIsInstance(errors, list) + if type(value) is not type(here): + self.assertNotEqual(errors, []) + + +class Aliases(unittest.TestCase): + def with_copy(self, change): + data = example() + twin = copy.deepcopy(data['records'][0]) + twin.update(id='EX-003', alias_of='EX-001') + data['records'].append(twin) + change(data['records']) + return data + + def test_canonical_alias_is_counted_separately(self): + data = self.with_copy(lambda r: None) + self.assertEqual(rc.validate(data), []) + counts = rc.aggregate([data])['counts'] + self.assertEqual((counts['records'], counts['aliases'], counts['evidence_items_non_alias']), (3, 1, 2)) + self.assertEqual(counts['distinct_evidence_items'], 2) + + def test_reordered_evidence_is_still_the_same_evidence(self): + data = self.with_copy(lambda r: r[2]['evidence'].reverse()) + self.assertEqual(rc.validate(data), []) + + def test_invalid_aliases(self): + cases = { + 'cycle': lambda r: r[0].update(alias_of='EX-003'), + 'chain': lambda r: r.append(dict(copy.deepcopy(r[2]), id='EX-004', alias_of='EX-003')), + 'unrelated evidence': lambda r: r[2].update( + evidence=[{'kind': 'github_comment', 'repository': 'main', 'ref': '6018658478'}]), + 'subset of evidence': lambda r: r[2]['evidence'].pop(), + 'other axis': lambda r: r[2].update(axis='source_review'), + 'other state': lambda r: r[2].update(state='unknown'), + 'alias of an unknown record': lambda r: r[1].update(alias_of='EX-001'), + 'dangling': lambda r: r[2].update(alias_of='EX-999'), + 'self': lambda r: r[2].update(alias_of='EX-003'), + 'not a string': lambda r: r[2].update(alias_of=['EX-001']), + } + for name, change in cases.items(): + with self.subTest(case=name): + self.assertNotEqual(rc.validate(self.with_copy(change)), []) + + +class Aggregate(unittest.TestCase): + def test_subject_identity_is_the_full_tuple(self): + data = example() + first, second = data['records'][0], copy.deepcopy(data['records'][0]) + second['id'] = 'EX-003' + for record, tail in ((first, 'a'), (second, 'b')): + record.update(delta=True) + record['subject'].update(commit='c' * 39 + tail, blob='d' * 39 + tail) + data['records'] = [first, second] + self.assertEqual(rc.validate(data), []) + subjects = rc.aggregate([data])['subjects'] + self.assertEqual(len(subjects), 2) + for key in subjects: + self.assertIn('c' * 39, key) + self.assertIn('d' * 39, key) + + def test_shard_must_match_directory(self): + self.assertEqual(rc.validate(example(), shard_dir='EXAMPLE'), []) + self.assertNotEqual(rc.validate(example(), shard_dir='B1_side24'), []) + + def test_strict_parsing(self): + text = EXAMPLE.read_text(encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'duplicate JSON key'): + rc.load_strict(text.replace('"shard": "EXAMPLE",', '"shard": "EXAMPLE", "shard": "OTHER",', 1)) + with self.assertRaisesRegex(ValueError, 'non-finite'): + rc.load_strict(text.replace('"independence_credit": 0', '"independence_credit": NaN', 1)) + + +def run_item(ref, job, head, conclusion, purpose='check', expected='success', attempt=1, checked=None): + return {'kind': 'workflow_run', 'repository': 'Math-', 'ref': ref, 'attempt': attempt, 'job': job, + 'run_head_sha': head, 'checked_commit': checked, 'purpose': purpose, 'conclusion': conclusion, + 'expected_conclusion': expected} + + +# Native objects read from the GitHub API on 2026-10-06. The two #343 runs (D2 square/Schur companion) are the +# failed and successful executions named in main#275 6019011163; the custody chain is the six records of +# main#275 6019173460 plus the landing push run of Math-#250. Their checkouts were not inspected, so checked_commit is +# null. The run-identity pair is from Math-#383 6019566432: PR run 37412915949 checked the merge commit 3e56e964 (its +# API parents are 532bc63f and the run head 89170cf0); push run 37458652010 checked its own head 3ab51620. +FAILED_RUN = run_item('37414215874', '112109136502', '3479b6e127b05e953f84feef3f3fb7e88de625ac', 'failure') +PASSED_RUN = run_item('37416061763', '112114814333', 'cc8b2d5bd7505b7f185607880bc0aa1403c8c148', 'success') +CUSTODY_CHAIN = [ + {'kind': 'github_review', 'repository': 'Math-', 'ref': '5365065925', + 'commit': '4e25b9159b7711f4625ec2f6b34b00f819fa29a6'}, + {'kind': 'github_review', 'repository': 'Math-', 'ref': '5401199597', + 'commit': '0efd00e64446f1859cd2c6bb1833068bb16e03a3'}, + {'kind': 'github_review', 'repository': 'Math-', 'ref': '5401287158', + 'commit': '5634edade1db84ddefa25f2cbf92e79da6cce7eb'}, + {'kind': 'github_comment', 'repository': 'Math-', 'ref': '5970327030'}, + {'kind': 'github_comment', 'repository': 'Math-', 'ref': '5970508391'}, + run_item('37131496772', None, '7858329974e28be79f29b22644370084ff43da4f', 'success'), + {'kind': 'github_comment', 'repository': 'Math-', 'ref': '5972612166'}, +] + + +class EvidenceBindings(unittest.TestCase): + """State records availability; each event keeps its own binding and outcome.""" + + def with_evidence(self, items): + data = example() + data['records'][0]['evidence'] = copy.deepcopy(items) + return data + + def test_failed_and_successful_runs_are_both_recorded(self): + data = self.with_evidence([FAILED_RUN, PASSED_RUN]) + self.assertEqual(rc.validate(data), []) + self.assertEqual(data['records'][0]['state'], 'recorded') + self.assertEqual([e['conclusion'] for e in data['records'][0]['evidence']], ['failure', 'success']) + self.assertEqual(rc.aggregate([data])['counts']['evidence_items_non_alias'], 2) + + def test_expected_failure_of_a_negative_control_is_well_formed(self): + control = dict(FAILED_RUN, purpose='negative_control', expected_conclusion='failure') + self.assertEqual(rc.validate(self.with_evidence([control])), []) + + def test_run_binding_is_closed(self): + cases = { + 'attempt zero': dict(FAILED_RUN, attempt=0), + 'attempt boolean': dict(FAILED_RUN, attempt=True), + 'attempt string': dict(FAILED_RUN, attempt='1'), + 'job integer': dict(FAILED_RUN, job=112109136502), + 'job text': dict(FAILED_RUN, job='d2-square-schur'), + 'run head short': dict(FAILED_RUN, run_head_sha='3479b6e1'), + 'old tested_commit key': dict({k: v for k, v in FAILED_RUN.items() if k != 'run_head_sha'}, + tested_commit=FAILED_RUN['run_head_sha']), + 'missing checked_commit': {k: v for k, v in FAILED_RUN.items() if k != 'checked_commit'}, + 'checked commit short': dict(FAILED_RUN, checked_commit='3e56e964'), + 'checked commit boolean': dict(FAILED_RUN, checked_commit=False), + 'purpose build': dict(FAILED_RUN, purpose='build'), + 'verdict as conclusion': dict(FAILED_RUN, conclusion='PASS'), + 'expected cancelled': dict(FAILED_RUN, expected_conclusion='cancelled'), + 'missing purpose': {k: v for k, v in FAILED_RUN.items() if k != 'purpose'}, + 'extra verdict': dict(FAILED_RUN, verdict='success'), + } + for name, item in cases.items(): + with self.subTest(case=name): + self.assertNotEqual(rc.validate(self.with_evidence([item])), []) + + def test_run_head_and_checked_commit_are_separate_roles(self): + merge_checkout = run_item('37412915949', None, '89170cf085d8789cf90a93ff9c2b17a23d23dc44', 'success', + checked='3e56e9644d89397b7eb3b1a96b4eff6fdd7c2b75') + push_checkout = run_item('37458652010', None, '3ab51620df9f095a77bb8aa92886428f6cb633ab', 'success', + checked='3ab51620df9f095a77bb8aa92886428f6cb633ab') + data = self.with_evidence([merge_checkout, push_checkout, FAILED_RUN]) + self.assertEqual(rc.validate(data), []) + items = data['records'][0]['evidence'] + self.assertNotEqual(items[0]['run_head_sha'], items[0]['checked_commit']) + self.assertEqual(items[1]['run_head_sha'], items[1]['checked_commit']) + self.assertIsNone(items[2]['checked_commit']) + + def test_review_commit_is_native_and_comments_carry_none(self): + data = self.with_evidence(CUSTODY_CHAIN) + self.assertEqual(rc.validate(data), []) + self.assertEqual(rc.aggregate([data])['counts']['evidence_items_non_alias'], 7) + cases = { + 'review without commit': (0, lambda e: e.pop('commit')), + 'review commit short': (0, lambda e: e.update(commit='4e25b915')), + 'comment with a commit': (3, lambda e: e.update(commit='5634edade1db84ddefa25f2cbf92e79da6cce7eb')), + 'comment with a verdict': (4, lambda e: e.update(verdict='PASS')), + } + for name, (index, change) in cases.items(): + with self.subTest(case=name): + items = copy.deepcopy(CUSTODY_CHAIN) + change(items[index]) + self.assertNotEqual(rc.validate(self.with_evidence(items)), []) + + +class GitVerification(unittest.TestCase): + def test_subject_and_file_blobs_are_checked_against_git(self): + head = git('rev-parse', 'HEAD') + path = 'tools/check_proof_reachability.py' + blob = git('rev-parse', 'HEAD:' + path) + data = example() + record = data['records'][0] + record.update(delta=True) + record['subject'].update(commit=head, path=path, blob=blob) + record['evidence'] = [{'kind': 'file', 'repository': 'Math-', 'commit': head, 'path': path, 'blob': blob}] + data['records'] = [record] + repos = {'Math-': ROOT} + self.assertEqual(rc.validate(data, repos=repos), []) + wrong = copy.deepcopy(data) + wrong['records'][0]['subject']['blob'] = 'f' * 40 + self.assertTrue(any('does not match git' in e for e in rc.validate(wrong, repos=repos))) + wrong = copy.deepcopy(data) + wrong['records'][0]['evidence'][0]['blob'] = 'f' * 40 + self.assertTrue(any('file evidence blob' in e for e in rc.validate(wrong, repos=repos))) + wrong = copy.deepcopy(data) + wrong['records'][0]['subject']['path'] = 'tools/absent_file.py' + self.assertTrue(any('does not match git' in e for e in rc.validate(wrong, repos=repos))) + + def test_commit_must_be_a_commit_and_path_a_blob(self): + head = git('rev-parse', 'HEAD') + root_tree = git('rev-parse', 'HEAD^{tree}') + path = 'tools/check_proof_reachability.py' + blob = git('rev-parse', 'HEAD:' + path) + tools_tree = git('rev-parse', 'HEAD:tools') + repos = {'Math-': ROOT} + data = example() + record = data['records'][0] + record.update(delta=True) + data['records'] = [record] + cases = { + 'tree as subject path': ('subject', {'commit': head, 'path': 'tools', 'blob': tools_tree}), + 'tree as subject commit': ('subject', {'commit': root_tree, 'path': path, 'blob': blob}), + 'tree as file evidence': ('evidence', {'commit': head, 'path': 'tools', 'blob': tools_tree}), + 'tree as evidence commit': ('evidence', {'commit': root_tree, 'path': path, 'blob': blob}), + } + good = {'commit': head, 'path': path, 'blob': blob} + for name, (where, fields) in cases.items(): + with self.subTest(case=name): + trial = copy.deepcopy(data) + target = trial['records'][0] + target['subject'].update(fields if where == 'subject' else good) + target['evidence'] = [dict({'kind': 'file', 'repository': 'Math-'}, + **(fields if where == 'evidence' else good))] + self.assertNotEqual(rc.validate(trial, repos=repos), []) + trial = copy.deepcopy(data) + trial['records'][0]['subject'].update(good) + trial['records'][0]['evidence'] = [dict({'kind': 'file', 'repository': 'Math-'}, **good)] + self.assertEqual(rc.validate(trial, repos=repos), []) + + +class CommandLine(unittest.TestCase): + def run_tool(self, *args): + flags = ['-O'] if sys.flags.optimize else [] + return subprocess.run([sys.executable, '-B', *flags, '-S', str(TOOL), *args], + capture_output=True, text=True, timeout=60) + + def test_valid_file_aggregates_and_invalid_file_fails(self): + run = self.run_tool('--aggregate', str(EXAMPLE)) + self.assertEqual((run.returncode, run.stderr), (0, '')) + report = json.loads(run.stdout) + self.assertTrue(report['passed']) + self.assertIs(report['aggregate']['status_authority'], False) + self.assertEqual(report['aggregate']['counts']['records'], 2) + with tempfile.TemporaryDirectory() as tmp: + shard = pathlib.Path(tmp) / 'B1_side24' + shard.mkdir() + bad = shard / 'RECORDS.json' + bad.write_text(EXAMPLE.read_text(encoding='utf-8')) + before = bad.read_bytes() + run = self.run_tool(str(bad)) + self.assertEqual(run.returncode, 1) + report = json.loads(run.stdout) + self.assertFalse(report['passed']) + self.assertTrue(any('does not match its directory' in e for e in report['files'][str(bad)]['errors'])) + self.assertEqual(bad.read_bytes(), before) + + def test_malformed_file_is_reported_and_later_files_are_still_read(self): + with tempfile.TemporaryDirectory() as tmp: + data = example() + data['records'][0]['subject']['repository'] = [] + data['records'][0]['evidence'][0]['repository'] = {} + bad = pathlib.Path(tmp) / 'malformed.json' + bad.write_text(json.dumps(data)) + run = self.run_tool('--repo', str(ROOT), str(bad), str(EXAMPLE)) + self.assertEqual((run.returncode, run.stderr), (1, '')) + report = json.loads(run.stdout) + errors = report['files'][str(bad)]['errors'] + self.assertTrue(any('subject.repository' in e for e in errors)) + self.assertTrue(any('evidence[0].repository' in e for e in errors)) + self.assertEqual(report['files'][str(EXAMPLE)], {'valid': True, 'errors': []}) + + def test_repeated_inputs_are_rejected_not_counted_twice(self): + relative = EXAMPLE.relative_to(ROOT) + spellings = { + 'same path twice': [str(EXAMPLE), str(EXAMPLE)], + 'equivalent spellings': [str(EXAMPLE), str(ROOT / 'reviews' / '.' / relative.relative_to('reviews'))], + } + for name, files in spellings.items(): + with self.subTest(case=name): + run = self.run_tool('--aggregate', *files) + self.assertEqual(run.returncode, 1) + report = json.loads(run.stdout) + self.assertFalse(report['passed']) + self.assertEqual(len(report['input_errors']), 1) + self.assertNotIn('aggregate', report) + with tempfile.TemporaryDirectory() as tmp: + copy_path = pathlib.Path(tmp) / 'example_copy.json' + copy_path.write_text(EXAMPLE.read_text(encoding='utf-8')) + run = self.run_tool('--aggregate', str(EXAMPLE), str(copy_path)) + self.assertEqual(run.returncode, 1) + report = json.loads(run.stdout) + self.assertTrue(report['files'][str(EXAMPLE)]['valid']) + self.assertTrue(any('already supplied' in e for e in report['files'][str(copy_path)]['errors'])) + self.assertNotIn('aggregate', report) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/retrofit_record_check.py b/tools/retrofit_record_check.py new file mode 100644 index 000000000..b95382013 --- /dev/null +++ b/tools/retrofit_record_check.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Validate retrofit evidence records against reviews/retrofit_20261006/CONTRACT.md v0.1 (standard library only). + +A RECORDS.json file states, for exact source objects, which evidence axes have a record, using a closed vocabulary +that cannot carry a scientific status. This checker parses strictly (duplicate keys and non-finite numbers are +rejected), enforces the closed keys and enums of the contract, and with --repo verifies every subject and file +evidence blob against git. --aggregate summarizes several valid files without writing anything. + +A record's state says only whether evidence is linked. A workflow_run item keeps its own attempt, run head, checked +commit (or null), purpose, native conclusion and expected conclusion, so a failed run stays recorded as failed, a +negative control that fails as expected is not mistaken for a failed check, and a pull request head is not mistaken for +the commit a job checked out. Nothing here selects a run as a success. + +Scientific effect: NONE. A valid record is not a review, an acceptance or a register transition. + +Usage: + python3 -B -S tools/retrofit_record_check.py [--repo PATH] RECORDS.json [...] + python3 -B -S tools/retrofit_record_check.py --aggregate [--repo PATH] RECORDS.json [...] +Exit status 0 iff every file is valid. +""" +import argparse +import json +import pathlib +import re +import subprocess +import sys + +SCHEMA = 'retrofit-records/v0.1' +BASELINE = { + 'Math-': '08f86862f859ac4804b4fd6c6477a7ed0421e37f', + 'main': 'cddbb7f6cf3f57f3b495277f7da148287e027b19', +} +AXES = ('source_review', 'provider_distinct_review', 'blind_reconstruction', 'adversarial_attack', 'formal_evidence', + 'numerical_reproduction', 'novelty', 'human_reading', 'custody', 'observable_statement') +STATES = ('recorded', 'not_recorded', 'unknown', 'not_applicable') +EVIDENCE_KINDS = ('github_comment', 'github_review', 'workflow_run', 'file') +EVIDENCE_KEYS = { + 'file': {'kind', 'repository', 'commit', 'path', 'blob'}, + 'github_comment': {'kind', 'repository', 'ref'}, + 'github_review': {'kind', 'repository', 'ref', 'commit'}, + 'workflow_run': {'kind', 'repository', 'ref', 'attempt', 'job', 'run_head_sha', 'checked_commit', 'purpose', + 'conclusion', 'expected_conclusion'}, +} +RUN_PURPOSES = ('check', 'negative_control') +RUN_CONCLUSIONS = ('success', 'failure', 'cancelled', 'skipped', 'timed_out', 'neutral', 'action_required', 'stale', + 'startup_failure') +EXPECTED_CONCLUSIONS = ('success', 'failure') +REPOSITORIES = ('Math-', 'main', 'Universal-Law-Workspace', 'd6g8k5htny-coder', 'google-drive', 'governance-', + 'meta-framework', 'query-', 'sandbox', 'trial') +TOP_KEYS = {'schema', 'shard', 'baseline', 'author', 'scientific_effect', 'status_authority', 'records'} +RECORD_KEYS = {'id', 'subject', 'delta', 'axis', 'state', 'evidence', 'performer', 'exposure', 'independence_credit', + 'alias_of', 'notes'} +SUBJECT_KEYS = {'repository', 'commit', 'path', 'blob'} +PERSON_KEYS = {'provider', 'model_or_agent', 'session'} +HEX40 = re.compile(r'[0-9a-f]{40}') +SHARD = re.compile(r'[A-Z][A-Za-z0-9_]{0,39}') +RECORD_ID = re.compile(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,63}') +DIGITS = re.compile(r'[1-9][0-9]{0,19}') + + +def load_strict(text): + """json.loads that rejects duplicate keys and NaN/Infinity (the R-2 rule).""" + def pairs(items): + out = {} + for key, value in items: + if key in out: + raise ValueError('duplicate JSON key: ' + key) + out[key] = value + return out + + def constant(name): + raise ValueError('non-finite JSON constant: ' + name) + + return json.loads(text, object_pairs_hook=pairs, parse_constant=constant) + + +def _is_text(value): + return isinstance(value, str) and value.strip() != '' + + +def _safe_path(value): + return (_is_text(value) and not value.startswith('/') and '\\' not in value + and all(part not in ('', '.', '..') for part in value.split('/'))) + + +def _git(repo, *args): + run = subprocess.run(['git', '-C', str(repo), *args], capture_output=True, text=True) + return run.stdout.strip() if run.returncode == 0 else None + + +def _git_blob(repo, commit, path): + """The blob id at commit:path; None unless commit names a commit object and path a blob in its tree.""" + if _git(repo, 'cat-file', '-t', commit) != 'commit': + return None + if _git(repo, 'cat-file', '-t', commit + ':' + path) != 'blob': + return None + return _git(repo, 'rev-parse', '--verify', '--quiet', commit + ':' + path) + + +def _person(value, where, errors): + if not isinstance(value, dict) or set(value) != PERSON_KEYS: + errors.append(where + ': requires exactly provider, model_or_agent, session') + return + for key in sorted(PERSON_KEYS): + if not _is_text(value[key]): + errors.append(where + '.' + key + ': non-empty string required (use UNKNOWN when unavailable)') + + +def validate(data, *, shard_dir=None, repos=None): + """Return the list of contract violations in one parsed RECORDS.json (empty when valid).""" + errors = [] + if not isinstance(data, dict) or set(data) != TOP_KEYS: + return ['top level requires exactly: ' + ', '.join(sorted(TOP_KEYS))] + if data['schema'] != SCHEMA: + errors.append('schema must be ' + SCHEMA) + if not isinstance(data['shard'], str) or not SHARD.fullmatch(data['shard']): + errors.append('shard must match ' + SHARD.pattern) + elif shard_dir is not None and shard_dir != data['shard']: + errors.append('shard %r does not match its directory %r' % (data['shard'], shard_dir)) + if data['baseline'] != BASELINE: + errors.append('baseline must be exactly the pinned v0.1 cut %r' % BASELINE) + _person(data['author'], 'author', errors) + if data['scientific_effect'] != 'NONE': + errors.append('scientific_effect must be NONE') + if data['status_authority'] is not False: + errors.append('status_authority must be the Boolean false') + records = data['records'] + if not isinstance(records, list) or not records: + return errors + ['records must be a non-empty list'] + by_id = {} + for record in records: + if isinstance(record, dict) and isinstance(record.get('id'), str) and record['id'] not in by_id: + by_id[record['id']] = record + seen = set() + for index, record in enumerate(records): + where = 'records[%d]' % index + if not isinstance(record, dict) or set(record) != RECORD_KEYS: + errors.append(where + ': requires exactly ' + ', '.join(sorted(RECORD_KEYS))) + continue + rid = record['id'] + if not isinstance(rid, str) or not RECORD_ID.fullmatch(rid): + errors.append(where + '.id: must match ' + RECORD_ID.pattern) + elif rid in seen: + errors.append(where + '.id: duplicate ' + rid) + else: + seen.add(rid) + subject = record['subject'] + if not isinstance(subject, dict) or set(subject) != SUBJECT_KEYS: + errors.append(where + '.subject: requires exactly repository, commit, path, blob') + else: + repo_name = subject['repository'] + if not isinstance(repo_name, str) or repo_name not in REPOSITORIES: + errors.append(where + '.subject.repository: unknown repository') + repo_name = None + if not isinstance(subject['commit'], str) or not HEX40.fullmatch(subject['commit']): + errors.append(where + '.subject.commit: 40 lowercase hex digits required') + if not _safe_path(subject['path']): + errors.append(where + '.subject.path: relative repository path required') + if not isinstance(subject['blob'], str) or not HEX40.fullmatch(subject['blob']): + errors.append(where + '.subject.blob: 40 lowercase hex digits required') + if type(record['delta']) is not bool: + errors.append(where + '.delta: Boolean required') + elif not record['delta'] and (repo_name is None or BASELINE.get(repo_name) != subject['commit']): + errors.append(where + '.subject.commit: not the pinned baseline; mark a candidate with delta true') + if repos and repo_name is not None and repo_name in repos and not errors_for(where, errors): + got = _git_blob(repos[repo_name], subject['commit'], subject['path']) + if got != subject['blob']: + errors.append(where + '.subject: blob does not match git (%s)' % (got or 'no blob at commit:path')) + if record['axis'] not in AXES: + errors.append(where + '.axis: one of ' + ', '.join(AXES)) + if record['state'] not in STATES: + errors.append(where + '.state: one of ' + ', '.join(STATES) + ' (no status words)') + evidence = record['evidence'] + if not isinstance(evidence, list): + errors.append(where + '.evidence: list required') + evidence = [] + if record['state'] == 'recorded' and not evidence: + errors.append(where + ': state recorded requires evidence') + if record['state'] in ('not_recorded', 'not_applicable') and evidence: + errors.append(where + ': state %s takes no evidence' % record['state']) + for k, item in enumerate(evidence): + _evidence(item, '%s.evidence[%d]' % (where, k), errors, repos) + _person(record['performer'], where + '.performer', errors) + if not _is_text(record['exposure']): + errors.append(where + '.exposure: non-empty string required (state what was read)') + if type(record['independence_credit']) is not int or record['independence_credit'] != 0: + errors.append(where + '.independence_credit: must be the integer 0') + _alias(record, by_id, where, errors) + if not isinstance(record['notes'], str): + errors.append(where + '.notes: string required (may be empty)') + return errors + + +def _canonical(evidence): + return sorted(json.dumps(item, sort_keys=True) for item in evidence) if isinstance(evidence, list) else None + + +def _alias(record, by_id, where, errors): + """alias_of names a canonical record (alias_of null) in this file with the same axis, state and evidence.""" + alias = record['alias_of'] + if alias is None: + return + target = by_id.get(alias) if isinstance(alias, str) else None + if target is None or target is record: + errors.append(where + '.alias_of: null or the id of another record in this file') + elif target.get('alias_of') is not None: + errors.append(where + '.alias_of: must name a canonical record (one whose alias_of is null); no chains') + elif (target.get('axis'), target.get('state')) != (record['axis'], record['state']): + errors.append(where + '.alias_of: an alias has the axis and state of its canonical record') + elif _canonical(target.get('evidence')) != _canonical(record['evidence']): + errors.append(where + '.alias_of: an alias restates exactly the evidence of its canonical record') + + +def errors_for(where, errors): + return any(e.startswith(where + '.subject') or e.startswith(where + '.delta') for e in errors) + + +def _evidence(item, where, errors, repos): + if not isinstance(item, dict) or 'kind' not in item: + errors.append(where + ': object with kind required') + return + kind = item['kind'] + if kind not in EVIDENCE_KINDS: + errors.append(where + '.kind: one of ' + ', '.join(EVIDENCE_KINDS)) + return + keys = EVIDENCE_KEYS[kind] + if set(item) != keys: + errors.append(where + ': %s evidence requires exactly %s' % (kind, ', '.join(sorted(keys)))) + return + known = isinstance(item['repository'], str) and item['repository'] in REPOSITORIES + if not known: + errors.append(where + '.repository: unknown repository') + if kind == 'file': + ok = (isinstance(item['commit'], str) and HEX40.fullmatch(item['commit']) + and isinstance(item['blob'], str) and HEX40.fullmatch(item['blob']) and _safe_path(item['path'])) + if not ok: + errors.append(where + ': file evidence needs 40-hex commit and blob and a relative path') + elif known and repos and item['repository'] in repos: + if _git_blob(repos[item['repository']], item['commit'], item['path']) != item['blob']: + errors.append(where + ': file evidence blob does not match git') + return + if not isinstance(item['ref'], str) or not DIGITS.fullmatch(item['ref']): + errors.append(where + '.ref: numeric GitHub id as a string required') + if kind == 'github_review' and not (isinstance(item['commit'], str) and HEX40.fullmatch(item['commit'])): + errors.append(where + '.commit: the review\'s native commit_id (40 hex) required') + if kind == 'workflow_run': + if type(item['attempt']) is not int or item['attempt'] < 1: + errors.append(where + '.attempt: positive integer required') + if item['job'] is not None and (not isinstance(item['job'], str) or not DIGITS.fullmatch(item['job'])): + errors.append(where + '.job: null or a numeric job id as a string') + if not isinstance(item['run_head_sha'], str) or not HEX40.fullmatch(item['run_head_sha']): + errors.append(where + '.run_head_sha: the run\'s native head_sha (40 hex) required') + if item['checked_commit'] is not None and (not isinstance(item['checked_commit'], str) + or not HEX40.fullmatch(item['checked_commit'])): + errors.append(where + '.checked_commit: null or the 40-hex commit the job checked out') + if item['purpose'] not in RUN_PURPOSES: + errors.append(where + '.purpose: one of ' + ', '.join(RUN_PURPOSES)) + if item['conclusion'] not in RUN_CONCLUSIONS: + errors.append(where + '.conclusion: the native GitHub conclusion, one of ' + ', '.join(RUN_CONCLUSIONS)) + if item['expected_conclusion'] not in EXPECTED_CONCLUSIONS: + errors.append(where + '.expected_conclusion: one of ' + ', '.join(EXPECTED_CONCLUSIONS)) + + +def aggregate(datasets): + """Per subject object and axis, the states recorded across files; aliases counted separately.""" + out = {} + counts = {'files': len(datasets), 'records': 0, 'aliases': 0, 'evidence_items_non_alias': 0} + distinct = set() + for data in datasets: + for record in data['records']: + counts['records'] += 1 + if record['alias_of'] is not None: + counts['aliases'] += 1 + else: + counts['evidence_items_non_alias'] += len(record['evidence']) + distinct.update(_canonical(record['evidence'])) + s = record['subject'] + key = '%s@%s:%s#%s' % (s['repository'], s['commit'], s['path'], s['blob']) + out.setdefault(key, {}).setdefault(record['axis'], []).append( + {'shard': data['shard'], 'id': record['id'], 'state': record['state'], + 'alias_of': record['alias_of']}) + counts['distinct_evidence_items'] = len(distinct) + return {'subjects': out, 'counts': counts, 'status_authority': False, 'scientific_effect': 'NONE', + 'meaning': 'derived summary of retrofit records; not a status record or acceptance'} + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('files', nargs='+') + ap.add_argument('--repo', help='local clone used to verify Math- blobs (main is verified only with --main-repo)') + ap.add_argument('--main-repo', help='local clone of main for main-repository blobs') + ap.add_argument('--aggregate', action='store_true') + args = ap.parse_args(argv) + repos = {} + if args.repo: + repos['Math-'] = pathlib.Path(args.repo) + if args.main_repo: + repos['main'] = pathlib.Path(args.main_repo) + report, datasets, ok = {'files': {}, 'input_errors': []}, [], True + resolved, shards = {}, {} + for name in args.files: + path = pathlib.Path(name) + real = path.resolve() + if real in resolved: + report['input_errors'].append('%s: same file as %s; each input is read once' % (name, resolved[real])) + ok = False + continue + resolved[real] = name + try: + data = load_strict(path.read_text(encoding='utf-8')) + shard_dir = path.parent.name if path.name == 'RECORDS.json' else None + errors = validate(data, shard_dir=shard_dir, repos=repos) + except (OSError, ValueError, UnicodeDecodeError) as exc: + data, errors = None, [str(exc)] + if not errors and data['shard'] in shards: + errors = ['shard %s already supplied by %s; one RECORDS.json per shard' % (data['shard'], + shards[data['shard']])] + report['files'][name] = {'valid': not errors, 'errors': errors} + ok &= not errors + if not errors: + shards[data['shard']] = name + datasets.append(data) + report['passed'] = ok + report['scientific_effect'] = 'NONE' + if args.aggregate and ok: + report['aggregate'] = aggregate(datasets) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if ok else 1 + + +if __name__ == '__main__': + sys.exit(main())