diff --git a/tests/test_evidence_profile.py b/tests/test_evidence_profile.py new file mode 100644 index 000000000..5e49e2737 --- /dev/null +++ b/tests/test_evidence_profile.py @@ -0,0 +1,249 @@ +"""Behavior of tools/evidence_profile.py (standard library only; the live graph is read, never written).""" +import contextlib +import hashlib +import importlib.util +import io +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +GRAPH = ROOT / 'frontiers' / 'downstream_gate_20260925' / 'GRAPH.json' +SPEC = importlib.util.spec_from_file_location('evidence_profile', ROOT / 'tools' / 'evidence_profile.py') +ep = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ep) +GATE = ep.load_gate() + + +def node(classification, **fields): + return dict(classification=classification, controlling=False, **fields) + + +def synthetic(): + return { + 'schema_version': 1, + 'object': 'SYNTHETIC', + 'nodes': { + 'math.top': node('OPEN_ACTIVE'), + 'math.mid': node('PROVED_REVIEWED', author_provider='OpenAI', + review_basis=[{'provider': 'xAI'}, {'provider': 'Anthropic/Claude'}]), + 'math.leaf': node('AUTHOR_SIDE_CANDIDATE', author_provider='Anthropic', + review_provider='Anthropic Claude session'), + 'math.shared': node('PROVED_REVIEWED', author_provider='OpenAI', review_providers=['nonauthor']), + 'math.boundary': node('REFUTED'), + }, + 'edges': [ + {'from': 'math.top', 'to': 'math.mid', 'required': True, 'relation': 'uses'}, + {'from': 'math.top', 'to': 'math.shared', 'required': True, 'relation': 'uses'}, + {'from': 'math.top', 'to': 'math.boundary', 'required': False, 'relation': 'boundary_only'}, + {'from': 'math.mid', 'to': 'math.leaf', 'required': True, 'relation': 'imports'}, + {'from': 'math.mid', 'to': 'math.shared', 'required': True, 'relation': 'imports'}, + ], + } + + +def run(argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = ep.main(argv) + return code, out.getvalue() + + +class SyntheticGraph(unittest.TestCase): + def setUp(self): + self.graph = synthetic() + GATE.validate_graph_fail_closed(self.graph) + + def test_unsatisfied_required_is_transitive_and_ignores_context_edges(self): + p = ep.profile(GATE, self.graph, 'math.top') + self.assertEqual(p['unsatisfied_required'], + [{'node': 'math.leaf', 'classification': 'AUTHOR_SIDE_CANDIDATE'}]) + self.assertEqual(ep.profile(GATE, self.graph, 'math.shared')['unsatisfied_required'], []) + + def test_provider_distinct_reads_only_recorded_families(self): + self.assertEqual(ep.provider_distinct(self.graph['nodes']['math.mid']), 'yes') + self.assertEqual(ep.provider_distinct(self.graph['nodes']['math.leaf']), 'no') + self.assertEqual(ep.provider_distinct(self.graph['nodes']['math.shared']), 'not recorded') + self.assertEqual(ep.provider_distinct(self.graph['nodes']['math.top']), 'not recorded') + self.assertIsNone(ep.family('OpenAI reviewed by xAI')) + self.assertEqual(ep.family('xAI/Grok via Cursor'), 'xAI') + + def test_unresolved_reviewer_never_yields_no(self): + # 6018400003 item 1: an unresolved or ambiguous reviewer must not be dropped before deciding 'no'. + for reviewers in (['OpenAI', 'unattributed'], ['OpenAI', 'OpenAI / xAI'], ['OpenAI', 'nonauthor']): + with self.subTest(reviewers=reviewers): + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', + review_providers=reviewers)), 'not recorded') + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', + review_providers=['unattributed', 'xAI'])), 'yes') + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', + review_providers=['OpenAI / GPT-6', 'Codex'])), 'no') + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', + review_providers=[])), 'not recorded') + # EP381-UNKNOWN-01 (6018473703): the same rule across fields, singular review_provider plus review_basis. + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', review_provider='OpenAI', + review_basis=[{'provider': 'unidentified'}])), 'not recorded') + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', review_provider='OpenAI', + review_basis=[{'provider': 'xAI'}])), 'yes') + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', + review_providers=['OpenAI or xAI'])), 'not recorded') + + def test_negated_or_undisclosed_provider_is_unresolved(self): + # EP381-02 (6018548621): non-identities must not be read as a provider. + for text in ('not OpenAI; provider not disclosed', 'no Anthropic involvement', 'other than xAI', + 'unidentified', 'provider not disclosed'): + with self.subTest(text=text): + self.assertIsNone(ep.family(text)) + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='Anthropic', + review_providers=['not OpenAI; provider not disclosed'])), + 'not recorded') + self.assertEqual(ep.provider_distinct(node('PROVED_REVIEWED', author_provider='OpenAI', + review_providers=['claudette'])), 'not recorded') + p = ep.profile(GATE, {'schema_version': 1, 'nodes': {'math.x': node( + 'PROVED_REVIEWED', author_provider='OpenAI', review_providers=['OpenAI', 'not disclosed'])}, + 'edges': []}, 'math.x') + self.assertEqual(p['provider_distinct'], 'not recorded') + self.assertEqual(p['unresolved_reviewer_providers'], ['not disclosed']) + + def test_default_selection_covers_context_only_cycles(self): + # 6018548621 nonblocking: a component without an edge-free root is still listed. + graph = synthetic() + graph['nodes']['math.a'] = node('OPEN_ACTIVE') + graph['nodes']['math.b'] = node('PROVED_REVIEWED') + graph['edges'] += [{'from': 'math.a', 'to': 'math.b', 'required': False, 'relation': 'context'}, + {'from': 'math.b', 'to': 'math.a', 'required': False, 'relation': 'context'}] + GATE.validate_graph_fail_closed(graph) + self.assertEqual(ep.roots(graph), ['math.top', 'math.a']) + + def test_positive_identity_grammar(self): + # EP381-02 residual (6018757501): only the documented positive form names a family. + for text, expected in (('OpenAI / GPT-6 Astra Pro', 'OpenAI'), ('Grok 4.7', 'xAI'), + ('xAI/Grok via Cursor (D1-A-E, section 9 v1.1, partial slices)', 'xAI'), + ('Claude Code session', 'Anthropic'), ('Google DeepMind Gemini', 'Google')): + with self.subTest(text=text): + self.assertEqual(ep.family(text), expected) + for text in ('possibly OpenAI', 'OpenAI or unspecified', 'review requested from OpenAI', + 'OpenAI reviewed by xAI', 'OpenAI (a) (b)', 'OpenAI, probably', 'Claude/Grok'): + with self.subTest(text=text): + self.assertIsNone(ep.family(text)) + + def test_family_matches_whole_tokens_only(self): + # 6018400003 item 2: substring hits such as 'notopenai' or 'claudette' name no family. + for text in ('notopenai', 'claudette', 'xairline', 'grokking-free', 'googleplex', ''): + with self.subTest(text=text): + self.assertIsNone(ep.family(text)) + for text, expected in (('OpenAI / GPT-6 Astra Pro', 'OpenAI'), ('Anthropic/Claude (A1-A7)', 'Anthropic'), + ('Claude Code session', 'Anthropic'), ('ChatGPT', 'OpenAI'), ('Gemini', 'Google')): + with self.subTest(text=text): + self.assertEqual(ep.family(text), expected) + + def test_tree_marks_context_edges_and_expands_shared_nodes_once(self): + lines = ep.tree_lines(GATE, self.graph, 'math.top') + text = '\n'.join(lines) + self.assertIn('context (boundary_only) math.boundary REFUTED', text) + self.assertEqual(sum('math.shared' in line for line in lines[1:-1]), 2) + self.assertEqual(sum('(expanded above)' in line for line in lines), 1) + self.assertEqual(lines[-1], 'Unsatisfied required dependencies (1): math.leaf (AUTHOR_SIDE_CANDIDATE)') + + def test_roots_are_math_nodes_without_incoming_edges(self): + self.assertEqual(ep.roots(self.graph), ['math.top']) + + def test_cli_on_file_json_and_text(self): + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / 'graph.json' + path.write_text(json.dumps(self.graph)) + code, text = run(['--graph', str(path)]) + self.assertEqual(code, 0) + self.assertTrue(text.startswith('math.top OPEN_ACTIVE')) + self.assertIn('no status authority; scientific effect NONE', text) + code, text = run(['--graph', str(path), '--node', 'math.mid', '--json']) + data = json.loads(text) + self.assertEqual([p['node'] for p in data['profiles']], ['math.mid']) + self.assertIs(data['status_authority'], False) + self.assertIs(data['lemma_closed'], False) + self.assertEqual(data['scientific_effect'], 'NONE') + self.assertIn('blind_reconstruction', data['axes_outside_graph']) + + def test_invalid_graphs_and_unknown_nodes_are_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / 'graph.json' + bad = synthetic() + bad['edges'].append({'from': 'math.leaf', 'to': 'math.top', 'required': True, 'relation': 'cycle'}) + path.write_text(json.dumps(bad)) + with self.assertRaisesRegex(ValueError, 'required dependency cycle'): + run(['--graph', str(path)]) + bad = synthetic() + bad['nodes']['math.top']['classification'] = 'CLOSED' + path.write_text(json.dumps(bad)) + with self.assertRaisesRegex(ValueError, 'unknown node classification'): + run(['--graph', str(path)]) + path.write_text(json.dumps(synthetic())) + with self.assertRaisesRegex(KeyError, 'unknown node'): + run(['--graph', str(path), '--node', 'math.absent']) + + +class ProviderAttributionCLI(unittest.TestCase): + """EP381-01/EP381-02 (review 5430026100) through the actual command line, in the current Python mode.""" + + CASES = { + 'math.mixed-unknown': ('OpenAI', ['OpenAI', 'not disclosed'], 'not recorded'), + 'math.mixed-ambiguous': ('OpenAI', ['OpenAI', 'OpenAI or Anthropic'], 'not recorded'), + 'math.distinct-plus-unknown': ('OpenAI', ['not disclosed', 'xAI'], 'yes'), + 'math.negated': ('Anthropic', ['not OpenAI; provider not disclosed'], 'not recorded'), + 'math.embedded-name': ('OpenAI', ['claudette'], 'not recorded'), + 'math.anthropic-claude': ('OpenAI', ['Anthropic/Claude'], 'yes'), + 'math.chatgpt-codex': ('OpenAI', ['ChatGPT/Codex'], 'no'), + 'math.xai-grok-cursor': ('OpenAI', ['xAI/Grok via Cursor'], 'yes'), + 'math.all-same': ('xAI', ['xAI/Grok via Cursor', 'Grok 4.7'], 'no'), + # Delta 6018757501: prose around a provider name is not a positive identity. + 'math.possibly': ('Anthropic', ['possibly OpenAI'], 'not recorded'), + 'math.or-unspecified': ('Anthropic', ['OpenAI or unspecified'], 'not recorded'), + 'math.invitation': ('Anthropic', ['review requested from OpenAI'], 'not recorded'), + 'math.astra-control': ('Anthropic', ['OpenAI / GPT-6 Astra Pro'], 'yes'), + } + + def test_cli_json_reports_conservative_provider_attribution(self): + graph = {'schema_version': 1, 'object': 'SYNTHETIC-CLI', 'edges': [], + 'nodes': {nid: node('PROVED_REVIEWED', author_provider=a, review_providers=r) + for nid, (a, r, _) in self.CASES.items()}} + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / 'graph.json' + path.write_text(json.dumps(graph)) + before = path.read_bytes() + flags = ['-O'] if sys.flags.optimize else [] + run = subprocess.run([sys.executable, '-B', *flags, '-S', str(ROOT / 'tools' / 'evidence_profile.py'), + '--graph', str(path), '--json'], capture_output=True, text=True, timeout=60) + self.assertEqual((run.returncode, run.stderr), (0, '')) + self.assertEqual(path.read_bytes(), before) + profiles = {p['node']: p for p in json.loads(run.stdout)['profiles']} + self.assertEqual(set(profiles), set(self.CASES)) + for nid, (_, reviewers, expected) in self.CASES.items(): + with self.subTest(node=nid): + self.assertEqual(profiles[nid]['provider_distinct'], expected) + self.assertEqual(profiles[nid]['reviewer_providers'], reviewers) + self.assertEqual(profiles['math.mixed-unknown'].get('unresolved_reviewer_providers'), ['not disclosed']) + self.assertEqual(profiles['math.negated'].get('unresolved_reviewer_providers'), + ['not OpenAI; provider not disclosed']) + + +class LiveGraph(unittest.TestCase): + def test_live_graph_renders_without_writing(self): + before = hashlib.sha256(GRAPH.read_bytes()).hexdigest() + code, text = run([]) + self.assertEqual(code, 0) + graph = GATE.load_graph(GRAPH) + for root in ep.roots(graph): + self.assertIn('\n' + root + ' ' if not text.startswith(root) else root, text) + code, text = run(['--json']) + data = json.loads(text) + self.assertEqual(len(data['profiles']), len(ep.roots(graph))) + for p in data['profiles']: + self.assertEqual(p['classification'], graph['nodes'][p['node']]['classification']) + self.assertEqual(hashlib.sha256(GRAPH.read_bytes()).hexdigest(), before) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/evidence_profile.py b/tools/evidence_profile.py new file mode 100644 index 000000000..a970f207e --- /dev/null +++ b/tools/evidence_profile.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Read-only dependency tree and evidence profile for the downstream gate graph (standard library only). + +Renders, for a node of frontiers/downstream_gate_20260925/GRAPH.json (or every math.* root), the tree of +its outgoing edges with each node's recorded classification, the providers its record names, and the +required dependencies that do not yet satisfy the hard gate. The graph is loaded and validated with the +gate's own strict loader; the dependency rule is the gate's (an edge is required unless it says +otherwise; only PROVED_REVIEWED satisfies a required premise). + +This is the derived view proposed in main#275 (decision record v1.1, rows PROPOSED) and main#276 +(governance/OP-CLOSURE-EVIDENCE-20261006.md, unmerged at publication): it writes +nothing, promotes nothing and is not a status record. Axes the graph does not record (blind +reconstruction, adversarial attack, formal evidence, numerical reproduction, novelty, human reading) +are listed as outside the graph rather than reported as passed or failed. Scientific effect: NONE. + +Usage: python3 -B -S tools/evidence_profile.py [--graph PATH] [--node ID] [--json] +""" +import argparse +import importlib.util +import json +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +GATE_DIR = ROOT / 'frontiers' / 'downstream_gate_20260925' +AXES_OUTSIDE_GRAPH = ( + 'blind_reconstruction', 'adversarial_attack', 'formal_evidence', + 'numerical_reproduction', 'novelty', 'human_reading', +) +# Provider attribution grammar (EP381-02, review 5430026100 and delta 6018757501). A record string names a provider +# family only in this positive form, compared case-insensitively: +# FAMILY_ALIAS ( ('/' | ' ') (FAMILY_ALIAS | DECORATION | VERSION) )* [ ' (' note ')' ] +# where every alias belongs to the same family, DECORATION is one of the listed product/agent/session words, VERSION is +# a model or version identifier containing a digit (gpt-6, 4.7, grok-4.7-high-fast), and one trailing parenthetical +# scope note is ignored. Anything else ('possibly OpenAI', 'OpenAI or Anthropic', 'review requested from OpenAI', +# 'not OpenAI', 'claudette') names no family and is reported verbatim as unresolved. +FAMILY_ALIASES = { + 'openai': 'OpenAI', 'chatgpt': 'OpenAI', 'codex': 'OpenAI', 'gpt': 'OpenAI', + 'anthropic': 'Anthropic', 'claude': 'Anthropic', + 'xai': 'xAI', 'grok': 'xAI', + 'google': 'Google', 'gemini': 'Google', +} +DECORATIONS = frozenset({'via', 'cursor', 'pro', 'code', 'session', 'agent', 'cloud', 'bot', 'model', 'astra', 'sol', + 'deepmind', 'high', 'fast'}) +VERSION = re.compile(r'[a-z]*-?\d[a-z0-9.\-]*') + + +def load_gate(): + spec = importlib.util.spec_from_file_location('downstream_hard_gate', GATE_DIR / 'hard_gate.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def family(text): + """Provider family named by a record string in the positive grammar above, else None (unresolved).""" + match = re.fullmatch(r'\s*([^()]*?)\s*(?:\([^()]*\))?\s*', text) + if match is None: + return None + tokens = [t for t in re.split(r'[\s/]+', match.group(1).lower()) if t] + if not tokens or tokens[0] not in FAMILY_ALIASES: + return None + name = FAMILY_ALIASES[tokens[0]] + for token in tokens[1:]: + if token in FAMILY_ALIASES: + if FAMILY_ALIASES[token] != name: + return None + elif token not in DECORATIONS and not VERSION.fullmatch(token): + return None + return name + + +def reviewer_strings(node): + out = [] + for item in node.get('review_basis', []) or []: + if isinstance(item, dict) and isinstance(item.get('provider'), str): + out.append(item['provider']) + providers = node.get('review_providers') + if isinstance(providers, list): + out.extend(p for p in providers if isinstance(p, str)) + if isinstance(node.get('review_provider'), str): + out.append(node['review_provider']) + return out + + +def provider_distinct(node): + """'yes' if some recorded reviewer resolves to a family other than the recorded author's; 'no' only if + every recorded reviewer resolves to the author's family; otherwise 'not recorded' (missing author or + reviewers, or a reviewer string naming no single family). Never organizational independence.""" + author = node.get('author_provider') + author_family = family(author) if isinstance(author, str) else None + reviewer_families = [family(s) for s in reviewer_strings(node)] + if author_family is None or not reviewer_families: + return 'not recorded' + if any(f is not None and f != author_family for f in reviewer_families): + return 'yes' + if all(f == author_family for f in reviewer_families): + return 'no' + return 'not recorded' + + +def profile(gate, graph, node_id): + node = gate.require_node(graph, node_id) + unsatisfied = [dep for dep in gate.transitive_required(graph, node_id) + if graph['nodes'][dep]['classification'] not in gate.REQUIRED_SATISFIED] + return { + 'node': node_id, + 'classification': node['classification'], + 'review_disposition': node.get('review_disposition', 'not recorded'), + 'author_provider': node.get('author_provider', 'not recorded'), + 'reviewer_providers': reviewer_strings(node) or ['not recorded'], + 'provider_distinct': provider_distinct(node), + 'unresolved_reviewer_providers': [s for s in reviewer_strings(node) if family(s) is None], + 'unsatisfied_required': [{'node': dep, 'classification': graph['nodes'][dep]['classification']} + for dep in unsatisfied], + 'axes_outside_graph': list(AXES_OUTSIDE_GRAPH), + } + + +def roots(graph): + """math.* nodes without an incoming edge, then, in sorted order, any math.* node not reached from those + (a component whose math nodes all have incoming edges, e.g. a context-only cycle), so none is omitted.""" + targets = {e['to'] for e in graph['edges']} + math_nodes = sorted(n for n in graph['nodes'] if n.startswith('math.')) + chosen = [n for n in math_nodes if n not in targets] + reached = set() + + def reach(nid): + stack = [nid] + while stack: + cur = stack.pop() + if cur in reached: + continue + reached.add(cur) + stack.extend(e['to'] for e in graph['edges'] if e['from'] == cur) + + for n in chosen: + reach(n) + for n in math_nodes: + if n not in reached: + chosen.append(n) + reach(n) + return chosen + + +def tree_lines(gate, graph, node_id): + lines = [] + seen = set() + + def label(nid): + p = profile(gate, graph, nid) + return '%s %s [author %s; reviewers %s; provider-distinct %s]' % ( + nid, p['classification'], p['author_provider'], ', '.join(p['reviewer_providers']), + p['provider_distinct']) + + def walk(nid, prefix): + edges = sorted((e for e in graph['edges'] if e['from'] == nid), key=lambda e: e['to']) + for i, e in enumerate(edges): + last = i == len(edges) - 1 + kind = 'requires' if e.get('required', True) else 'context (%s)' % e.get('relation', '?') + child = e['to'] + again = child in seen + lines.append('%s%s %s %s%s' % (prefix, '└──' if last else '├──', kind, label(child), + ' (expanded above)' if again else '')) + if not again: + seen.add(child) + walk(child, prefix + (' ' if last else '│ ')) + + seen.add(node_id) + lines.append(label(node_id)) + walk(node_id, '') + unsatisfied = profile(gate, graph, node_id)['unsatisfied_required'] + if unsatisfied: + lines.append('Unsatisfied required dependencies (%d): %s' % (len(unsatisfied), ', '.join( + '%s (%s)' % (u['node'], u['classification']) for u in unsatisfied))) + else: + lines.append('Unsatisfied required dependencies: none') + return lines + + +def report(gate, graph, node_ids): + return { + 'object': graph.get('object'), + 'profiles': [profile(gate, graph, nid) for nid in node_ids], + 'axes_outside_graph': list(AXES_OUTSIDE_GRAPH), + 'meaning': 'derived read-only view of recorded graph fields; not a status record or acceptance', + 'status_authority': False, + 'lemma_closed': False, + 'scientific_effect': 'NONE', + } + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--graph', default=str(GATE_DIR / 'GRAPH.json')) + ap.add_argument('--node', action='append', help='node id (repeatable); default: every math.* root') + ap.add_argument('--json', action='store_true', help='print the profiles as JSON instead of trees') + args = ap.parse_args(argv) + gate = load_gate() + graph = gate.load_graph(pathlib.Path(args.graph)) + node_ids = args.node or roots(graph) + for nid in node_ids: + gate.require_node(graph, nid) + if args.json: + print(json.dumps(report(gate, graph, node_ids), indent=2, sort_keys=True)) + return 0 + for index, nid in enumerate(node_ids): + if index: + print() + print('\n'.join(tree_lines(gate, graph, nid))) + print() + print('Not in the graph (consult review records): ' + ', '.join(AXES_OUTSIDE_GRAPH)) + print('Derived view only: no status authority; scientific effect NONE.') + return 0 + + +if __name__ == '__main__': + sys.exit(main())