From 87f89cf017b294e89ac37ae373f456a0c75304f9 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Sat, 17 Oct 2026 16:05:54 -0500 Subject: [PATCH 1/2] Require complete D5 reconciliation mutant reports Preserve the scientific packet and authenticate stdout, status, and stderr for each native mutant in both child modes. Cover the complete Bash verifier with real-shell regression controls. Task: Math-#322 / pickup6240886901 Scientific effect: NONE --- .github/workflows/d5-reconciliation.yml | 34 ++- tests/test_d5_reconciliation_workflow.py | 274 +++++++++++++++++++++++ 2 files changed, 302 insertions(+), 6 deletions(-) create mode 100644 tests/test_d5_reconciliation_workflow.py diff --git a/.github/workflows/d5-reconciliation.yml b/.github/workflows/d5-reconciliation.yml index 2474267a9..b5d14f6e1 100644 --- a/.github/workflows/d5-reconciliation.yml +++ b/.github/workflows/d5-reconciliation.yml @@ -5,6 +5,7 @@ name: D5 reconciliation checks paths: - 'reviews/d5_reconciliation_20260929/**' - '.github/workflows/d5-reconciliation.yml' + - 'tests/test_d5_reconciliation_workflow.py' - 'reviews/d5_*/**' - 'frontiers/intermediate_window_20260928/**' - 'frontiers/remote_window_20260924/**' @@ -26,6 +27,10 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: '3.11.16' + - name: Exercise rejection protocol controls + run: | + python -B -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v + python -B -O -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v - name: Verify sources, replay both modes, reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -49,16 +54,33 @@ jobs: script = str(root / 'reconciliation_check.py') expected = (root / 'RESULTS.json').read_bytes() mutants = ['optional-continuum', 'allow-symlink', 'no-hash', 'drop-dependency', 'drop-edge', 'stale-fingerprint', 'drop-collar', 'close-c6'] + # Complete native report differences, characterized before this repair. + failed_checks = { + 'optional-continuum': 'NEGATIVES', + 'allow-symlink': 'NEGATIVES', + 'no-hash': 'NEGATIVES', + 'drop-dependency': 'TRANSITIONS', + 'drop-edge': 'TRANSITIONS', + 'stale-fingerprint': 'TRANSITIONS', + 'drop-collar': 'TILING', + 'close-c6': 'OPEN_C6', + } for flags in (['-B', '-S'], ['-B', '-O', '-S']): out = subprocess.run([sys.executable, *flags, script], - check=True, capture_output=True, timeout=300).stdout - if out != expected: + check=True, capture_output=True, timeout=300) + if out.stdout != expected: raise RuntimeError('output differs from RESULTS.json under ' + ' '.join(flags)) + if out.stderr: + raise RuntimeError('unexpected baseline stderr under ' + ' '.join(flags)) for m in mutants: - rc = subprocess.run([sys.executable, *flags, script, '--mutant', m], - capture_output=True, timeout=300).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + report = json.loads(expected) + report['passed'] = False + report['checks'][failed_checks[m]] = False + wanted = (json.dumps(report, indent=2, sort_keys=True) + '\n').encode('utf-8') + rejected = subprocess.run([sys.executable, *flags, script, '--mutant', m], + capture_output=True, timeout=300) + if rejected.returncode != 1 or rejected.stdout != wanted or rejected.stderr: + raise RuntimeError('mutant ' + m + ' did not emit its expected rejection report') print(json.dumps({'scientific_effect': 'NONE', 'mathematical_acceptance': False})) PY git diff --exit-code diff --git a/tests/test_d5_reconciliation_workflow.py b/tests/test_d5_reconciliation_workflow.py new file mode 100644 index 000000000..b383166bc --- /dev/null +++ b/tests/test_d5_reconciliation_workflow.py @@ -0,0 +1,274 @@ +"""Real-shell d5-reconciliation protocol tests with committed SYNTHETIC fixtures. + +No mathematical packet is replayed here. Real checker children record their +interpreter flags; the full workflow block retains its source and Git guards. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +WORKFLOW = Path(__file__).resolve().parents[1] / '.github/workflows/d5-reconciliation.yml' +PACKET = Path('reviews/d5_reconciliation_20260929') +# Characterized from reconciliation_check.py and RESULTS.json; native replay is separate. +# Labels sharing a check intentionally share report bytes; event traces bind +# every actual invocation independently of those equivalent reports. +FAILED_CHECKS = { + 'optional-continuum': 'NEGATIVES', + 'allow-symlink': 'NEGATIVES', + 'no-hash': 'NEGATIVES', + 'drop-dependency': 'TRANSITIONS', + 'drop-edge': 'TRANSITIONS', + 'stale-fingerprint': 'TRANSITIONS', + 'drop-collar': 'TILING', + 'close-c6': 'OPEN_C6', +} +LABELS = tuple(FAILED_CHECKS) +BASELINE = { + 'checks': {'IDENTITIES': True, 'NEGATIVES': True, 'OPEN_C6': True, + 'TILING': True, 'TRANSITIONS': True, 'VERDICTS': True}, + 'inventory_files': 17, + 'object': 'SYNTHETIC-D5-RECONCILIATION-PROTOCOL-FIXTURE', + 'passed': True, + 'scope': 'SYNTHETIC workflow fixture only; no mathematical assertion', +} + +# This is a real child process, never a mocked subprocess API or interpreter. +CHILD = r'''import json, os, pathlib, sys +label = sys.argv[-1] if '--mutant' in sys.argv else 'baseline' +with open(os.environ['RECONCILIATION_EVENTS'], 'a', encoding='utf-8') as stream: + stream.write(json.dumps([label, sys.flags.optimize, sys.flags.no_site, sys.dont_write_bytecode])+'\n') +selected = label == os.environ.get('RECONCILIATION_TARGET') and str(sys.flags.optimize) == os.environ.get('RECONCILIATION_MODE') +kind = os.environ.get('RECONCILIATION_KIND', 'valid') if selected else 'valid' +if kind == 'crash': raise RuntimeError('synthetic unrelated crash') +if kind == 'silent': sys.exit(1) +if kind == 'wrong-stdout': print('{}'); sys.exit(1) +if kind == 'baseline-wrong': print('synthetic wrong bytes'); sys.exit(0) +value = json.loads((pathlib.Path(__file__).parent/'RESULTS.json').read_bytes()) +key = FAILED_CHECKS.get(label) +if key is not None: + value['passed'] = False + value['checks'][key] = False + if kind == 'wrong-check': + value['checks'][key] = True + value['checks']['IDENTITIES'] = False + if kind == 'passed-true': value['passed'] = True + if kind == 'numeric-false': value['checks'][key] = 0 + if kind == 'string-false': value['checks'][key] = 'false' + if kind == 'numeric-passed': value['passed'] = 0 + if kind == 'extra-field': value['extra'] = 'unexpected' + if kind == 'missing-check': del value['checks'][key] + if kind == 'metadata-object': value['object'] = 'OTHER-SYNTHETIC-OBJECT' + if kind == 'metadata-scope': value['scope'] = 'changed scope' + if kind == 'metadata-count': value['inventory_files'] += 1 + if kind == 'metadata-numeric-alias': value['inventory_files'] = float(value['inventory_files']) +if kind == 'noise': print('synthetic unexpected stderr', file=sys.stderr) +print(json.dumps(value, sort_keys=True, **({} if kind == 'format' else {'indent': 2}))) +sys.exit({'exit-zero': 0, 'exit-two': 2, 'baseline-exit': 1}.get(kind, 0 if label == 'baseline' else 1)) +''' +CHILD = 'FAILED_CHECKS = ' + repr(FAILED_CHECKS) + '\n' + CHILD + + +def report_bytes(value): + return (json.dumps(value, indent=2, sort_keys=True) + '\n').encode() + + +def workflow_shell(text): + name = ' - name: Verify sources, replay both modes, reject mutants\n' + if text.count(name) != 1: + raise ValueError('expected one named protocol step') + tail = text.split(name, 1)[1].split(' - ', 1)[0] + marker = ' run: |\n' + if tail.count(marker) != 1: + raise ValueError('expected named literal run block') + lines = tail.split(marker, 1)[1].splitlines(keepends=True) + if any(line.strip() and not line.startswith(' ') for line in lines): + raise ValueError('unexpected run-block indentation') + shell = ''.join(line[10:] if line.startswith(' ') else line for line in lines) + if not shell.startswith('set -euo pipefail\n') or not shell.endswith('git diff --exit-code\n'): + raise ValueError('incomplete protocol shell') + return shell + + +class D5ReconciliationWorkflow(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not shutil.which('git') or not shutil.which('bash'): + raise RuntimeError('these workflow tests require Git and Bash') + cls.workflow = WORKFLOW.read_text(encoding='utf-8') + cls.shell = workflow_shell(cls.workflow) + + def run_fixture(self, kind='valid', target='baseline', mode=0): + with tempfile.TemporaryDirectory(prefix='d5-reconciliation-protocol-') as temp: + outer = Path(temp) + repo = outer / 'repo' + packet = repo / PACKET + packet.mkdir(parents=True) + executable_dir = outer / 'bin' + executable_dir.mkdir() + (executable_dir / 'python').symlink_to(sys.executable) + env = {key: value for key, value in os.environ.items() + if not key.startswith(('GIT_', 'PYTHON', 'RECONCILIATION_')) and key not in ('BASH_ENV', 'ENV')} + env.update(PATH=str(executable_dir) + os.pathsep + os.environ.get('PATH', ''), + GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, + GIT_TERMINAL_PROMPT='0', PYTHONDONTWRITEBYTECODE='1', + RECONCILIATION_KIND=kind, RECONCILIATION_TARGET=target, RECONCILIATION_MODE=str(mode), + RECONCILIATION_EVENTS=str(outer / 'events.jsonl')) + + def git(*args): + proc = subprocess.run(['git', '-c', 'gc.auto=0', *args], cwd=repo, + env=env, capture_output=True, timeout=15) + self.assertEqual(proc.returncode, 0, proc.stderr.decode(errors='replace')) + return proc.stdout + + payloads = {'reconciliation_check.py': CHILD.encode(), 'RESULTS.json': report_bytes(BASELINE), + 'RECONCILIATION.md': b'SYNTHETIC fixture; not a reconciliation verdict.\n'} + for name, data in payloads.items(): + (packet / name).write_bytes(data) + manifest = {'synthetic_fixture': True, 'files': [ + {'path': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()} + for name, data in sorted(payloads.items())]} + (packet / 'SOURCE_FILES.json').write_bytes(report_bytes(manifest)) + (repo / 'marker.txt').write_bytes(b'original\n') + if kind == 'tamper': + (packet / 'reconciliation_check.py').write_bytes(CHILD.encode() + b'\n# unbound\n') + elif kind == 'missing': + (packet / 'RECONCILIATION.md').unlink() + elif kind == 'extra': + (packet / 'extra.txt').write_bytes(b'extra\n') + elif kind == 'symlink': + (packet / 'RECONCILIATION.md').unlink() + (outer / 'reconciliation.txt').write_bytes(payloads['RECONCILIATION.md']) + (packet / 'RECONCILIATION.md').symlink_to(outer / 'reconciliation.txt') + git('init', '-q') + git('add', '.') + git('-c', 'user.name=Protocol fixture', '-c', 'user.email=fixture@example.invalid', + 'commit', '-qm', 'Synthetic protocol fixture only') + head = git('rev-parse', 'HEAD') + if kind == 'dirty': + (repo / 'marker.txt').write_bytes(b'intentionally dirty\n') + before = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + proc = subprocess.run(['bash', '-c', self.shell], cwd=repo, env=env, + capture_output=True, timeout=30) + after = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + self.assertEqual(before, after, 'workflow changed fixture payload bytes') + self.assertEqual(git('rev-parse', 'HEAD'), head) + events_path = outer / 'events.jsonl' + events = [json.loads(line) for line in events_path.read_text().splitlines()] if events_path.exists() else [] + return proc, events + + @staticmethod + def sequence(): + return [[label, mode, 1, True] for mode in (0, 1) for label in ('baseline', *LABELS)] + + def assert_refusal(self, kind, target, mode, diagnostic): + proc, events = self.run_fixture(kind, target, mode) + self.assertNotEqual(proc.returncode, 0, (kind, target, mode, proc.stdout, proc.stderr)) + self.assertNotIn(b'"mathematical_acceptance": false', proc.stdout) + self.assertIn(diagnostic, proc.stderr) + index = self.sequence().index([target, mode, 1, True]) + self.assertEqual(events, self.sequence()[:index + 1]) + + def test_valid_baselines_and_eight_label_reports_reach_both_modes(self): + proc, events = self.run_fixture() + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout, b'{"scientific_effect": "NONE", "mathematical_acceptance": false}\n') + self.assertEqual(proc.stderr, b'') + self.assertEqual(events, self.sequence()) + + def test_unrelated_crash_is_not_a_mutant_rejection(self): + self.check_mutants('crash') + + def test_silent_exit_one_is_not_a_mutant_rejection(self): + self.check_mutants('silent') + + def test_wrong_stdout_is_not_a_mutant_rejection(self): + self.check_mutants('wrong-stdout') + + def test_complete_flat_label_report_and_boolean_types_are_required(self): + for kind in ('wrong-check', 'passed-true', 'numeric-false', 'string-false', + 'numeric-passed', 'extra-field', 'missing-check', 'format'): + self.check_mutants(kind) + + def test_all_noncheck_report_metadata_remains_exact(self): + for kind in ('metadata-object', 'metadata-scope', 'metadata-count', 'metadata-numeric-alias'): + self.check_mutants(kind) + + def check_mutants(self, kind): + for mode in (0, 1): + for label in LABELS: + with self.subTest(kind=kind, mode=mode, label=label): + self.assert_refusal(kind, label, mode, b'mutant '+label.encode()) + + def test_mutant_stderr_is_not_accepted(self): + self.check_mutants('noise') + + def test_wrong_mutant_exit_is_not_accepted(self): + for kind in ('exit-zero', 'exit-two'): + self.check_mutants(kind) + + def test_baseline_stderr_is_not_accepted(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('noise', 'baseline', mode, b'baseline stderr') + + def test_baseline_stdout_remains_exact(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-wrong', 'baseline', mode, b'output differs from RESULTS.json') + + def test_baseline_nonzero_stops_before_mutants(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-exit', 'baseline', mode, b'CalledProcessError') + + def test_manifest_and_source_guards_precede_checker_execution(self): + for kind, diagnostic in [('tamper', b'source identity mismatch'), + ('missing', b'packet tree differs'), ('extra', b'packet tree differs'), + ('symlink', b'flat regular source required')]: + with self.subTest(kind=kind): + proc, events = self.run_fixture(kind) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(diagnostic, proc.stderr) + self.assertEqual(events, []) + + def test_final_git_guard_still_refuses_dirty_fixture(self): + proc, events = self.run_fixture('dirty') + self.assertNotEqual(proc.returncode, 0) + self.assertIn(b'diff --git', proc.stdout) + self.assertEqual(events, self.sequence()) + + def test_existing_workflow_execution_contract_is_preserved(self): + for path in ('reviews/d5_reconciliation_20260929/**', '.github/workflows/d5-reconciliation.yml', + 'reviews/d5_*/**', 'frontiers/intermediate_window_20260928/**', + 'frontiers/remote_window_20260924/**', 'frontiers/remote_collision_20260928/**', + 'frontiers/rn_annulus_bridge_20260925/**'): + with self.subTest(path=path): + self.assertIn(" - '" + path + "'\n", self.workflow) + self.assertIn(" workflow_dispatch:\n", self.workflow) + self.assertIn("permissions:\n contents: read\n", self.workflow) + self.assertIn("timeout-minutes: 10\n", self.workflow) + self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", self.workflow) + self.assertIn("persist-credentials: false", self.workflow) + self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", self.workflow) + self.assertIn("PYTHONDONTWRITEBYTECODE: '1'", self.workflow) + + def test_protocol_tests_are_wired_for_both_outer_modes(self): + self.assertIn(" - 'tests/test_d5_reconciliation_workflow.py'\n", self.workflow) + self.assertIn("python -B -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v", self.workflow) + self.assertIn("python -B -O -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v", self.workflow) + self.assertIn("python-version: '3.11.16'", self.workflow) + self.assertEqual(self.shell.count('timeout=300'), 2) + + +if __name__ == '__main__': + unittest.main() From e01e346f660812734edc7d4c968b05c51c74fb7e Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Sat, 17 Oct 2026 16:37:08 -0500 Subject: [PATCH 2/2] Require exact elder failures and extend the full-suite budget --- .github/workflows/elder-lower-all-d.yml | 25 +- .github/workflows/full-regression-suite.yml | 2 +- tests/test_elder_lower_all_d_workflow.py | 264 ++++++++++++++++++++ tests/test_full_suite_routing.py | 38 ++- 4 files changed, 315 insertions(+), 14 deletions(-) create mode 100644 tests/test_elder_lower_all_d_workflow.py diff --git a/.github/workflows/elder-lower-all-d.yml b/.github/workflows/elder-lower-all-d.yml index e036ccb64..89cd0618d 100644 --- a/.github/workflows/elder-lower-all-d.yml +++ b/.github/workflows/elder-lower-all-d.yml @@ -5,6 +5,7 @@ name: Elder-failure lower bound (every d) checks paths: - 'frontiers/elder_lower_all_d_20260929/**' - '.github/workflows/elder-lower-all-d.yml' + - 'tests/test_elder_lower_all_d_workflow.py' workflow_dispatch: permissions: @@ -44,16 +45,28 @@ jobs: script = str(root / 'lower_all_d_check.py') expected = (root / 'RESULTS.json').read_bytes() mutants = ['M1', 'M2', 'M3'] + expected_failures = { + 'M1': b'FAIL L_ledger_m2\n', + 'M2': b'FAIL P2_hessian_M\n', + 'M3': b'FAIL E6_margin_below\n', + } for flags in (['-B', '-S'], ['-B', '-O', '-S']): out = subprocess.run([sys.executable, *flags, script], - check=True, capture_output=True, timeout=300).stdout - if out != expected: + check=True, capture_output=True, timeout=300) + if out.stdout != expected: raise RuntimeError('output differs from RESULTS.json under ' + ' '.join(flags)) + if out.stderr: + raise RuntimeError('baseline stderr under ' + ' '.join(flags)) for m in mutants: - rc = subprocess.run([sys.executable, *flags, script, '--mutant', m], - capture_output=True, timeout=300).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + out = subprocess.run([sys.executable, *flags, script, '--mutant', m], + capture_output=True, timeout=300) + if out.returncode != 1 or out.stdout != expected_failures[m] or out.stderr: + raise RuntimeError('mutant ' + m + ' not rejected with expected failure') print(json.dumps({'scientific_effect': 'NONE', 'scientific_acceptance': False})) PY git diff --exit-code + - name: Full-shell protocol regression tests in both modes + run: | + set -euo pipefail + python -B -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v + python -B -O -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v diff --git a/.github/workflows/full-regression-suite.yml b/.github/workflows/full-regression-suite.yml index ce56218e2..e003c3fe5 100644 --- a/.github/workflows/full-regression-suite.yml +++ b/.github/workflows/full-regression-suite.yml @@ -29,7 +29,7 @@ concurrency: jobs: full-suite: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: diff --git a/tests/test_elder_lower_all_d_workflow.py b/tests/test_elder_lower_all_d_workflow.py new file mode 100644 index 000000000..5a52984b5 --- /dev/null +++ b/tests/test_elder_lower_all_d_workflow.py @@ -0,0 +1,264 @@ +"""Real-shell elder-lower-all-d protocol tests with committed SYNTHETIC fixtures. + +No mathematical packet is replayed here. Real checker children record their +interpreter flags; the full workflow block retains its source and Git guards. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +WORKFLOW = Path(__file__).resolve().parents[1] / '.github/workflows/elder-lower-all-d.yml' +PACKET = Path('frontiers/elder_lower_all_d_20260929') +# Source-derived first failed checks; native characterization is separate. +FAILURE_LINES = { + 'M1': 'FAIL L_ledger_m2\n', + 'M2': 'FAIL P2_hessian_M\n', + 'M3': 'FAIL E6_margin_below\n', +} +LABELS = tuple(FAILURE_LINES) +BASELINE = { + 'object': 'SYNTHETIC-ELDER-ALL-D-PROTOCOL-FIXTURE', + 'passed': True, + 'results': {'synthetic': 'workflow engineering fixture; no mathematical assertion'}, + 'scope': 'SYNTHETIC source and child-protocol controls only', +} + +# This is a real child process, never a mocked subprocess API or interpreter. +CHILD = r'''import json, os, pathlib, sys +args = sys.argv[1:] +label = args[-1] if '--mutant' in args else 'baseline' +event = {'label': label, 'argv': args, + 'interpreter_flags': sys.orig_argv[1:sys.orig_argv.index(sys.argv[0])], + 'optimize': sys.flags.optimize, 'no_site': sys.flags.no_site, + 'dont_write_bytecode': sys.dont_write_bytecode} +with open(os.environ['ELDER_ALL_D_EVENTS'], 'a', encoding='utf-8') as stream: + stream.write(json.dumps(event)+'\n') +selected = label == os.environ.get('ELDER_ALL_D_TARGET') and str(sys.flags.optimize) == os.environ.get('ELDER_ALL_D_MODE') +kind = os.environ.get('ELDER_ALL_D_KIND', 'valid') if selected else 'valid' +if kind == 'crash': raise RuntimeError('synthetic unrelated crash') +if kind == 'silent': sys.exit(1) +expected = (pathlib.Path(__file__).parent/'RESULTS.json').read_text() +out = expected if label == 'baseline' else FAILURE_LINES[label] +if kind == 'baseline-wrong': out = 'synthetic wrong baseline bytes\n' +if kind == 'wrong-label': out = FAILURE_LINES['M2' if label == 'M1' else 'M1'] +if kind == 'baseline-json': out = expected +if kind == 'missing-newline': out = out.rstrip('\n') +if kind == 'prefix-noise': out = 'unexpected prefix\n' + out +if kind == 'suffix-noise': out += 'unexpected suffix\n' +if kind == 'stderr': print('synthetic unexpected stderr', file=sys.stderr) +sys.stdout.write(out) +sys.exit({'exit-zero': 0, 'exit-two': 2, 'baseline-exit': 1}.get(kind, 0 if label == 'baseline' else 1)) +''' +CHILD = 'FAILURE_LINES = ' + repr(FAILURE_LINES) + '\n' + CHILD + + +def report_bytes(value): + return (json.dumps(value, indent=2, sort_keys=True) + '\n').encode() + + +def workflow_shell(text): + name = ' - name: Verify sources, replay both modes, reject mutants\n' + if text.count(name) != 1: + raise ValueError('expected one named protocol step') + tail = text.split(name, 1)[1].split(' - ', 1)[0] + marker = ' run: |\n' + if tail.count(marker) != 1: + raise ValueError('expected named literal run block') + lines = tail.split(marker, 1)[1].splitlines(keepends=True) + if any(line.strip() and not line.startswith(' ') for line in lines): + raise ValueError('unexpected run-block indentation') + shell = ''.join(line[10:] if line.startswith(' ') else line for line in lines) + if not shell.startswith('set -euo pipefail\n') or not shell.endswith('git diff --exit-code\n'): + raise ValueError('incomplete protocol shell') + return shell + + +class ElderLowerAllDWorkflow(unittest.TestCase): + shell_runs = 0 + + @classmethod + def setUpClass(cls): + if not shutil.which('git') or not shutil.which('bash'): + raise RuntimeError('these workflow tests require Git and Bash') + cls.shell_runs = 0 + cls.workflow = WORKFLOW.read_text(encoding='utf-8') + cls.shell = workflow_shell(cls.workflow) + + def run_fixture(self, kind='valid', target='baseline', mode=0): + with tempfile.TemporaryDirectory(prefix='elder-lower-all-d-protocol-') as temp: + outer = Path(temp) + repo = outer / 'repo' + packet = repo / PACKET + packet.mkdir(parents=True) + executable_dir = outer / 'bin' + executable_dir.mkdir() + (executable_dir / 'python').symlink_to(sys.executable) + env = {key: value for key, value in os.environ.items() + if not key.startswith(('GIT_', 'PYTHON', 'ELDER_ALL_D_')) and key not in ('BASH_ENV', 'ENV')} + env.update(PATH=str(executable_dir) + os.pathsep + os.environ.get('PATH', ''), + GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, + GIT_TERMINAL_PROMPT='0', PYTHONDONTWRITEBYTECODE='1', + ELDER_ALL_D_KIND=kind, ELDER_ALL_D_TARGET=target, ELDER_ALL_D_MODE=str(mode), + ELDER_ALL_D_EVENTS=str(outer / 'events.jsonl')) + + def git(*args): + proc = subprocess.run(['git', '-c', 'gc.auto=0', *args], cwd=repo, + env=env, capture_output=True, timeout=15) + self.assertEqual(proc.returncode, 0, proc.stderr.decode(errors='replace')) + return proc.stdout + + payloads = {'lower_all_d_check.py': CHILD.encode(), 'RESULTS.json': report_bytes(BASELINE), + 'PROOF.md': b'SYNTHETIC fixture; not a mathematical proof.\n'} + for name, data in payloads.items(): + (packet / name).write_bytes(data) + manifest = {'synthetic_fixture': True, 'files': [ + {'path': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()} + for name, data in sorted(payloads.items())]} + (packet / 'SOURCE_FILES.json').write_bytes(report_bytes(manifest)) + (repo / 'marker.txt').write_bytes(b'original\n') + if kind == 'hash-only': + proof = payloads['PROOF.md'] + (packet / 'PROOF.md').write_bytes(b'X' + proof[1:]) # same length, different hash + elif kind == 'size-only': + proof_entry = next(row for row in manifest['files'] if row['path'] == 'PROOF.md') + proof_entry['bytes'] += 1 # actual bytes and their expected SHA256 stay unchanged + (packet / 'SOURCE_FILES.json').write_bytes(report_bytes(manifest)) + elif kind == 'missing': + (packet / 'PROOF.md').unlink() + elif kind == 'extra': + (packet / 'extra.txt').write_bytes(b'extra\n') + elif kind == 'symlink': + (packet / 'PROOF.md').unlink() + (outer / 'proof.txt').write_bytes(payloads['PROOF.md']) + (packet / 'PROOF.md').symlink_to(outer / 'proof.txt') + git('init', '-q') + git('add', '.') + git('-c', 'user.name=Protocol fixture', '-c', 'user.email=fixture@example.invalid', + 'commit', '-qm', 'Synthetic protocol fixture only') + head = git('rev-parse', 'HEAD') + if kind == 'dirty': + (repo / 'marker.txt').write_bytes(b'intentionally dirty\n') + before = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + type(self).shell_runs += 1 + proc = subprocess.run(['bash', '-c', self.shell], cwd=repo, env=env, + capture_output=True, timeout=30) + after = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + self.assertEqual(before, after, 'workflow changed fixture payload bytes') + self.assertEqual(git('rev-parse', 'HEAD'), head) + events_path = outer / 'events.jsonl' + events = [json.loads(line) for line in events_path.read_text().splitlines()] if events_path.exists() else [] + return proc, events + + @staticmethod + def event(label, mode): + return {'label': label, 'argv': [] if label == 'baseline' else ['--mutant', label], + 'interpreter_flags': ['-B', '-S'] if mode == 0 else ['-B', '-O', '-S'], + 'optimize': mode, 'no_site': 1, 'dont_write_bytecode': True} + + @classmethod + def sequence(cls): + return [cls.event(label, mode) + for mode in (0, 1) for label in ('baseline', *LABELS)] + + def assert_refusal(self, kind, target, mode, diagnostic): + proc, events = self.run_fixture(kind, target, mode) + self.assertNotEqual(proc.returncode, 0, (kind, target, mode, proc.stdout, proc.stderr)) + self.assertNotIn(b'"scientific_acceptance": false', proc.stdout) + self.assertIn(diagnostic, proc.stderr) + index = self.sequence().index(self.event(target, mode)) + self.assertEqual(events, self.sequence()[:index + 1]) + + def test_valid_baseline_and_three_failure_lines_reach_both_modes(self): + proc, events = self.run_fixture() + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout, b'{"scientific_effect": "NONE", "scientific_acceptance": false}\n') + self.assertEqual(proc.stderr, b'') + self.assertEqual(events, self.sequence()) + + def test_unrelated_crash_is_not_a_mutant_rejection(self): + self.check_mutants('crash') + + def test_silent_exit_one_is_not_a_mutant_rejection(self): + self.check_mutants('silent') + + def test_complete_label_failure_bytes_are_required(self): + for kind in ('wrong-label', 'baseline-json', 'missing-newline', 'prefix-noise', 'suffix-noise'): + self.check_mutants(kind) + + def check_mutants(self, kind): + for mode in (0, 1): + for label in LABELS: + with self.subTest(kind=kind, mode=mode, label=label): + self.assert_refusal(kind, label, mode, b'mutant '+label.encode()) + + def test_mutant_stderr_is_not_accepted(self): + self.check_mutants('stderr') + + def test_wrong_mutant_exit_is_not_accepted(self): + for kind in ('exit-zero', 'exit-two'): + self.check_mutants(kind) + + def test_baseline_stderr_is_not_accepted(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('stderr', 'baseline', mode, b'baseline stderr') + + def test_baseline_stdout_remains_exact(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-wrong', 'baseline', mode, b'output differs from RESULTS.json') + + def test_baseline_nonzero_stops_before_mutants(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-exit', 'baseline', mode, b'CalledProcessError') + + def test_manifest_and_source_guards_precede_checker_execution(self): + for kind, diagnostic in [('hash-only', b'source identity mismatch'), + ('size-only', b'source identity mismatch'), + ('missing', b'packet tree differs'), ('extra', b'packet tree differs'), + ('symlink', b'flat regular source required')]: + with self.subTest(kind=kind): + proc, events = self.run_fixture(kind) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(diagnostic, proc.stderr) + self.assertEqual(events, []) + + def test_final_git_guard_still_refuses_dirty_fixture(self): + proc, events = self.run_fixture('dirty') + self.assertNotEqual(proc.returncode, 0) + self.assertIn(b'diff --git', proc.stdout) + self.assertEqual(events, self.sequence()) + + def test_existing_workflow_execution_contract_is_preserved(self): + for path in ('frontiers/elder_lower_all_d_20260929/**', '.github/workflows/elder-lower-all-d.yml'): + with self.subTest(path=path): + self.assertIn(" - '" + path + "'\n", self.workflow) + self.assertIn(" workflow_dispatch:\n", self.workflow) + self.assertIn("permissions:\n contents: read\n", self.workflow) + self.assertIn("timeout-minutes: 10\n", self.workflow) + self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", self.workflow) + self.assertIn("persist-credentials: false", self.workflow) + self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", self.workflow) + self.assertIn("PYTHONDONTWRITEBYTECODE: '1'", self.workflow) + + def test_protocol_tests_are_wired_for_both_outer_modes(self): + self.assertIn(" - 'tests/test_elder_lower_all_d_workflow.py'\n", self.workflow) + self.assertIn("python -B -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v", self.workflow) + self.assertIn("python -B -O -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v", self.workflow) + self.assertIn("python-version: '3.11.16'", self.workflow) + self.assertEqual(self.shell.count('timeout=300'), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_full_suite_routing.py b/tests/test_full_suite_routing.py index 5a9e9160d..07f058659 100644 --- a/tests/test_full_suite_routing.py +++ b/tests/test_full_suite_routing.py @@ -248,10 +248,12 @@ def strip_concurrency_insertion(text): def reconstruct_original_central(text): """Historical central workflow at 49fb197c / blob ea291276, authenticated. - Remove only the approved concurrency insertion, then revert only the suite - body; the result must hash to the unchanged historical digest. + Remove only the approved concurrency insertion, restore the historical + 20-minute budget from the current 30-minute budget, then revert only the + suite body; the result must hash to the unchanged historical digest. """ text = strip_concurrency_insertion(text) + text = mutate(text, '\n timeout-minutes: 30\n', '\n timeout-minutes: 20\n') block = named(parse(text)[1], REGRESSION)['run'] rendered = lambda body: ''.join(' ' + line + '\n' for line in body.splitlines()) @@ -613,7 +615,7 @@ def violations(workflow, text): if bind != central_bind: problems.append('tested-commit binding step differs from the central workflow') if workflow == CENTRAL: - if job['id'] != 'full-suite' or job.get('timeout-minutes') != '20': + if job['id'] != 'full-suite' or job.get('timeout-minutes') != '30': problems.append('central job id/budget changed') if names != [BIND, REGRESSION, CONFIRM]: problems.append('central step order changed: ' + repr(names)) @@ -691,8 +693,8 @@ def mutate(text, old, new, count=1): class FullSuiteRoutingTests(unittest.TestCase): def original_central(self): - # Remove only the authenticated concurrency insertion, reconstruct only - # the suite body and authenticate the complete original workflow at + # Remove only the authenticated concurrency insertion, restore only the + # historical budget and suite body, and authenticate the original at # 49fb197c / blob ea291276, not a hand-written shell surrogate. original = reconstruct_original_central(source(CENTRAL)) self.assertEqual(hashlib.sha256(original.encode()).hexdigest(), ORIGINAL_CENTRAL_SHA256) @@ -776,7 +778,7 @@ def test_historical_bytes_outside_insertion_are_authenticated(self): central = source(CENTRAL) self.assertEqual(reconstruct_original_central(central), self.original_central()) for old, new in (('# This check is not required', '# This check is now required'), - (' timeout-minutes: 20\n', ' timeout-minutes: 21\n'), + (' timeout-minutes: 30\n', ' timeout-minutes: 31\n'), ('permissions:\n contents: read\n\nconcurrency:', 'permissions:\n contents: read\n\n\nconcurrency:'), (CONCURRENCY_INSERTION + 'jobs:\n', CONCURRENCY_INSERTION + '\njobs:\n'), @@ -786,6 +788,28 @@ def test_historical_bytes_outside_insertion_are_authenticated(self): with self.assertRaises((ValueError, AssertionError)): reconstruct_original_central(mutate(central, old, new)) + def test_current_and_historical_budgets_have_separate_exact_contracts(self): + central = source(CENTRAL) + self.assertEqual(parse(central)[1]['timeout-minutes'], '30') + original = self.original_central() + self.assertEqual(parse(original)[1]['timeout-minutes'], '20') + for budget in ('0', '20', '29', '31', '360', "'30'"): + with self.subTest(budget=budget): + changed = mutate(central, ' timeout-minutes: 30\n', + ' timeout-minutes: ' + budget + '\n') + self.assertIn('central job id/budget changed', violations(CENTRAL, changed)) + with self.assertRaises((ValueError, AssertionError)): + reconstruct_original_central(changed) + relocated = mutate(central, '\n timeout-minutes: 30\n', + '\n timeout-minutes: 20\n') + relocated = mutate(relocated, ' log_dir="$(mktemp -d)"\n', + ' # timeout-minutes: 30\n' + ' log_dir="$(mktemp -d)"\n') + self.assertEqual(parse(relocated)[1]['timeout-minutes'], '20') + self.assertIn('central job id/budget changed', violations(CENTRAL, relocated)) + with self.assertRaises((ValueError, AssertionError)): + reconstruct_original_central(relocated) + def test_original_accepts_synthetic_empty_success_but_preserves_failures(self): original = self.original_central() for event in ('pull_request', 'workflow_dispatch'): @@ -1186,7 +1210,7 @@ def test_meaningful_negatives_are_detected(self): 'central gains a condition': (CENTRAL, central, ' - name: Full repository regression suite in both Python modes\n', ' - name: Full repository regression suite in both Python modes\n' " if: github.event_name == 'workflow_dispatch'\n"), - 'central continue-on-error': (CENTRAL, central, ' timeout-minutes: 20\n', ' timeout-minutes: 20\n continue-on-error: true\n'), + 'central continue-on-error': (CENTRAL, central, ' timeout-minutes: 30\n', ' timeout-minutes: 30\n continue-on-error: true\n'), 'central unbinds commit': (CENTRAL, central, ' if [ "$checked_out" != "$TESTED_SHA" ]; then\n', ' if false; then\n'), }