diff --git a/.github/workflows/d5-reconciliation.yml b/.github/workflows/d5-reconciliation.yml index 2474267a9..b5d14f6e1 100644 --- a/.github/workflows/d5-reconciliation.yml +++ b/.github/workflows/d5-reconciliation.yml @@ -5,6 +5,7 @@ name: D5 reconciliation checks paths: - 'reviews/d5_reconciliation_20260929/**' - '.github/workflows/d5-reconciliation.yml' + - 'tests/test_d5_reconciliation_workflow.py' - 'reviews/d5_*/**' - 'frontiers/intermediate_window_20260928/**' - 'frontiers/remote_window_20260924/**' @@ -26,6 +27,10 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: '3.11.16' + - name: Exercise rejection protocol controls + run: | + python -B -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v + python -B -O -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v - name: Verify sources, replay both modes, reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -49,16 +54,33 @@ jobs: script = str(root / 'reconciliation_check.py') expected = (root / 'RESULTS.json').read_bytes() mutants = ['optional-continuum', 'allow-symlink', 'no-hash', 'drop-dependency', 'drop-edge', 'stale-fingerprint', 'drop-collar', 'close-c6'] + # Complete native report differences, characterized before this repair. + failed_checks = { + 'optional-continuum': 'NEGATIVES', + 'allow-symlink': 'NEGATIVES', + 'no-hash': 'NEGATIVES', + 'drop-dependency': 'TRANSITIONS', + 'drop-edge': 'TRANSITIONS', + 'stale-fingerprint': 'TRANSITIONS', + 'drop-collar': 'TILING', + 'close-c6': 'OPEN_C6', + } for flags in (['-B', '-S'], ['-B', '-O', '-S']): out = subprocess.run([sys.executable, *flags, script], - check=True, capture_output=True, timeout=300).stdout - if out != expected: + check=True, capture_output=True, timeout=300) + if out.stdout != expected: raise RuntimeError('output differs from RESULTS.json under ' + ' '.join(flags)) + if out.stderr: + raise RuntimeError('unexpected baseline stderr under ' + ' '.join(flags)) for m in mutants: - rc = subprocess.run([sys.executable, *flags, script, '--mutant', m], - capture_output=True, timeout=300).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + report = json.loads(expected) + report['passed'] = False + report['checks'][failed_checks[m]] = False + wanted = (json.dumps(report, indent=2, sort_keys=True) + '\n').encode('utf-8') + rejected = subprocess.run([sys.executable, *flags, script, '--mutant', m], + capture_output=True, timeout=300) + if rejected.returncode != 1 or rejected.stdout != wanted or rejected.stderr: + raise RuntimeError('mutant ' + m + ' did not emit its expected rejection report') print(json.dumps({'scientific_effect': 'NONE', 'mathematical_acceptance': False})) PY git diff --exit-code diff --git a/.github/workflows/elder-lower-all-d.yml b/.github/workflows/elder-lower-all-d.yml index e036ccb64..89cd0618d 100644 --- a/.github/workflows/elder-lower-all-d.yml +++ b/.github/workflows/elder-lower-all-d.yml @@ -5,6 +5,7 @@ name: Elder-failure lower bound (every d) checks paths: - 'frontiers/elder_lower_all_d_20260929/**' - '.github/workflows/elder-lower-all-d.yml' + - 'tests/test_elder_lower_all_d_workflow.py' workflow_dispatch: permissions: @@ -44,16 +45,28 @@ jobs: script = str(root / 'lower_all_d_check.py') expected = (root / 'RESULTS.json').read_bytes() mutants = ['M1', 'M2', 'M3'] + expected_failures = { + 'M1': b'FAIL L_ledger_m2\n', + 'M2': b'FAIL P2_hessian_M\n', + 'M3': b'FAIL E6_margin_below\n', + } for flags in (['-B', '-S'], ['-B', '-O', '-S']): out = subprocess.run([sys.executable, *flags, script], - check=True, capture_output=True, timeout=300).stdout - if out != expected: + check=True, capture_output=True, timeout=300) + if out.stdout != expected: raise RuntimeError('output differs from RESULTS.json under ' + ' '.join(flags)) + if out.stderr: + raise RuntimeError('baseline stderr under ' + ' '.join(flags)) for m in mutants: - rc = subprocess.run([sys.executable, *flags, script, '--mutant', m], - capture_output=True, timeout=300).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + out = subprocess.run([sys.executable, *flags, script, '--mutant', m], + capture_output=True, timeout=300) + if out.returncode != 1 or out.stdout != expected_failures[m] or out.stderr: + raise RuntimeError('mutant ' + m + ' not rejected with expected failure') print(json.dumps({'scientific_effect': 'NONE', 'scientific_acceptance': False})) PY git diff --exit-code + - name: Full-shell protocol regression tests in both modes + run: | + set -euo pipefail + python -B -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v + python -B -O -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v diff --git a/.github/workflows/full-regression-suite.yml b/.github/workflows/full-regression-suite.yml index ce56218e2..e003c3fe5 100644 --- a/.github/workflows/full-regression-suite.yml +++ b/.github/workflows/full-regression-suite.yml @@ -29,7 +29,7 @@ concurrency: jobs: full-suite: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: diff --git a/tests/test_d5_reconciliation_workflow.py b/tests/test_d5_reconciliation_workflow.py new file mode 100644 index 000000000..b383166bc --- /dev/null +++ b/tests/test_d5_reconciliation_workflow.py @@ -0,0 +1,274 @@ +"""Real-shell d5-reconciliation protocol tests with committed SYNTHETIC fixtures. + +No mathematical packet is replayed here. Real checker children record their +interpreter flags; the full workflow block retains its source and Git guards. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +WORKFLOW = Path(__file__).resolve().parents[1] / '.github/workflows/d5-reconciliation.yml' +PACKET = Path('reviews/d5_reconciliation_20260929') +# Characterized from reconciliation_check.py and RESULTS.json; native replay is separate. +# Labels sharing a check intentionally share report bytes; event traces bind +# every actual invocation independently of those equivalent reports. +FAILED_CHECKS = { + 'optional-continuum': 'NEGATIVES', + 'allow-symlink': 'NEGATIVES', + 'no-hash': 'NEGATIVES', + 'drop-dependency': 'TRANSITIONS', + 'drop-edge': 'TRANSITIONS', + 'stale-fingerprint': 'TRANSITIONS', + 'drop-collar': 'TILING', + 'close-c6': 'OPEN_C6', +} +LABELS = tuple(FAILED_CHECKS) +BASELINE = { + 'checks': {'IDENTITIES': True, 'NEGATIVES': True, 'OPEN_C6': True, + 'TILING': True, 'TRANSITIONS': True, 'VERDICTS': True}, + 'inventory_files': 17, + 'object': 'SYNTHETIC-D5-RECONCILIATION-PROTOCOL-FIXTURE', + 'passed': True, + 'scope': 'SYNTHETIC workflow fixture only; no mathematical assertion', +} + +# This is a real child process, never a mocked subprocess API or interpreter. +CHILD = r'''import json, os, pathlib, sys +label = sys.argv[-1] if '--mutant' in sys.argv else 'baseline' +with open(os.environ['RECONCILIATION_EVENTS'], 'a', encoding='utf-8') as stream: + stream.write(json.dumps([label, sys.flags.optimize, sys.flags.no_site, sys.dont_write_bytecode])+'\n') +selected = label == os.environ.get('RECONCILIATION_TARGET') and str(sys.flags.optimize) == os.environ.get('RECONCILIATION_MODE') +kind = os.environ.get('RECONCILIATION_KIND', 'valid') if selected else 'valid' +if kind == 'crash': raise RuntimeError('synthetic unrelated crash') +if kind == 'silent': sys.exit(1) +if kind == 'wrong-stdout': print('{}'); sys.exit(1) +if kind == 'baseline-wrong': print('synthetic wrong bytes'); sys.exit(0) +value = json.loads((pathlib.Path(__file__).parent/'RESULTS.json').read_bytes()) +key = FAILED_CHECKS.get(label) +if key is not None: + value['passed'] = False + value['checks'][key] = False + if kind == 'wrong-check': + value['checks'][key] = True + value['checks']['IDENTITIES'] = False + if kind == 'passed-true': value['passed'] = True + if kind == 'numeric-false': value['checks'][key] = 0 + if kind == 'string-false': value['checks'][key] = 'false' + if kind == 'numeric-passed': value['passed'] = 0 + if kind == 'extra-field': value['extra'] = 'unexpected' + if kind == 'missing-check': del value['checks'][key] + if kind == 'metadata-object': value['object'] = 'OTHER-SYNTHETIC-OBJECT' + if kind == 'metadata-scope': value['scope'] = 'changed scope' + if kind == 'metadata-count': value['inventory_files'] += 1 + if kind == 'metadata-numeric-alias': value['inventory_files'] = float(value['inventory_files']) +if kind == 'noise': print('synthetic unexpected stderr', file=sys.stderr) +print(json.dumps(value, sort_keys=True, **({} if kind == 'format' else {'indent': 2}))) +sys.exit({'exit-zero': 0, 'exit-two': 2, 'baseline-exit': 1}.get(kind, 0 if label == 'baseline' else 1)) +''' +CHILD = 'FAILED_CHECKS = ' + repr(FAILED_CHECKS) + '\n' + CHILD + + +def report_bytes(value): + return (json.dumps(value, indent=2, sort_keys=True) + '\n').encode() + + +def workflow_shell(text): + name = ' - name: Verify sources, replay both modes, reject mutants\n' + if text.count(name) != 1: + raise ValueError('expected one named protocol step') + tail = text.split(name, 1)[1].split(' - ', 1)[0] + marker = ' run: |\n' + if tail.count(marker) != 1: + raise ValueError('expected named literal run block') + lines = tail.split(marker, 1)[1].splitlines(keepends=True) + if any(line.strip() and not line.startswith(' ') for line in lines): + raise ValueError('unexpected run-block indentation') + shell = ''.join(line[10:] if line.startswith(' ') else line for line in lines) + if not shell.startswith('set -euo pipefail\n') or not shell.endswith('git diff --exit-code\n'): + raise ValueError('incomplete protocol shell') + return shell + + +class D5ReconciliationWorkflow(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not shutil.which('git') or not shutil.which('bash'): + raise RuntimeError('these workflow tests require Git and Bash') + cls.workflow = WORKFLOW.read_text(encoding='utf-8') + cls.shell = workflow_shell(cls.workflow) + + def run_fixture(self, kind='valid', target='baseline', mode=0): + with tempfile.TemporaryDirectory(prefix='d5-reconciliation-protocol-') as temp: + outer = Path(temp) + repo = outer / 'repo' + packet = repo / PACKET + packet.mkdir(parents=True) + executable_dir = outer / 'bin' + executable_dir.mkdir() + (executable_dir / 'python').symlink_to(sys.executable) + env = {key: value for key, value in os.environ.items() + if not key.startswith(('GIT_', 'PYTHON', 'RECONCILIATION_')) and key not in ('BASH_ENV', 'ENV')} + env.update(PATH=str(executable_dir) + os.pathsep + os.environ.get('PATH', ''), + GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, + GIT_TERMINAL_PROMPT='0', PYTHONDONTWRITEBYTECODE='1', + RECONCILIATION_KIND=kind, RECONCILIATION_TARGET=target, RECONCILIATION_MODE=str(mode), + RECONCILIATION_EVENTS=str(outer / 'events.jsonl')) + + def git(*args): + proc = subprocess.run(['git', '-c', 'gc.auto=0', *args], cwd=repo, + env=env, capture_output=True, timeout=15) + self.assertEqual(proc.returncode, 0, proc.stderr.decode(errors='replace')) + return proc.stdout + + payloads = {'reconciliation_check.py': CHILD.encode(), 'RESULTS.json': report_bytes(BASELINE), + 'RECONCILIATION.md': b'SYNTHETIC fixture; not a reconciliation verdict.\n'} + for name, data in payloads.items(): + (packet / name).write_bytes(data) + manifest = {'synthetic_fixture': True, 'files': [ + {'path': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()} + for name, data in sorted(payloads.items())]} + (packet / 'SOURCE_FILES.json').write_bytes(report_bytes(manifest)) + (repo / 'marker.txt').write_bytes(b'original\n') + if kind == 'tamper': + (packet / 'reconciliation_check.py').write_bytes(CHILD.encode() + b'\n# unbound\n') + elif kind == 'missing': + (packet / 'RECONCILIATION.md').unlink() + elif kind == 'extra': + (packet / 'extra.txt').write_bytes(b'extra\n') + elif kind == 'symlink': + (packet / 'RECONCILIATION.md').unlink() + (outer / 'reconciliation.txt').write_bytes(payloads['RECONCILIATION.md']) + (packet / 'RECONCILIATION.md').symlink_to(outer / 'reconciliation.txt') + git('init', '-q') + git('add', '.') + git('-c', 'user.name=Protocol fixture', '-c', 'user.email=fixture@example.invalid', + 'commit', '-qm', 'Synthetic protocol fixture only') + head = git('rev-parse', 'HEAD') + if kind == 'dirty': + (repo / 'marker.txt').write_bytes(b'intentionally dirty\n') + before = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + proc = subprocess.run(['bash', '-c', self.shell], cwd=repo, env=env, + capture_output=True, timeout=30) + after = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + self.assertEqual(before, after, 'workflow changed fixture payload bytes') + self.assertEqual(git('rev-parse', 'HEAD'), head) + events_path = outer / 'events.jsonl' + events = [json.loads(line) for line in events_path.read_text().splitlines()] if events_path.exists() else [] + return proc, events + + @staticmethod + def sequence(): + return [[label, mode, 1, True] for mode in (0, 1) for label in ('baseline', *LABELS)] + + def assert_refusal(self, kind, target, mode, diagnostic): + proc, events = self.run_fixture(kind, target, mode) + self.assertNotEqual(proc.returncode, 0, (kind, target, mode, proc.stdout, proc.stderr)) + self.assertNotIn(b'"mathematical_acceptance": false', proc.stdout) + self.assertIn(diagnostic, proc.stderr) + index = self.sequence().index([target, mode, 1, True]) + self.assertEqual(events, self.sequence()[:index + 1]) + + def test_valid_baselines_and_eight_label_reports_reach_both_modes(self): + proc, events = self.run_fixture() + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout, b'{"scientific_effect": "NONE", "mathematical_acceptance": false}\n') + self.assertEqual(proc.stderr, b'') + self.assertEqual(events, self.sequence()) + + def test_unrelated_crash_is_not_a_mutant_rejection(self): + self.check_mutants('crash') + + def test_silent_exit_one_is_not_a_mutant_rejection(self): + self.check_mutants('silent') + + def test_wrong_stdout_is_not_a_mutant_rejection(self): + self.check_mutants('wrong-stdout') + + def test_complete_flat_label_report_and_boolean_types_are_required(self): + for kind in ('wrong-check', 'passed-true', 'numeric-false', 'string-false', + 'numeric-passed', 'extra-field', 'missing-check', 'format'): + self.check_mutants(kind) + + def test_all_noncheck_report_metadata_remains_exact(self): + for kind in ('metadata-object', 'metadata-scope', 'metadata-count', 'metadata-numeric-alias'): + self.check_mutants(kind) + + def check_mutants(self, kind): + for mode in (0, 1): + for label in LABELS: + with self.subTest(kind=kind, mode=mode, label=label): + self.assert_refusal(kind, label, mode, b'mutant '+label.encode()) + + def test_mutant_stderr_is_not_accepted(self): + self.check_mutants('noise') + + def test_wrong_mutant_exit_is_not_accepted(self): + for kind in ('exit-zero', 'exit-two'): + self.check_mutants(kind) + + def test_baseline_stderr_is_not_accepted(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('noise', 'baseline', mode, b'baseline stderr') + + def test_baseline_stdout_remains_exact(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-wrong', 'baseline', mode, b'output differs from RESULTS.json') + + def test_baseline_nonzero_stops_before_mutants(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-exit', 'baseline', mode, b'CalledProcessError') + + def test_manifest_and_source_guards_precede_checker_execution(self): + for kind, diagnostic in [('tamper', b'source identity mismatch'), + ('missing', b'packet tree differs'), ('extra', b'packet tree differs'), + ('symlink', b'flat regular source required')]: + with self.subTest(kind=kind): + proc, events = self.run_fixture(kind) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(diagnostic, proc.stderr) + self.assertEqual(events, []) + + def test_final_git_guard_still_refuses_dirty_fixture(self): + proc, events = self.run_fixture('dirty') + self.assertNotEqual(proc.returncode, 0) + self.assertIn(b'diff --git', proc.stdout) + self.assertEqual(events, self.sequence()) + + def test_existing_workflow_execution_contract_is_preserved(self): + for path in ('reviews/d5_reconciliation_20260929/**', '.github/workflows/d5-reconciliation.yml', + 'reviews/d5_*/**', 'frontiers/intermediate_window_20260928/**', + 'frontiers/remote_window_20260924/**', 'frontiers/remote_collision_20260928/**', + 'frontiers/rn_annulus_bridge_20260925/**'): + with self.subTest(path=path): + self.assertIn(" - '" + path + "'\n", self.workflow) + self.assertIn(" workflow_dispatch:\n", self.workflow) + self.assertIn("permissions:\n contents: read\n", self.workflow) + self.assertIn("timeout-minutes: 10\n", self.workflow) + self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", self.workflow) + self.assertIn("persist-credentials: false", self.workflow) + self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", self.workflow) + self.assertIn("PYTHONDONTWRITEBYTECODE: '1'", self.workflow) + + def test_protocol_tests_are_wired_for_both_outer_modes(self): + self.assertIn(" - 'tests/test_d5_reconciliation_workflow.py'\n", self.workflow) + self.assertIn("python -B -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v", self.workflow) + self.assertIn("python -B -O -S -m unittest discover -s tests -p 'test_d5_reconciliation_workflow.py' -v", self.workflow) + self.assertIn("python-version: '3.11.16'", self.workflow) + self.assertEqual(self.shell.count('timeout=300'), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_elder_lower_all_d_workflow.py b/tests/test_elder_lower_all_d_workflow.py new file mode 100644 index 000000000..5a52984b5 --- /dev/null +++ b/tests/test_elder_lower_all_d_workflow.py @@ -0,0 +1,264 @@ +"""Real-shell elder-lower-all-d protocol tests with committed SYNTHETIC fixtures. + +No mathematical packet is replayed here. Real checker children record their +interpreter flags; the full workflow block retains its source and Git guards. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +WORKFLOW = Path(__file__).resolve().parents[1] / '.github/workflows/elder-lower-all-d.yml' +PACKET = Path('frontiers/elder_lower_all_d_20260929') +# Source-derived first failed checks; native characterization is separate. +FAILURE_LINES = { + 'M1': 'FAIL L_ledger_m2\n', + 'M2': 'FAIL P2_hessian_M\n', + 'M3': 'FAIL E6_margin_below\n', +} +LABELS = tuple(FAILURE_LINES) +BASELINE = { + 'object': 'SYNTHETIC-ELDER-ALL-D-PROTOCOL-FIXTURE', + 'passed': True, + 'results': {'synthetic': 'workflow engineering fixture; no mathematical assertion'}, + 'scope': 'SYNTHETIC source and child-protocol controls only', +} + +# This is a real child process, never a mocked subprocess API or interpreter. +CHILD = r'''import json, os, pathlib, sys +args = sys.argv[1:] +label = args[-1] if '--mutant' in args else 'baseline' +event = {'label': label, 'argv': args, + 'interpreter_flags': sys.orig_argv[1:sys.orig_argv.index(sys.argv[0])], + 'optimize': sys.flags.optimize, 'no_site': sys.flags.no_site, + 'dont_write_bytecode': sys.dont_write_bytecode} +with open(os.environ['ELDER_ALL_D_EVENTS'], 'a', encoding='utf-8') as stream: + stream.write(json.dumps(event)+'\n') +selected = label == os.environ.get('ELDER_ALL_D_TARGET') and str(sys.flags.optimize) == os.environ.get('ELDER_ALL_D_MODE') +kind = os.environ.get('ELDER_ALL_D_KIND', 'valid') if selected else 'valid' +if kind == 'crash': raise RuntimeError('synthetic unrelated crash') +if kind == 'silent': sys.exit(1) +expected = (pathlib.Path(__file__).parent/'RESULTS.json').read_text() +out = expected if label == 'baseline' else FAILURE_LINES[label] +if kind == 'baseline-wrong': out = 'synthetic wrong baseline bytes\n' +if kind == 'wrong-label': out = FAILURE_LINES['M2' if label == 'M1' else 'M1'] +if kind == 'baseline-json': out = expected +if kind == 'missing-newline': out = out.rstrip('\n') +if kind == 'prefix-noise': out = 'unexpected prefix\n' + out +if kind == 'suffix-noise': out += 'unexpected suffix\n' +if kind == 'stderr': print('synthetic unexpected stderr', file=sys.stderr) +sys.stdout.write(out) +sys.exit({'exit-zero': 0, 'exit-two': 2, 'baseline-exit': 1}.get(kind, 0 if label == 'baseline' else 1)) +''' +CHILD = 'FAILURE_LINES = ' + repr(FAILURE_LINES) + '\n' + CHILD + + +def report_bytes(value): + return (json.dumps(value, indent=2, sort_keys=True) + '\n').encode() + + +def workflow_shell(text): + name = ' - name: Verify sources, replay both modes, reject mutants\n' + if text.count(name) != 1: + raise ValueError('expected one named protocol step') + tail = text.split(name, 1)[1].split(' - ', 1)[0] + marker = ' run: |\n' + if tail.count(marker) != 1: + raise ValueError('expected named literal run block') + lines = tail.split(marker, 1)[1].splitlines(keepends=True) + if any(line.strip() and not line.startswith(' ') for line in lines): + raise ValueError('unexpected run-block indentation') + shell = ''.join(line[10:] if line.startswith(' ') else line for line in lines) + if not shell.startswith('set -euo pipefail\n') or not shell.endswith('git diff --exit-code\n'): + raise ValueError('incomplete protocol shell') + return shell + + +class ElderLowerAllDWorkflow(unittest.TestCase): + shell_runs = 0 + + @classmethod + def setUpClass(cls): + if not shutil.which('git') or not shutil.which('bash'): + raise RuntimeError('these workflow tests require Git and Bash') + cls.shell_runs = 0 + cls.workflow = WORKFLOW.read_text(encoding='utf-8') + cls.shell = workflow_shell(cls.workflow) + + def run_fixture(self, kind='valid', target='baseline', mode=0): + with tempfile.TemporaryDirectory(prefix='elder-lower-all-d-protocol-') as temp: + outer = Path(temp) + repo = outer / 'repo' + packet = repo / PACKET + packet.mkdir(parents=True) + executable_dir = outer / 'bin' + executable_dir.mkdir() + (executable_dir / 'python').symlink_to(sys.executable) + env = {key: value for key, value in os.environ.items() + if not key.startswith(('GIT_', 'PYTHON', 'ELDER_ALL_D_')) and key not in ('BASH_ENV', 'ENV')} + env.update(PATH=str(executable_dir) + os.pathsep + os.environ.get('PATH', ''), + GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, + GIT_TERMINAL_PROMPT='0', PYTHONDONTWRITEBYTECODE='1', + ELDER_ALL_D_KIND=kind, ELDER_ALL_D_TARGET=target, ELDER_ALL_D_MODE=str(mode), + ELDER_ALL_D_EVENTS=str(outer / 'events.jsonl')) + + def git(*args): + proc = subprocess.run(['git', '-c', 'gc.auto=0', *args], cwd=repo, + env=env, capture_output=True, timeout=15) + self.assertEqual(proc.returncode, 0, proc.stderr.decode(errors='replace')) + return proc.stdout + + payloads = {'lower_all_d_check.py': CHILD.encode(), 'RESULTS.json': report_bytes(BASELINE), + 'PROOF.md': b'SYNTHETIC fixture; not a mathematical proof.\n'} + for name, data in payloads.items(): + (packet / name).write_bytes(data) + manifest = {'synthetic_fixture': True, 'files': [ + {'path': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()} + for name, data in sorted(payloads.items())]} + (packet / 'SOURCE_FILES.json').write_bytes(report_bytes(manifest)) + (repo / 'marker.txt').write_bytes(b'original\n') + if kind == 'hash-only': + proof = payloads['PROOF.md'] + (packet / 'PROOF.md').write_bytes(b'X' + proof[1:]) # same length, different hash + elif kind == 'size-only': + proof_entry = next(row for row in manifest['files'] if row['path'] == 'PROOF.md') + proof_entry['bytes'] += 1 # actual bytes and their expected SHA256 stay unchanged + (packet / 'SOURCE_FILES.json').write_bytes(report_bytes(manifest)) + elif kind == 'missing': + (packet / 'PROOF.md').unlink() + elif kind == 'extra': + (packet / 'extra.txt').write_bytes(b'extra\n') + elif kind == 'symlink': + (packet / 'PROOF.md').unlink() + (outer / 'proof.txt').write_bytes(payloads['PROOF.md']) + (packet / 'PROOF.md').symlink_to(outer / 'proof.txt') + git('init', '-q') + git('add', '.') + git('-c', 'user.name=Protocol fixture', '-c', 'user.email=fixture@example.invalid', + 'commit', '-qm', 'Synthetic protocol fixture only') + head = git('rev-parse', 'HEAD') + if kind == 'dirty': + (repo / 'marker.txt').write_bytes(b'intentionally dirty\n') + before = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + type(self).shell_runs += 1 + proc = subprocess.run(['bash', '-c', self.shell], cwd=repo, env=env, + capture_output=True, timeout=30) + after = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + self.assertEqual(before, after, 'workflow changed fixture payload bytes') + self.assertEqual(git('rev-parse', 'HEAD'), head) + events_path = outer / 'events.jsonl' + events = [json.loads(line) for line in events_path.read_text().splitlines()] if events_path.exists() else [] + return proc, events + + @staticmethod + def event(label, mode): + return {'label': label, 'argv': [] if label == 'baseline' else ['--mutant', label], + 'interpreter_flags': ['-B', '-S'] if mode == 0 else ['-B', '-O', '-S'], + 'optimize': mode, 'no_site': 1, 'dont_write_bytecode': True} + + @classmethod + def sequence(cls): + return [cls.event(label, mode) + for mode in (0, 1) for label in ('baseline', *LABELS)] + + def assert_refusal(self, kind, target, mode, diagnostic): + proc, events = self.run_fixture(kind, target, mode) + self.assertNotEqual(proc.returncode, 0, (kind, target, mode, proc.stdout, proc.stderr)) + self.assertNotIn(b'"scientific_acceptance": false', proc.stdout) + self.assertIn(diagnostic, proc.stderr) + index = self.sequence().index(self.event(target, mode)) + self.assertEqual(events, self.sequence()[:index + 1]) + + def test_valid_baseline_and_three_failure_lines_reach_both_modes(self): + proc, events = self.run_fixture() + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout, b'{"scientific_effect": "NONE", "scientific_acceptance": false}\n') + self.assertEqual(proc.stderr, b'') + self.assertEqual(events, self.sequence()) + + def test_unrelated_crash_is_not_a_mutant_rejection(self): + self.check_mutants('crash') + + def test_silent_exit_one_is_not_a_mutant_rejection(self): + self.check_mutants('silent') + + def test_complete_label_failure_bytes_are_required(self): + for kind in ('wrong-label', 'baseline-json', 'missing-newline', 'prefix-noise', 'suffix-noise'): + self.check_mutants(kind) + + def check_mutants(self, kind): + for mode in (0, 1): + for label in LABELS: + with self.subTest(kind=kind, mode=mode, label=label): + self.assert_refusal(kind, label, mode, b'mutant '+label.encode()) + + def test_mutant_stderr_is_not_accepted(self): + self.check_mutants('stderr') + + def test_wrong_mutant_exit_is_not_accepted(self): + for kind in ('exit-zero', 'exit-two'): + self.check_mutants(kind) + + def test_baseline_stderr_is_not_accepted(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('stderr', 'baseline', mode, b'baseline stderr') + + def test_baseline_stdout_remains_exact(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-wrong', 'baseline', mode, b'output differs from RESULTS.json') + + def test_baseline_nonzero_stops_before_mutants(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-exit', 'baseline', mode, b'CalledProcessError') + + def test_manifest_and_source_guards_precede_checker_execution(self): + for kind, diagnostic in [('hash-only', b'source identity mismatch'), + ('size-only', b'source identity mismatch'), + ('missing', b'packet tree differs'), ('extra', b'packet tree differs'), + ('symlink', b'flat regular source required')]: + with self.subTest(kind=kind): + proc, events = self.run_fixture(kind) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(diagnostic, proc.stderr) + self.assertEqual(events, []) + + def test_final_git_guard_still_refuses_dirty_fixture(self): + proc, events = self.run_fixture('dirty') + self.assertNotEqual(proc.returncode, 0) + self.assertIn(b'diff --git', proc.stdout) + self.assertEqual(events, self.sequence()) + + def test_existing_workflow_execution_contract_is_preserved(self): + for path in ('frontiers/elder_lower_all_d_20260929/**', '.github/workflows/elder-lower-all-d.yml'): + with self.subTest(path=path): + self.assertIn(" - '" + path + "'\n", self.workflow) + self.assertIn(" workflow_dispatch:\n", self.workflow) + self.assertIn("permissions:\n contents: read\n", self.workflow) + self.assertIn("timeout-minutes: 10\n", self.workflow) + self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", self.workflow) + self.assertIn("persist-credentials: false", self.workflow) + self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", self.workflow) + self.assertIn("PYTHONDONTWRITEBYTECODE: '1'", self.workflow) + + def test_protocol_tests_are_wired_for_both_outer_modes(self): + self.assertIn(" - 'tests/test_elder_lower_all_d_workflow.py'\n", self.workflow) + self.assertIn("python -B -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v", self.workflow) + self.assertIn("python -B -O -S -m unittest discover -s tests -p 'test_elder_lower_all_d_workflow.py' -v", self.workflow) + self.assertIn("python-version: '3.11.16'", self.workflow) + self.assertEqual(self.shell.count('timeout=300'), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_full_suite_routing.py b/tests/test_full_suite_routing.py index 5a9e9160d..07f058659 100644 --- a/tests/test_full_suite_routing.py +++ b/tests/test_full_suite_routing.py @@ -248,10 +248,12 @@ def strip_concurrency_insertion(text): def reconstruct_original_central(text): """Historical central workflow at 49fb197c / blob ea291276, authenticated. - Remove only the approved concurrency insertion, then revert only the suite - body; the result must hash to the unchanged historical digest. + Remove only the approved concurrency insertion, restore the historical + 20-minute budget from the current 30-minute budget, then revert only the + suite body; the result must hash to the unchanged historical digest. """ text = strip_concurrency_insertion(text) + text = mutate(text, '\n timeout-minutes: 30\n', '\n timeout-minutes: 20\n') block = named(parse(text)[1], REGRESSION)['run'] rendered = lambda body: ''.join(' ' + line + '\n' for line in body.splitlines()) @@ -613,7 +615,7 @@ def violations(workflow, text): if bind != central_bind: problems.append('tested-commit binding step differs from the central workflow') if workflow == CENTRAL: - if job['id'] != 'full-suite' or job.get('timeout-minutes') != '20': + if job['id'] != 'full-suite' or job.get('timeout-minutes') != '30': problems.append('central job id/budget changed') if names != [BIND, REGRESSION, CONFIRM]: problems.append('central step order changed: ' + repr(names)) @@ -691,8 +693,8 @@ def mutate(text, old, new, count=1): class FullSuiteRoutingTests(unittest.TestCase): def original_central(self): - # Remove only the authenticated concurrency insertion, reconstruct only - # the suite body and authenticate the complete original workflow at + # Remove only the authenticated concurrency insertion, restore only the + # historical budget and suite body, and authenticate the original at # 49fb197c / blob ea291276, not a hand-written shell surrogate. original = reconstruct_original_central(source(CENTRAL)) self.assertEqual(hashlib.sha256(original.encode()).hexdigest(), ORIGINAL_CENTRAL_SHA256) @@ -776,7 +778,7 @@ def test_historical_bytes_outside_insertion_are_authenticated(self): central = source(CENTRAL) self.assertEqual(reconstruct_original_central(central), self.original_central()) for old, new in (('# This check is not required', '# This check is now required'), - (' timeout-minutes: 20\n', ' timeout-minutes: 21\n'), + (' timeout-minutes: 30\n', ' timeout-minutes: 31\n'), ('permissions:\n contents: read\n\nconcurrency:', 'permissions:\n contents: read\n\n\nconcurrency:'), (CONCURRENCY_INSERTION + 'jobs:\n', CONCURRENCY_INSERTION + '\njobs:\n'), @@ -786,6 +788,28 @@ def test_historical_bytes_outside_insertion_are_authenticated(self): with self.assertRaises((ValueError, AssertionError)): reconstruct_original_central(mutate(central, old, new)) + def test_current_and_historical_budgets_have_separate_exact_contracts(self): + central = source(CENTRAL) + self.assertEqual(parse(central)[1]['timeout-minutes'], '30') + original = self.original_central() + self.assertEqual(parse(original)[1]['timeout-minutes'], '20') + for budget in ('0', '20', '29', '31', '360', "'30'"): + with self.subTest(budget=budget): + changed = mutate(central, ' timeout-minutes: 30\n', + ' timeout-minutes: ' + budget + '\n') + self.assertIn('central job id/budget changed', violations(CENTRAL, changed)) + with self.assertRaises((ValueError, AssertionError)): + reconstruct_original_central(changed) + relocated = mutate(central, '\n timeout-minutes: 30\n', + '\n timeout-minutes: 20\n') + relocated = mutate(relocated, ' log_dir="$(mktemp -d)"\n', + ' # timeout-minutes: 30\n' + ' log_dir="$(mktemp -d)"\n') + self.assertEqual(parse(relocated)[1]['timeout-minutes'], '20') + self.assertIn('central job id/budget changed', violations(CENTRAL, relocated)) + with self.assertRaises((ValueError, AssertionError)): + reconstruct_original_central(relocated) + def test_original_accepts_synthetic_empty_success_but_preserves_failures(self): original = self.original_central() for event in ('pull_request', 'workflow_dispatch'): @@ -1186,7 +1210,7 @@ def test_meaningful_negatives_are_detected(self): 'central gains a condition': (CENTRAL, central, ' - name: Full repository regression suite in both Python modes\n', ' - name: Full repository regression suite in both Python modes\n' " if: github.event_name == 'workflow_dispatch'\n"), - 'central continue-on-error': (CENTRAL, central, ' timeout-minutes: 20\n', ' timeout-minutes: 20\n continue-on-error: true\n'), + 'central continue-on-error': (CENTRAL, central, ' timeout-minutes: 30\n', ' timeout-minutes: 30\n continue-on-error: true\n'), 'central unbinds commit': (CENTRAL, central, ' if [ "$checked_out" != "$TESTED_SHA" ]; then\n', ' if false; then\n'), }