diff --git a/.github/workflows/sard-g-r3r4-review.yml b/.github/workflows/sard-g-r3r4-review.yml index 8ea00c717..97b87e137 100644 --- a/.github/workflows/sard-g-r3r4-review.yml +++ b/.github/workflows/sard-g-r3r4-review.yml @@ -5,6 +5,7 @@ name: SARD-G R3-R4 nonauthor review checks paths: - 'reviews/sard_g_r3r4_claude_20260929/**' - '.github/workflows/sard-g-r3r4-review.yml' + - 'tests/test_sard_r3r4_workflow.py' workflow_dispatch: permissions: @@ -21,6 +22,10 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: '3.11.16' + - name: Exercise rejection protocol controls + run: | + python -B -S -m unittest discover -s tests -p 'test_sard_r3r4_workflow.py' -v + python -B -O -S -m unittest discover -s tests -p 'test_sard_r3r4_workflow.py' -v - name: Verify sources, replay both modes, reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -44,16 +49,28 @@ jobs: script = str(root / 'r3r4_review_check.py') expected = (root / 'RESULTS.json').read_bytes() mutants = ['wrong-residual', 'keep-degenerate', 'span-deficient'] + # Fixed native report differences characterized independently of children. + failed_check = { + 'wrong-residual': 'R3c_residual_uncorrelated', + 'keep-degenerate': 'degenerate_direction_removed', + 'span-deficient': 'span_of_point_evaluation_images_is_full', + } for flags in (['-B', '-S'], ['-B', '-O', '-S']): out = subprocess.run([sys.executable, *flags, script], - check=True, capture_output=True, timeout=300).stdout - if out != expected: + check=True, capture_output=True, timeout=300) + if out.stdout != expected: raise RuntimeError('output differs from RESULTS.json under ' + ' '.join(flags)) + if out.stderr: + raise RuntimeError('unexpected baseline stderr under ' + ' '.join(flags)) for m in mutants: - rc = subprocess.run([sys.executable, *flags, script, '--mutant', m], - capture_output=True, timeout=300).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + report = json.loads(expected) + report['passed'] = False + report['checks'][failed_check[m]] = False + wanted = (json.dumps(report, indent=2, sort_keys=True) + '\n').encode('utf-8') + rejected = subprocess.run([sys.executable, *flags, script, '--mutant', m], + capture_output=True, timeout=300) + if rejected.returncode != 1 or rejected.stdout != wanted or rejected.stderr: + raise RuntimeError('mutant ' + m + ' did not emit its expected rejection report') print(json.dumps({'scientific_effect': 'NONE', 'mathematical_acceptance': False})) PY git diff --exit-code diff --git a/tests/test_sard_r3r4_workflow.py b/tests/test_sard_r3r4_workflow.py new file mode 100644 index 000000000..a475272ed --- /dev/null +++ b/tests/test_sard_r3r4_workflow.py @@ -0,0 +1,257 @@ +"""Real-shell SARD protocol controls using committed SYNTHETIC packet fixtures. + +These are not mathematical counterexamples or replays of the reviewed packet. +The complete named workflow block runs, including its source and final Git guards. +Its parent stays normal Python; its real checker children explicitly run both modes. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +WORKFLOW = Path(__file__).resolve().parents[1] / '.github/workflows/sard-g-r3r4-review.yml' +PACKET = Path('reviews/sard_g_r3r4_claude_20260929') +LABELS = ('wrong-residual', 'keep-degenerate', 'span-deficient') +# Independently characterized native reports: Math-322/6194150658. +FAILED_CHECKS = { + 'wrong-residual': 'R3c_residual_uncorrelated', + 'keep-degenerate': 'degenerate_direction_removed', + 'span-deficient': 'span_of_point_evaluation_images_is_full', +} +BASELINE = { + 'checks': { + 'R3a_rkhs_identity': True, + 'R3c_residual_uncorrelated': True, + 'degenerate_direction_removed': True, + 'fewer_points_rank_deficient_control': True, + 'regular_zeros_have_nonzero_derivative': True, + 'span_of_point_evaluation_images_is_full': True, + }, + 'object': 'SYNTHETIC-SARD-PROTOCOL-FIXTURE', + 'passed': True, + 'scope': 'workflow engineering fixture only; no mathematical assertion', +} + +# Faults affect only one explicitly selected child. No subprocess API is mocked. +CHILD = r'''import json, os, pathlib, sys +label = sys.argv[-1] if '--mutant' in sys.argv else 'baseline' +selected = label == os.environ.get('SARD_TARGET') and str(sys.flags.optimize) == os.environ.get('SARD_MODE') +with open(os.environ['SARD_EVENTS'], 'a', encoding='utf-8') as stream: + stream.write(json.dumps([label, sys.flags.optimize, sys.flags.no_site, sys.dont_write_bytecode])+'\n') +kind = os.environ.get('SARD_KIND', 'valid') if selected else 'valid' +if kind == 'crash': raise RuntimeError('synthetic unrelated crash') +if kind == 'silent': sys.exit(1) +if kind == 'baseline-wrong': print('synthetic wrong bytes'); sys.exit(0) +value = json.loads((pathlib.Path(__file__).parent/'RESULTS.json').read_bytes()) +if label != 'baseline': + key = {'wrong-residual': 'R3c_residual_uncorrelated', + 'keep-degenerate': 'degenerate_direction_removed', + 'span-deficient': 'span_of_point_evaluation_images_is_full'}[label] + value['passed'] = False + value['checks'][key] = False + if kind == 'wrong-check': + value['checks'][key] = True + value['checks']['R3a_rkhs_identity'] = False + if kind == 'passed-true': value['passed'] = True + if kind == 'numeric-false': value['checks'][key] = 0 + if kind == 'extra-field': value['extra'] = 'unexpected' +if kind == 'noise': print('synthetic unexpected stderr', file=sys.stderr) +print(json.dumps(value, indent=2, sort_keys=True)) +sys.exit({'exit-zero': 0, 'exit-two': 2, 'baseline-exit': 1}.get(kind, 0 if label == 'baseline' else 1)) +''' + + +def report_bytes(value: dict) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + '\n').encode() + + +def workflow_shell(text: str) -> str: + """Extract the named literal block, not arbitrary YAML or another run step.""" + name = ' - name: Verify sources, replay both modes, reject mutants\n' + if text.count(name) != 1: + raise ValueError('expected one named protocol step') + tail = text.split(name, 1)[1] + tail = tail.split(' - ', 1)[0] + marker = ' run: |\n' + if tail.count(marker) != 1: + raise ValueError('expected named literal run block') + lines = tail.split(marker, 1)[1].splitlines(keepends=True) + if any(line.strip() and not line.startswith(' ') for line in lines): + raise ValueError('unexpected run-block indentation') + shell = ''.join(line[10:] if line.startswith(' ') else line for line in lines) + if not shell.startswith('set -euo pipefail\n') or not shell.endswith('git diff --exit-code\n'): + raise ValueError('incomplete protocol shell') + return shell + + +class SardR3R4Workflow(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not shutil.which('git') or not shutil.which('bash'): + raise RuntimeError('these workflow tests require Git and Bash') + cls.workflow = WORKFLOW.read_text(encoding='utf-8') + cls.shell = workflow_shell(cls.workflow) + + def run_fixture(self, kind='valid', target='baseline', mode=0): + with tempfile.TemporaryDirectory(prefix='sard-r3r4-protocol-') as temp: + outer = Path(temp) + repo = outer / 'repo' + packet = repo / PACKET + packet.mkdir(parents=True) + executable_dir = outer / 'bin' + executable_dir.mkdir() + # A real interpreter, not a fake success-output executable. + (executable_dir / 'python').symlink_to(sys.executable) + env = {key: value for key, value in os.environ.items() + if not key.startswith(('GIT_', 'PYTHON', 'SARD_')) and key not in ('BASH_ENV', 'ENV')} + env.update(PATH=str(executable_dir) + os.pathsep + os.environ.get('PATH', ''), + GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, + GIT_TERMINAL_PROMPT='0', PYTHONDONTWRITEBYTECODE='1', + SARD_KIND=kind, SARD_TARGET=target, SARD_MODE=str(mode), + SARD_EVENTS=str(outer / 'events.jsonl')) + + def git(*args): + proc = subprocess.run(['git', '-c', 'gc.auto=0', *args], cwd=repo, + env=env, capture_output=True, timeout=15) + self.assertEqual(proc.returncode, 0, proc.stderr.decode(errors='replace')) + return proc.stdout + + payloads = {'r3r4_review_check.py': CHILD.encode(), + 'RESULTS.json': report_bytes(BASELINE), + 'REVIEW.md': b'SYNTHETIC fixture; not a review verdict.\n'} + for name, data in payloads.items(): + (packet / name).write_bytes(data) + manifest = {'synthetic_fixture': True, 'files': [ + {'path': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()} + for name, data in sorted(payloads.items())]} + (packet / 'SOURCE_FILES.json').write_bytes(report_bytes(manifest)) + (repo / 'marker.txt').write_bytes(b'original\n') + if kind == 'tamper': + (packet / 'r3r4_review_check.py').write_bytes(CHILD.encode() + b'\n# unbound\n') + elif kind == 'missing': + (packet / 'REVIEW.md').unlink() + elif kind == 'extra': + (packet / 'extra.txt').write_bytes(b'extra\n') + elif kind == 'symlink': + (packet / 'REVIEW.md').unlink() + (outer / 'review.txt').write_bytes(payloads['REVIEW.md']) + (packet / 'REVIEW.md').symlink_to(outer / 'review.txt') + git('init', '-q') + git('add', '.') + git('-c', 'user.name=Protocol fixture', '-c', 'user.email=fixture@example.invalid', + 'commit', '-qm', 'Synthetic protocol fixture only') + head = git('rev-parse', 'HEAD') + if kind == 'dirty': + (repo / 'marker.txt').write_bytes(b'intentionally dirty\n') + before = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + proc = subprocess.run(['bash', '-c', self.shell], cwd=repo, env=env, + capture_output=True, timeout=30) + after = {p.relative_to(repo).as_posix(): p.read_bytes() + for p in repo.rglob('*') if p.is_file() and '.git' not in p.parts} + self.assertEqual(before, after, 'workflow changed fixture payload bytes') + self.assertEqual(git('rev-parse', 'HEAD'), head) + events_path = outer / 'events.jsonl' + events = [json.loads(line) for line in events_path.read_text().splitlines()] if events_path.exists() else [] + return proc, events + + @staticmethod + def sequence(): + return [[label, mode, 1, True] + for mode in (0, 1) for label in ('baseline', *LABELS)] + + def assert_refusal(self, kind, target, mode, diagnostic): + proc, events = self.run_fixture(kind, target, mode) + self.assertNotEqual(proc.returncode, 0, (kind, target, mode, proc.stdout, proc.stderr)) + self.assertNotIn(b'"mathematical_acceptance": false', proc.stdout) + self.assertIn(diagnostic, proc.stderr) + index = self.sequence().index([target, mode, 1, True]) + self.assertEqual(events, self.sequence()[:index + 1]) + + def test_valid_baselines_and_label_reports_reach_both_modes(self): + proc, events = self.run_fixture() + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout, b'{"scientific_effect": "NONE", "mathematical_acceptance": false}\n') + self.assertEqual(proc.stderr, b'') + self.assertEqual(events, self.sequence()) + + def test_unrelated_crash_is_not_a_mutant_rejection(self): + for mode in (0, 1): + for label in LABELS: + with self.subTest(mode=mode, label=label): + self.assert_refusal('crash', label, mode, b'mutant '+label.encode()) + + def test_silent_exit_one_is_not_a_mutant_rejection(self): + for mode in (0, 1): + for label in LABELS: + with self.subTest(mode=mode, label=label): + self.assert_refusal('silent', label, mode, b'mutant '+label.encode()) + + def test_complete_label_report_and_boolean_types_are_required(self): + for kind in ('wrong-check', 'passed-true', 'numeric-false', 'extra-field'): + for mode in (0, 1): + for label in LABELS: + with self.subTest(kind=kind, mode=mode, label=label): + self.assert_refusal(kind, label, mode, b'mutant '+label.encode()) + + def test_mutant_stderr_is_not_accepted(self): + for mode in (0, 1): + for label in LABELS: + with self.subTest(mode=mode, label=label): + self.assert_refusal('noise', label, mode, b'mutant '+label.encode()) + + def test_wrong_mutant_exit_is_not_accepted(self): + for kind in ('exit-zero', 'exit-two'): + for mode in (0, 1): + for label in LABELS: + with self.subTest(kind=kind, mode=mode, label=label): + self.assert_refusal(kind, label, mode, b'mutant '+label.encode()) + + def test_baseline_stderr_is_not_accepted(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('noise', 'baseline', mode, b'baseline stderr') + + def test_baseline_stdout_remains_exact(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-wrong', 'baseline', mode, b'output differs from RESULTS.json') + + def test_baseline_nonzero_stops_before_mutants(self): + for mode in (0, 1): + with self.subTest(mode=mode): + self.assert_refusal('baseline-exit', 'baseline', mode, b'CalledProcessError') + + def test_manifest_and_source_guards_precede_checker_execution(self): + for kind, diagnostic in [('tamper', b'source identity mismatch'), + ('missing', b'packet tree differs'), ('extra', b'packet tree differs'), + ('symlink', b'flat regular source required')]: + with self.subTest(kind=kind): + proc, events = self.run_fixture(kind) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(diagnostic, proc.stderr) + self.assertEqual(events, []) + + def test_final_git_guard_still_refuses_dirty_fixture(self): + proc, events = self.run_fixture('dirty') + self.assertNotEqual(proc.returncode, 0) + self.assertIn(b'diff --git', proc.stdout) + self.assertEqual(events, self.sequence()) + + def test_protocol_tests_are_wired_for_both_outer_modes(self): + self.assertIn(" - 'tests/test_sard_r3r4_workflow.py'\n", self.workflow) + self.assertIn("python -B -S -m unittest discover -s tests -p 'test_sard_r3r4_workflow.py' -v", self.workflow) + self.assertIn("python -B -O -S -m unittest discover -s tests -p 'test_sard_r3r4_workflow.py' -v", self.workflow) + self.assertIn("python-version: '3.11.16'", self.workflow) + self.assertEqual(self.shell.count('timeout=300'), 2) + + +if __name__ == '__main__': + unittest.main()