diff --git a/.github/workflows/c6-cluster-coefficients-numerics.yml b/.github/workflows/c6-cluster-coefficients-numerics.yml index 268ca3fb8..e4f49dcb4 100644 --- a/.github/workflows/c6-cluster-coefficients-numerics.yml +++ b/.github/workflows/c6-cluster-coefficients-numerics.yml @@ -5,6 +5,8 @@ name: C6 cluster coefficient numerics (replay) paths: - 'frontiers/c6_cluster_coefficients_numerics_20260930/**' - '.github/workflows/c6-cluster-coefficients-numerics.yml' + - 'tools/c6_coefficient_numerics_replay.py' + - 'tests/test_c6_coefficient_numerics_workflow.py' workflow_dispatch: permissions: @@ -21,6 +23,13 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: '3.11.16' + - name: Verify full-shell output contracts in both Python modes + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + set -euo pipefail + python -B -S -m unittest discover -s tests -p test_c6_coefficient_numerics_workflow.py -v + python -B -O -S -m unittest discover -s tests -p test_c6_coefficient_numerics_workflow.py -v - name: Verify manifest and main-resident pins, replay exact controls and GH40, reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -48,15 +57,9 @@ jobs: blob = hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest() if len(data) != entry['bytes'] or hashlib.sha256(data).hexdigest() != entry['sha256'] or blob != entry['git_blob']: raise ValueError('consumed source identity mismatch on this tree: ' + entry['path']) - script = str(root / 'coefficients.py') - for flags in (['-B', '-S'], ['-B', '-O', '-S']): - out = subprocess.run([sys.executable, *flags, script, '--check'], cwd=root, capture_output=True, timeout=900) - if out.returncode != 0: - raise RuntimeError('check failed under ' + ' '.join(flags) + ': ' + out.stdout.decode()[-400:]) - for m in ['cubic-sign', 'antiderivative', 'typed-boundary']: - rc = subprocess.run([sys.executable, *flags, script, '--check', '--mutant', m], cwd=root, capture_output=True, timeout=900).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + sys.path.insert(0, str(pathlib.Path('tools').resolve())) + from c6_coefficient_numerics_replay import replay + replay(root) print(json.dumps({'scientific_effect': 'NONE', 'certified': False})) PY git diff --exit-code diff --git a/reviews/c6_numerics_protocol_20261006/SCOPE.md b/reviews/c6_numerics_protocol_20261006/SCOPE.md new file mode 100644 index 000000000..b7900ffd5 --- /dev/null +++ b/reviews/c6_numerics_protocol_20261006/SCOPE.md @@ -0,0 +1,95 @@ +# C6 numerical-workflow report contracts — AUD-C6NUM-PROTOCOL-01 + +Dylan Roy — delegated AI engineering. Author: OpenAI / GPT-6 Astra Pro, +`qs-parent-c6num-repair-20261006`, pickup Math-#334/6009005443. +Scientific effect NONE. Same-account/provider organizational-independence credit 0. +This is an engineering repair, not a new mathematical or numerical acceptance. + +## Source and ownership + +Original source read at `516235290e12decdcd223cfb20928a2d97b824f1`: + +| Source | Git blob | Bytes | SHA256 | +|---|---|---:|---| +| workflow | `268ca3fb856b818022784f87bea96d0714c146aa` | 3180 | `06ee6f3beed34eb85ee45e0bbc19ce5e94aeab474a0dd31a492f48b93ca5f78c` | +| coefficients.py | `bd45a9fdecf300111b646644384dcead20c2e529` | 18825 | `bcd63f0806a58a76a85189372bc6eee99fc3847c74dcc34906ef63464a64b97b` | +| RESULTS.json | `2648314f8c11f0637ce301ba76360142a8309e8f` | 4095 | `9c88ea85b6ba08199996c9b5ff74ecd06afc0f458e08c28b4c3af6191f437fcb` | + +The original finding belongs to `pr321-followthrough-wrapper-audit-20261006`. +Its report and later authentic-characterization contribution #334/6009064997 +remain separately attributed. A crossed later writer pickup6009007450 was +acknowledged in6009140749; earlier6009005443 retains the single production route. +No other writer's implementation or source review is imported here. + +## Measured fixed contracts + +Local CPython 3.13.5, unchanged checker and RESULTS, both `-B -S` and `-B -O -S`: + +| Invocation suffix | Exit | Exact structural report | Stdout bytes / SHA256 | +|---|---:|---|---| +| --check | 0 | `{"mode":"check","passed":true,"scientific_effect":"NONE"}` | 63 / `c54d375daf9298c6a7cc591191dd9e3508b80213ddd24e5320859b429acefff6` | +| --check --mutant cubic-sign | 1 | `{"error":"classifier agrees with the direct critical-point count on the typed domain","passed":false}` | 105 / `b0271f45b76f6b28702624c2e6efce69bbd54e1d2246fdca3d09aa553d9095b0` | +| --check --mutant antiderivative | 1 | `{"error":"[LM] cubic mass 48 k^5 on n = 2","passed":false}` | 62 / `2f60570838650467fb3deeadf8138cf1809f355da1db0511e68cabca7f263ad5` | +| --check --mutant typed-boundary | 1 | Same classifier report as cubic-sign | 105 / `b0271f45b76f6b28702624c2e6efce69bbd54e1d2246fdca3d09aa553d9095b0` | + +All eight original characterization processes completed with empty stderr; +all four mode pairs were byte-identical. Actual original stdout uses sorted +keys, default JSON spaces and one trailing newline; the table compacts spacing. +The validator intentionally allows harmless JSON whitespace/key ordering but +requires the complete key set, exact Boolean/string types and exact values. +It rejects duplicate keys, numeric values, nonfinite values, extra or missing +fields, trailing documents, malformed UTF-8, silence, wrong reports, wrong +exits, signals and unexpected stderr. Expectations are fixed source, never +learned from the invocation under test. Timeout/launch exceptions propagate. + +The two classifier mutants really share one diagnostic. Output validation +cannot distinguish a malicious replacement emitting that same report. The +unchanged source/membership and consumed-pin guards remain essential. The +helper is not a sandbox or a replacement for those guards. + +## Test-first execution and preserved workflow + +Twelve full-shell regression methods execute 132 distinct committed synthetic +Git-repository scenarios. Each fixture runs the actual named replay Bash block, +including manifest/source/current-pin checks and final `git diff --exit-code`. +The trace is outside the checkout; children are real processes, not mocks. + +Against the ORIGINAL workflow: **100 intended assertion failures, zero errors**; +109 fixtures exit 0 (100 invalid, nine valid/formatting), 23 reject as intended; +952 child invocations. Against the repair: all 132 scenarios behave correctly, +both normal and optimized outer test runs; each run has 594 child invocations, +nine valid/formatting admissions and 123 rejections. These are 132 distinct +scenarios, not 264 kinds of evidence. They are not scientific checker results. + +The genuine repaired helper then executes all eight original checker invocations +with the same 900-second child budget. It exits 0 with empty stderr; every +reported output hash/exit equals the independent characterization. The full +numerical packet's source-manifest shell was NOT replayed locally: only the +three authenticated original files and engineering sources were available. +Hosted dedicated execution must verify the actual complete repository tree. +No no-argument generation, Monte Carlo rerun, RESULTS edit, or tolerance change +occurred. The original numerical comparisons remain noncertified. + +The workflow's complete manifest/pin preflight and final JSON/Git postflight +are byte-identical. Only the intervening exit-only loop is replaced by the +helper; two focused test invocations and their trigger paths are added. Runtime +3.11.16, action pins, permissions, child 900 seconds and job 20 minutes are +unchanged. Fixtures isolate their own Git settings and disable automatic +maintenance; this is not a repair claim about #321's separate cleanup race. + +Python's primary reference contracts are `subprocess.CompletedProcess` and +`json.loads`/`object_pairs_hook`/`parse_constant` in the official Python 3.11 +documentation. Numeric and duplicate rejection is explicit, not reliance on +default JSON parsing. No third-party package is introduced. + +## Remaining review and integration requirements + +This four-file proposal changes no byte in the numerical packet, mathematical +sources, RESULTS, manifests, source maps, other workflows, formal gate, or +scientific-status surfaces. A separate nonauthor engineering review, actual +pinned-runtime dedicated/general checks and eligible expected-head integration +are required. Author local execution is not a review or merge receipt. Old +failed records remain valid historical failures. No full repository local +suite, local Lean execution, independent whole-program alignment or scientific +closure is claimed. Retained in-memory/printed diagnostics do not guarantee raw +child-output persistence after process or filesystem failure. diff --git a/tests/test_c6_coefficient_numerics_workflow.py b/tests/test_c6_coefficient_numerics_workflow.py new file mode 100644 index 000000000..ec54de6d8 --- /dev/null +++ b/tests/test_c6_coefficient_numerics_workflow.py @@ -0,0 +1,233 @@ +"""Real full-shell C6 numerical workflow contracts; fixtures are not mathematics.""" +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / '.github/workflows/c6-cluster-coefficients-numerics.yml' +HELPER = ROOT / 'tools/c6_coefficient_numerics_replay.py' +PACKET = 'frontiers/c6_cluster_coefficients_numerics_20260930' +STAGES = ('baseline', 'cubic-sign', 'antiderivative', 'typed-boundary') +REPORTS = { + 'baseline': {'mode': 'check', 'passed': True, 'scientific_effect': 'NONE'}, + 'cubic-sign': {'error': 'classifier agrees with the direct critical-point count on the typed domain', 'passed': False}, + 'antiderivative': {'error': '[LM] cubic mass 48 k^5 on n = 2', 'passed': False}, + 'typed-boundary': {'error': 'classifier agrees with the direct critical-point count on the typed domain', 'passed': False}, +} + + +def replay_shell(text): + """Extract only the actual replay step, not its preceding test invocation.""" + marker = ' - name: Verify manifest and main-resident pins, replay exact controls and GH40, reject mutants\n' + if text.count(marker) != 1: + raise ValueError('expected one named replay step') + tail = text.split(marker, 1)[1].split(' run: |\n', 1)[1] + lines = [] + for line in tail.splitlines(): + if line and not line.startswith(' '): + break + lines.append(line[10:] if line else '') + if not lines or lines[0] != 'set -euo pipefail': + raise ValueError('strict replay shell missing') + return '\n'.join(lines) + '\n' + + +def identity(path, data): + return {'path': path, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()} + + +def fixture_script(): + # The fixture emits independently specified reports. The production helper is + # never imported here, so a broken helper cannot define its own oracle. + return '''import json, os, sys +from pathlib import Path +reports = ''' + repr(REPORTS) + ''' +a = sys.argv[1:] +if not a or a[0] != '--check': raise RuntimeError('unsafe/non-check invocation') +name = a[2] if len(a) == 3 and a[1] == '--mutant' else 'baseline' +if a != (['--check'] if name == 'baseline' else ['--check','--mutant',name]): + raise RuntimeError('unexpected invocation') +mode = sys.flags.optimize +with open(os.environ['C6NUM_TEST_TRACE'], 'a') as f: + f.write(json.dumps([mode, name])+'\\n') +row = dict(reports[name]); code = 0 if name == 'baseline' else 1 +out = (json.dumps(row, sort_keys=True)+'\\n').encode(); err = b'' +if os.environ['C6NUM_TEST_CASE'] == 'dirty': + Path('RESULTS.json').write_text('changed after preflight\\n') +if mode == int(os.environ['C6NUM_TEST_MODE']) and name == os.environ['C6NUM_TEST_STAGE']: + case = os.environ['C6NUM_TEST_CASE'] + if case == 'crash': raise RuntimeError('unrelated synthetic crash') + if case == 'silent': out = b'' + elif case == 'garbage': out = b'unrelated failure\\n' + elif case == 'stderr': err = b'unexpected stderr\\n' + elif case == 'wrong-exit': code = 1 if name == 'baseline' else 0 + elif case == 'signalled': os.kill(os.getpid(), 15) + elif case == 'wrong-report': row = {'passed': not row['passed'], 'error':'unrelated'} + elif case == 'wrong-reason': row['error'] = 'unrelated mathematical check' + elif case == 'wrong-effect': row['scientific_effect'] = 'ACCEPT' + elif case == 'wrong-mode': row['mode'] = 'full' + elif case == 'integer-bool': row['passed'] = int(row['passed']) + elif case == 'float-bool': row['passed'] = float(row['passed']) + elif case == 'null-bool': row['passed'] = None + elif case == 'missing': del row['passed'] + elif case == 'extra': row['extra'] = 'not in measured report' + elif case == 'array': row = [row] + if case in ('wrong-report','wrong-reason','wrong-effect','wrong-mode', + 'integer-bool','float-bool','null-bool','missing','extra','array'): + out = (json.dumps(row, sort_keys=True)+'\\n').encode() + if case == 'duplicate': + out = out.rstrip()[:-1] + b', "passed": false}\\n' + elif case == 'trailing': out += b'{}\\n' + elif case == 'nonfinite': out = out.replace(b'false', b'NaN').replace(b'true', b'Infinity') + elif case == 'bad-utf8': out = b'\\xff'+out + elif case == 'formatted': out = (' \\n'+json.dumps(row, indent=4)+' \\n').encode() +sys.stdout.buffer.write(out); sys.stderr.buffer.write(err); sys.exit(code) +''' + + +class WorkflowContractTests(unittest.TestCase): + def run_case(self, case='valid', stage='baseline', mode=0, guard=None): + with tempfile.TemporaryDirectory(prefix='c6num-test-') as tmp: + home = Path(tmp); repo = home/'repo'; repo.mkdir() + packet = repo/PACKET; packet.mkdir(parents=True) + trace = home/'trace.jsonl' + (packet/'coefficients.py').write_text(fixture_script()) + (packet/'RESULTS.json').write_text('{}\n') + consumed = repo/'consumed.txt'; consumed.write_bytes(b'fixed synthetic upstream\n') + pin = identity('consumed.txt', consumed.read_bytes()) + pin.update(current_required=True, git_blob=hashlib.sha1( + b'blob %d\0'%len(consumed.read_bytes())+consumed.read_bytes()).hexdigest()) + (packet/'SOURCE_MAP.json').write_text(json.dumps({'sources':[pin]})) + manifest = [identity(p.name, p.read_bytes()) for p in sorted(packet.iterdir())] + (packet/'SOURCE_FILES.json').write_text(json.dumps({'files':manifest})) + if HELPER.exists(): + (repo/'tools').mkdir(); shutil.copyfile(HELPER, repo/'tools'/HELPER.name) + # Isolate every real synthetic Git repo from user signing/hooks and + # asynchronous maintenance; do not suppress postflight or cleanup. + env = {k:v for k,v in os.environ.items() if not k.startswith('GIT_')} + env.update(HOME=str(home), GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, + GIT_AUTHOR_NAME='Fixture', GIT_AUTHOR_EMAIL='fixture@example.invalid', + GIT_COMMITTER_NAME='Fixture', GIT_COMMITTER_EMAIL='fixture@example.invalid', + PYTHONDONTWRITEBYTECODE='1', C6NUM_TEST_TRACE=str(trace), + C6NUM_TEST_CASE=case, C6NUM_TEST_STAGE=stage, C6NUM_TEST_MODE=str(mode)) + env['PATH'] = str(Path(sys.executable).parent)+os.pathsep+env.get('PATH','') + for args in (['init','-q'], ['config','gc.auto','0'], ['config','maintenance.auto','false'], + ['config','core.hooksPath',os.devnull], ['config','commit.gpgsign','false'], + ['add','.'], ['commit','-q','-m','synthetic fixture']): + subprocess.run(['git',*args],cwd=repo,env=env,capture_output=True,check=True,timeout=20) + if guard == 'source': + (packet/'coefficients.py').write_text(fixture_script()+'\n# tamper\n') + elif guard == 'pin': consumed.write_bytes(b'wrong upstream\n') + elif guard == 'extra': (packet/'extra.txt').write_text('unexpected') + elif guard == 'missing': (packet/'RESULTS.json').unlink() + elif guard == 'symlink': + target = home/'RESULTS.json'; shutil.copyfile(packet/'RESULTS.json',target) + (packet/'RESULTS.json').unlink();(packet/'RESULTS.json').symlink_to(target) + elif guard == 'nested': + source = packet/'RESULTS.json'; (packet/'subdir').mkdir();source.rename(packet/'subdir/RESULTS.json') + for row in manifest: + if row['path']=='RESULTS.json': row['path']='subdir/RESULTS.json' + (packet/'SOURCE_FILES.json').write_text(json.dumps({'files':manifest})) + shell = replay_shell(WORKFLOW.read_text()) + result = subprocess.run(['bash','--noprofile','--norc','-c',shell],cwd=repo,env=env, + capture_output=True,timeout=30) + calls = [json.loads(line) for line in trace.read_text().splitlines()] if trace.exists() else [] + saved = os.environ.get('C6NUM_TEST_SAVE') + if saved: + dest=Path(saved)/(case+'-'+stage+'-'+str(mode)+'-'+str(guard));dest.mkdir(parents=True,exist_ok=True) + (dest/'stdout').write_bytes(result.stdout);(dest/'stderr').write_bytes(result.stderr) + (dest/'result.json').write_text(json.dumps({'case':case,'stage':stage,'mode':mode,'guard':guard, + 'returncode':result.returncode,'calls':calls},sort_keys=True)+'\n') + return result, calls + + def rejected(self, case, stage, mode): + result,calls=self.run_case(case,stage,mode) + target=[mode,stage] + self.assertIn(target,calls, 'target never reached: '+result.stderr.decode(errors='replace')) + self.assertEqual(calls.count(target),1) + self.assertNotEqual(result.returncode,0,'invalid report admitted: '+repr((case,stage,mode,calls))) + + def test_valid_complete_inventory(self): + result,calls=self.run_case() + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')) + self.assertEqual(calls,[[m,s]for m in(0,1)for s in STAGES]) + + def test_harmless_json_formatting(self): + for mode in(0,1): + for stage in STAGES: + with self.subTest(mode=mode,stage=stage): + result,calls=self.run_case('formatted',stage,mode) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')) + self.assertEqual(len(calls),8) + + def test_silent_and_garbage_reports(self): + for case in('silent','garbage'): + for mode in(0,1): + for stage in STAGES: + with self.subTest(case=case,mode=mode,stage=stage):self.rejected(case,stage,mode) + + def test_wrong_reports_and_reasons(self): + for mode in(0,1): + for stage in STAGES: + with self.subTest(mode=mode,stage=stage):self.rejected('wrong-report',stage,mode) + for stage in STAGES[1:]: + with self.subTest(reason=stage,mode=mode):self.rejected('wrong-reason',stage,mode) + for case in('wrong-effect','wrong-mode'): + with self.subTest(case=case,mode=mode):self.rejected(case,'baseline',mode) + + def test_exact_boolean_types(self): + for case in('integer-bool','float-bool','null-bool'): + for mode in(0,1): + for stage in STAGES: + with self.subTest(case=case,mode=mode,stage=stage):self.rejected(case,stage,mode) + + def test_duplicate_nonfinite_and_trailing_json(self): + for case in('duplicate','nonfinite','trailing','bad-utf8'): + for mode in(0,1): + for stage in('baseline','antiderivative'): + with self.subTest(case=case,mode=mode,stage=stage):self.rejected(case,stage,mode) + + def test_complete_shape(self): + for case in('missing','extra','array'): + for mode in(0,1): + for stage in('baseline','typed-boundary'): + with self.subTest(case=case,mode=mode,stage=stage):self.rejected(case,stage,mode) + + def test_stderr_is_not_ignored(self): + for mode in(0,1): + for stage in STAGES: + with self.subTest(mode=mode,stage=stage):self.rejected('stderr',stage,mode) + + def test_runtime_crash_is_not_intended_rejection(self): + for mode in(0,1): + for stage in STAGES[1:]: + with self.subTest(mode=mode,stage=stage):self.rejected('crash',stage,mode) + + def test_wrong_exit_and_signal(self): + for case in('wrong-exit','signalled'): + for mode in(0,1): + for stage in STAGES: + with self.subTest(case=case,mode=mode,stage=stage):self.rejected(case,stage,mode) + + def test_source_guards_run_before_children(self): + for guard in('source','pin','extra','missing','symlink','nested'): + with self.subTest(guard=guard): + result,calls=self.run_case(guard=guard) + self.assertNotEqual(result.returncode,0) + self.assertEqual(calls,[]) + + def test_tracked_mutation_fails_postflight(self): + result,calls=self.run_case('dirty') + self.assertNotEqual(result.returncode,0) + self.assertEqual(len(calls),8) + self.assertIn(b'changed after preflight',result.stdout) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/c6_coefficient_numerics_replay.py b/tools/c6_coefficient_numerics_replay.py new file mode 100644 index 000000000..d7cd12725 --- /dev/null +++ b/tools/c6_coefficient_numerics_replay.py @@ -0,0 +1,82 @@ +"""Exact output contracts for the C6 numerical replay, not numerical certification. + +The workflow authenticates the packet and consumed pins before calling replay. +Expectations below were measured from unchanged coefficients.py blob +bd45a9fdecf300111b646644384dcead20c2e529 in normal and optimized Python modes. +They are fixed source, never learned from the invocation being validated. +""" +import hashlib +import json +from pathlib import Path +import subprocess +import sys + +MUTANT_REASONS = { + 'cubic-sign': 'classifier agrees with the direct critical-point count on the typed domain', + 'antiderivative': '[LM] cubic mass 48 k^5 on n = 2', + 'typed-boundary': 'classifier agrees with the direct critical-point count on the typed domain', +} + + +class ProtocolError(RuntimeError): + """A completed child did not satisfy its fixed report contract.""" + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate JSON key: ' + key) + result[key] = value + return result + + +def reject_number(value): + raise ValueError('numerical JSON value is outside the report schema: ' + value) + + +def verify_report(result: subprocess.CompletedProcess, mutant: str | None = None) -> None: + """Require the whole typed report, actual exit and captured empty stderr.""" + if mutant is not None and mutant not in MUTANT_REASONS: + raise ValueError('unrecognized C6 numerical mutant: ' + repr(mutant)) + expected = ({'mode': 'check', 'passed': True, 'scientific_effect': 'NONE'} + if mutant is None else {'error': MUTANT_REASONS[mutant], 'passed': False}) + code = 0 if mutant is None else 1 + label = 'baseline' if mutant is None else mutant + if type(result.returncode) is not int or result.returncode != code: + raise ProtocolError(label + ': unexpected return code ' + repr(result.returncode)) + if type(result.stdout) is not bytes or type(result.stderr) is not bytes: + raise ProtocolError(label + ': both raw output streams must be captured') + if result.stderr != b'': + raise ProtocolError(label + ': unexpected stderr, SHA256=' + hashlib.sha256(result.stderr).hexdigest()) + try: + report = json.loads(result.stdout.decode('utf-8'), object_pairs_hook=unique_object, + parse_int=reject_number, parse_float=reject_number, parse_constant=reject_number) + except (ValueError, UnicodeError, RecursionError) as exc: + raise ProtocolError(label + ': invalid UTF-8/JSON report') from exc + if type(report) is not dict or report.keys() != expected.keys(): + raise ProtocolError(label + ': report fields differ from the fixed contract') + for key, value in expected.items(): + if type(report[key]) is not type(value) or report[key] != value: + raise ProtocolError(label + ': wrong type or value for ' + key) + + +def replay(root: Path) -> None: + """Execute the original two-mode, four-stage inventory with strict reports. + + Caller must first validate the full packet and current consumed-source pins. + Timeout and process-launch exceptions propagate; they are never rejections. + This function never calls the RESULTS-regenerating no-argument path. + """ + root = Path(root).resolve() + script = str(root / 'coefficients.py') + for flags in (['-B', '-S'], ['-B', '-O', '-S']): + for mutant in (None, *MUTANT_REASONS): + args = ['--check'] + ([] if mutant is None else ['--mutant', mutant]) + result = subprocess.run([sys.executable, *flags, script, *args], cwd=root, + capture_output=True, timeout=900) + verify_report(result, mutant) + print(json.dumps({'flags': flags, 'mutant': mutant, 'returncode': result.returncode, + 'stdout_sha256': hashlib.sha256(result.stdout).hexdigest(), + 'stderr_sha256': hashlib.sha256(result.stderr).hexdigest(), + 'scientific_effect': 'NONE'}, sort_keys=True), flush=True)