Skip to content

ci: reject empty central full-suite discovery in both modes #438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
d6g8k5htny-coder merged 3 commits into from
Oct 12, 2026
Merged

ci: reject empty central full-suite discovery in both modes #438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f5e5ce3
test(ci): expose successful empty central discovery before guard
d6g8k5htny-coder Oct 12, 2026
0837ba0
ci: require nonempty central unittest discovery after meaningful host…
d6g8k5htny-coder Oct 12, 2026
dd1cbaf
Merge current main after PR #437 into central discovery guard; preser…
d6g8k5htny-coder Oct 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions .github/workflows/full-regression-suite.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,12 @@ jobs:
- name: Full repository regression suite in both Python modes
run: |
set -euo pipefail
python -B -S -m unittest discover -s tests -v
python -B -O -S -m unittest discover -s tests -v
log_dir="$(mktemp -d)"
trap 'rm -rf "$log_dir"' EXIT
python -B -S -m unittest discover -s tests -v 2>&1 | tee "$log_dir/normal.log"
grep -Eq '^Ran [1-9][0-9]* tests? in [0-9]+([.][0-9]+)?s$' "$log_dir/normal.log"
python -B -O -S -m unittest discover -s tests -v 2>&1 | tee "$log_dir/optimized.log"
grep -Eq '^Ran [1-9][0-9]* tests? in [0-9]+([.][0-9]+)?s$' "$log_dir/optimized.log"
- name: Confirm the suite left the tested commit unchanged
env:
TESTED_SHA: ${{ github.sha }}
Expand Down
198 changes: 186 additions & 12 deletions tests/test_full_suite_routing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import os
import re
import subprocess
import sys
import shutil
import tempfile
import unittest
from unittest import mock
Expand All @@ -31,9 +33,18 @@
BIND = 'Bind tested commit'
CONFIRM = 'Confirm the suite left the tested commit unchanged'
MANUAL_GUARD = "github.event_name == 'workflow_dispatch'"
SUITE_BLOCK = ('set -euo pipefail\n'
MANUAL_SUITE_BLOCK = ('set -euo pipefail\n'
'python -B -S -m unittest discover -s tests -v\n'
'python -B -O -S -m unittest discover -s tests -v\n')
CENTRAL_SUITE_BLOCK = (
"set -euo pipefail\n"
"log_dir=\"$(mktemp -d)\"\n"
"trap 'rm -rf \"$log_dir\"' EXIT\n"
"python -B -S -m unittest discover -s tests -v 2>&1 | tee \"$log_dir/normal.log\"\n"
"grep -Eq '^Ran [1-9][0-9]* tests? in [0-9]+([.][0-9]+)?s$' \"$log_dir/normal.log\"\n"
"python -B -O -S -m unittest discover -s tests -v 2>&1 | tee \"$log_dir/optimized.log\"\n"
"grep -Eq '^Ran [1-9][0-9]* tests? in [0-9]+([.][0-9]+)?s$' \"$log_dir/optimized.log\"\n"
)
NORMAL = ['-B', '-S', '-m', 'unittest', 'discover', '-s', 'tests', '-v']
OPTIMIZED = ['-B', '-O', '-S', '-m', 'unittest', 'discover', '-s', 'tests', '-v']
# The complete PR trigger union of the three workflows at Math- 7014efec.
Expand Down Expand Up @@ -202,8 +213,13 @@ def condition(expr, event, failed):
raise AssertionError('unexpected step condition: ' + expr)


def run_job(text, event, fail=(), sha_override=None):
"""Run the job's steps locally with a fake python; packet -> sentinel."""
def run_job(text, event, fail=(), sha_override=None, counts=(2, 2),
summaries=None, fail_capture=(), real_cases=None):
"""Run actual shell steps with controlled children; packet -> sentinel.

Default children emit synthetic unittest summaries. real_cases instead runs
this interpreter on tiny generated fixtures, not the repository test suite.
"""
_, job = parse(text)
with tempfile.TemporaryDirectory(prefix='full-suite-routing-') as td:
root = Path(td)
Expand All @@ -229,12 +245,49 @@ def run_job(text, event, fail=(), sha_override=None):
bindir.mkdir()
log = root / 'python.log'
fake = bindir / 'python'
if summaries is None:
summaries = tuple('Ran ' + str(count) + ' test' +
('' if count == 1 else 's') + ' in 0.001s'
for count in counts)
fake.write_text('#!/bin/sh\n'
'printf "%s\\n" "$*" >> "$FAKE_LOG"\n'
'case " $* " in *" -O "*) mode=optimized ;; *) mode=normal ;; esac\n'
'if [ "$FAKE_REAL" = 1 ]; then\n'
' cd "$REAL_FIXTURE_ROOT/$mode" || exit 9\n'
' exec "$REAL_PYTHON" "$@"\n'
'fi\n'
'case "$mode" in\n'
' normal) summary=$NORMAL_SUMMARY ;;\n'
' optimized) summary=$OPTIMIZED_SUMMARY ;;\n'
'esac\n'
'if [ -n "$summary" ]; then printf "%s\\n" "$summary" >&2; fi\n'
'case " $FAKE_FAIL " in *" $mode "*) exit 7 ;; esac\n'
'exit 0\n')
fake.chmod(0o755)
temp = root / 'temp'
temp.mkdir()
real_root = root / 'real-discovery'
if real_cases is not None:
for mode, case in zip(('normal', 'optimized'), real_cases):
tests = real_root / mode / 'tests'
tests.mkdir(parents=True)
if case not in ('empty', 'pass', 'fail'):
raise ValueError('unsupported real discovery case: ' + case)
if case != 'empty':
(tests / 'test_control.py').write_text(
'import unittest\nclass Control(unittest.TestCase):\n'
' def test_control(self):\n'
' self.assertEqual(1, ' + ('1' if case == 'pass' else '2') + ')\n')
if fail_capture:
capture = bindir / 'tee'
capture.write_text('#!/bin/sh\n'
'n=0\n'
'[ ! -f "$CAPTURE_COUNT" ] || read -r n < "$CAPTURE_COUNT"\n'
'n=$((n + 1))\n'
'printf "%s\\n" "$n" > "$CAPTURE_COUNT"\n'
'case " $CAPTURE_FAIL " in *" $n "*) cat; exit 8 ;; esac\n'
'exec "$REAL_TEE" "$@"\n')
capture.chmod(0o755)
sentinel = root / 'packet.sentinel'
context = {
'github.sha': sha_override or head,
Expand All @@ -245,6 +298,7 @@ def run_job(text, event, fail=(), sha_override=None):
}
summary = root / 'summary.txt'
outcomes = []
streams = []
failed = False
for step in job['steps']:
name = step.get('name') or step.get('uses')
Expand All @@ -256,7 +310,14 @@ def run_job(text, event, fail=(), sha_override=None):
continue
step_env = dict(env, PATH=str(bindir) + os.pathsep + env.get('PATH', ''),
FAKE_LOG=str(log), FAKE_FAIL=' '.join(fail),
GITHUB_STEP_SUMMARY=str(summary))
GITHUB_STEP_SUMMARY=str(summary), TMPDIR=str(temp),
RUNNER_TEMP=str(temp), NORMAL_SUMMARY=summaries[0],
OPTIMIZED_SUMMARY=summaries[1],
FAKE_REAL='1' if real_cases is not None else '0',
REAL_PYTHON=sys.executable, REAL_FIXTURE_ROOT=str(real_root),
CAPTURE_COUNT=str(root / 'capture-count'),
CAPTURE_FAIL=' '.join(map(str, fail_capture)),
REAL_TEE=shutil.which('tee') or '')
for key, value in step.get('env', {}).items():
m = re.fullmatch(r'\$\{\{ ([a-z_.]+) \}\}', value)
if m:
Expand All @@ -274,12 +335,14 @@ def run_job(text, event, fail=(), sha_override=None):
raise AssertionError('inline expression in run block: ' + name)
cp = subprocess.run(['bash', '-e', '-c', block], cwd=repo, env=step_env,
capture_output=True, timeout=30)
streams.append((name, cp.returncode, cp.stdout, cp.stderr))
outcomes.append((name, 'success' if cp.returncode == 0 else 'failure'))
failed = failed or cp.returncode != 0
calls = [line.split() for line in log.read_text().splitlines()] if log.exists() else []
return {
'conclusion': 'failure' if failed else 'success',
'outcomes': outcomes,
'streams': streams,
'calls': calls,
'sentinel': sentinel.exists(),
'summary': summary.read_text() if summary.exists() else '',
Expand Down Expand Up @@ -314,8 +377,9 @@ def violations(workflow, text):
suite = named(job, REGRESSION)
except ValueError as exc:
return problems + [str(exc)]
if suite.get('run') != SUITE_BLOCK:
problems.append('suite block must be both literal discovery modes under set -euo pipefail')
expected_suite = CENTRAL_SUITE_BLOCK if workflow == CENTRAL else MANUAL_SUITE_BLOCK
if suite.get('run') != expected_suite:
problems.append('suite block must preserve its exact central/manual discovery contract')
central_bind = named(parse((WORKFLOWS / (CENTRAL + '.yml')).read_text())[1], BIND)
if bind != central_bind:
problems.append('tested-commit binding step differs from the central workflow')
Expand Down Expand Up @@ -397,6 +461,114 @@ def mutate(text, old, new, count=1):


class FullSuiteRoutingTests(unittest.TestCase):
def original_central(self):
# Reconstruct only the suite body and authenticate the complete original
# workflow at 49fb197c / blob ea291276, not a hand-written shell surrogate.
text = source(CENTRAL)
block = named(parse(text)[1], REGRESSION)['run']
rendered = lambda body: ''.join(' ' + line + '\n'
for line in body.splitlines())
original = mutate(text, rendered(block), rendered(MANUAL_SUITE_BLOCK))
self.assertEqual(hashlib.sha256(original.encode()).hexdigest(),
'505e20940473c5a1fefaf5559e3f2ec2425fdb0e03de7c16d05e5371235e3e26')
return original

def test_original_accepts_synthetic_empty_success_but_preserves_failures(self):
original = self.original_central()
for event in ('pull_request', 'workflow_dispatch'):
for counts in ((0, 2), (2, 0), (0, 0), (1, 1), (2, 3)):
with self.subTest(event=event, counts=counts):
result = run_job(original, event, counts=counts)
self.assertEqual(result['conclusion'], 'success', result)
self.assertEqual(result['calls'], [NORMAL, OPTIMIZED])
self.assertEqual(result['outcomes'][-1], (CONFIRM, 'success'))
for mode, calls in (('normal', [NORMAL]), ('optimized', [NORMAL, OPTIMIZED])):
with self.subTest(event=event, failure=mode):
result = run_job(original, event, fail=(mode,))
self.assertEqual(result['conclusion'], 'failure', result)
self.assertEqual(result['calls'], calls)
self.assertEqual(result['outcomes'][-1], (CONFIRM, 'skipped'))

def test_central_rejects_empty_success_before_confirmation(self):
for event in ('pull_request', 'workflow_dispatch'):
for counts, calls in (((0, 2), [NORMAL]), ((2, 0), [NORMAL, OPTIMIZED]),
((0, 0), [NORMAL])):
with self.subTest(event=event, counts=counts):
result = run_job(source(CENTRAL), event, counts=counts)
self.assertEqual(result['conclusion'], 'failure', result)
self.assertEqual(result['calls'], calls)
self.assertEqual(result['outcomes'][-1], (CONFIRM, 'skipped'))

def test_central_accepts_singular_and_multiple_successful_tests(self):
for event in ('pull_request', 'workflow_dispatch'):
for counts in ((1, 1), (2, 13), (13, 2)):
with self.subTest(event=event, counts=counts):
result = run_job(source(CENTRAL), event, counts=counts)
self.assertEqual(result['conclusion'], 'success', result)
self.assertEqual(result['calls'], [NORMAL, OPTIMIZED])
self.assertEqual(result['outcomes'][-1], (CONFIRM, 'success'))

def test_central_requires_well_formed_positive_summary_in_each_mode(self):
good = 'Ran 2 tests in 0.001s'
for bad in ('', 'Ran 00 tests in 0.001s', 'Ran 01 tests in 0.001s',
'Ran -1 tests in 0.001s', 'Ran 1.0 tests in 0.001s',
'Ran 1 test in unknown', 'prefix Ran 1 test in 0.001s',
'Ran 1 test in 0.001s suffix'):
for index in (0, 1):
with self.subTest(summary=bad, mode=index):
summaries = [good, good]
summaries[index] = bad
result = run_job(source(CENTRAL), 'pull_request', summaries=summaries)
self.assertEqual(result['conclusion'], 'failure', result)
self.assertEqual(result['calls'], [NORMAL] if index == 0
else [NORMAL, OPTIMIZED])
self.assertEqual(result['outcomes'][-1], (CONFIRM, 'skipped'))

def test_positive_summary_cannot_hide_child_or_capture_failure(self):
for event in ('pull_request', 'workflow_dispatch'):
for mode, calls in (('normal', [NORMAL]), ('optimized', [NORMAL, OPTIMIZED])):
with self.subTest(event=event, failed_child=mode):
result = run_job(source(CENTRAL), event, fail=(mode,))
self.assertEqual(result['conclusion'], 'failure', result)
self.assertEqual(result['calls'], calls)
self.assertEqual(result['outcomes'][-1], (CONFIRM, 'skipped'))
self.assertEqual(next(x[1] for x in result['streams']
if x[0] == REGRESSION), 7)
for capture, calls in ((1, [NORMAL]), (2, [NORMAL, OPTIMIZED])):
with self.subTest(event=event, failed_capture=capture):
result = run_job(source(CENTRAL), event, fail_capture=(capture,))
self.assertEqual(result['conclusion'], 'failure', result)
self.assertEqual(result['calls'], calls)
self.assertEqual(result['outcomes'][-1], (CONFIRM, 'skipped'))
self.assertEqual(next(x[1] for x in result['streams']
if x[0] == REGRESSION), 8)

@unittest.skipUnless(sys.version_info[:2] == (3, 11),
'real empty-discovery contract requires hosted CPython 3.11')
def test_real_python311_original_and_candidate_zero_pass_fail_controls(self):
original = self.original_central()
for cases in (('empty', 'pass'), ('pass', 'empty'), ('empty', 'empty'),
('pass', 'pass'), ('fail', 'pass'), ('pass', 'fail')):
for candidate, text in ((False, original), (True, source(CENTRAL))):
with self.subTest(candidate=candidate, cases=cases):
result = run_job(text, 'pull_request', real_cases=cases)
fails = 'fail' in cases or (candidate and 'empty' in cases)
expected = 'failure' if fails else 'success'
self.assertEqual(result['conclusion'], expected, result)
first_stops = cases[0] == 'fail' or (candidate and cases[0] == 'empty')
self.assertEqual(result['calls'], [NORMAL] if first_stops
else [NORMAL, OPTIMIZED])
self.assertEqual(result['outcomes'][-1],
(CONFIRM, 'skipped' if fails else 'success'))
regression = next(x for x in result['streams'] if x[0] == REGRESSION)
output = (regression[2] + regression[3]).decode()
self.assertIn('Ran ' + ('0 tests' if cases[0] == 'empty' else '1 test'),
output)
if cases[0] == 'pass' and cases[1] == 'empty':
self.assertIn('Ran 0 tests', output)
if 'fail' in cases:
self.assertIn('FAILED (failures=1)', output)

def test_job_preserves_foreign_git_state(self):
# Real foreign repositories, never a caller's checkout. Setup and
# observations use their own clean environment, independent of run_job.
Expand Down Expand Up @@ -569,9 +741,11 @@ def test_trigger_split_routes_tests_only_changes_centrally(self):
def test_meaningful_negatives_are_detected(self):
central = source(CENTRAL)
cases = {
'central drops normal mode': (CENTRAL, central, ' python -B -S -m unittest discover -s tests -v\n', ''),
'central drops optimized mode': (CENTRAL, central, ' python -B -O -S -m unittest discover -s tests -v\n', ''),
'central swallows failure': (CENTRAL, central, 'discover -s tests -v\n', 'discover -s tests -v || true\n', 2),
'central drops normal mode': (CENTRAL, central,
' python -B -S -m unittest discover -s tests -v 2>&1 | tee "$log_dir/normal.log"\n', ''),
'central drops optimized mode': (CENTRAL, central,
' python -B -O -S -m unittest discover -s tests -v 2>&1 | tee "$log_dir/optimized.log"\n', ''),
'central swallows failure': (CENTRAL, central, '.log"\n', '.log" || true\n', 4),
'central loses tests trigger': (CENTRAL, central, " - 'tests/**'\n", ''),
'central loses tools trigger': (CENTRAL, central, " - 'tools/**'\n", ''),
'central loses a frontier trigger': (CENTRAL, central, " - 'frontiers/c6_palm_route_20260929/**'\n", ''),
Expand All @@ -597,7 +771,7 @@ def test_meaningful_negatives_are_detected(self):
workflow + ' removes suite step': (workflow, text,
" - name: " + REGRESSION + "\n"
" if: github.event_name == 'workflow_dispatch'\n"
" run: |\n" + ''.join(' ' + l + '\n' for l in SUITE_BLOCK.splitlines()),
" run: |\n" + ''.join(' ' + l + '\n' for l in MANUAL_SUITE_BLOCK.splitlines()),
''),
workflow + ' drops optimized mode': (workflow, text, ' python -B -O -S -m unittest discover -s tests -v\n', ''),
workflow + ' swallows failure': (workflow, text, 'python -B -S -m unittest discover -s tests -v\n',
Expand All @@ -619,8 +793,8 @@ def test_parser_fails_closed(self):
text = source(CENTRAL)
for old, new in ((" - 'tools/**'\n", " - tools/**\n"),
(' steps:\n', ' strategy: {}\n steps:\n'),
(' run: |\n set -euo pipefail\n python',
' run: set -euo pipefail\n shell: bash\n run2: |\n python')):
(' run: |\n set -euo pipefail\n log_dir=',
' run: set -euo pipefail\n shell: bash\n run2: |\n log_dir=')):
with self.subTest(old=old):
with self.assertRaises(ValueError):
parse(mutate(text, old, new))
Expand Down
Loading