From b1ef9f99fd4bd376e8c7f37aaaa038cd9fc33c71 Mon Sep 17 00:00:00 2001 From: argszero Date: Fri, 23 Oct 2026 04:34:53 +0800 Subject: [PATCH] emrg: a probe is read against a floor, not against the best score --- emrg/server/competition_prompt.md | 2 +- tests/test_competition_prompt.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/emrg/server/competition_prompt.md b/emrg/server/competition_prompt.md index 0e8d2085..cba45679 100644 --- a/emrg/server/competition_prompt.md +++ b/emrg/server/competition_prompt.md @@ -143,7 +143,7 @@ One improvement per round: local validation → submit → record the score delt 1. **Classify** the cases that are not right, and keep the classification where the next round can read it. 2. **Draw a small sample from one class and submit it as a probe** — not the whole class. -3. **Read the probe's outcome as that class's answer.** A probe whose result is not read has bought nothing; a class whose answer is established is a fact the remaining cases can be judged against. +3. **Read the probe's outcome as that class's answer, against an absolute baseline rather than against your best score so far.** A probe whose result is not read has bought nothing; a class whose answer is established is a fact the remaining cases can be judged against. Comparing a probe with the best submission makes a zero delta undecidable: the change either restored the cases it was meant to restore or touched none of them, and both print the same number, so the submission bought no fact. A baseline that is a known floor turns every probe into a yes/no question instead. Pin that floor first, with a deliberately wrong answer in the same submission format so it is graded rather than rejected, and hold everything else fixed while probing, so the score can move only for the thing that changed. The floor costs one submission and every later probe is read against it. 4. **Turn the answer into one concrete change** — a detector, a re-read rule, a corrected output format — recording what the answer was, where the previous behaviour went wrong, and what changed. Step 4 is the one most easily left out. A round that classified, counted or observed and stopped has not improved anything, whatever it wrote down. Widen from the sample to the whole class only when the sample's reading cannot be interpreted. diff --git a/tests/test_competition_prompt.py b/tests/test_competition_prompt.py index 11d7b4b3..e9c13769 100644 --- a/tests/test_competition_prompt.py +++ b/tests/test_competition_prompt.py @@ -1092,6 +1092,31 @@ def test_phase_d_establishes_the_answer_on_a_sample_before_changing_the_class(): ) +def test_a_probe_is_read_against_an_absolute_baseline(): + """§Phase D must say what a probe is read *against*, not only that it is read. + + The four steps read as complete while naming no baseline, and the baseline decides + whether the reading is decidable at all. A probe compared with the best submission + leaves a zero delta with two opposite meanings — the cases it targeted came back, or + it touched none of them — and one number is printed for both, so the submission + bought no fact. An absolute floor makes each probe a yes/no question. Asserted by + name for the same reason step 4 is: the rule is reasoning, and a reader who is not + given the reasoning has no ground for spending a submission on a floor. + """ + text = PROMPT.read_text(encoding="utf-8") + start = text.index("#### Phase D — Iteration") + end = text.index("#### Phase E — Deadline wrap-up") + phase_d = text[start:end] + assert "against an absolute baseline rather than against your best score so far" in phase_d, ( + "§Phase D does not name the baseline a probe is read against, so a zero delta " + "reads the same for a case that came back and one that was never touched" + ) + assert "deliberately wrong answer in the same submission format" in phase_d, ( + "§Phase D must say the floor submission is graded rather than rejected, or the " + "floor itself buys nothing and the round is back to an ambiguous zero" + ) + + def test_a_discouraging_reading_is_written_as_the_gap_not_as_a_verdict(rendered): """§5 asks how likely the goal is; the answer has one accepted form.