Six models: Qwen3.8 and DeepSeek V4 Pro, Modal repeat, two grader fixes

docs/preprint/make_run_figures.py

@@ -18,9 +18,11 @@
 OUT = ROOT / "docs/preprint/figures"
 OUT.mkdir(parents=True, exist_ok=True)
 
-ORDER = ["claude-sonnet-5.5", "claude-opus-5.5", "claude-fable-5.1", "gpt-6.1-sol"]
-NAME = {"claude-sonnet-5.5": "Sonnet 5.5", "claude-opus-5.5": "Opus 5.5", "claude-fable-5.1": "Fable 5.1", "gpt-6.1-sol": "GPT-6.1 Sol"}
-COL = {"claude-opus-5.5": "#2a78d6", "claude-sonnet-5.5": "#eb6834", "claude-fable-5.1": "#1baf7a", "gpt-6.1-sol": "#4a3aa7"}
+ORDER = ["claude-sonnet-5.5", "claude-opus-5.5", "claude-fable-5.1", "gpt-6.1-sol", "qwen3.8-2.4t-a95b", "deepseek-v4-pro-0813"]
+NAME = {"claude-sonnet-5.5": "Sonnet 5.5", "claude-opus-5.5": "Opus 5.5", "claude-fable-5.1": "Fable 5.1", "gpt-6.1-sol": "GPT-6.1 Sol",
+        "qwen3.8-2.4t-a95b": "Qwen3.8 2.4T", "deepseek-v4-pro-0813": "DeepSeek V4 Pro"}
+COL = {"claude-opus-5.5": "#2a78d6", "claude-sonnet-5.5": "#eb6834", "claude-fable-5.1": "#1baf7a", "gpt-6.1-sol": "#4a3aa7",
+       "qwen3.8-2.4t-a95b": "#eda100", "deepseek-v4-pro-0813": "#e87ba4"}
 MODELS = [m for m in ORDER if (RUN / m).is_dir()]
 OFF = lambda j, width: (j - (len(MODELS) - 1) / 2) * width   # bar offset of model j in a group
 INK, INK2, MUTED, GRID, REFUSE = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#d9d7cf"
@@ -79,7 +81,7 @@ def xgrid(ax): ax.xaxis.grid(True, color=GRID, lw=0.6); ax.set_axisbelow(True)
 
 
 # Figure 8: how the mean depends on refusals, and what it cost
-fig, (a, b) = plt.subplots(1, 2, figsize=(9.2, 3.4), gridspec_kw={"width_ratios": [1.35, 1]})
+fig, (a, b) = plt.subplots(1, 2, figsize=(10.5, 3.8), gridspec_kw={"width_ratios": [1.6, 1]})
 rules = [(f"The {len(common)} tasks\nall answered", lambda m: mean([reward(m, k) for k in common])),
          ("All tasks the\nmodel answered", lambda m: mean([reward(m, k) for k in answered[m]])),
          ("Easy tasks\nanswered", lambda m: mean([reward(m, k) for k in answered[m] if k in EASY])),
@@ -90,32 +92,33 @@ def xgrid(ax): ax.xaxis.grid(True, color=GRID, lw=0.6); ax.set_axisbelow(True)
     xs = [i + OFF(j, w) for i in range(len(rules))]
     a.bar(xs, vals, w * .92, color=COL[m], label=NAME[m])
     for x, v in zip(xs, vals):
-        a.text(x, v - .012, f"{v:.2f}", ha="center", va="top", fontsize=6.5, color="white", fontweight="bold")
+        a.text(x, v + .006, f"{v:.2f}".lstrip("0"), ha="center", va="bottom", fontsize=5.5, color=INK2, rotation=90)
 a.set_xticks(range(len(rules)), [r for r, _ in rules], fontsize=8)
-a.set_ylim(.5, 1.0); a.set_ylabel("Mean reward"); ygrid(a)
-a.legend(loc="upper center", bbox_to_anchor=(.5, -.2), ncol=len(MODELS), fontsize=7.5)
+a.set_ylim(.5, 1.07); a.set_yticks([.5, .6, .7, .8, .9, 1.0]); a.set_ylabel("Mean reward"); ygrid(a)
+a.legend(loc="upper center", bbox_to_anchor=(.5, -.2), ncol=(len(MODELS) + 1) // 2 if len(MODELS) > 4 else len(MODELS), fontsize=7.5)
 a.set_title("Refused tasks are not scored"); panel(a, "a")
 for m in MODELS:
     cost = sum(trials[(m, k)].get("cost_usd") or 0 for k in TASKS)
     v = rules[0][1](m)
     b.scatter(cost, v, s=90, color=COL[m], zorder=3)
     nref = sum(refused(m, k) for k in TASKS)
-    b.annotate(f"{NAME[m]}\n{nref} refused, not scored" if nref else NAME[m], (cost, v), xytext=(8, -4), textcoords="offset points", fontsize=8, color=INK2)
+    b.annotate(f"{NAME[m]}, {nref} refused" if nref else NAME[m], (cost, v), xytext=(8, -3), textcoords="offset points", fontsize=7.5, color=INK2)
 b.set_xlabel("Agent cost for the tasks it answered (USD)"); b.set_ylabel(f"Mean over the {len(common)} tasks all answered")
 costs = [sum(trials[(m, k)].get("cost_usd") or 0 for k in TASKS) for m in MODELS]
 vals = [rules[0][1](m) for m in MODELS]
-b.set_xlim(0, max(costs) * 1.45); b.set_ylim(min(vals) - .05, min(1.02, max(vals) + .05)); ygrid(b); xgrid(b)
+b.set_xlim(0, max(costs) * 1.75); b.set_ylim(min(vals) - .05, min(1.02, max(vals) + .05)); ygrid(b); xgrid(b)
 b.set_title("Cost against reward"); panel(b, "b")
 fig.tight_layout(); fig.savefig(OUT / "run_means_cost.png"); plt.close(fig)
 
 
 # Figure 9: where points were lost: deterministic risks vs judge rubric items
-RISKS = [("simulator_ran", "Runs in the simulator"), ("end_state", "End-state volumes"),
+RISKS = [("simulator_ran", "Runs in the simulator"), ("end_state", "End-state volumes\n(or paper-range mix)"),
          ("tip_before_aspirate", "Tip before pipetting"), ("no_overdispense", "No overdispense"),
          ("no_aspirate_from_empty_well", "No empty-well draw"), ("tip_dropped_at_end", "Tip dropped at end"),
          ("no_cross_contamination", "No cross-contamination"), ("rna", "RNA run-log checks")]
 def risk_of(name):
     base = name.split(":")[0].split("[")[0]       # deck_labware is recorded only when it fails: none did
+    base = "end_state" if base == "composition" else base   # colony-hard's plate: inputs, equal wells, paper's range
     return base if base in dict(RISKS) else "rna"
 passed, ran = collections.Counter(), collections.Counter()
 for (m, k), t in trials.items():
@@ -124,22 +127,18 @@ def risk_of(name):
         checks = checks.get("checks", [])
     for c in checks:
         passed[(m, risk_of(c["name"]))] += bool(c["pass"]); ran[(m, risk_of(c["name"]))] += 1
-fig, (a, b) = plt.subplots(1, 2, figsize=(10, 3.9), gridspec_kw={"width_ratios": [1, 1]})
-ys = range(len(RISKS))
-for j, m in enumerate(MODELS):
-    vals = [passed[(m, r)] / ran[(m, r)] if ran[(m, r)] else None for r, _ in RISKS]
-    for y, v, (r, _) in zip(ys, vals, RISKS):
-        if v is not None:
-            a.barh(y + OFF(j, .8 / len(MODELS)), v, .72 / len(MODELS), color=COL[m], label=NAME[m] if y == 0 else None)
-
-for y, (r, _) in zip(ys, RISKS):   # one line per risk that any model missed: every model's passed/run
-    if any(ran[(m, r)] and passed[(m, r)] < ran[(m, r)] for m in MODELS):
-        a.text(1.02, y, "\n".join(f"{NAME[m]} {passed[(m, r)]}/{ran[(m, r)]}" for m in MODELS if ran[(m, r)]),
-               va="center", fontsize=5.8, color=INK2, linespacing=1.0)
-a.set_yticks(list(ys), [l for _, l in RISKS], fontsize=8); a.invert_yaxis()
-a.set_xlim(0, 1.3); a.set_xticks([0, .2, .4, .6, .8, 1]); a.set_xlabel("Share of checks passed"); xgrid(a)
-a.set_title("Deterministic checks: almost nothing fails"); panel(a, "a")
-a.legend(loc="lower left", fontsize=7.5, ncol=len(MODELS), bbox_to_anchor=(0, -.3))
+fig, (a, b) = plt.subplots(1, 2, figsize=(10.5, 3.9), gridspec_kw={"width_ratios": [1.1, 1]})
+for i, (r, _) in enumerate(RISKS):     # one cell per risk and model: checks passed / run, red where any failed
+    for j, m in enumerate(MODELS):
+        n, ok = ran[(m, r)], passed[(m, r)]
+        face = "#f4f3ee" if not n else ("#2a78d6" if ok == n else "#b8321e")
+        a.add_patch(plt.Rectangle((j - .5, i - .5), 1, 1, facecolor=face, edgecolor="white", lw=1.5))
+        a.text(j, i, f"{ok}/{n}" if n else "-", ha="center", va="center", fontsize=7, color="white" if n else MUTED)
+a.set_xlim(-.5, len(MODELS) - .5); a.set_ylim(len(RISKS) - .5, -.5)
+a.set_xticks(range(len(MODELS)), [NAME[m] for m in MODELS], rotation=30, ha="right", fontsize=7.5)
+a.set_yticks(range(len(RISKS)), [l for _, l in RISKS], fontsize=7.5)
+a.tick_params(length=0); [sp.set_visible(False) for sp in a.spines.values()]
+a.set_title("Deterministic checks passed / run"); panel(a, "a")
 fails = collections.Counter()
 for (m, k), t in trials.items():
     if m not in MODELS:
@@ -157,8 +156,8 @@ def risk_of(name):
 b.set_yticks(range(len(ITEMS)), ITEMS, fontsize=8.5, family="monospace"); b.invert_yaxis()
 b.set_xlabel(f"Failed rubric items (all {len(MODELS)} models)"); b.xaxis.set_major_locator(matplotlib.ticker.MaxNLocator(integer=True)); xgrid(b)
 b.set_title("LLM judge: every lost point"); panel(b, "b")
-b.text(.98, .03, f"{sum(fails.values())} failed items in all; every other item passed",
-       transform=b.transAxes, ha="right", fontsize=7.5, color=MUTED)
+b.set_xlabel(f"Failed rubric items ({sum(fails.values())} in all; every other item passed)")
+b.legend(loc="lower right", fontsize=7)
 fig.tight_layout(); fig.savefig(OUT / "run_where_lost.png"); plt.close(fig)
 
 
@@ -178,21 +177,23 @@ def risk_of(name):
             ax.text(j, i, "n/a", ha="center", va="center", fontsize=6, color=MUTED)
             continue
         v = r.get("reward") or 0.0
-        ctrl = a_.startswith("control")
+        ctrl = a_.startswith("control") or bool(r.get("valid"))   # a valid variant (e.g. the paper's own volumes) must score 1 too
         face = ("#2a78d6" if v == 1 else "#b8321e") if ctrl else ("#b8321e" if v == 1 else SCORE(.5 + v * .4) if v > 0 else "#e7e5dc")
-        ax.add_patch(plt.Rectangle((j - .5, i - .5), 1, 1, facecolor=face, edgecolor="white", lw=1.5))
+        ax.add_patch(plt.Rectangle((j - .5, i - .5), 1, 1, facecolor=face, edgecolor=INK if r.get("valid") else "white", lw=1.5))
         ax.text(j, i, "1" if v == 1 else ("0" if v == 0 else f"{v:.2f}".lstrip("0")), ha="center", va="center", fontsize=6.5,
                 color="white" if ctrl or v == 1 else INK)
 ax.axvline(2.5, color=INK, lw=1.2)
 ax.set_xlim(-.5, len(ATT) - .5); ax.set_ylim(len(tasks) - .5, -.5)
 ax.set_xticks(range(len(ATT)), [SHORT[a_] for a_ in ATT], rotation=55, ha="right", fontsize=7.5)
 ax.set_yticks(range(len(tasks)), tasks, fontsize=7.5)
 ax.tick_params(length=0); [s.set_visible(False) for s in ax.spines.values()]
-holes = sum(1 for v in adv.values() for a_, r in v.items() if not a_.startswith("control") and r.get("reward") == 1.0)
+holes = sum(1 for v in adv.values() for a_, r in v.items() if not a_.startswith("control") and not r.get("valid") and r.get("reward") == 1.0)
+valid = sum(1 for v in adv.values() for r in v.values() if r.get("valid"))
 ctrl_ok = sum(1 for v in adv.values() for a_, r in v.items() if a_.startswith("control") and r.get("reward") == 1.0)
 ctrl_n = sum(1 for v in adv.values() for a_ in v if a_.startswith("control"))
-ran = sum(1 for v in adv.values() for a_, r in v.items() if not a_.startswith("control") and "skipped" not in r)
-ax.set_title(f"Controls: {ctrl_ok}/{ctrl_n} score 1.0    Attacks: {ran} run, {holes} scored 1.0", loc="left", fontsize=9.5)
+ran = sum(1 for v in adv.values() for a_, r in v.items() if not a_.startswith("control") and "skipped" not in r and not r.get("valid"))
+ax.set_title(f"Controls: {ctrl_ok}/{ctrl_n} score 1.0    Attacks: {ran} run, {holes} scored 1.0"
+             + (f"    Valid variants: {valid} (score 1.0, outlined)" if valid else ""), loc="left", fontsize=9.5)
 ax.legend(handles=[Patch(color="#2a78d6", label="reference scores 1 (must)"), Patch(color="#e7e5dc", label="attack scores 0"),
                    Patch(color=SCORE(.6), label="attack: partial credit"), Patch(color="#b8321e", label="hole (attack scores 1)")],
           loc="upper left", bbox_to_anchor=(0, -.42), ncol=4, fontsize=7.5)

eval/protocol_lint.py

@@ -18,12 +18,20 @@
 
 
 FLOW_RATE_FIELDS = {"aspirate", "dispense", "blow_out"}
+# Settable properties of a pipette (InstrumentContext) in the Opentrons API docs: ordinary protocol code, not patching.
+PIPETTE_SETTINGS = {"tip_racks", "starting_tip", "default_speed"}
+CLEARANCE_FIELDS = {"aspirate", "dispense"}
 
 
 def _is_flow_rate(t: ast.Attribute) -> bool:
-    """pipette.flow_rate.aspirate/dispense/blow_out = ... is ordinary Opentrons API use (pipetting speed)."""
-    return (t.attr in FLOW_RATE_FIELDS and isinstance(t.value, ast.Attribute) and t.value.attr == "flow_rate"
-            and isinstance(t.value.value, ast.Name))
+    """A documented pipette setting on a local variable: p.flow_rate.aspirate/dispense/blow_out,
+    p.well_bottom_clearance.aspirate/dispense, p.tip_racks, p.starting_tip, p.default_speed. Assigning to anything else,
+    or to these on a module or class (protocol_api.InstrumentContext.tip_racks = ...), is still refused."""
+    if isinstance(t.value, ast.Name):
+        return t.attr in PIPETTE_SETTINGS
+    return (isinstance(t.value, ast.Attribute) and isinstance(t.value.value, ast.Name)
+            and ((t.value.attr == "flow_rate" and t.attr in FLOW_RATE_FIELDS)
+                 or (t.value.attr == "well_bottom_clearance" and t.attr in CLEARANCE_FIELDS)))
 
 
 def violations(source: str) -> list[str]: