From cddd7e17d510835ff350f72f7d8ee64a95d3fe91 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 28 Jun 2026 16:30:19 +0700 Subject: [PATCH 1/2] feat: AUPRC benchmark, two new seeds, assay pre-registration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AUPRC (Area Under Precision-Recall Curve): - Add _auprc() to retrospective.py (trapezoidal integration) - run_retrospective_benchmark() now returns auprc, auprc_random_baseline, auprc_above_random alongside existing AUROC + bootstrap CI - 10 new tests in tests/test_auprc.py (perfect/worst/random/edge cases + integration test verifying all three keys appear in benchmark output) - METHODS.md Section 8 updated with AUPRC explanation and rationale New seeds: - SEED-009: RRLPRPPYLPRP — Bac2A bovine proline-rich cathelicidin (intracellular DnaK-targeting mechanism, genuinely distinct from all 8 prior membrane-disruption seeds; Gram-negative selective) - SEED-010: KRIVQRIKDFLR — KR-12, shortest active fragment of human LL-37 cathelicidin (clinical relevance; good selectivity vs MRSA + E. coli) - Increases effective independent scaffolds: 8 → 10 - Calibrated breaking-news probability: 15-30% → 18-30% Pre-registration: - docs/ASSAY_PREREGISTRATION.md: full assay pre-registration template including EUCAST/CLSI broth microdilution protocol, HC50 protocol, selectivity index definition, pre-registered hit criteria, statistical plan, deviation log, and signature block --- docs/ASSAY_PREREGISTRATION.md | 193 ++++++++++++++++++ docs/DISCOVERY_PREDICTION.md | 30 +-- docs/METHODS.md | 22 +- examples/sequences/amp_seeds.csv | 2 + .../benchmark/retrospective.py | 37 ++++ tests/test_auprc.py | 98 +++++++++ 6 files changed, 362 insertions(+), 20 deletions(-) create mode 100644 docs/ASSAY_PREREGISTRATION.md create mode 100644 tests/test_auprc.py diff --git a/docs/ASSAY_PREREGISTRATION.md b/docs/ASSAY_PREREGISTRATION.md new file mode 100644 index 00000000..a7c18da2 --- /dev/null +++ b/docs/ASSAY_PREREGISTRATION.md @@ -0,0 +1,193 @@ +# Assay Pre-Registration — Wave 1 + +**Status:** PRE-REGISTERED (lock before synthesis order is placed) +**Pipeline version:** 0.5.x +**Manifest SHA-256:** see `outputs/phase3_manifest.json` +**Locked by:** [PI name + date] +**Document version:** 1.0 (2026-06-28) + +--- + +## Purpose + +This document pre-registers the primary assay plan for Wave 1 of the OpenAMP Foundry pilot panel. +Pre-registration means the analysis plan, pass/fail thresholds, and reporting obligations are locked +**before** any wet-lab result is seen. This prevents p-hacking, outcome switching, and selective +non-reporting of negative results. + +> **This document must be signed by the PI and filed before synthesis is ordered.** +> Any deviation from this plan must be logged in a deviation record and reported in the methods. + +--- + +## 1. Candidate Set + +| Field | Value | +|-------|-------| +| Number of candidates | 20 (pilot panel) | +| Selection algorithm | `pilot-panel` with similarity_threshold=0.75, max_per_seed=4 | +| Selection date | [date of `make pilot` run] | +| Selection locked at manifest | `outputs/phase3_manifest.json` | +| Confident panel (post-external-predictor filter) | TBD after running CAMPR4 / AMPScanner / dbAMP | + +--- + +## 2. Primary Assay — Broth Microdilution MIC + +### Protocol + +- **Method:** Broth microdilution (EUCAST ISO 20776-1 or CLSI M07) +- **Broth:** Mueller-Hinton Broth II (MHB-II; cation-adjusted for Gram-negative) +- **Volume:** 100 µL per well (96-well plate, flat-bottom) +- **Inoculum:** 5 × 10⁵ CFU/mL (0.5 McFarland ± 10%) +- **Peptide concentration range:** 0.25, 0.5, 1, 2, 4, 8, 16, 32, 64, 128 µg/mL (serial 2-fold) +- **Incubation:** 37 °C, 18–20 h (aerobic, static) +- **Readout:** Optical density at 600 nm (OD600); visual confirmation of turbidity +- **MIC definition:** Lowest concentration with no visible growth (OD600 ≤ 10% of growth control) + +### Target Organisms + +| Priority | Organism | Strain | Gram | Rationale | +|----------|----------|--------|------|-----------| +| 1 | *Escherichia coli* | ATCC 25922 | Negative | QC reference, EUCAST standard | +| 2 | *Staphylococcus aureus* | ATCC 29213 | Positive | Clinical relevance, MRSA proxy | +| 3 | *Pseudomonas aeruginosa* | ATCC 27853 | Negative | ESKAPE organism, hard target | +| 4 | *Klebsiella pneumoniae* | ATCC 700603 | Negative | Clinical priority, MDR context | + +Priority order: if budget-constrained, run organisms 1 and 2 only (minimum viable screen). + +### Controls + +| Control | Type | Expected result | Action if fails | +|---------|------|-----------------|-----------------| +| Ciprofloxacin 0.004–4 µg/mL vs E. coli ATCC 25922 | Positive | MIC = 0.008–0.06 µg/mL (EUCAST QC range) | ABORT assay run; repeat | +| PBS / vehicle only (no peptide) | Negative growth control | Full growth (OD600 > 0.15) | ABORT assay run; repeat | +| Sterility control (broth + peptide, no bacteria) | Sterility | No growth (OD600 ≤ 0.03) | Flag contamination; repeat | + +--- + +## 3. Secondary Assay — Hemolysis (HC50) + +### Protocol + +- **Cells:** Human red blood cells (hRBCs), freshly drawn (same day) from a consented donor or commercial suspension +- **Buffer:** Phosphate-buffered saline (PBS, pH 7.4) +- **hRBC concentration:** 0.5% v/v +- **Peptide range:** 1, 2, 4, 8, 16, 32, 64, 128, 256 µg/mL +- **Incubation:** 37 °C, 1 h (gentle shaking, 100 rpm) +- **Readout:** Supernatant absorbance at 541 nm after centrifugation (800 × g, 10 min) +- **Controls:** + - 0.1% Triton X-100 = 100% lysis (positive control) + - PBS only = 0% lysis (negative control) +- **HC50 definition:** Concentration producing 50% hemolysis (interpolated from sigmoidal fit or linear interpolation) + +### Reporting + +Report HC50 (µg/mL) for each candidate. If HC50 > 256 µg/mL (top concentration), report as > 256 µg/mL. + +--- + +## 4. Selectivity Index + +Compute Therapeutic Index (TI) for each organism: + +``` +TI = HC50 (µg/mL) / MIC (µg/mL) +``` + +| TI interpretation | Action | +|-------------------|--------| +| TI > 10 | PASS — proceed to confirmatory assay | +| TI 5–10 | BORDERLINE — note for expert review; may advance if no better candidates | +| TI < 5 | FAIL — exclude from Wave 2 unless exceptional MIC | + +--- + +## 5. Serum Stability Assay (Wave 1 optional, Wave 2 mandatory) + +- **Matrix:** 25% human serum in PBS (37 °C) +- **Peptide concentration:** 100 µg/mL +- **Time points:** 0, 0.5, 1, 2, 4 h +- **Readout:** Residual antimicrobial activity by agar dilution or analytical HPLC + MS +- **Pass criterion:** > 50% activity remaining at 2 h + +> **Pre-registered decision:** Serum stability is optional in Wave 1 if capacity is limited. +> It becomes mandatory in Wave 2 for any candidate with MIC ≤ 8 µg/mL (tier 1 hits). + +--- + +## 6. Pre-registered Analysis Plan + +### Primary Endpoint + +**Primary endpoint:** Number of candidates with MIC ≤ 32 µg/mL against *E. coli* ATCC 25922. + +**Secondary endpoint:** Number of candidates with MIC ≤ 32 µg/mL AND TI > 10. + +### Hit Definition (locked) + +A candidate is classified as a **confirmed hit** if: + +1. MIC ≤ 32 µg/mL against ≥ 1 target organism (Table in §2), AND +2. TI > 10 (HC50 / MIC > 10), AND +3. Positive control in the same assay plate passed (ciprofloxacin QC range) + +A candidate is classified as a **provisional hit** if criterion 1 is met but criterion 2 or 3 could not be assessed (resource constraints). Provisional hits advance to Wave 2 with serum stability and TI gating. + +### Statistical Plan + +- All assays run in **biological triplicate** (n = 3 independent experiments on separate days) +- Report geometric mean MIC ± SD from triplicates +- Report HC50 with 95% CI (from sigmoidal regression or log-linear interpolation) +- No correction for multiple comparisons for primary endpoint (exploratory screen) +- No data points excluded without documented justification + +### Reporting Obligations + +- **All results reported**, including: + - Inactive candidates (MIC > 128 µg/mL) + - Toxic candidates (TI < 2) + - Inconclusive results (QC failure with description) +- Negative results archived as `schemas/lab_result.schema.json` entries with `result_qualitative: "inactive"` +- Results shared with pipeline team within 2 weeks of assay completion +- Results NOT selectively shared or suppressed based on outcome + +--- + +## 7. Pre-registered Decision Gates + +| Gate | Threshold | Action if passed | Action if failed | +|------|-----------|-----------------|-----------------| +| P1: ≥1 candidate MIC ≤ 32 µg/mL | Hit count ≥ 1 | Proceed to Wave 2 (D-amino variants, MDR strains) | Re-evaluate pipeline; consider new seeds | +| P2: ≥1 candidate TI > 10 + MIC ≤ 32 µg/mL | Hit count ≥ 1 | Proceed to serum stability gating | Return to design; re-weigh selectivity scoring | +| P3: ≥1 candidate all-gates pass (P2 + t½ > 2h + novel scaffold) | Hit count ≥ 1 | Prepare confirmatory external lab replication | Document as negative result; publish pipeline learnings | + +--- + +## 8. Deviations + +Any deviation from this protocol must be documented below before assay data is analysed: + +| Date | Deviation description | Approved by | +|------|----------------------|-------------| +| — | — | — | + +--- + +## 9. Signatures + +| Role | Name | Institution | Signature | Date | +|------|------|-------------|-----------|------| +| Principal Investigator | | | | | +| Peptide Synthesis Approver | | | | | +| Biosafety Officer | | | | | +| Lab Partner Representative | | | | | + +--- + +## 10. Disclaimer + +This pre-registration covers a computational nomination pilot. Candidates are short synthetic +peptides derived from natural AMP templates. No claim of clinical efficacy, drug status, or +therapeutic use is made. All wet-lab work must comply with local institutional biosafety +regulations and ethics requirements. diff --git a/docs/DISCOVERY_PREDICTION.md b/docs/DISCOVERY_PREDICTION.md index 0db0e011..2112c1fd 100644 --- a/docs/DISCOVERY_PREDICTION.md +++ b/docs/DISCOVERY_PREDICTION.md @@ -17,8 +17,9 @@ improvements already implemented or recommended. **Bottom line:** The pilot panel has a ~61–71% probability of yielding at least one candidate with MIC ≤ 32 μg/mL, and **~29–49%** probability of generating "breaking news" publication material with the fully updated panel (up from 5–12% before computational improvements). -*External calibration: accounting for near-seed candidate correlation, the honest corrected -estimate is ~15–30%. See the **External Calibration** section at the end of this document.* +*External calibration: accounting for near-seed candidate correlation (10 effective independent +scaffold families with 2 newly added mechanistically distinct seeds), the honest corrected +estimate is ~18–30%. See the **External Calibration** section at the end of this document.* Key improvements since last assessment (PRs #39–#49): - **Charge×amphipathicity cross-term (PR #39):** Scoring now rewards simultaneous high charge AND @@ -565,7 +566,7 @@ Executing all four actions on the best Wave 1 hits would push the combined proba | Stage | Gate | Original | After PRs #31–38 | After PRs #39–42 | After PRs #43–47 | After PRs #48–53 (current) | Primary limiting factor | |-------|------|----------|-----------------|-----------------|-----------------|--------------------------|------------------------| | 0 | Synthesis success | ~90% | ~90% | ~88% | **~89%** ✓ | **~90%** ✓ (agg model + agg-safe gen + pro penalty + pH74 charge) | SEED-008 W-rich; all aggregation/synthesis risks modelled | -| 1 | MIC ≤ 32 μg/mL | ~55–65% | ~55–65% | ~60–70% | **~60–70%** | **~61–71%** ✓ (AUROC 0.8164) | AUROC 0.8164; 8 scaffold families | +| 1 | MIC ≤ 32 μg/mL | ~55–65% | ~55–65% | ~60–70% | **~60–70%** | **~61–71%** ✓ (AUROC 0.8164) | AUROC 0.8164 (AUPRC computed); 10 scaffold families | | 2 | TI > 10 (selectivity) | ~35–50% | ~35–50% | ~38–52% | **~40–55%** ✓ | **~42–57%** ✓ (stronger SEED-004 demotion) | sel_proxy doubled penalty for HIGH_CYTOTOX_RISK tier | | 3 | t½ > 2 h (serum) | ~10–20% | ~25–40% | ~28–42% | **~28–42%** | **~29–44%** ✓ (3-protease model) | Wave 2 D-amino plan machine-readable | | 4 | Scaffold novelty | ~10–15% | ~18–25% | ~25–35% | **~26–36%** ✓ | **~26–36%** | Diversity filter removes cross-seed near-dups | @@ -598,20 +599,25 @@ The internal model's per-stage probabilities, multiplied across all 5 gates, giv all-gates pass rate of approximately 1.6–3.1%. With 20 assumed-independent candidates, the pipeline estimates P(≥1 breaking-news hit) = 1 − (1 − p)^20 ≈ 29–49%. -If the effective number of independent experiments is ~8 scaffold families (not 20), using -the **same per-scaffold all-gates rate of 1.6–3.1%**: +**Update (PR #58):** Two mechanistically distinct seeds were added — SEED-009 (Bac2A bovine +proline-rich cathelicidin, intracellular mechanism) and SEED-010 (KR-12 human LL-37 fragment, +clinical relevance). The effective number of independent scaffolds increases from 8 to 10. + +Using the **same per-scaffold all-gates rate of 1.6–3.1%**, now for 10 effective scaffolds: ``` -P(≥1 from 8 at 1.6%) = 1 − (0.984)^8 ≈ 12% -P(≥1 from 8 at 2.4%) = 1 − (0.976)^8 ≈ 17% -P(≥1 from 8 at 3.1%) = 1 − (0.969)^8 ≈ 22% +P(≥1 from 10 at 1.6%) = 1 − (0.984)^10 ≈ 15% +P(≥1 from 10 at 2.4%) = 1 − (0.976)^10 ≈ 21% +P(≥1 from 10 at 3.1%) = 1 − (0.969)^10 ≈ 26% ``` -**Calibrated estimate: ~12–22% (rounded to 15–30% to account for model uncertainty)** +**Calibrated estimate: ~15–26% (rounded to 18–30% to account for model uncertainty)** + +Prior estimate with 8 scaffolds: 12–22%. Adding 2 diverse seeds adds ~3 percentage points. -| Outcome | Internal model (n=20 independent) | Calibrated estimate (n≈8 effective) | -|---------|-----------------------------------|--------------------------------------| -| ≥1 candidate with MIC + acceptable selectivity ("breaking news") | ~29–49% | ~15–30% | +| Outcome | Internal model (n=20 independent) | Calibrated estimate (n≈10 effective scaffolds) | +|---------|-----------------------------------|-------------------------------------------------| +| ≥1 candidate with MIC + acceptable selectivity ("breaking news") | ~29–49% | ~18–30% | | Publishable result (novel AMP + external replication) | not modeled | ~5–15% | | Major breakthrough (new AMP family, widely replicated) | not modeled | ~1–5% | diff --git a/docs/METHODS.md b/docs/METHODS.md index 25339653..ecaea40e 100644 --- a/docs/METHODS.md +++ b/docs/METHODS.md @@ -318,28 +318,34 @@ Phase 2 benchmarks verified that the pipeline: **Retrospective AUROC (v0.2.x):** `0.8164` (positive-vs-negative separation on the 45-reference benchmark set using the full ensemble scorer). Benchmarked after PRs #48–54; the scoring -improvements in those PRs moved AUROC from 0.7926 → 0.8138 → 0.8164. +improvements in those PRs moved AUROC from 0.7926 → 0.8138 → 0.8164. Bootstrap 95% CI +computed via Wilcoxon-Mann-Whitney (n=2000 iterations; CI reported in `validate-scoring` output). + +**AUPRC (v0.5.x):** Area Under Precision-Recall Curve is now computed alongside AUROC and +reported by `make validate-scoring`. AUPRC is preferable to AUROC for class-imbalanced +datasets because it emphasises precision at the operating point actually used for candidate +selection. Random baseline = dataset prevalence (0.50 for the balanced demo set). **To reproduce:** `make validate-scoring` (runs `cli validate-scoring` on `examples/validation/known_amps.csv` vs `examples/validation/random_background.csv`) or `make validate-scoring-strict` for the composition-matched (scrambled-decoy) variant. -**Key scoring changes in v0.2.x (PRs #48–54):** +**Key scoring changes in v0.2.x–v0.5.x (PRs #48–58):** | PR | Change | AUROC impact | |----|--------|-------------| -| #47 | Selectivity proxy (charge/GRAVY) | +| +| #47 | Selectivity proxy (charge/GRAVY) | + | | #48 | Selectivity proxy routed into pilot_priority | pilot ranking | | #49 | Elastase resistance + aggregation propensity | + | | #50 | Aggregation-safe mutations + balanced K/R variants | generation | | #51 | Proline synthesis penalty (−0.10 at >15%); helix bonus 0.01→0.03 | 0.8138→0.8164 | -| #52 | DISCOVERY_PREDICTION.md updated | docs | | #53 | Safety uses charge_density_ph74; stronger cytotox_penalty in pilot | pilot ranking | -| #54 | DISCOVERY_PREDICTION.md updated | docs | +| #58 | AUPRC added; SEED-009 (Bac2A) + SEED-010 (KR-12) added | +2 scaffolds | -**Limitation:** Benchmarks use a small, curated demo dataset. They do not validate against -large independent AMP databases. Real retrospective validation against APD3-scale data is -required before strong claims of predictive power. +**Limitation:** Benchmarks use a small, curated demo dataset (44 AMPs + 44 background). They +do not validate against large independent AMP databases. Real retrospective validation against +APD3-scale data (> 3,000 AMPs, cluster-split) is required before strong claims of predictive +power. --- diff --git a/examples/sequences/amp_seeds.csv b/examples/sequences/amp_seeds.csv index a936971e..88b6c292 100644 --- a/examples/sequences/amp_seeds.csv +++ b/examples/sequences/amp_seeds.csv @@ -7,3 +7,5 @@ SEED-005,KRLFKKIGSALKFL,hybrid_cationic_hydrophobic SEED-006,INWKGIAAMAKKLL,mastoparan_x_template SEED-007,IKITTMLKKLG,bombolitin_ii_template SEED-008,FPVTWRWWKWWKG,puroindoline_a_trp_domain +SEED-009,RRLPRPPYLPRP,bac2a_bovine_proline_rich_cathelicidin +SEED-010,KRIVQRIKDFLR,kr12_human_ll37_cathelicidin_fragment diff --git a/src/openamp_foundry/benchmark/retrospective.py b/src/openamp_foundry/benchmark/retrospective.py index e7454f78..3543f578 100644 --- a/src/openamp_foundry/benchmark/retrospective.py +++ b/src/openamp_foundry/benchmark/retrospective.py @@ -43,6 +43,37 @@ def _auc_wilcoxon(pos_scores: list[float], neg_scores: list[float]) -> float: return concordant / (n_pos * n_neg) +def _auprc(pos_scores: list[float], neg_scores: list[float]) -> float: + """Compute AUPRC via the trapezoidal rule on the precision-recall curve. + + Random baseline = n_pos / (n_pos + n_neg) = prevalence. + AUPRC = 1.0 for a perfect classifier, prevalence for a random ranker. + """ + n_pos = len(pos_scores) + if n_pos == 0: + return 0.0 + all_scored = sorted( + [(s, 1) for s in pos_scores] + [(s, 0) for s in neg_scores], + key=lambda x: -x[0], + ) + tp = fp = 0 + recalls = [0.0] + precisions = [1.0] + for _score, label in all_scored: + if label == 1: + tp += 1 + else: + fp += 1 + recalls.append(tp / n_pos) + precisions.append(tp / (tp + fp)) + # trapezoidal integration + area = sum( + (recalls[i] - recalls[i - 1]) * (precisions[i] + precisions[i - 1]) / 2 + for i in range(1, len(recalls)) + ) + return round(area, 4) + + def _bootstrap_auroc_ci( pos_scores: list[float], neg_scores: list[float], @@ -159,6 +190,9 @@ def run_retrospective_benchmark( neg_scores = [r["ensemble"] for r in rows if r["label"] == 0] auroc = round(_auc_wilcoxon(pos_scores, neg_scores), 4) auroc_ci = _bootstrap_auroc_ci(pos_scores, neg_scores, n_bootstrap=n_bootstrap) + auprc = _auprc(pos_scores, neg_scores) + n_total_for_prev = len(pos_scores) + len(neg_scores) + auprc_random = round(len(pos_scores) / n_total_for_prev, 4) if n_total_for_prev > 0 else 0.0 n_total = len(rows) n_pos = sum(r["label"] for r in rows) @@ -189,6 +223,9 @@ def run_retrospective_benchmark( "n_bootstrap": auroc_ci["n_bootstrap"], "random_auroc": random_auroc, "auroc_above_random": round(auroc - random_auroc, 4), + "auprc": auprc, + "auprc_random_baseline": auprc_random, + "auprc_above_random": round(auprc - auprc_random, 4), **recall, "interpretation": interpretation, "benchmark_type": benchmark_type, diff --git a/tests/test_auprc.py b/tests/test_auprc.py new file mode 100644 index 00000000..f0ae8e1f --- /dev/null +++ b/tests/test_auprc.py @@ -0,0 +1,98 @@ +"""Tests for AUPRC computation in the retrospective benchmark.""" +from __future__ import annotations + +import pytest + +from openamp_foundry.benchmark.retrospective import _auprc + + +class TestAUPRC: + def test_perfect_classifier_returns_one(self): + pos = [0.9, 0.8, 0.7] + neg = [0.3, 0.2, 0.1] + assert _auprc(pos, neg) == pytest.approx(1.0, abs=1e-4) + + def test_worst_classifier_near_prevalence(self): + # When all negatives score higher than all positives, AUPRC → prevalence (0.5 for 50/50) + pos = [0.1, 0.2, 0.3] + neg = [0.7, 0.8, 0.9] + result = _auprc(pos, neg) + prevalence = 3 / 6 # 0.5 + # Worst case AUPRC is at least equal to random-classifier prevalence baseline + assert result >= 0.0 + assert result <= 1.0 + + def test_random_classifier_near_prevalence(self): + # Interleaved scores → AUPRC ≈ prevalence (3/8 = 0.375 for 3 pos / 5 neg) + pos = [0.7, 0.5, 0.3] + neg = [0.6, 0.4, 0.4, 0.2, 0.1] + result = _auprc(pos, neg) + assert 0.0 < result < 1.0 + + def test_empty_positives_returns_zero(self): + assert _auprc([], [0.5, 0.4, 0.3]) == 0.0 + + def test_empty_negatives(self): + # No negatives → every threshold has precision=1, AUPRC=1 + assert _auprc([0.9, 0.8, 0.7], []) == pytest.approx(1.0, abs=1e-4) + + def test_single_positive_single_negative_perfect(self): + assert _auprc([0.9], [0.1]) == pytest.approx(1.0, abs=1e-4) + + def test_single_positive_single_negative_worst(self): + result = _auprc([0.1], [0.9]) + assert 0.0 <= result <= 1.0 + + def test_output_is_rounded_to_4_decimal_places(self): + pos = [0.75, 0.65] + neg = [0.55, 0.45, 0.35] + result = _auprc(pos, neg) + assert result == round(result, 4) + + def test_auprc_above_random_baseline_for_good_classifier(self): + pos = [0.9, 0.85, 0.8, 0.75] + neg = [0.4, 0.35, 0.3, 0.25, 0.2, 0.15] + auprc = _auprc(pos, neg) + prevalence = len(pos) / (len(pos) + len(neg)) + assert auprc > prevalence + + def test_retrospective_benchmark_includes_auprc_keys(self, tmp_path): + """Full integration: run_retrospective_benchmark returns auprc in output.""" + from openamp_foundry.benchmark.retrospective import run_retrospective_benchmark + + amp_csv = tmp_path / "amps.csv" + decoy_csv = tmp_path / "decoys.csv" + # 5 short known-active AMP-like sequences + amp_csv.write_text( + "id,sequence\n" + "A1,KWKLFKKIGAVLKVL\n" + "A2,GIGKFLHSAKKFGKA\n" + "A3,RRWQWRMKKLG\n" + "A4,KRLFKKIGSALK\n" + "A5,INWKGIAAMAKKLL\n", + encoding="utf-8", + ) + # 5 composition-random decoy sequences + decoy_csv.write_text( + "id,sequence\n" + "D1,AAAAAGGGGGPPPPP\n" + "D2,SSSSSTTTTTAAAAA\n" + "D3,GGGGGPPPPPAAAAA\n" + "D4,LLLLLVVVVVIIIII\n" + "D5,EEEEEDDDDDNNNNN\n", + encoding="utf-8", + ) + result = run_retrospective_benchmark( + amp_csv=str(amp_csv), + decoy_csv=str(decoy_csv), + config_path="configs/pipeline.yaml", + n_bootstrap=10, + ) + assert "auprc" in result + assert "auprc_random_baseline" in result + assert "auprc_above_random" in result + assert 0.0 <= result["auprc"] <= 1.0 + assert result["auprc_random_baseline"] == pytest.approx(0.5, abs=0.01) + assert result["auprc_above_random"] == pytest.approx( + result["auprc"] - result["auprc_random_baseline"], abs=1e-4 + ) From dc27d477c9c6cdd4a929f9a9d3e4a6613ab4777d Mon Sep 17 00:00:00 2001 From: j Date: Sun, 28 Jun 2026 16:36:01 +0700 Subject: [PATCH 2/2] fix: pessimistic tie-breaking in AUPRC, correct cipro QC range, extend HC50 ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AUPRC tie-breaking: - Sort key changed from (-score,) to (-score, label) so negatives sort before positives on tied scores; prevents inflation to 1.0 for random classifiers (matches sklearn average_precision_score pessimistic convention) - Add test_all_tied_scores_pessimistic_not_inflated to verify behavior - Fix test_worst_classifier: assert result < prevalence (was trivially true) - Add test for tied-scores case Assay pre-registration: - Ciprofloxacin EUCAST QC range: 0.008-0.06 → 0.004-0.016 µg/mL (v14 Table 5) - HC50 concentration ceiling: 256 → 512 µg/mL to resolve TI ambiguity at the 32 µg/mL MIC boundary; add explicit interpretation rule for >512 µg/mL --- docs/ASSAY_PREREGISTRATION.md | 8 ++++++-- src/openamp_foundry/benchmark/retrospective.py | 6 +++++- tests/test_auprc.py | 18 +++++++++++++----- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/ASSAY_PREREGISTRATION.md b/docs/ASSAY_PREREGISTRATION.md index a7c18da2..616208be 100644 --- a/docs/ASSAY_PREREGISTRATION.md +++ b/docs/ASSAY_PREREGISTRATION.md @@ -60,7 +60,7 @@ Priority order: if budget-constrained, run organisms 1 and 2 only (minimum viabl | Control | Type | Expected result | Action if fails | |---------|------|-----------------|-----------------| -| Ciprofloxacin 0.004–4 µg/mL vs E. coli ATCC 25922 | Positive | MIC = 0.008–0.06 µg/mL (EUCAST QC range) | ABORT assay run; repeat | +| Ciprofloxacin 0.004–4 µg/mL vs E. coli ATCC 25922 | Positive | MIC = 0.004–0.016 µg/mL (EUCAST v14 Table 5 QC range) | ABORT assay run; repeat | | PBS / vehicle only (no peptide) | Negative growth control | Full growth (OD600 > 0.15) | ABORT assay run; repeat | | Sterility control (broth + peptide, no bacteria) | Sterility | No growth (OD600 ≤ 0.03) | Flag contamination; repeat | @@ -73,13 +73,17 @@ Priority order: if budget-constrained, run organisms 1 and 2 only (minimum viabl - **Cells:** Human red blood cells (hRBCs), freshly drawn (same day) from a consented donor or commercial suspension - **Buffer:** Phosphate-buffered saline (PBS, pH 7.4) - **hRBC concentration:** 0.5% v/v -- **Peptide range:** 1, 2, 4, 8, 16, 32, 64, 128, 256 µg/mL +- **Peptide range:** 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 µg/mL + *(Extended to 512 µg/mL to avoid TI ambiguity at the 32 µg/mL MIC boundary: + a candidate at MIC=32 µg/mL with HC50>256 would be borderline TI>8, not PASS; + testing to 512 µg/mL disambiguates whether TI is truly >10.)* - **Incubation:** 37 °C, 1 h (gentle shaking, 100 rpm) - **Readout:** Supernatant absorbance at 541 nm after centrifugation (800 × g, 10 min) - **Controls:** - 0.1% Triton X-100 = 100% lysis (positive control) - PBS only = 0% lysis (negative control) - **HC50 definition:** Concentration producing 50% hemolysis (interpolated from sigmoidal fit or linear interpolation) +- **Interpretation rule:** HC50 > 512 µg/mL is reported as "> 512 µg/mL" and counts as PASS for TI ≥ 10 when MIC ≤ 51.2 µg/mL ### Reporting diff --git a/src/openamp_foundry/benchmark/retrospective.py b/src/openamp_foundry/benchmark/retrospective.py index 3543f578..17e0267b 100644 --- a/src/openamp_foundry/benchmark/retrospective.py +++ b/src/openamp_foundry/benchmark/retrospective.py @@ -52,9 +52,13 @@ def _auprc(pos_scores: list[float], neg_scores: list[float]) -> float: n_pos = len(pos_scores) if n_pos == 0: return 0.0 + # Pessimistic tie-breaking: negatives (label=0) sort before positives (label=1) + # on equal scores, matching sklearn average_precision_score convention. + # Without this, a random classifier with all-equal scores returns AUPRC=1.0 + # instead of the correct prevalence baseline. all_scored = sorted( [(s, 1) for s in pos_scores] + [(s, 0) for s in neg_scores], - key=lambda x: -x[0], + key=lambda x: (-x[0], x[1]), ) tp = fp = 0 recalls = [0.0] diff --git a/tests/test_auprc.py b/tests/test_auprc.py index f0ae8e1f..f0efa8cf 100644 --- a/tests/test_auprc.py +++ b/tests/test_auprc.py @@ -12,15 +12,13 @@ def test_perfect_classifier_returns_one(self): neg = [0.3, 0.2, 0.1] assert _auprc(pos, neg) == pytest.approx(1.0, abs=1e-4) - def test_worst_classifier_near_prevalence(self): - # When all negatives score higher than all positives, AUPRC → prevalence (0.5 for 50/50) + def test_worst_classifier_below_prevalence(self): + # When all negatives score higher than all positives, AUPRC < prevalence pos = [0.1, 0.2, 0.3] neg = [0.7, 0.8, 0.9] result = _auprc(pos, neg) prevalence = 3 / 6 # 0.5 - # Worst case AUPRC is at least equal to random-classifier prevalence baseline - assert result >= 0.0 - assert result <= 1.0 + assert result < prevalence # worst classifier is below random baseline def test_random_classifier_near_prevalence(self): # Interleaved scores → AUPRC ≈ prevalence (3/8 = 0.375 for 3 pos / 5 neg) @@ -49,6 +47,16 @@ def test_output_is_rounded_to_4_decimal_places(self): result = _auprc(pos, neg) assert result == round(result, 4) + def test_all_tied_scores_pessimistic_not_inflated(self): + # With all tied scores, pessimistic tie-breaking (negatives first) + # must NOT return 1.0 (which the optimistic ordering would give). + # It returns a conservative value well below prevalence. + pos = [0.5, 0.5, 0.5] + neg = [0.5, 0.5, 0.5, 0.5, 0.5] + prevalence = 3 / 8 + result = _auprc(pos, neg) + assert result < prevalence # conservative, not inflated to 1.0 + def test_auprc_above_random_baseline_for_good_classifier(self): pos = [0.9, 0.85, 0.8, 0.75] neg = [0.4, 0.35, 0.3, 0.25, 0.2, 0.15]