diff --git a/src/openamp_foundry/evidence/hit_confirmation_report.py b/src/openamp_foundry/evidence/hit_confirmation_report.py new file mode 100644 index 00000000..4340855c --- /dev/null +++ b/src/openamp_foundry/evidence/hit_confirmation_report.py @@ -0,0 +1,197 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from typing import List, Optional + + +VALID_CONFIRMATION_VERDICTS = frozenset({ + "confirmed_hit", + "partial_hit", + "not_confirmed", + "inconclusive", +}) + +VALID_DIVERGENCE_TYPES = frozenset({ + "mic_above_predicted_range", + "mic_below_predicted_range", + "activity_spectrum_mismatch", + "selectivity_mismatch", + "hemolysis_above_predicted", + "no_divergence", + "multiple_divergences", +}) + +VALID_PREDICTION_QUALITY_GRADES = frozenset({ + "A", # prediction matched outcome within expected range + "B", # minor deviation, within 2x + "C", # significant deviation, 2-10x + "D", # major divergence, >10x or qualitative mismatch + "N/A", # insufficient data to grade +}) + + +@dataclass +class HitConfirmationReport: + hcr_id: str + candidate_id: str + sequence: str + whr_ids: List[str] + pre_registration_id: Optional[str] + dry_lab_score: float + confirmation_verdict: str + divergence_types: List[str] + prediction_quality_grade: str + n_experiments: int + n_confirmed: int + n_inactive: int + n_inconclusive: int + dry_lab_only: bool + limitations: str + notes: str = "" + + +@dataclass +class HCRValidationResult: + valid: bool + violations: List[str] = field(default_factory=list) + + +def validate_hit_confirmation_report(report: HitConfirmationReport) -> HCRValidationResult: + violations = [] + + if not report.hcr_id.startswith("HCR-"): + violations.append("hcr_id must start with 'HCR-'") + + if not report.candidate_id: + violations.append("candidate_id is required") + + if report.candidate_id.startswith("TOY-"): + violations.append("TOY- candidate IDs are not allowed in real HCR- records") + + if not report.sequence or len(report.sequence) < 5: + violations.append("sequence must be at least 5 amino acids") + + if not report.whr_ids: + violations.append("at least one whr_id is required") + + for wid in report.whr_ids: + if not wid.startswith("WHR-"): + violations.append(f"whr_id '{wid}' must start with 'WHR-'") + + if report.dry_lab_score < 0.0 or report.dry_lab_score > 1.0: + violations.append("dry_lab_score must be in [0.0, 1.0]") + + if report.confirmation_verdict not in VALID_CONFIRMATION_VERDICTS: + violations.append( + f"confirmation_verdict '{report.confirmation_verdict}' must be one of {sorted(VALID_CONFIRMATION_VERDICTS)}" + ) + + for dt in report.divergence_types: + if dt not in VALID_DIVERGENCE_TYPES: + violations.append( + f"divergence_type '{dt}' must be one of {sorted(VALID_DIVERGENCE_TYPES)}" + ) + + if report.prediction_quality_grade not in VALID_PREDICTION_QUALITY_GRADES: + violations.append( + f"prediction_quality_grade '{report.prediction_quality_grade}' must be one of {sorted(VALID_PREDICTION_QUALITY_GRADES)}" + ) + + if report.n_experiments < 1: + violations.append("n_experiments must be >= 1") + + if report.n_confirmed < 0: + violations.append("n_confirmed must be >= 0") + + if report.n_inactive < 0: + violations.append("n_inactive must be >= 0") + + if report.n_inconclusive < 0: + violations.append("n_inconclusive must be >= 0") + + total = report.n_confirmed + report.n_inactive + report.n_inconclusive + if total > report.n_experiments: + violations.append( + f"n_confirmed ({report.n_confirmed}) + n_inactive ({report.n_inactive}) + n_inconclusive ({report.n_inconclusive}) = {total} exceeds n_experiments ({report.n_experiments})" + ) + + if report.dry_lab_only: + violations.append("dry_lab_only must be False for HCR- records (real experimental outcomes required)") + + if not report.limitations or len(report.limitations.strip()) < 10: + violations.append("limitations must be a non-empty string (at least 10 characters)") + + # Consistency: confirmed_hit requires at least one confirmed experiment + if report.confirmation_verdict == "confirmed_hit" and report.n_confirmed < 1: + violations.append("confirmation_verdict='confirmed_hit' requires n_confirmed >= 1") + + # Consistency: not_confirmed requires n_inactive >= 1 + if report.confirmation_verdict == "not_confirmed" and report.n_inactive < 1: + violations.append("confirmation_verdict='not_confirmed' requires n_inactive >= 1") + + # Consistency: no_divergence cannot coexist with other divergence types + if "no_divergence" in report.divergence_types and len(report.divergence_types) > 1: + violations.append("divergence_type 'no_divergence' cannot coexist with other divergence types") + + return HCRValidationResult(valid=len(violations) == 0, violations=violations) + + +def build_hit_confirmation_report( + hcr_id: str, + candidate_id: str, + sequence: str, + whr_ids: List[str], + dry_lab_score: float, + confirmation_verdict: str, + divergence_types: List[str], + prediction_quality_grade: str, + n_experiments: int, + n_confirmed: int, + n_inactive: int, + n_inconclusive: int, + limitations: str, + pre_registration_id: Optional[str] = None, + notes: str = "", +) -> HitConfirmationReport: + report = HitConfirmationReport( + hcr_id=hcr_id, + candidate_id=candidate_id, + sequence=sequence, + whr_ids=whr_ids, + pre_registration_id=pre_registration_id, + dry_lab_score=dry_lab_score, + confirmation_verdict=confirmation_verdict, + divergence_types=divergence_types, + prediction_quality_grade=prediction_quality_grade, + n_experiments=n_experiments, + n_confirmed=n_confirmed, + n_inactive=n_inactive, + n_inconclusive=n_inconclusive, + dry_lab_only=False, + limitations=limitations, + notes=notes, + ) + result = validate_hit_confirmation_report(report) + if not result.valid: + raise ValueError(f"Invalid HCR: {result.violations}") + return report + + +def format_hit_confirmation_report(report: HitConfirmationReport) -> str: + lines = [ + f"Hit Confirmation Report — {report.hcr_id}", + f"Candidate: {report.candidate_id} | Sequence: {report.sequence}", + f"Dry-Lab Score: {report.dry_lab_score:.3f}", + f"Verdict: {report.confirmation_verdict}", + f"Prediction Quality: {report.prediction_quality_grade}", + f"Experiments: {report.n_experiments} total ({report.n_confirmed} confirmed, {report.n_inactive} inactive, {report.n_inconclusive} inconclusive)", + f"WHR Records: {', '.join(report.whr_ids)}", + ] + if report.pre_registration_id: + lines.append(f"Pre-Registration: {report.pre_registration_id}") + if report.divergence_types: + lines.append(f"Divergences: {', '.join(report.divergence_types)}") + lines.append(f"Limitations: {report.limitations}") + if report.notes: + lines.append(f"Notes: {report.notes}") + lines.append("dry_lab_only: False (real experimental outcomes)") + return "\n".join(lines) diff --git a/tests/evidence/test_hit_confirmation_report.py b/tests/evidence/test_hit_confirmation_report.py new file mode 100644 index 00000000..beca7b66 --- /dev/null +++ b/tests/evidence/test_hit_confirmation_report.py @@ -0,0 +1,490 @@ +import pytest +from src.openamp_foundry.evidence.hit_confirmation_report import ( + VALID_CONFIRMATION_VERDICTS, + VALID_DIVERGENCE_TYPES, + VALID_PREDICTION_QUALITY_GRADES, + HitConfirmationReport, + HCRValidationResult, + validate_hit_confirmation_report, + build_hit_confirmation_report, + format_hit_confirmation_report, +) + + +def _valid_report(**kwargs): + defaults = dict( + hcr_id="HCR-0001", + candidate_id="AMPF-0001", + sequence="KWKLFKKIEK", + whr_ids=["WHR-0001"], + pre_registration_id="PRE-0001", + dry_lab_score=0.85, + confirmation_verdict="confirmed_hit", + divergence_types=["no_divergence"], + prediction_quality_grade="A", + n_experiments=3, + n_confirmed=2, + n_inactive=1, + n_inconclusive=0, + dry_lab_only=False, + limitations="Single lab, single strain, not generalizable.", + notes="", + ) + defaults.update(kwargs) + return HitConfirmationReport(**defaults) + + +# ── Constants ────────────────────────────────────────────────────────────────── + +class TestConstants: + def test_valid_confirmation_verdicts_is_frozenset(self): + assert isinstance(VALID_CONFIRMATION_VERDICTS, frozenset) + + def test_valid_confirmation_verdicts_count(self): + assert len(VALID_CONFIRMATION_VERDICTS) == 4 + + def test_confirmed_hit_in_verdicts(self): + assert "confirmed_hit" in VALID_CONFIRMATION_VERDICTS + + def test_partial_hit_in_verdicts(self): + assert "partial_hit" in VALID_CONFIRMATION_VERDICTS + + def test_not_confirmed_in_verdicts(self): + assert "not_confirmed" in VALID_CONFIRMATION_VERDICTS + + def test_inconclusive_in_verdicts(self): + assert "inconclusive" in VALID_CONFIRMATION_VERDICTS + + def test_valid_divergence_types_is_frozenset(self): + assert isinstance(VALID_DIVERGENCE_TYPES, frozenset) + + def test_valid_divergence_types_count(self): + assert len(VALID_DIVERGENCE_TYPES) == 7 + + def test_no_divergence_in_divergence_types(self): + assert "no_divergence" in VALID_DIVERGENCE_TYPES + + def test_mic_above_predicted_range_in_divergence_types(self): + assert "mic_above_predicted_range" in VALID_DIVERGENCE_TYPES + + def test_multiple_divergences_in_divergence_types(self): + assert "multiple_divergences" in VALID_DIVERGENCE_TYPES + + def test_valid_prediction_quality_grades_is_frozenset(self): + assert isinstance(VALID_PREDICTION_QUALITY_GRADES, frozenset) + + def test_prediction_quality_grades_count(self): + assert len(VALID_PREDICTION_QUALITY_GRADES) == 5 + + def test_grade_a_in_quality_grades(self): + assert "A" in VALID_PREDICTION_QUALITY_GRADES + + def test_grade_na_in_quality_grades(self): + assert "N/A" in VALID_PREDICTION_QUALITY_GRADES + + +# ── Dataclass ────────────────────────────────────────────────────────────────── + +class TestHitConfirmationReportDataclass: + def test_instantiation(self): + r = _valid_report() + assert r.hcr_id == "HCR-0001" + + def test_whr_ids_list(self): + r = _valid_report(whr_ids=["WHR-0001", "WHR-0002"]) + assert len(r.whr_ids) == 2 + + def test_dry_lab_only_false(self): + r = _valid_report() + assert r.dry_lab_only is False + + def test_pre_registration_id_optional(self): + r = _valid_report(pre_registration_id=None) + assert r.pre_registration_id is None + + def test_notes_default_empty(self): + r = _valid_report(notes="") + assert r.notes == "" + + def test_divergence_types_list(self): + r = _valid_report(divergence_types=["mic_above_predicted_range"]) + assert "mic_above_predicted_range" in r.divergence_types + + +# ── Validation ───────────────────────────────────────────────────────────────── + +class TestValidateHitConfirmationReport: + def test_valid_report_passes(self): + r = _valid_report() + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_invalid_hcr_id_prefix(self): + r = _valid_report(hcr_id="BAD-0001") + result = validate_hit_confirmation_report(r) + assert not result.valid + assert any("hcr_id" in v for v in result.violations) + + def test_empty_candidate_id_blocked(self): + r = _valid_report(candidate_id="") + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_toy_candidate_id_blocked(self): + r = _valid_report(candidate_id="TOY-0001") + result = validate_hit_confirmation_report(r) + assert not result.valid + assert any("TOY-" in v for v in result.violations) + + def test_short_sequence_blocked(self): + r = _valid_report(sequence="KWK") + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_empty_whr_ids_blocked(self): + r = _valid_report(whr_ids=[]) + result = validate_hit_confirmation_report(r) + assert not result.valid + assert any("whr_id" in v for v in result.violations) + + def test_invalid_whr_id_prefix_blocked(self): + r = _valid_report(whr_ids=["BAD-0001"]) + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_dry_lab_score_above_1_blocked(self): + r = _valid_report(dry_lab_score=1.5) + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_dry_lab_score_negative_blocked(self): + r = _valid_report(dry_lab_score=-0.1) + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_invalid_confirmation_verdict_blocked(self): + r = _valid_report(confirmation_verdict="definitely_active") + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_invalid_divergence_type_blocked(self): + r = _valid_report(divergence_types=["unknown_divergence"]) + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_invalid_prediction_quality_grade_blocked(self): + r = _valid_report(prediction_quality_grade="E") + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_n_experiments_zero_blocked(self): + r = _valid_report(n_experiments=0) + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_n_confirmed_negative_blocked(self): + r = _valid_report(n_confirmed=-1) + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_counts_exceed_n_experiments_blocked(self): + r = _valid_report(n_experiments=2, n_confirmed=2, n_inactive=1, n_inconclusive=0) + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_dry_lab_only_true_blocked(self): + r = _valid_report(dry_lab_only=True) + result = validate_hit_confirmation_report(r) + assert not result.valid + assert any("dry_lab_only" in v for v in result.violations) + + def test_empty_limitations_blocked(self): + r = _valid_report(limitations="") + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_short_limitations_blocked(self): + r = _valid_report(limitations="Too short") + result = validate_hit_confirmation_report(r) + assert not result.valid + + def test_confirmed_hit_requires_n_confirmed_ge_1(self): + r = _valid_report(confirmation_verdict="confirmed_hit", n_confirmed=0, n_inactive=2, n_inconclusive=1) + result = validate_hit_confirmation_report(r) + assert not result.valid + assert any("n_confirmed" in v for v in result.violations) + + def test_not_confirmed_requires_n_inactive_ge_1(self): + r = _valid_report(confirmation_verdict="not_confirmed", n_confirmed=0, n_inactive=0, n_inconclusive=3) + result = validate_hit_confirmation_report(r) + assert not result.valid + assert any("n_inactive" in v for v in result.violations) + + def test_no_divergence_cannot_coexist_with_other_types(self): + r = _valid_report(divergence_types=["no_divergence", "mic_above_predicted_range"]) + result = validate_hit_confirmation_report(r) + assert not result.valid + assert any("no_divergence" in v for v in result.violations) + + def test_multiple_whr_ids_valid(self): + r = _valid_report(whr_ids=["WHR-0001", "WHR-0002", "WHR-0003"]) + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_dry_lab_score_boundary_0(self): + r = _valid_report(dry_lab_score=0.0) + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_dry_lab_score_boundary_1(self): + r = _valid_report(dry_lab_score=1.0) + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_partial_hit_verdict_valid(self): + r = _valid_report(confirmation_verdict="partial_hit", n_confirmed=1, n_inactive=1, n_inconclusive=1) + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_inconclusive_verdict_valid(self): + r = _valid_report(confirmation_verdict="inconclusive", n_confirmed=0, n_inactive=0, n_inconclusive=3) + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_multiple_divergence_types_valid(self): + r = _valid_report(divergence_types=["mic_above_predicted_range", "selectivity_mismatch"]) + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_grade_b_valid(self): + r = _valid_report(prediction_quality_grade="B") + result = validate_hit_confirmation_report(r) + assert result.valid + + def test_grade_na_valid(self): + r = _valid_report(prediction_quality_grade="N/A") + result = validate_hit_confirmation_report(r) + assert result.valid + + +# ── Build ────────────────────────────────────────────────────────────────────── + +class TestBuildHitConfirmationReport: + def test_build_valid(self): + r = build_hit_confirmation_report( + hcr_id="HCR-0001", + candidate_id="AMPF-0001", + sequence="KWKLFKKIEK", + whr_ids=["WHR-0001"], + dry_lab_score=0.85, + confirmation_verdict="confirmed_hit", + divergence_types=["no_divergence"], + prediction_quality_grade="A", + n_experiments=3, + n_confirmed=2, + n_inactive=1, + n_inconclusive=0, + limitations="Single lab, single strain, not generalizable.", + ) + assert r.hcr_id == "HCR-0001" + assert r.dry_lab_only is False + + def test_build_enforces_dry_lab_only_false(self): + r = build_hit_confirmation_report( + hcr_id="HCR-0002", + candidate_id="AMPF-0002", + sequence="FLPKLKKLLK", + whr_ids=["WHR-0002"], + dry_lab_score=0.72, + confirmation_verdict="partial_hit", + divergence_types=["mic_above_predicted_range"], + prediction_quality_grade="C", + n_experiments=2, + n_confirmed=1, + n_inactive=1, + n_inconclusive=0, + limitations="Limited to E. coli K-12, results not extrapolatable.", + ) + assert r.dry_lab_only is False + + def test_build_raises_on_toy_id(self): + with pytest.raises(ValueError): + build_hit_confirmation_report( + hcr_id="HCR-0003", + candidate_id="TOY-0001", + sequence="KWKLFKKIEK", + whr_ids=["WHR-0001"], + dry_lab_score=0.85, + confirmation_verdict="confirmed_hit", + divergence_types=["no_divergence"], + prediction_quality_grade="A", + n_experiments=1, + n_confirmed=1, + n_inactive=0, + n_inconclusive=0, + limitations="Single lab, single strain.", + ) + + def test_build_raises_on_invalid_verdict(self): + with pytest.raises(ValueError): + build_hit_confirmation_report( + hcr_id="HCR-0004", + candidate_id="AMPF-0001", + sequence="KWKLFKKIEK", + whr_ids=["WHR-0001"], + dry_lab_score=0.85, + confirmation_verdict="probably_active", + divergence_types=["no_divergence"], + prediction_quality_grade="A", + n_experiments=1, + n_confirmed=1, + n_inactive=0, + n_inconclusive=0, + limitations="Single lab, single strain.", + ) + + def test_build_with_pre_registration_id(self): + r = build_hit_confirmation_report( + hcr_id="HCR-0005", + candidate_id="AMPF-0005", + sequence="GIKCKILKKLR", + whr_ids=["WHR-0005"], + dry_lab_score=0.91, + confirmation_verdict="confirmed_hit", + divergence_types=["no_divergence"], + prediction_quality_grade="A", + n_experiments=4, + n_confirmed=4, + n_inactive=0, + n_inconclusive=0, + limitations="Single lab, reproducibility not independently verified.", + pre_registration_id="PRE-0001", + ) + assert r.pre_registration_id == "PRE-0001" + + def test_build_raises_on_empty_whr_ids(self): + with pytest.raises(ValueError): + build_hit_confirmation_report( + hcr_id="HCR-0006", + candidate_id="AMPF-0006", + sequence="KWKLFKKIEK", + whr_ids=[], + dry_lab_score=0.85, + confirmation_verdict="confirmed_hit", + divergence_types=["no_divergence"], + prediction_quality_grade="A", + n_experiments=1, + n_confirmed=1, + n_inactive=0, + n_inconclusive=0, + limitations="Single lab, single strain.", + ) + + def test_build_not_confirmed_verdict(self): + r = build_hit_confirmation_report( + hcr_id="HCR-0007", + candidate_id="AMPF-0007", + sequence="KWKLFKKIEK", + whr_ids=["WHR-0007"], + dry_lab_score=0.60, + confirmation_verdict="not_confirmed", + divergence_types=["mic_above_predicted_range"], + prediction_quality_grade="D", + n_experiments=3, + n_confirmed=0, + n_inactive=3, + n_inconclusive=0, + limitations="Multiple labs, consistent result, generalizability unknown.", + ) + assert r.confirmation_verdict == "not_confirmed" + + +# ── Format ───────────────────────────────────────────────────────────────────── + +class TestFormatHitConfirmationReport: + def _build(self, **kwargs): + defaults = dict( + hcr_id="HCR-0001", + candidate_id="AMPF-0001", + sequence="KWKLFKKIEK", + whr_ids=["WHR-0001"], + dry_lab_score=0.85, + confirmation_verdict="confirmed_hit", + divergence_types=["no_divergence"], + prediction_quality_grade="A", + n_experiments=3, + n_confirmed=2, + n_inactive=1, + n_inconclusive=0, + limitations="Single lab, single strain, not generalizable.", + ) + defaults.update(kwargs) + return build_hit_confirmation_report(**defaults) + + def test_format_returns_string(self): + r = self._build() + assert isinstance(format_hit_confirmation_report(r), str) + + def test_format_contains_hcr_id(self): + r = self._build() + assert "HCR-0001" in format_hit_confirmation_report(r) + + def test_format_contains_candidate_id(self): + r = self._build() + assert "AMPF-0001" in format_hit_confirmation_report(r) + + def test_format_contains_verdict(self): + r = self._build() + assert "confirmed_hit" in format_hit_confirmation_report(r) + + def test_format_contains_dry_lab_score(self): + r = self._build() + assert "0.850" in format_hit_confirmation_report(r) + + def test_format_contains_dry_lab_only_false(self): + r = self._build() + assert "dry_lab_only: False" in format_hit_confirmation_report(r) + + def test_format_contains_prediction_quality(self): + r = self._build() + assert "A" in format_hit_confirmation_report(r) + + def test_format_contains_limitations(self): + r = self._build() + assert "Single lab" in format_hit_confirmation_report(r) + + def test_format_contains_whr_ids(self): + r = self._build() + assert "WHR-0001" in format_hit_confirmation_report(r) + + def test_format_contains_pre_registration_id(self): + r = self._build(pre_registration_id="PRE-0001") + assert "PRE-0001" in format_hit_confirmation_report(r) + + def test_format_omits_pre_registration_when_none(self): + r = self._build() + out = format_hit_confirmation_report(r) + assert "Pre-Registration" not in out + + def test_format_multiline(self): + r = self._build() + lines = format_hit_confirmation_report(r).split("\n") + assert len(lines) >= 7 + + def test_format_not_confirmed_verdict(self): + r = self._build( + hcr_id="HCR-0007", + candidate_id="AMPF-0007", + sequence="KWKLFKKIEK", + whr_ids=["WHR-0007"], + dry_lab_score=0.60, + confirmation_verdict="not_confirmed", + divergence_types=["mic_above_predicted_range"], + prediction_quality_grade="D", + n_experiments=3, + n_confirmed=0, + n_inactive=3, + n_inconclusive=0, + limitations="Multiple labs, consistent result, generalizability unknown.", + ) + assert "not_confirmed" in format_hit_confirmation_report(r) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index f234d46c..2bd8c1a7 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -3,7 +3,7 @@ import subprocess import math -BASELINE = 8774 +BASELINE = 8837 def test_test_count_regression():