diff --git a/src/openamp_foundry/evidence/selection_transparency_ledger.py b/src/openamp_foundry/evidence/selection_transparency_ledger.py new file mode 100644 index 00000000..b09d6fd7 --- /dev/null +++ b/src/openamp_foundry/evidence/selection_transparency_ledger.py @@ -0,0 +1,246 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from typing import List, Optional + + +VALID_SELECTION_BASES = frozenset({ + "score_rank", # selected primarily by pipeline score rank + "safety_gate_passed", # selected because it passed all safety screens + "novelty_evidence", # selected due to novelty evidence (CFC/FNR) + "benchmark_performance", # selected due to benchmark performance vs baselines + "expert_review", # selected following expert human review + "calibration_signal", # selected based on calibration-adjusted confidence + "diversity_criteria", # selected to maximize batch diversity + "not_specified", # selection basis not documented (should be avoided) +}) + +VALID_LEDGER_COMPLETENESS_GRADES = frozenset({ + "complete", # all expected artifact references are present + "partial", # some expected artifacts are missing but core selection documented + "incomplete", # major artifacts missing; selection not fully traceable + "not_assessed", # completeness not checked +}) + +REQUIRED_SELECTION_ARTIFACT_TYPES = frozenset({ + "BSP", # batch selection proposal + "SEG", # score vs enemy gap report + "SRG", # scientific review readiness gate +}) + +VALID_SELECTION_OUTCOME_TYPES = frozenset({ + "selected_for_pilot", + "selected_for_next_batch", + "selected_for_external_review", + "deselected_safety", + "deselected_low_score", + "deselected_insufficient_evidence", + "held_pending_review", +}) + + +@dataclass +class ArtifactReference: + artifact_type: str + artifact_id: str + influence_description: str + + +@dataclass +class SelectionTransparencyLedger: + stl_id: str + run_id: str + candidate_family_id: str + selection_basis: str + selection_outcome: str + completeness_grade: str + artifact_references: List[ArtifactReference] + n_artifacts_referenced: int + has_required_artifacts: bool + missing_required_artifact_types: List[str] + seg_id: Optional[str] + srg_id: Optional[str] + bsp_id: Optional[str] + dry_lab_only: bool + limitations: str + notes: str = "" + + +@dataclass +class STLValidationResult: + valid: bool + violations: List[str] = field(default_factory=list) + + +def validate_selection_transparency_ledger(ledger: SelectionTransparencyLedger) -> STLValidationResult: + violations = [] + + if not ledger.stl_id.startswith("STL-"): + violations.append("stl_id must start with 'STL-'") + + if not ledger.run_id: + violations.append("run_id is required") + + if not ledger.candidate_family_id: + violations.append("candidate_family_id is required") + + if ledger.candidate_family_id.startswith("TOY-"): + violations.append("TOY- candidate_family_id is not allowed in real STL- records") + + if ledger.selection_basis not in VALID_SELECTION_BASES: + violations.append( + f"selection_basis '{ledger.selection_basis}' must be one of {sorted(VALID_SELECTION_BASES)}" + ) + + if ledger.selection_outcome not in VALID_SELECTION_OUTCOME_TYPES: + violations.append( + f"selection_outcome '{ledger.selection_outcome}' must be one of {sorted(VALID_SELECTION_OUTCOME_TYPES)}" + ) + + if ledger.completeness_grade not in VALID_LEDGER_COMPLETENESS_GRADES: + violations.append( + f"completeness_grade '{ledger.completeness_grade}' must be one of {sorted(VALID_LEDGER_COMPLETENESS_GRADES)}" + ) + + if ledger.n_artifacts_referenced != len(ledger.artifact_references): + violations.append( + f"n_artifacts_referenced ({ledger.n_artifacts_referenced}) must match len(artifact_references) ({len(ledger.artifact_references)})" + ) + + if ledger.n_artifacts_referenced < 1: + violations.append("at least one artifact_reference is required") + + # Validate each artifact reference + for i, ref in enumerate(ledger.artifact_references): + if not ref.artifact_type: + violations.append(f"artifact_references[{i}].artifact_type is required") + if not ref.artifact_id: + violations.append(f"artifact_references[{i}].artifact_id is required") + if not ref.influence_description or len(ref.influence_description.strip()) < 5: + violations.append(f"artifact_references[{i}].influence_description must be at least 5 chars") + + # Check presence of required artifacts + present_types = {ref.artifact_type for ref in ledger.artifact_references} + missing = sorted(REQUIRED_SELECTION_ARTIFACT_TYPES - present_types) + if sorted(ledger.missing_required_artifact_types) != missing: + violations.append( + f"missing_required_artifact_types {ledger.missing_required_artifact_types} does not match computed {missing}" + ) + + expected_has_required = len(missing) == 0 + if ledger.has_required_artifacts != expected_has_required: + violations.append( + f"has_required_artifacts ({ledger.has_required_artifacts}) inconsistent with missing types ({missing})" + ) + + # complete grade requires has_required_artifacts + if ledger.completeness_grade == "complete" and not ledger.has_required_artifacts: + violations.append( + "completeness_grade='complete' requires has_required_artifacts=True" + ) + + # incomplete grade must not have all required artifacts + if ledger.completeness_grade == "incomplete" and ledger.has_required_artifacts and not missing: + violations.append( + "completeness_grade='incomplete' should not have all required artifacts present" + ) + + # not_specified selection_basis is a weak signal + if ledger.selection_basis == "not_specified": + violations.append( + "selection_basis='not_specified' is not allowed; document the actual selection basis" + ) + + # ID field prefix checks + if ledger.seg_id is not None and not ledger.seg_id.startswith("SEG-"): + violations.append("seg_id must start with 'SEG-' when provided") + + if ledger.srg_id is not None and not ledger.srg_id.startswith("SRG-"): + violations.append("srg_id must start with 'SRG-' when provided") + + if ledger.bsp_id is not None and not ledger.bsp_id.startswith("BSP-"): + violations.append("bsp_id must start with 'BSP-' when provided") + + if not ledger.dry_lab_only: + violations.append("dry_lab_only must be True for STL- records") + + if not ledger.limitations or len(ledger.limitations.strip()) < 10: + violations.append("limitations must be a non-empty string (at least 10 characters)") + + return STLValidationResult(valid=len(violations) == 0, violations=violations) + + +def build_selection_transparency_ledger( + stl_id: str, + run_id: str, + candidate_family_id: str, + selection_basis: str, + selection_outcome: str, + artifact_references: List[ArtifactReference], + limitations: str, + notes: str = "", + seg_id: Optional[str] = None, + srg_id: Optional[str] = None, + bsp_id: Optional[str] = None, +) -> SelectionTransparencyLedger: + present_types = {ref.artifact_type for ref in artifact_references} + missing = sorted(REQUIRED_SELECTION_ARTIFACT_TYPES - present_types) + has_required = len(missing) == 0 + + if has_required and len(artifact_references) >= len(REQUIRED_SELECTION_ARTIFACT_TYPES): + completeness_grade = "complete" + elif missing and len(missing) < len(REQUIRED_SELECTION_ARTIFACT_TYPES): + completeness_grade = "partial" + elif not artifact_references: + completeness_grade = "incomplete" + else: + completeness_grade = "partial" + + ledger = SelectionTransparencyLedger( + stl_id=stl_id, + run_id=run_id, + candidate_family_id=candidate_family_id, + selection_basis=selection_basis, + selection_outcome=selection_outcome, + completeness_grade=completeness_grade, + artifact_references=artifact_references, + n_artifacts_referenced=len(artifact_references), + has_required_artifacts=has_required, + missing_required_artifact_types=missing, + seg_id=seg_id, + srg_id=srg_id, + bsp_id=bsp_id, + dry_lab_only=True, + limitations=limitations, + notes=notes, + ) + result = validate_selection_transparency_ledger(ledger) + if not result.valid: + raise ValueError(f"Invalid STL: {result.violations}") + return ledger + + +def format_selection_transparency_ledger(ledger: SelectionTransparencyLedger) -> str: + lines = [ + f"Selection Transparency Ledger — {ledger.stl_id}", + f"Run: {ledger.run_id} | Family: {ledger.candidate_family_id}", + f"Selection Basis: {ledger.selection_basis}", + f"Outcome: {ledger.selection_outcome}", + f"Completeness Grade: {ledger.completeness_grade}", + f"Artifacts Referenced: {ledger.n_artifacts_referenced}", + f"Required Artifacts Present: {ledger.has_required_artifacts}", + ] + if ledger.missing_required_artifact_types: + lines.append(f"Missing Required: {', '.join(ledger.missing_required_artifact_types)}") + for ref in ledger.artifact_references: + lines.append(f" [{ref.artifact_type}] {ref.artifact_id}: {ref.influence_description}") + if ledger.seg_id: + lines.append(f"SEG Link: {ledger.seg_id}") + if ledger.srg_id: + lines.append(f"SRG Link: {ledger.srg_id}") + if ledger.bsp_id: + lines.append(f"BSP Link: {ledger.bsp_id}") + lines.append(f"Limitations: {ledger.limitations}") + if ledger.notes: + lines.append(f"Notes: {ledger.notes}") + lines.append("dry_lab_only: True") + return "\n".join(lines) diff --git a/tests/evidence/test_selection_transparency_ledger.py b/tests/evidence/test_selection_transparency_ledger.py new file mode 100644 index 00000000..c4ebf5ab --- /dev/null +++ b/tests/evidence/test_selection_transparency_ledger.py @@ -0,0 +1,307 @@ +"""Tests for selection_transparency_ledger module — Phase S S2 (STL-).""" +import pytest +from openamp_foundry.evidence.selection_transparency_ledger import ( + SelectionTransparencyLedger, + ArtifactReference, + VALID_SELECTION_BASES, + VALID_LEDGER_COMPLETENESS_GRADES, + VALID_SELECTION_OUTCOME_TYPES, + REQUIRED_SELECTION_ARTIFACT_TYPES, + validate_selection_transparency_ledger, + build_selection_transparency_ledger, + format_selection_transparency_ledger, +) + + +def _make_ref(artifact_type="BSP", artifact_id="BSP-001", desc="Batch selection basis"): + return ArtifactReference(artifact_type=artifact_type, artifact_id=artifact_id, influence_description=desc) + + +def _all_required_refs(): + return [ + _make_ref("BSP", "BSP-001", "Batch selection basis"), + _make_ref("SEG", "SEG-001", "Score vs enemy gap"), + _make_ref("SRG", "SRG-001", "Review readiness gate"), + ] + + +def _make_ledger(**kwargs): + refs = _all_required_refs() + defaults = dict( + stl_id="STL-001", run_id="RUN-001", candidate_family_id="FAM-001", + selection_basis="score_rank", selection_outcome="selected_for_pilot", + completeness_grade="complete", artifact_references=refs, + n_artifacts_referenced=3, has_required_artifacts=True, + missing_required_artifact_types=[], + seg_id=None, srg_id=None, bsp_id=None, + dry_lab_only=True, + limitations="Dry-lab only. Selection based on computational scores.", + notes="", + ) + defaults.update(kwargs) + return SelectionTransparencyLedger(**defaults) + + +class TestSTLIDValidation: + def test_valid_stl_id(self): + assert validate_selection_transparency_ledger(_make_ledger(stl_id="STL-001")).valid + + def test_invalid_stl_id_missing_prefix(self): + r = validate_selection_transparency_ledger(_make_ledger(stl_id="001")) + assert not r.valid and any("stl_id" in v for v in r.violations) + + def test_invalid_stl_id_wrong_prefix(self): + assert not validate_selection_transparency_ledger(_make_ledger(stl_id="SEG-001")).valid + + def test_toy_family_id_blocked(self): + r = validate_selection_transparency_ledger(_make_ledger(candidate_family_id="TOY-001")) + assert not r.valid and any("TOY-" in v for v in r.violations) + + def test_run_id_required(self): + r = validate_selection_transparency_ledger(_make_ledger(run_id="")) + assert not r.valid and any("run_id" in v for v in r.violations) + + def test_candidate_family_id_required(self): + assert not validate_selection_transparency_ledger(_make_ledger(candidate_family_id="")).valid + + def test_seg_id_prefix_required_when_provided(self): + r = validate_selection_transparency_ledger(_make_ledger(seg_id="001")) + assert not r.valid and any("seg_id" in v for v in r.violations) + + def test_srg_id_prefix_required_when_provided(self): + r = validate_selection_transparency_ledger(_make_ledger(srg_id="001")) + assert not r.valid and any("srg_id" in v for v in r.violations) + + def test_bsp_id_prefix_required_when_provided(self): + r = validate_selection_transparency_ledger(_make_ledger(bsp_id="001")) + assert not r.valid and any("bsp_id" in v for v in r.violations) + + def test_all_optional_ids_valid_when_correct_prefix(self): + ledger = _make_ledger(seg_id="SEG-001", srg_id="SRG-001", bsp_id="BSP-001") + assert validate_selection_transparency_ledger(ledger).valid + + +class TestSelectionBasisValidation: + def test_all_valid_bases_accepted(self): + for basis in VALID_SELECTION_BASES - {"not_specified"}: + assert validate_selection_transparency_ledger(_make_ledger(selection_basis=basis)).valid, f"Expected valid for basis={basis}" + + def test_invalid_basis_rejected(self): + r = validate_selection_transparency_ledger(_make_ledger(selection_basis="gut_feeling")) + assert not r.valid and any("selection_basis" in v for v in r.violations) + + def test_not_specified_basis_blocked(self): + r = validate_selection_transparency_ledger(_make_ledger(selection_basis="not_specified")) + assert not r.valid and any("not_specified" in v for v in r.violations) + + def test_score_rank_basis_valid(self): + assert validate_selection_transparency_ledger(_make_ledger(selection_basis="score_rank")).valid + + def test_expert_review_basis_valid(self): + assert validate_selection_transparency_ledger(_make_ledger(selection_basis="expert_review")).valid + + +class TestSelectionOutcomeValidation: + def test_all_valid_outcomes_accepted(self): + for outcome in VALID_SELECTION_OUTCOME_TYPES: + assert validate_selection_transparency_ledger(_make_ledger(selection_outcome=outcome)).valid, f"Expected valid for outcome={outcome}" + + def test_invalid_outcome_rejected(self): + r = validate_selection_transparency_ledger(_make_ledger(selection_outcome="maybe_selected")) + assert not r.valid and any("selection_outcome" in v for v in r.violations) + + def test_selected_for_pilot_valid(self): + assert validate_selection_transparency_ledger(_make_ledger(selection_outcome="selected_for_pilot")).valid + + def test_deselected_safety_valid(self): + assert validate_selection_transparency_ledger(_make_ledger(selection_outcome="deselected_safety")).valid + + +class TestCompletenessValidation: + def test_complete_grade_with_all_required_valid(self): + ledger = _make_ledger(completeness_grade="complete", has_required_artifacts=True, missing_required_artifact_types=[]) + assert validate_selection_transparency_ledger(ledger).valid + + def test_complete_grade_without_required_rejected(self): + refs = [_make_ref("BSP", "BSP-001", "Batch basis")] # missing SEG, SRG + ledger = _make_ledger( + completeness_grade="complete", artifact_references=refs, + n_artifacts_referenced=1, has_required_artifacts=False, + missing_required_artifact_types=["SEG", "SRG"] + ) + r = validate_selection_transparency_ledger(ledger) + assert not r.valid and any("complete" in v for v in r.violations) + + def test_partial_grade_with_some_missing_valid(self): + refs = [_make_ref("BSP", "BSP-001", "Batch basis"), _make_ref("SEG", "SEG-001", "Gap")] + ledger = _make_ledger( + completeness_grade="partial", artifact_references=refs, + n_artifacts_referenced=2, has_required_artifacts=False, + missing_required_artifact_types=["SRG"] + ) + assert validate_selection_transparency_ledger(ledger).valid + + def test_incomplete_grade_with_all_required_rejected(self): + refs = _all_required_refs() + ledger = _make_ledger( + completeness_grade="incomplete", artifact_references=refs, + n_artifacts_referenced=3, has_required_artifacts=True, + missing_required_artifact_types=[] + ) + r = validate_selection_transparency_ledger(ledger) + assert not r.valid + + def test_all_valid_grades_in_constant(self): + assert len(VALID_LEDGER_COMPLETENESS_GRADES) == 4 + + def test_n_artifacts_must_match_refs(self): + refs = _all_required_refs() + r = validate_selection_transparency_ledger(_make_ledger(artifact_references=refs, n_artifacts_referenced=5)) + assert not r.valid and any("n_artifacts_referenced" in v for v in r.violations) + + def test_at_least_one_artifact_required(self): + r = validate_selection_transparency_ledger(_make_ledger( + artifact_references=[], n_artifacts_referenced=0, + has_required_artifacts=False, missing_required_artifact_types=["BSP","SEG","SRG"], + completeness_grade="incomplete" + )) + assert not r.valid + + def test_missing_required_types_computed_correctly(self): + refs = [_make_ref("BSP", "BSP-001", "Batch basis")] + r = validate_selection_transparency_ledger(_make_ledger( + artifact_references=refs, n_artifacts_referenced=1, + has_required_artifacts=False, missing_required_artifact_types=["SEG"], # wrong — SRG also missing + completeness_grade="partial" + )) + assert not r.valid and any("missing_required_artifact_types" in v for v in r.violations) + + def test_has_required_inconsistency_rejected(self): + refs = _all_required_refs() + r = validate_selection_transparency_ledger(_make_ledger( + artifact_references=refs, n_artifacts_referenced=3, + has_required_artifacts=False, # wrong — all required present + missing_required_artifact_types=[] + )) + assert not r.valid + + def test_required_artifact_types_count(self): + assert len(REQUIRED_SELECTION_ARTIFACT_TYPES) == 3 + + +class TestArtifactReferenceValidation: + def test_empty_artifact_type_rejected(self): + bad_ref = ArtifactReference(artifact_type="", artifact_id="BSP-001", influence_description="Batch basis") + refs = [bad_ref, _make_ref("SEG", "SEG-001", "Gap"), _make_ref("SRG", "SRG-001", "Gate")] + r = validate_selection_transparency_ledger(_make_ledger( + artifact_references=refs, n_artifacts_referenced=3, + has_required_artifacts=False, missing_required_artifact_types=["BSP"], + completeness_grade="partial" + )) + assert not r.valid + + def test_empty_artifact_id_rejected(self): + bad_ref = ArtifactReference(artifact_type="BSP", artifact_id="", influence_description="Batch basis") + refs = [bad_ref, _make_ref("SEG", "SEG-001", "Gap"), _make_ref("SRG", "SRG-001", "Gate")] + r = validate_selection_transparency_ledger(_make_ledger( + artifact_references=refs, n_artifacts_referenced=3, + has_required_artifacts=True, missing_required_artifact_types=[], + completeness_grade="complete" + )) + assert not r.valid + + def test_too_short_influence_description_rejected(self): + bad_ref = ArtifactReference(artifact_type="BSP", artifact_id="BSP-001", influence_description="hi") + refs = [bad_ref, _make_ref("SEG", "SEG-001", "Gap"), _make_ref("SRG", "SRG-001", "Gate")] + r = validate_selection_transparency_ledger(_make_ledger( + artifact_references=refs, n_artifacts_referenced=3, + has_required_artifacts=True, missing_required_artifact_types=[], + completeness_grade="complete" + )) + assert not r.valid and any("influence_description" in v for v in r.violations) + + +class TestDryLabAndLimitations: + def test_dry_lab_only_false_rejected(self): + r = validate_selection_transparency_ledger(_make_ledger(dry_lab_only=False)) + assert not r.valid and any("dry_lab_only" in v for v in r.violations) + + def test_limitations_required(self): + r = validate_selection_transparency_ledger(_make_ledger(limitations="")) + assert not r.valid and any("limitations" in v for v in r.violations) + + def test_limitations_too_short_rejected(self): + assert not validate_selection_transparency_ledger(_make_ledger(limitations="Too short")).valid + + def test_limitations_sufficient_valid(self): + assert validate_selection_transparency_ledger(_make_ledger(limitations="Dry-lab only. Selection based on computational scores.")).valid + + +class TestBuildAndFormat: + def test_build_auto_computes_complete_grade(self): + refs = _all_required_refs() + ledger = build_selection_transparency_ledger( + stl_id="STL-001", run_id="RUN-001", candidate_family_id="FAM-001", + selection_basis="score_rank", selection_outcome="selected_for_pilot", + artifact_references=refs, + limitations="Dry-lab only. Selection based on computational scores.", + ) + assert ledger.completeness_grade == "complete" + assert ledger.has_required_artifacts is True + assert ledger.dry_lab_only is True + + def test_build_auto_computes_partial_grade(self): + refs = [_make_ref("BSP", "BSP-001", "Batch basis"), _make_ref("SEG", "SEG-001", "Gap")] + ledger = build_selection_transparency_ledger( + stl_id="STL-001", run_id="RUN-001", candidate_family_id="FAM-001", + selection_basis="score_rank", selection_outcome="selected_for_pilot", + artifact_references=refs, + limitations="Dry-lab only. Selection based on computational scores.", + ) + assert ledger.completeness_grade == "partial" + assert ledger.has_required_artifacts is False + assert "SRG" in ledger.missing_required_artifact_types + + def test_build_raises_on_invalid(self): + with pytest.raises(ValueError): + build_selection_transparency_ledger( + stl_id="INVALID", run_id="RUN-001", candidate_family_id="FAM-001", + selection_basis="score_rank", selection_outcome="selected_for_pilot", + artifact_references=_all_required_refs(), + limitations="Dry-lab only. Selection based on computational scores.", + ) + + def test_build_raises_on_not_specified_basis(self): + with pytest.raises(ValueError): + build_selection_transparency_ledger( + stl_id="STL-001", run_id="RUN-001", candidate_family_id="FAM-001", + selection_basis="not_specified", selection_outcome="selected_for_pilot", + artifact_references=_all_required_refs(), + limitations="Dry-lab only. Selection based on computational scores.", + ) + + def test_format_includes_stl_id(self): + assert "STL-001" in format_selection_transparency_ledger(_make_ledger()) + + def test_format_includes_selection_basis(self): + assert "score_rank" in format_selection_transparency_ledger(_make_ledger()) + + def test_format_includes_completeness_grade(self): + assert "complete" in format_selection_transparency_ledger(_make_ledger()) + + def test_format_includes_artifact_count(self): + assert "3" in format_selection_transparency_ledger(_make_ledger()) + + def test_format_includes_seg_link_when_present(self): + assert "SEG-001" in format_selection_transparency_ledger(_make_ledger(seg_id="SEG-001")) + + def test_format_is_multiline(self): + assert "\n" in format_selection_transparency_ledger(_make_ledger()) + + def test_valid_ledger_passes_validation(self): + r = validate_selection_transparency_ledger(_make_ledger()) + assert r.valid and r.violations == [] + + def test_multiple_violations_collected(self): + r = validate_selection_transparency_ledger(_make_ledger(stl_id="INVALID", selection_basis="bad_basis")) + assert not r.valid and len(r.violations) >= 2 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index e9e347cc..d96126da 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -3,7 +3,7 @@ import subprocess import math -BASELINE = 9278 +BASELINE = 9341 def test_test_count_regression():