diff --git a/backend/api.py b/backend/api.py index 36eacb3..80e2cf7 100644 --- a/backend/api.py +++ b/backend/api.py @@ -43,7 +43,7 @@ configure_rate_limiting, rate_limit) import requests from routes.analytics import analytics_bp, record_scan -from text_preparation import prepare_text +from text_preparation import PREPARATION_VERSION, prepare_text from utils.spamSeverity import calculate_spam_severity @@ -584,11 +584,23 @@ def handle_internal_error(e): def _build_model_metadata(): """Fingerprint the currently-on-disk classifier artifacts (issue #1007).""" - return model_registry.build_metadata( + metadata = model_registry.build_metadata( model_path=str(MODEL_PATH), vectorizer_path=str(VECTORIZER_PATH), label_encoder_path=str(LABEL_ENCODER_PATH), ) + # Surfaced loudly rather than fatally: a contract mismatch degrades accuracy + # but the model still answers, and refusing to boot would take the API down + # over a metadata disagreement an operator may already be mid-way through + # resolving with a retrain. + if not metadata.preparation_matches(PREPARATION_VERSION): + app.logger.warning( + "served model was trained under text-preparation contract %s but %s " + "is in force; retrain to restore train-serve parity", + metadata.preparation_version, + PREPARATION_VERSION, + ) + return metadata def _load_serving_objects(): diff --git a/backend/model_registry.py b/backend/model_registry.py index 1b39b79..edda77e 100644 --- a/backend/model_registry.py +++ b/backend/model_registry.py @@ -8,8 +8,9 @@ This module fingerprints those artifacts. :func:`build_metadata` reads each ``.pkl`` and captures its SHA-256, size and mtime, and -- when a -``model_card.json`` sits next to the model -- folds in the human-authored -provenance fields (``trained_at``, ``metrics``, ``labels``). The immutable +``model_card.json`` sits next to the model -- folds in the provenance fields +``retrain.py`` records there (``trained_at``, ``metrics``, ``labels`` and the +``preparation_version`` the artifacts were trained under). The immutable :class:`ModelMetadata` it returns is stored alongside the serving objects in ``serving_state`` and surfaced at ``GET /model-info``; its :attr:`ModelMetadata.short_checksum` tags predictions and reload audit logs. @@ -87,6 +88,19 @@ class ModelMetadata: trained_at: str | None = None metrics: dict | None = None labels: list | None = None + preparation_version: str | None = None + + def preparation_matches(self, serving_version: str) -> bool: + """Whether these artifacts were trained under ``serving_version``. + + An unrecorded version (``None``) counts as a match: artifacts predating + the model card carry no claim about their preparation, and refusing to + serve them would break existing deployments over missing metadata rather + than over a known conflict. + """ + if self.preparation_version is None: + return True + return self.preparation_version == serving_version @property def short_checksum(self) -> str: @@ -111,6 +125,7 @@ def to_dict(self) -> dict: "trained_at": self.trained_at, "metrics": self.metrics, "labels": self.labels, + "preparation_version": self.preparation_version, } @@ -142,6 +157,7 @@ def build_metadata( trained_at=card.get("trained_at"), metrics=card.get("metrics"), labels=card.get("labels"), + preparation_version=card.get("preparation_version"), ) diff --git a/backend/retrain.py b/backend/retrain.py index 46bb322..e8cc7de 100644 --- a/backend/retrain.py +++ b/backend/retrain.py @@ -21,7 +21,10 @@ - linear_svm_model.pkl - tfidf_vectorizer.pkl - label_encoder.pkl - 7. Triggers a live model reload only AFTER a successful save. + 7. Writes model_card.json alongside them, recording when the model was + trained, how it scored, its label set, and the text-preparation contract + it was trained under. + 8. Triggers a live model reload only AFTER a successful save. Run this from the backend/ directory: cd backend @@ -227,6 +230,36 @@ def save_artifacts( print(f"Saved: {label_encoder_path}") +def write_model_card( + result, + *, + model_path=MODEL_PATH, + card_path=None, +): + """Emit the provenance sidecar the registry reads for ``GET /model-info``. + + Records the preparation contract the artifacts were trained under so a later + mismatch between trained and serving text handling is detectable instead of + silently degrading predictions. Written after the artifacts, so a card can + never describe a model that failed to persist. + """ + path = card_path or os.path.join( + os.path.dirname(os.path.abspath(model_path)), MODEL_CARD_FILENAME + ) + card = { + "trained_at": datetime.now(timezone.utc).isoformat(), + "metrics": { + "holdout_accuracy": round(float(result.holdout.accuracy), 4), + "training_rows": result.n_rows, + }, + "labels": [str(label) for label in result.label_encoder.classes_], + "preparation_version": PREPARATION_VERSION, + } + _atomic_write_json(card, path) + print(f"Saved: {path}") + return path + + def backup_existing_files(): """Copy existing .pkl files to a timestamped backup folder before overwriting.""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") @@ -316,6 +349,7 @@ def main(argv=None): backup_existing_files() save_artifacts(result) + write_model_card(result) print("\nRetraining complete. Triggering live model reload...") trigger_model_reload() @@ -356,6 +390,23 @@ def _evaluate_holdout( ) +def _atomic_write_json(payload, path): + """Write JSON through a temp file in the destination directory, then replace, + so a reader never sees a half-written card.""" + directory = os.path.dirname(os.path.abspath(path)) + os.makedirs(directory, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(dir=directory, suffix=".tmp") + os.close(fd) + try: + with open(tmp_path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, sort_keys=True) + os.replace(tmp_path, path) + except BaseException: + if os.path.exists(tmp_path): + os.remove(tmp_path) + raise + + def _atomic_joblib_dump(obj, path): """joblib.dump to a temp file in the destination directory, then os.replace so readers never observe a half-written artifact.""" diff --git a/backend/tests/test_preparation_provenance.py b/backend/tests/test_preparation_provenance.py new file mode 100644 index 0000000..9eeef95 --- /dev/null +++ b/backend/tests/test_preparation_provenance.py @@ -0,0 +1,101 @@ +"""Preparation contract recorded in, and read back from, model provenance (#1037). + +Covers both ends of the sidecar: what ``retrain.py`` writes after a successful +save, and what ``model_registry`` reports for the artifacts being served. +""" + +import json +from types import SimpleNamespace + +import model_registry +import retrain +from text_preparation import PREPARATION_VERSION + + +def _artifact(name): + return model_registry.ArtifactInfo( + path=name, sha256="ab" * 32, size_bytes=1, mtime=0.0 + ) + + +def _metadata(**overrides): + fields = { + "model": _artifact("m.pkl"), + "vectorizer": _artifact("v.pkl"), + "label_encoder": _artifact("l.pkl"), + } + fields.update(overrides) + return model_registry.ModelMetadata(**fields) + + +class TestPreparationMatching: + def test_same_version_matches(self): + assert _metadata(preparation_version="1").preparation_matches("1") + + def test_different_version_does_not_match(self): + assert not _metadata(preparation_version="1").preparation_matches("2") + + def test_unrecorded_version_is_treated_as_compatible(self): + # Artifacts predating the card make no claim; they must stay servable. + assert _metadata().preparation_matches("2") + + def test_version_is_reported_in_the_payload(self): + assert ( + _metadata(preparation_version="1").to_dict()["preparation_version"] == "1" + ) + + +class TestCardIsReadBack: + def test_registry_surfaces_the_recorded_version(self, tmp_path): + for name in ("m.pkl", "v.pkl", "l.pkl"): + (tmp_path / name).write_bytes(b"x") + (tmp_path / model_registry.MODEL_CARD_FILENAME).write_text( + json.dumps({"preparation_version": "7"}) + ) + + metadata = model_registry.build_metadata( + model_path=str(tmp_path / "m.pkl"), + vectorizer_path=str(tmp_path / "v.pkl"), + label_encoder_path=str(tmp_path / "l.pkl"), + ) + + assert metadata.preparation_version == "7" + + +class TestCardEmission: + def test_training_records_the_contract_in_force(self, tmp_path): + result = SimpleNamespace( + holdout=SimpleNamespace(accuracy=0.9375), + label_encoder=SimpleNamespace(classes_=["ham", "spam"]), + n_rows=120, + ) + + path = retrain.write_model_card( + result, card_path=str(tmp_path / model_registry.MODEL_CARD_FILENAME) + ) + card = json.loads(open(path, encoding="utf-8").read()) + + assert card["preparation_version"] == PREPARATION_VERSION + assert card["metrics"]["holdout_accuracy"] == 0.9375 + assert card["metrics"]["training_rows"] == 120 + assert card["labels"] == ["ham", "spam"] + assert card["trained_at"] + + def test_card_is_readable_by_the_registry(self, tmp_path): + result = SimpleNamespace( + holdout=SimpleNamespace(accuracy=1.0), + label_encoder=SimpleNamespace(classes_=["ham"]), + n_rows=10, + ) + for name in ("m.pkl", "v.pkl", "l.pkl"): + (tmp_path / name).write_bytes(b"x") + + retrain.write_model_card(result, model_path=str(tmp_path / "m.pkl")) + metadata = model_registry.build_metadata( + model_path=str(tmp_path / "m.pkl"), + vectorizer_path=str(tmp_path / "v.pkl"), + label_encoder_path=str(tmp_path / "l.pkl"), + ) + + assert metadata.preparation_version == PREPARATION_VERSION + assert metadata.preparation_matches(PREPARATION_VERSION) diff --git a/backend/text_preparation.py b/backend/text_preparation.py index 4a6bc55..9df8cb9 100644 --- a/backend/text_preparation.py +++ b/backend/text_preparation.py @@ -26,7 +26,15 @@ from utils.text_normalizer import normalizer -__all__ = ["prepare_text"] +__all__ = ["PREPARATION_VERSION", "prepare_text"] + +# Identifies the canonical form :func:`prepare_text` produces. Bump it whenever a +# change to the preparation steps alters the output for any input: artifacts +# trained under an older version learned a different vocabulary, and serving them +# under the newer one silently reintroduces the train-serve skew this contract +# exists to prevent. Recorded in the model card at training time and compared +# against the served artifacts when they are loaded. +PREPARATION_VERSION = "1" def prepare_text(text):