From 7a07d660a8553550a0a0390e1bdf8a5358f2d44a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 13:36:30 +0000 Subject: [PATCH 01/11] Add atomic JSON I/O and fix lab row append dedup skip - Add atomic_write_text, load_json, write_json helpers in corpus_io - Use atomic writes in write_manifest and write_labs - Remove existing_keys skip in _append_lab_rows; always append rows - Route dedup_labs, apply_loinc_map, merge_labs through load_labs/write_labs Co-authored-by: apodobe --- medbots/corpus_io.py | 51 ++++++++++++++++++++--------- medbots/corpus_writers.py | 22 +++---------- medbots/dedup_labs.py | 10 ++---- medbots/merge_labs_corpus.py | 6 ++-- medbots/pipeline/apply_loinc_map.py | 10 +++--- 5 files changed, 51 insertions(+), 48 deletions(-) diff --git a/medbots/corpus_io.py b/medbots/corpus_io.py index 0b87772..af70ba2 100644 --- a/medbots/corpus_io.py +++ b/medbots/corpus_io.py @@ -4,6 +4,7 @@ import json import os +import tempfile from pathlib import Path from typing import Any @@ -43,35 +44,55 @@ def resolve_corpus(path: Path | str | None = None) -> Path: return Path(path).expanduser().resolve() +def atomic_write_text(path: Path, text: str) -> None: + """Write text atomically via temp file in the same directory.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + tmp_path = Path(tmp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(text) + os.replace(tmp_path, path) + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + +def load_json(path: Path, default: Any) -> Any: + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return default + + +def write_json(path: Path, data: Any, *, trailing_newline: bool = False) -> None: + text = json.dumps(data, ensure_ascii=False, indent=2) + if trailing_newline and not text.endswith("\n"): + text += "\n" + atomic_write_text(path, text) + + def empty_manifest() -> dict[str, Any]: return {"version": 1, "pdfs": [], "images": [], "meta": {}} def load_manifest(corpus: Path) -> dict[str, Any]: - p = corpus / "manifest.json" - if p.exists(): - return json.loads(p.read_text(encoding="utf-8")) - return empty_manifest() + return load_json(corpus / "manifest.json", empty_manifest()) def write_manifest(corpus: Path, data: dict[str, Any]) -> None: - (corpus / "manifest.json").write_text( - json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8" - ) + write_json(corpus / "manifest.json", data) def load_labs(corpus: Path) -> dict[str, Any]: - p = corpus / "LABS_NORMALIZED.json" - if p.exists(): - return json.loads(p.read_text(encoding="utf-8")) - return {"rows": []} + return load_json(corpus / "LABS_NORMALIZED.json", {"rows": []}) def write_labs(corpus: Path, data: dict[str, Any]) -> None: - text = json.dumps(data, ensure_ascii=False, indent=2) - if not text.endswith("\n"): - text += "\n" - (corpus / "LABS_NORMALIZED.json").write_text(text, encoding="utf-8") + write_json(corpus / "LABS_NORMALIZED.json", data, trailing_newline=True) def load_patient_dob(corpus: Path) -> str: diff --git a/medbots/corpus_writers.py b/medbots/corpus_writers.py index 50077dc..4212717 100644 --- a/medbots/corpus_writers.py +++ b/medbots/corpus_writers.py @@ -7,6 +7,8 @@ from pathlib import Path from typing import Any +from medbots.corpus_io import load_labs, write_labs + def _utc_date_slug() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%d") @@ -146,29 +148,15 @@ def _write_doc_text_md( def _append_lab_rows(corpus: Path, lab_rows: list[dict], source_path_rel: str) -> int: if not lab_rows: return 0 - labs_path = corpus / "LABS_NORMALIZED.json" - if labs_path.exists(): - labs = json.loads(labs_path.read_text(encoding="utf-8")) - else: - labs = {"rows": []} + labs = load_labs(corpus) existing_rows: list[dict] = list(labs.get("rows") or []) - existing_keys = { - (r.get("canonical_key", ""), r.get("specimen_date", "")) - for r in existing_rows - } - added = 0 for row in lab_rows: row = dict(row) row["source_path"] = source_path_rel - key = (row.get("canonical_key", ""), row.get("specimen_date", "")) - if key in existing_keys: - continue existing_rows.append(row) - existing_keys.add(key) - added += 1 labs["rows"] = existing_rows - labs_path.write_text(json.dumps(labs, ensure_ascii=False, indent=2), encoding="utf-8") - return added + write_labs(corpus, labs) + return len(lab_rows) def _append_supplement_mentions(corpus: Path, mentions: list[dict], source_path_rel: str) -> int: diff --git a/medbots/dedup_labs.py b/medbots/dedup_labs.py index 48bc714..299a59c 100644 --- a/medbots/dedup_labs.py +++ b/medbots/dedup_labs.py @@ -3,13 +3,12 @@ from __future__ import annotations import argparse -import json import re import sys from pathlib import Path from typing import Any -from medbots.corpus_io import manifest_vendor_index +from medbots.corpus_io import load_labs, manifest_vendor_index, write_labs _SOURCE_TIER = { "medsi": 0, @@ -61,7 +60,7 @@ def dedup_labs(corpus: Path, *, apply: bool = True) -> dict[str, int]: raise FileNotFoundError(labs_path) manifest_index = manifest_vendor_index(corpus) - data: dict[str, Any] = json.loads(labs_path.read_text(encoding="utf-8")) + data = load_labs(corpus) rows: list[dict[str, Any]] = data.get("rows") or [] groups: dict[tuple[str, str | None], list[dict[str, Any]]] = {} @@ -90,10 +89,7 @@ def dedup_labs(corpus: Path, *, apply: bool = True) -> dict[str, int]: if apply: data["rows"] = kept - labs_path.write_text( - json.dumps(data, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) + write_labs(corpus, data) return stats diff --git a/medbots/merge_labs_corpus.py b/medbots/merge_labs_corpus.py index 7c10626..ad29436 100644 --- a/medbots/merge_labs_corpus.py +++ b/medbots/merge_labs_corpus.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any -from medbots.corpus_io import default_corpus_root, load_manifest +from medbots.corpus_io import default_corpus_root, load_labs, load_manifest, write_labs from medbots.corpus_writers import _append_lab_rows from medbots.local_structure_pdfs import _entry_title, _parse_entry, _pdf_text_path @@ -44,13 +44,13 @@ def _remove_rows_for_sources(corpus: Path, source_paths: set[str]) -> int: labs_path = corpus / "LABS_NORMALIZED.json" if not labs_path.exists(): return 0 - labs = json.loads(labs_path.read_text(encoding="utf-8")) + labs = load_labs(corpus) rows: list[dict[str, Any]] = labs.get("rows") or [] kept = [r for r in rows if (r.get("source_path") or "") not in source_paths] removed = len(rows) - len(kept) if removed: labs["rows"] = kept - labs_path.write_text(json.dumps(labs, ensure_ascii=False, indent=2), encoding="utf-8") + write_labs(corpus, labs) return removed diff --git a/medbots/pipeline/apply_loinc_map.py b/medbots/pipeline/apply_loinc_map.py index e2e4a9f..34a6f51 100644 --- a/medbots/pipeline/apply_loinc_map.py +++ b/medbots/pipeline/apply_loinc_map.py @@ -4,11 +4,12 @@ import argparse import csv -import json import sys from pathlib import Path from typing import Any +from medbots.corpus_io import load_labs, write_labs + def _load_loinc_map(path: Path) -> dict[str, str]: mapping: dict[str, str] = {} @@ -31,7 +32,7 @@ def apply_loinc_map(corpus: Path, *, apply: bool = True) -> dict[str, int]: raise FileNotFoundError(map_path) loinc_map = _load_loinc_map(map_path) - data: dict[str, Any] = json.loads(labs_path.read_text(encoding="utf-8")) + data = load_labs(corpus) rows: list[dict[str, Any]] = data.get("rows") or [] stats = { @@ -59,10 +60,7 @@ def apply_loinc_map(corpus: Path, *, apply: bool = True) -> dict[str, int]: stats["loinc_null_after"] = sum(1 for r in rows if not r.get("loinc")) if apply: - labs_path.write_text( - json.dumps(data, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) + write_labs(corpus, data) return stats From 8304ae32ad74223cb995ab90218bf48d0b439cfc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 13:37:37 +0000 Subject: [PATCH 02/11] refactor: pass patient_dob explicitly instead of global in local_structure_pdfs Remove module-level _skip_patient_dob and _configure_patient_dob. Thread patient_dob parameter through _first_iso_date, parse helpers, _parse_entry, and run(). run() loads DOB from corpus when not provided. Co-authored-by: apodobe --- medbots/local_structure_pdfs.py | 134 ++++++++++++++++++++++++-------- 1 file changed, 100 insertions(+), 34 deletions(-) diff --git a/medbots/local_structure_pdfs.py b/medbots/local_structure_pdfs.py index c1594cb..cfaf498 100644 --- a/medbots/local_structure_pdfs.py +++ b/medbots/local_structure_pdfs.py @@ -12,6 +12,7 @@ from typing import Any, Optional from medbots.corpus_io import bot_root, default_corpus_root, load_manifest, load_patient_dob, write_manifest +from medbots.time_util import utc_now_iso from medbots.corpus_writers import ( _append_lab_rows, _safe_txt_name, @@ -36,7 +37,6 @@ r"дата:\s*\n\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE ) _FILENAME_ISO_DATE = re.compile(r"(?:^|/)(20\d{2}-\d{2}-\d{2})__") -_skip_patient_dob: str = "" _SECTION_HEADERS = frozenset( { "Биохимия 19 показателей (расширенная)", @@ -103,7 +103,7 @@ def _utc_ts() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return utc_now_iso() def _dmy_to_iso(d: str, m: str, y: str) -> str: @@ -120,7 +120,13 @@ def _date_from_source_path(source_pdf: str) -> Optional[str]: return iso -def _first_iso_date(text: str, *, source_pdf: str = "", fallback_iso: Optional[str] = None) -> Optional[str]: +def _first_iso_date( + text: str, + *, + source_pdf: str = "", + fallback_iso: Optional[str] = None, + patient_dob: str = "", +) -> Optional[str]: path_date = _date_from_source_path(source_pdf) if path_date: return path_date @@ -131,14 +137,14 @@ def _first_iso_date(text: str, *, source_pdf: str = "", fallback_iso: Optional[s m = pat.search(text) if m: iso = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - if iso != _skip_patient_dob: + if iso != patient_dob: return iso m = _DATE_DMY.search(text) if m: iso = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - if iso != _skip_patient_dob: + if iso != patient_dob: return iso - if fallback_iso and fallback_iso != _skip_patient_dob: + if fallback_iso and fallback_iso != patient_dob: return fallback_iso return None @@ -279,8 +285,17 @@ def _extract_conclusion(text: str) -> str: return "—" -def parse_emias_lab(text: str, title: str, *, source_pdf: str = "", fallback_iso: Optional[str] = None) -> dict[str, Any]: - doc_date = _first_iso_date(text, source_pdf=source_pdf, fallback_iso=fallback_iso) +def parse_emias_lab( + text: str, + title: str, + *, + source_pdf: str = "", + fallback_iso: Optional[str] = None, + patient_dob: str = "", +) -> dict[str, Any]: + doc_date = _first_iso_date( + text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) facility = _extract_emias_facility(text) lab_rows: list[dict[str, Any]] = [] title_l = title.lower() @@ -308,7 +323,7 @@ def parse_emias_lab(text: str, title: str, *, source_pdf: str = "", fallback_iso ) ) elif re.search(r"Исследование - \(L", text): - lab_date = doc_date or _medsi_iso_date(text) + lab_date = doc_date or _medsi_iso_date(text, patient_dob=patient_dob) if lab_date: lab_rows = _parse_medsi_lab_rows(text, lab_date, facility) else: @@ -368,10 +383,18 @@ def parse_emias_lab(text: str, title: str, *, source_pdf: str = "", fallback_iso def parse_emias_consult_or_imaging( - text: str, title: str, doc_type: str, *, source_pdf: str = "", fallback_iso: Optional[str] = None, + text: str, + title: str, + doc_type: str, + *, + source_pdf: str = "", + fallback_iso: Optional[str] = None, facility_override: str | None = None, + patient_dob: str = "", ) -> dict[str, Any]: - doc_date = _first_iso_date(text, source_pdf=source_pdf, fallback_iso=fallback_iso) + doc_date = _first_iso_date( + text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) facility = facility_override or _extract_emias_facility(text) conclusion = _extract_conclusion(text) if doc_type == "imaging": @@ -724,10 +747,13 @@ def parse_gemotest( source_pdf: str, *, fallback_iso: Optional[str] = None, + patient_dob: str = "", ) -> dict[str, Any]: subtype = _gemotest_subtype(source_pdf) facility = _gemotest_facility(text) - doc_date = _first_iso_date(text, source_pdf=source_pdf, fallback_iso=fallback_iso) + doc_date = _first_iso_date( + text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) if subtype == "certificate": md = _markdown_header( @@ -842,11 +868,11 @@ def _is_medsi_value_line(line: str) -> bool: return _parse_float(line.strip()) is not None -def _medsi_iso_date(text: str) -> Optional[str]: +def _medsi_iso_date(text: str, *, patient_dob: str = "") -> Optional[str]: m = _EMIAS_DATE.search(text) if m: return _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - return _first_iso_date(text) + return _first_iso_date(text, patient_dob=patient_dob) def _parse_medsi_lab_rows(text: str, doc_date: str, facility: str) -> list[dict[str, Any]]: @@ -944,8 +970,9 @@ def parse_medsi_lab( *, source_pdf: str = "", fallback_iso: Optional[str] = None, + patient_dob: str = "", ) -> dict[str, Any]: - doc_date = _medsi_iso_date(text) or fallback_iso + doc_date = _medsi_iso_date(text, patient_dob=patient_dob) or fallback_iso facility = "Медси" if "мичуринск" in text.lower(): facility = 'Медси "Мичуринский"' @@ -1074,7 +1101,7 @@ def _pdf_text_path(corpus: Path, entry: dict[str, Any]) -> Path: return corpus / "pdf_text" / _safe_txt_name(source_pdf) -def _parse_entry(text: str, entry: dict[str, Any]) -> dict[str, Any]: +def _parse_entry(text: str, entry: dict[str, Any], *, patient_dob: str = "") -> dict[str, Any]: title = _entry_title(entry) source_pdf = entry.get("source_pdf") or "" source_system = (entry.get("source_system") or "").lower() @@ -1087,63 +1114,102 @@ def _parse_entry(text: str, entry: dict[str, Any]) -> dict[str, Any]: if is_legacy and doc_type == "lab": if "Показатель" in text and "Референсные значения" in text: - return parse_gemotest(text, title, source_pdf, fallback_iso=fallback_iso) - return parse_emias_lab(text, title, source_pdf=source_pdf, fallback_iso=fallback_iso) + return parse_gemotest( + text, title, source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) + return parse_emias_lab( + text, title, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) if source_system == "medsi" or "sources/medsi" in source_pdf: facility = _extract_medsi_facility(text) if doc_type == "lab" or "анализ крови" in title_l or "биохим" in title_l: parsed = parse_medsi_lab( - text, title, source_pdf=source_pdf, fallback_iso=fallback_iso + text, + title, + source_pdf=source_pdf, + fallback_iso=fallback_iso, + patient_dob=patient_dob, ) parsed["institution"] = facility return parsed if doc_type == "functional" or "электрокарди" in title_l or "эхокг" in title_l: return parse_emias_consult_or_imaging( - text, title, "functional", source_pdf=source_pdf, fallback_iso=fallback_iso, + text, + title, + "functional", + source_pdf=source_pdf, + fallback_iso=fallback_iso, facility_override=facility, + patient_dob=patient_dob, ) if doc_type == "imaging" or "ультразвуков" in title_l or "дуплекс" in title_l: return parse_emias_consult_or_imaging( - text, title, "imaging", source_pdf=source_pdf, fallback_iso=fallback_iso, + text, + title, + "imaging", + source_pdf=source_pdf, + fallback_iso=fallback_iso, facility_override=facility, + patient_dob=patient_dob, ) return parse_emias_consult_or_imaging( - text, title, "consultation", source_pdf=source_pdf, fallback_iso=fallback_iso, + text, + title, + "consultation", + source_pdf=source_pdf, + fallback_iso=fallback_iso, facility_override=facility, + patient_dob=patient_dob, ) if source_system == "gemotest" or "sources/gemotest" in source_pdf: - return parse_gemotest(text, title, source_pdf, fallback_iso=fallback_iso) + return parse_gemotest( + text, title, source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) if doc_type == "lab" or "коронавирус" in title.lower() or "covid" in title.lower(): - return parse_emias_lab(text, title, source_pdf=source_pdf, fallback_iso=fallback_iso) + return parse_emias_lab( + text, title, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) if doc_type in ("consultation", "consult"): return parse_emias_consult_or_imaging( - text, title, "consultation", source_pdf=source_pdf, fallback_iso=fallback_iso + text, + title, + "consultation", + source_pdf=source_pdf, + fallback_iso=fallback_iso, + patient_dob=patient_dob, ) if doc_type == "imaging": return parse_emias_consult_or_imaging( - text, title, "imaging", source_pdf=source_pdf, fallback_iso=fallback_iso + text, + title, + "imaging", + source_pdf=source_pdf, + fallback_iso=fallback_iso, + patient_dob=patient_dob, ) return parse_emias_consult_or_imaging( - text, title, doc_type, source_pdf=source_pdf, fallback_iso=fallback_iso + text, + title, + doc_type, + source_pdf=source_pdf, + fallback_iso=fallback_iso, + patient_dob=patient_dob, ) -def _configure_patient_dob(corpus: Path) -> None: - global _skip_patient_dob - _skip_patient_dob = load_patient_dob(corpus) - - def run( corpus: Path, *, dry_run: bool = False, force: bool = False, sources: set[str] | None = None, + patient_dob: str = "", ) -> dict[str, Any]: - _configure_patient_dob(corpus) + if not patient_dob: + patient_dob = load_patient_dob(corpus) manifest = load_manifest(corpus) pdfs: list[dict[str, Any]] = list(manifest.get("pdfs") or []) @@ -1171,7 +1237,7 @@ def run( continue try: - extracted = _parse_entry(text, entry) + extracted = _parse_entry(text, entry, patient_dob=patient_dob) except Exception as exc: errors.append(f"parse {source_pdf}: {exc}") skipped += 1 From 8a5481f8ef846a38ac16b628c54f5a73f7667989 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 13:37:38 +0000 Subject: [PATCH 03/11] Add shared time_util and cli_args utilities for corpus pipeline Introduce utc_now_iso() and corpus argparse helpers, then replace duplicated inline datetime formatting and --corpus parsing across scan/extract/pipeline modules. Co-authored-by: apodobe --- medbots/cli_args.py | 37 +++++++++++++++++++ medbots/dedup_labs.py | 15 ++------ medbots/extract_pdf_text.py | 4 +- medbots/pipeline/apply_loinc_map.py | 16 ++------ .../pipeline/extract_goals_from_doc_text.py | 10 ++--- .../pipeline/extract_protocols_from_corpus.py | 12 +++--- .../extract_supplements_from_corpus.py | 14 +++---- medbots/pipeline/generate_discrepancies.py | 6 +-- medbots/pipeline/generate_lhm.py | 22 +++-------- medbots/pipeline/reconcile_goals.py | 8 ++-- medbots/scan_sources.py | 6 +-- medbots/time_util.py | 8 ++++ 12 files changed, 82 insertions(+), 76 deletions(-) create mode 100644 medbots/cli_args.py create mode 100644 medbots/time_util.py diff --git a/medbots/cli_args.py b/medbots/cli_args.py new file mode 100644 index 0000000..c124420 --- /dev/null +++ b/medbots/cli_args.py @@ -0,0 +1,37 @@ +"""Shared argparse helpers for corpus CLI entrypoints.""" +from __future__ import annotations + +import argparse +import sys +from collections.abc import Callable +from pathlib import Path + +from medbots.corpus_io import default_corpus_root + + +def add_corpus_argument(parser: argparse.ArgumentParser, default: Path | None = None) -> None: + parser.add_argument( + "--corpus", + type=Path, + default=default, + help="structured_database path override", + ) + + +def resolve_corpus_from_args(args: argparse.Namespace) -> Path: + corpus_arg = getattr(args, "corpus", None) + if corpus_arg is None: + corpus_arg = default_corpus_root() + corpus = corpus_arg.expanduser().resolve() + if not corpus.is_dir(): + print(f"ERROR: corpus not found: {corpus}", file=sys.stderr) + sys.exit(1) + return corpus + + +def corpus_main_wrapper(description: str, run_fn: Callable[[Path], int]) -> int: + ap = argparse.ArgumentParser(description=description) + add_corpus_argument(ap) + args = ap.parse_args() + corpus = resolve_corpus_from_args(args) + return run_fn(corpus) diff --git a/medbots/dedup_labs.py b/medbots/dedup_labs.py index 299a59c..572bef5 100644 --- a/medbots/dedup_labs.py +++ b/medbots/dedup_labs.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any +from medbots.cli_args import add_corpus_argument, resolve_corpus_from_args from medbots.corpus_io import load_labs, manifest_vendor_index, write_labs _SOURCE_TIER = { @@ -95,20 +96,10 @@ def dedup_labs(corpus: Path, *, apply: bool = True) -> dict[str, int]: def main() -> int: ap = argparse.ArgumentParser(description="Deduplicate LABS_NORMALIZED.json") - ap.add_argument( - "--corpus", - type=Path, - default=None, - ) + add_corpus_argument(ap) ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() - from medbots.corpus_io import default_corpus_root - if getattr(args, "corpus", None) is None: - args.corpus = default_corpus_root() - corpus = args.corpus.expanduser().resolve() - if not corpus.is_dir(): - print(f"ERROR: corpus not found: {corpus}", file=sys.stderr) - return 1 + corpus = resolve_corpus_from_args(args) try: stats = dedup_labs(corpus, apply=not args.dry_run) except FileNotFoundError as exc: diff --git a/medbots/extract_pdf_text.py b/medbots/extract_pdf_text.py index e7ddc53..72f1ee5 100644 --- a/medbots/extract_pdf_text.py +++ b/medbots/extract_pdf_text.py @@ -5,13 +5,13 @@ import json import re import sys -from datetime import datetime, timezone from pathlib import Path from typing import Any import fitz from medbots.corpus_io import load_manifest, resolve_corpus, write_manifest +from medbots.time_util import utc_now_iso def _safe_txt_name(bot_root: Path, pdf_path: Path) -> str: @@ -83,7 +83,7 @@ def run(bot_root: Path, corpus: Path | None = None) -> dict[str, Any]: manifest["pdfs"] = updated meta = dict(manifest.get("meta") or {}) meta["pdf_count"] = len(updated) - meta["pdf_text_extracted"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + meta["pdf_text_extracted"] = utc_now_iso() manifest["meta"] = meta write_manifest(corp, manifest) diff --git a/medbots/pipeline/apply_loinc_map.py b/medbots/pipeline/apply_loinc_map.py index 34a6f51..9949a0a 100644 --- a/medbots/pipeline/apply_loinc_map.py +++ b/medbots/pipeline/apply_loinc_map.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import Any +from medbots.cli_args import add_corpus_argument, resolve_corpus_from_args + from medbots.corpus_io import load_labs, write_labs @@ -66,20 +68,10 @@ def apply_loinc_map(corpus: Path, *, apply: bool = True) -> dict[str, int]: def main() -> int: ap = argparse.ArgumentParser(description="Apply LOINC_MAP.tsv to LABS_NORMALIZED.json") - ap.add_argument( - "--corpus", - type=Path, - default=None, - ) + add_corpus_argument(ap) ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() - from medbots.corpus_io import default_corpus_root - if getattr(args, "corpus", None) is None: - args.corpus = default_corpus_root() - corpus = args.corpus.expanduser().resolve() - if not corpus.is_dir(): - print(f"ERROR: corpus not found: {corpus}", file=sys.stderr) - return 1 + corpus = resolve_corpus_from_args(args) try: stats = apply_loinc_map(corpus, apply=not args.dry_run) except FileNotFoundError as exc: diff --git a/medbots/pipeline/extract_goals_from_doc_text.py b/medbots/pipeline/extract_goals_from_doc_text.py index cd88522..002b2ff 100644 --- a/medbots/pipeline/extract_goals_from_doc_text.py +++ b/medbots/pipeline/extract_goals_from_doc_text.py @@ -7,10 +7,12 @@ import json import re import sys -from datetime import datetime, timezone +import sys from pathlib import Path from typing import Any +from medbots.time_util import utc_now_iso + _CONSULT_TYPE = re.compile(r"^type:\s*(consult|consultation)\s*$", re.M | re.I) _CONSULT_TITLE = re.compile(r"осмотр|консультац|прием\s+врач", re.I) _DATE_FRONT = re.compile(r"^date:\s*(\d{4}-\d{2}-\d{2})\s*$", re.M) @@ -50,10 +52,6 @@ ) -def _utc_now() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - def _item_id(doc_stem: str, kind: str, text: str) -> str: digest = hashlib.sha256(f"{doc_stem}|{kind}|{text}".encode()).hexdigest()[:12] return f"goal_{kind}_{digest}" @@ -207,7 +205,7 @@ def extract_goals(corpus: Path, *, apply: bool = True) -> dict[str, int]: data["items"] = list(data.get("items") or []) data["items"].extend(new_items) meta = data.setdefault("meta", {}) - meta["last_extract"] = _utc_now() + meta["last_extract"] = utc_now_iso() meta["extract_note"] = "scripts/extract_goals_from_doc_text.py" meta["extracted_by"] = "composer" meta["review_status"] = "pending_gemini" diff --git a/medbots/pipeline/extract_protocols_from_corpus.py b/medbots/pipeline/extract_protocols_from_corpus.py index d3a9275..d89431e 100644 --- a/medbots/pipeline/extract_protocols_from_corpus.py +++ b/medbots/pipeline/extract_protocols_from_corpus.py @@ -7,10 +7,12 @@ import json import re import sys -from datetime import datetime, timezone +import sys from pathlib import Path from typing import Any +from medbots.time_util import utc_now_iso + _DATE_FRONT = re.compile(r"^date:\s*(\d{4}-\d{2}-\d{2})\s*$", re.M) _TITLE_FRONT = re.compile(r"^title:\s*(.+?)\s*$", re.M) @@ -40,10 +42,6 @@ ) -def _utc_now() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - def _proto_id(category: str, snippet: str) -> str: digest = hashlib.sha256(f"{category}|{snippet}".encode()).hexdigest()[:10] return f"proto_{category}_{digest}" @@ -118,7 +116,7 @@ def extract_protocols(corpus: Path, *, apply: bool = True) -> dict[str, int]: "source_path": rel, "why_ru": f"Извлечено из «{title}»", "extracted_by": "composer", - "extracted_at": _utc_now(), + "extracted_at": utc_now_iso(), } ) existing_ids.add(pid) @@ -126,7 +124,7 @@ def extract_protocols(corpus: Path, *, apply: bool = True) -> dict[str, int]: new_count += 1 meta = data.setdefault("meta", {}) - meta["updated_at"] = _utc_now()[:10] + meta["updated_at"] = utc_now_iso()[:10] meta["extracted_by"] = "composer" meta["review_status"] = "pending_gemini" diff --git a/medbots/pipeline/extract_supplements_from_corpus.py b/medbots/pipeline/extract_supplements_from_corpus.py index cd7895b..7309542 100644 --- a/medbots/pipeline/extract_supplements_from_corpus.py +++ b/medbots/pipeline/extract_supplements_from_corpus.py @@ -7,10 +7,12 @@ import json import re import sys -from datetime import datetime, timezone +import sys from pathlib import Path from typing import Any +from medbots.time_util import utc_now_iso + _DATE_FRONT = re.compile(r"^date:\s*(\d{4}-\d{2}-\d{2})\s*$", re.M) _TITLE_FRONT = re.compile(r"^title:\s*(.+?)\s*$", re.M) @@ -46,10 +48,6 @@ ) -def _utc_now() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - def _item_id(name: str, source: str, snippet: str) -> str: digest = hashlib.sha256(f"{name}|{source}|{snippet}".encode()).hexdigest()[:12] return f"sup_{digest}" @@ -117,7 +115,7 @@ def _parse_mention(line: str, *, doc_date: str | None, title: str, rel_path: str "active": True, "status": "mentioned_in_document", "extracted_by": "composer", - "extracted_at": _utc_now(), + "extracted_at": utc_now_iso(), } @@ -149,7 +147,7 @@ def _from_goals(goals_path: Path) -> list[dict[str, Any]]: "active": bool(g.get("active", True)), "status": "from_goals_reminders", "extracted_by": "composer", - "extracted_at": _utc_now(), + "extracted_at": utc_now_iso(), } ) return out @@ -193,7 +191,7 @@ def extract_supplements(corpus: Path, *, apply: bool = True) -> dict[str, int]: "meta": { "note": "Composer v0 draft — review with Gemini/Opus before clinical use", "extracted_by": "composer", - "extracted_at": _utc_now(), + "extracted_at": utc_now_iso(), "extract_script": "scripts/extract_supplements_from_corpus.py", "review_status": "pending_gemini", }, diff --git a/medbots/pipeline/generate_discrepancies.py b/medbots/pipeline/generate_discrepancies.py index 14e1b11..37563cc 100644 --- a/medbots/pipeline/generate_discrepancies.py +++ b/medbots/pipeline/generate_discrepancies.py @@ -10,9 +10,7 @@ from pathlib import Path from typing import Any - -def _utc_now() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +from medbots.time_util import utc_now_iso def _item_id(category: str, payload: str) -> str: @@ -193,7 +191,7 @@ def generate_discrepancies(corpus: Path, *, apply: bool = True) -> dict[str, int payload = { "version": 1, - "generated_at": _utc_now(), + "generated_at": utc_now_iso(), "items": items, "meta": { "generator": "scripts/generate_discrepancies.py", diff --git a/medbots/pipeline/generate_lhm.py b/medbots/pipeline/generate_lhm.py index 2ef9bc4..9118cdd 100644 --- a/medbots/pipeline/generate_lhm.py +++ b/medbots/pipeline/generate_lhm.py @@ -6,13 +6,11 @@ import json import sys from collections import Counter -from datetime import datetime, timezone from pathlib import Path from typing import Any - -def _utc_now() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +from medbots.cli_args import add_corpus_argument, resolve_corpus_from_args +from medbots.time_util import utc_now_iso def _read_excerpt(path: Path, max_lines: int) -> str: @@ -62,7 +60,7 @@ def generate_lhm(corpus: Path) -> Path: lines: list[str] = [ "# Живая сводка здоровья (LHM)", "", - f"**Сгенерировано:** {_utc_now()}", + f"**Сгенерировано:** {utc_now_iso()}", f"**Пациент:** {patient}", "", "## Лаборатория — топ-30 показателей (последние 5 значений)", @@ -128,19 +126,9 @@ def generate_lhm(corpus: Path) -> Path: def main() -> int: ap = argparse.ArgumentParser(description="Generate LIVING_HEALTH_SUMMARY.md") - ap.add_argument( - "--corpus", - type=Path, - default=None, - ) + add_corpus_argument(ap) args = ap.parse_args() - from medbots.corpus_io import default_corpus_root - if getattr(args, "corpus", None) is None: - args.corpus = default_corpus_root() - corpus = args.corpus.expanduser().resolve() - if not corpus.is_dir(): - print(f"ERROR: corpus not found: {corpus}", file=sys.stderr) - return 1 + corpus = resolve_corpus_from_args(args) try: out = generate_lhm(corpus) except FileNotFoundError as exc: diff --git a/medbots/pipeline/reconcile_goals.py b/medbots/pipeline/reconcile_goals.py index 5e4c691..7149454 100644 --- a/medbots/pipeline/reconcile_goals.py +++ b/medbots/pipeline/reconcile_goals.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any +from medbots.time_util import utc_now_iso + _FILENAME_DATE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})") @@ -26,10 +28,6 @@ def _file_doc_date(path: Path) -> float | None: return None -def _utc_now() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - def _load_goals(corpus: Path) -> dict[str, Any]: p = corpus / "GOALS_REMINDERS.json" if not p.exists(): @@ -103,7 +101,7 @@ def reconcile( changed.append(str(item.get("id", "?"))) meta = data.setdefault("meta", {}) - meta["last_auto_sync"] = _utc_now() + meta["last_auto_sync"] = utc_now_iso() if apply and changed: out = corpus / "GOALS_REMINDERS.json" out.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") diff --git a/medbots/scan_sources.py b/medbots/scan_sources.py index 4a7f08a..7151a17 100644 --- a/medbots/scan_sources.py +++ b/medbots/scan_sources.py @@ -6,11 +6,11 @@ import json import re import sys -from datetime import datetime, timezone from pathlib import Path from typing import Any from medbots.corpus_io import empty_manifest, load_manifest, resolve_corpus, write_manifest +from medbots.time_util import utc_now_iso _VENDORS = ("emias", "medsi", "gemotest") @@ -63,7 +63,7 @@ def scan( "source_pdf": rel, "source_system": vendor, "doc_type": "lab" if vendor != "emias" else "other", - "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "created_at": utc_now_iso(), "sha256": digest, "user_drop_title": title, **({f"{vendor}_title": title} if vendor == "gemotest" else {}), @@ -78,7 +78,7 @@ def scan( manifest["pdfs"] = pdfs meta = dict(manifest.get("meta") or {}) meta["pdf_count"] = len(pdfs) - meta["scanned_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + meta["scanned_at"] = utc_now_iso() manifest["meta"] = meta write_manifest(corp, manifest) diff --git a/medbots/time_util.py b/medbots/time_util.py new file mode 100644 index 0000000..b511c4b --- /dev/null +++ b/medbots/time_util.py @@ -0,0 +1,8 @@ +"""Shared UTC timestamp helpers.""" +from __future__ import annotations + +from datetime import datetime, timezone + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") From f22afe099994e354ed2cf7ffcf3f3c1bc2d17850 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 13:38:14 +0000 Subject: [PATCH 04/11] Fix double parsing in merge_labs_corpus and add medsi/emias dedup test Parse each lab document once and store the extracted dict in targets. Load patient_dob once via load_patient_dob and pass to _parse_entry. Reuse stored extraction in dry_run and write paths. Add test_dedup_vendor_priority for medsi-over-emias vendor priority. Add test_merge_parses_each_document_once to guard against re-parsing. Co-authored-by: apodobe --- medbots/merge_labs_corpus.py | 17 ++++------ tests/test_dedup_vendor_priority.py | 51 +++++++++++++++++++++++++++++ tests/test_merge_labs_corpus.py | 31 ++++++++++++++++++ 3 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 tests/test_dedup_vendor_priority.py diff --git a/medbots/merge_labs_corpus.py b/medbots/merge_labs_corpus.py index ad29436..aa90357 100644 --- a/medbots/merge_labs_corpus.py +++ b/medbots/merge_labs_corpus.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any -from medbots.corpus_io import default_corpus_root, load_labs, load_manifest, write_labs +from medbots.corpus_io import default_corpus_root, load_labs, load_manifest, load_patient_dob, write_labs from medbots.corpus_writers import _append_lab_rows from medbots.local_structure_pdfs import _entry_title, _parse_entry, _pdf_text_path @@ -55,10 +55,11 @@ def _remove_rows_for_sources(corpus: Path, source_paths: set[str]) -> int: def run(corpus: Path, *, dry_run: bool = False, replace: bool = True) -> dict[str, Any]: + patient_dob = load_patient_dob(corpus) manifest = load_manifest(corpus) pdfs: list[dict[str, Any]] = list(manifest.get("pdfs") or []) - targets: list[tuple[dict[str, Any], str, str]] = [] + targets: list[tuple[dict[str, Any], str, str, dict[str, Any]]] = [] errors: list[str] = [] source_paths: set[str] = set() @@ -75,7 +76,7 @@ def run(corpus: Path, *, dry_run: bool = False, replace: bool = True) -> dict[st errors.append(f"empty text: {source_pdf}") continue try: - extracted = _parse_entry(text, entry) + extracted = _parse_entry(text, entry, patient_dob=patient_dob) except Exception as exc: errors.append(f"parse {source_pdf}: {exc}") continue @@ -84,7 +85,7 @@ def run(corpus: Path, *, dry_run: bool = False, replace: bool = True) -> dict[st continue doc_rel = _doc_text_rel(entry) source_paths.add(doc_rel) - targets.append((entry, doc_rel, source_pdf)) + targets.append((entry, doc_rel, source_pdf, extracted)) before_count = 0 labs_path = corpus / "LABS_NORMALIZED.json" @@ -98,9 +99,7 @@ def run(corpus: Path, *, dry_run: bool = False, replace: bool = True) -> dict[st labs_added_total = 0 per_doc: list[dict[str, Any]] = [] if not dry_run: - for entry, doc_rel, source_pdf in targets: - text = _pdf_text_path(corpus, entry).read_text(encoding="utf-8").strip() - extracted = _parse_entry(text, entry) + for _entry, doc_rel, source_pdf, extracted in targets: added = _append_lab_rows(corpus, extracted.get("lab_rows") or [], doc_rel) labs_added_total += added per_doc.append( @@ -112,9 +111,7 @@ def run(corpus: Path, *, dry_run: bool = False, replace: bool = True) -> dict[st } ) else: - for entry, doc_rel, source_pdf in targets: - text = _pdf_text_path(corpus, entry).read_text(encoding="utf-8").strip() - extracted = _parse_entry(text, entry) + for _entry, doc_rel, source_pdf, extracted in targets: n = len(extracted.get("lab_rows") or []) labs_added_total += n per_doc.append( diff --git a/tests/test_dedup_vendor_priority.py b/tests/test_dedup_vendor_priority.py new file mode 100644 index 0000000..f0ec8d4 --- /dev/null +++ b/tests/test_dedup_vendor_priority.py @@ -0,0 +1,51 @@ +"""Vendor priority in dedup_labs: medsi wins over emias for duplicate keys.""" +from __future__ import annotations + +import json +from pathlib import Path + +from medbots.dedup_labs import dedup_labs + + +def _write_labs(corpus: Path, rows: list[dict]) -> None: + path = corpus / "LABS_NORMALIZED.json" + path.write_text(json.dumps({"rows": rows}, ensure_ascii=False), encoding="utf-8") + + +def _read_labs(corpus: Path) -> list[dict]: + data = json.loads((corpus / "LABS_NORMALIZED.json").read_text(encoding="utf-8")) + return data["rows"] + + +def test_dedup_medsi_wins_over_emias(tmp_corpus: Path) -> None: + rows = [ + { + "canonical_key": "glucose", + "specimen_date": "2024-03-15", + "source_path": "sources/emias/2024-03-15__blood__abc123.pdf", + "value": "5.0", + "unit": "mmol/L", + }, + { + "canonical_key": "glucose", + "specimen_date": "2024-03-15", + "source_path": "sources/medsi/2024-03-15__blood__def456.pdf", + "value": "5.2", + "unit": "mmol/L", + "ref_low": 3.9, + "ref_high": 6.1, + }, + ] + _write_labs(tmp_corpus, rows) + + stats = dedup_labs(tmp_corpus, apply=True) + + assert stats["rows_before"] == 2 + assert stats["rows_after"] == 1 + assert stats["removed"] == 1 + assert stats["dup_groups"] == 1 + + kept = _read_labs(tmp_corpus) + assert len(kept) == 1 + assert "medsi" in kept[0]["source_path"] + assert kept[0]["value"] == "5.2" diff --git a/tests/test_merge_labs_corpus.py b/tests/test_merge_labs_corpus.py index e21d23a..809243f 100644 --- a/tests/test_merge_labs_corpus.py +++ b/tests/test_merge_labs_corpus.py @@ -3,9 +3,11 @@ import json from pathlib import Path +from unittest.mock import patch import pytest +from medbots.local_structure_pdfs import _parse_entry as real_parse_entry from medbots.merge_labs_corpus import run @@ -169,3 +171,32 @@ def test_merge_legacy_flat_lab_rows( assert stats["errors"] == [] assert stats["documents"] == 1 assert stats["lab_rows_added"] > 0 + + +def test_merge_parses_each_document_once( + tmp_corpus: Path, + load_pdf_text_fixture, +) -> None: + txt_name = ( + "sources__gemotest__2021-02-14__gemotest_69371781__" + "Биохимия_19_показателей__d6c4ee6e.pdf.txt" + ) + text = load_pdf_text_fixture(txt_name) + source_pdf = ( + "sources/gemotest/2021-02-14/gemotest_69371781__" + "Биохимия_19_показателей__d6c4ee6e.pdf" + ) + _write_lab_manifest( + tmp_corpus, + source_system="gemotest", + source_pdf=source_pdf, + txt_filename=txt_name, + text=text, + ) + + with patch("medbots.merge_labs_corpus._parse_entry", wraps=real_parse_entry) as parse_mock: + stats = run(tmp_corpus, dry_run=False) + + assert stats["errors"] == [] + assert stats["documents"] == 1 + assert parse_mock.call_count == 1 From f79367059526e73ba2c50a1992802c38523b7be8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 13:41:10 +0000 Subject: [PATCH 05/11] Replace subprocess pipeline steps with direct function calls Refactor medbots/pipeline/run.py to import and invoke module functions directly instead of spawning python -m subprocesses for merge labs, LOINC mapping, dedup, goals/supplements/protocols extraction, discrepancies, LHM generation, goal reconciliation, validation, and corpus index writing. Keep subprocess only for optional legacy scripts (bridge_legacy_flat_pdfs, reconcile_weekly_pending). Update cli.py structure and validate commands to call local_structure_pdfs.run() and validate_corpus.main() directly. Co-authored-by: apodobe --- medbots/cli.py | 38 ++++++++++-------- medbots/pipeline/run.py | 89 +++++++++++++++++++++++++++++------------ 2 files changed, 84 insertions(+), 43 deletions(-) diff --git a/medbots/cli.py b/medbots/cli.py index 9300f00..403f36d 100644 --- a/medbots/cli.py +++ b/medbots/cli.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse +import json +import os import subprocess import sys from pathlib import Path @@ -12,7 +14,9 @@ from medbots.extract_pdf_text import run as run_extract from medbots.import_apple_health import run_import from medbots.init_instance import init as run_init +from medbots.local_structure_pdfs import run as run_structure from medbots.pipeline.run import run_pipeline +from medbots.pipeline.validate_corpus import main as validate_corpus_main from medbots.scan_sources import scan as run_scan @@ -59,20 +63,18 @@ def _cmd_validate_apple_health(args: argparse.Namespace) -> int: def _cmd_structure(args: argparse.Namespace) -> int: root = Path(args.bot_root).resolve() if args.bot_root else find_bot_root() corpus = resolve_corpus(args.corpus) if args.corpus else resolve_corpus(root / "structured_database") - cmd = [ - sys.executable, - "-m", - "medbots.local_structure_pdfs", - "--corpus", - str(corpus), - ] - if args.force: - cmd.append("--force") - for src in args.source: - cmd.extend(["--source", src]) - if args.dry_run: - cmd.append("--dry-run") - return subprocess.call(cmd) + source_filter = {s.strip().lower() for s in args.source if s.strip()} or None + stats = run_structure( + corpus, + dry_run=args.dry_run, + force=args.force, + sources=source_filter, + ) + print(json.dumps(stats, ensure_ascii=False, indent=2)) + if stats.get("errors"): + for err in stats["errors"]: + print(f"WARN: {err}", file=sys.stderr) + return 0 def _cmd_pipeline(args: argparse.Namespace) -> int: @@ -80,14 +82,16 @@ def _cmd_pipeline(args: argparse.Namespace) -> int: corpus = resolve_corpus(args.corpus) if args.corpus else None try: run_pipeline(bot_root=root, corpus=corpus) - except subprocess.CalledProcessError as exc: - return exc.returncode or 1 + except SystemExit as exc: + code = exc.code + return code if isinstance(code, int) else 1 return 0 def _cmd_validate(args: argparse.Namespace) -> int: corpus = resolve_corpus(args.corpus) - return subprocess.call([sys.executable, "-m", "medbots.pipeline.validate_corpus", "--corpus", str(corpus)]) + os.environ["MEDBOTS_CORPUS_PATH"] = str(corpus) + return validate_corpus_main() def _cmd_patient_dob(args: argparse.Namespace) -> int: diff --git a/medbots/pipeline/run.py b/medbots/pipeline/run.py index 7f27e0b..fbd741c 100644 --- a/medbots/pipeline/run.py +++ b/medbots/pipeline/run.py @@ -2,6 +2,7 @@ """Run post-ingest corpus pipeline with bot_config feature flags.""" from __future__ import annotations +import json import os import subprocess import sys @@ -9,14 +10,35 @@ from medbots.config import BotConfig, corpus_from_config, feature_enabled, load_config from medbots.corpus_io import resolve_corpus - - -def _run_module(module: str, corpus: Path, extra: list[str] | None = None) -> None: - cmd = [sys.executable, "-m", module, "--corpus", str(corpus)] - if extra: - cmd.extend(extra) - print(f"==> {module}") - subprocess.run(cmd, check=True) +from medbots.dedup_labs import dedup_labs +from medbots.merge_labs_corpus import run as merge_labs_corpus +from medbots.pipeline.apply_loinc_map import apply_loinc_map +from medbots.pipeline.extract_goals_from_doc_text import extract_goals +from medbots.pipeline.extract_protocols_from_corpus import extract_protocols +from medbots.pipeline.extract_supplements_from_corpus import extract_supplements +from medbots.pipeline.generate_discrepancies import generate_discrepancies +from medbots.pipeline.generate_lhm import generate_lhm +from medbots.pipeline.reconcile_goals import reconcile +from medbots.pipeline.validate_corpus import main as validate_corpus_main +from medbots.pipeline.write_corpus_index import build_index + + +def _exit_on_failure(step: str, exc: BaseException | None = None, code: int = 1) -> None: + if exc is not None: + print(f"ERROR: {step} failed: {exc}", file=sys.stderr) + raise SystemExit(code) from exc + print(f"ERROR: {step} failed with exit code {code}", file=sys.stderr) + raise SystemExit(code) + + +def _run_step(step: str, fn) -> None: + print(f"==> {step}") + try: + result = fn() + except Exception as exc: + _exit_on_failure(step, exc) + if isinstance(result, int) and result != 0: + _exit_on_failure(step, code=result) def _run_script(script: Path, corpus: Path, extra: list[str] | None = None) -> None: @@ -24,7 +46,10 @@ def _run_script(script: Path, corpus: Path, extra: list[str] | None = None) -> N if extra: cmd.extend(extra) print(f"==> {script.name}") - subprocess.run(cmd, check=True) + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as exc: + _exit_on_failure(script.name, code=exc.returncode or 1) def _corpus_has(corpus: Path, rel: str) -> bool: @@ -52,19 +77,28 @@ def run_pipeline( if bridge.is_file(): _run_script(bridge, corp, ["--apply"]) - _run_module("medbots.merge_labs_corpus", corp) - _run_module("medbots.pipeline.apply_loinc_map", corp) - _run_module("medbots.dedup_labs", corp) + _run_step("medbots.merge_labs_corpus", lambda: merge_labs_corpus(corp, dry_run=False, replace=True)) + _run_step("medbots.pipeline.apply_loinc_map", lambda: apply_loinc_map(corp, apply=True)) + _run_step("medbots.dedup_labs", lambda: dedup_labs(corp, apply=True)) if feature_enabled("goals_reminders", cfg): - _run_module("medbots.pipeline.extract_goals_from_doc_text", corp) + _run_step( + "medbots.pipeline.extract_goals_from_doc_text", + lambda: extract_goals(corp, apply=True), + ) if nutrition or _corpus_has(corp, "nutrition/NUTRITION.json"): - _run_module("medbots.pipeline.extract_supplements_from_corpus", corp) - _run_module("medbots.pipeline.extract_protocols_from_corpus", corp) - - _run_module("medbots.pipeline.generate_discrepancies", corp) - _run_module("medbots.pipeline.generate_lhm", corp) + _run_step( + "medbots.pipeline.extract_supplements_from_corpus", + lambda: extract_supplements(corp, apply=True), + ) + _run_step( + "medbots.pipeline.extract_protocols_from_corpus", + lambda: extract_protocols(corp, apply=True), + ) + + _run_step("medbots.pipeline.generate_discrepancies", lambda: generate_discrepancies(corp, apply=True)) + _run_step("medbots.pipeline.generate_lhm", lambda: generate_lhm(corp)) if feature_enabled("weekly_pending", cfg): weekly = scripts / "reconcile_weekly_pending.py" @@ -74,15 +108,18 @@ def run_pipeline( print("==> reconcile_weekly_pending (skipped: script not found)") if feature_enabled("goals_reminders", cfg): - _run_module("medbots.pipeline.reconcile_goals", corp, ["--apply"]) + _run_step( + "medbots.pipeline.reconcile_goals", + lambda: reconcile(corp, apply=True), + ) + + _run_step("medbots.pipeline.validate_corpus", validate_corpus_main) - _run_module("medbots.pipeline.validate_corpus", corp) + def _write_index() -> None: + index = build_index(corp, cfg.bot_id, cfg.vps_corpus_path) + out = corp / "CORPUS_INDEX.json" + out.write_text(json.dumps(index, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - index_args: list[str] = ["--bot-root", str(root)] - if corp: - index_args.extend(["--corpus", str(corp)]) - cmd = [sys.executable, "-m", "medbots.pipeline.write_corpus_index", *index_args] - print("==> write_corpus_index") - subprocess.run(cmd, check=True, cwd=str(root), env={**os.environ, "MEDBOTS_CORPUS_PATH": str(corp)}) + _run_step("write_corpus_index", _write_index) print("Pipeline OK.") From 78b36b234303b3340b0f627680473535500a92f1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 13:41:19 +0000 Subject: [PATCH 06/11] Add unified vendor_registry module and consolidate detection logic Extract vendor detection, lab-source filtering, and structure-target selection into medbots/vendor_registry.py. Update merge_labs_corpus and local_structure_pdfs to use the shared helpers, and add unit tests. Co-authored-by: apodobe --- medbots/local_structure_pdfs.py | 36 +------- medbots/merge_labs_corpus.py | 20 +---- medbots/vendor_registry.py | 78 ++++++++++++++++ tests/test_vendor_registry.py | 155 ++++++++++++++++++++++++++++++++ 4 files changed, 238 insertions(+), 51 deletions(-) create mode 100644 medbots/vendor_registry.py create mode 100644 tests/test_vendor_registry.py diff --git a/medbots/local_structure_pdfs.py b/medbots/local_structure_pdfs.py index cfaf498..26b567b 100644 --- a/medbots/local_structure_pdfs.py +++ b/medbots/local_structure_pdfs.py @@ -13,6 +13,7 @@ from medbots.corpus_io import bot_root, default_corpus_root, load_manifest, load_patient_dob, write_manifest from medbots.time_util import utc_now_iso +from medbots.vendor_registry import is_legacy_flat_entry, is_structure_target_entry from medbots.corpus_writers import ( _append_lab_rows, _safe_txt_name, @@ -1021,14 +1022,6 @@ def _append_extracted_section_if_new( _write_to_extracted_images_md(corpus, section_id, extracted) -def _is_legacy_flat_entry(entry: dict[str, Any]) -> bool: - source_system = (entry.get("source_system") or "").lower() - source_pdf = entry.get("source_pdf") or "" - if source_system == "legacy_flat" and entry.get("extracted_txt"): - return True - return bool(source_pdf and not source_pdf.startswith("sources/") and entry.get("extracted_txt")) - - def _infer_legacy_doc_type(source_pdf: str) -> str: s = source_pdf.lower() if any( @@ -1050,29 +1043,6 @@ def _infer_legacy_doc_type(source_pdf: str) -> str: return "other" -def _is_target_entry(entry: dict[str, Any], *, force: bool = False, sources: set[str] | None = None) -> bool: - if entry.get("grok_ingested_at"): - return False - source_pdf = entry.get("source_pdf") or "" - source_system = (entry.get("source_system") or "").lower() - if sources and source_system not in sources: - if not any(s in source_pdf for s in sources): - if not (_is_legacy_flat_entry(entry) and "legacy_flat" in sources): - return False - if entry.get("structured_locally_at") and not force: - return False - if _is_legacy_flat_entry(entry): - return True - if source_system in ("emias", "gemotest"): - return True - if "sources/emias" in source_pdf or "sources/gemotest" in source_pdf: - return True - if source_system == "medsi" or "sources/medsi" in source_pdf: - if entry.get("user_drop_batch") or entry.get("ingest_note") == "ALL-NEW-FILES drop": - return True - return False - - def _entry_title(entry: dict[str, Any]) -> str: return ( entry.get("emias_title") @@ -1108,7 +1078,7 @@ def _parse_entry(text: str, entry: dict[str, Any], *, patient_dob: str = "") -> doc_type = entry.get("doc_type") or "other" fallback_iso = (entry.get("created_at") or "")[:10] or None title_l = title.lower() - is_legacy = _is_legacy_flat_entry(entry) + is_legacy = is_legacy_flat_entry(entry) if is_legacy and not doc_type: doc_type = _infer_legacy_doc_type(source_pdf) @@ -1220,7 +1190,7 @@ def run( now = _utc_ts() for idx, entry in enumerate(pdfs): - if not _is_target_entry(entry, force=force, sources=sources): + if not is_structure_target_entry(entry, force=force, sources=sources): continue source_pdf = entry.get("source_pdf") or "" diff --git a/medbots/merge_labs_corpus.py b/medbots/merge_labs_corpus.py index aa90357..acd79e0 100644 --- a/medbots/merge_labs_corpus.py +++ b/medbots/merge_labs_corpus.py @@ -11,23 +11,7 @@ from medbots.corpus_io import default_corpus_root, load_labs, load_manifest, load_patient_dob, write_labs from medbots.corpus_writers import _append_lab_rows from medbots.local_structure_pdfs import _entry_title, _parse_entry, _pdf_text_path - - -def _is_lab_source(entry: dict[str, Any]) -> bool: - source_system = (entry.get("source_system") or "").lower() - source_pdf = entry.get("source_pdf") or "" - doc_type = entry.get("doc_type") or "" - if source_system == "legacy_flat": - return doc_type == "lab" - if doc_type == "lab" and ( - source_system == "medsi" or "sources/medsi" in source_pdf - ): - return True - if source_system in ("emias", "gemotest"): - return True - if not source_pdf.startswith("sources/") and doc_type == "lab": - return True - return "sources/emias" in source_pdf or "sources/gemotest" in source_pdf +from medbots.vendor_registry import is_lab_source_entry def _doc_text_rel(entry: dict[str, Any]) -> str: @@ -64,7 +48,7 @@ def run(corpus: Path, *, dry_run: bool = False, replace: bool = True) -> dict[st source_paths: set[str] = set() for entry in pdfs: - if not _is_lab_source(entry): + if not is_lab_source_entry(entry): continue source_pdf = entry.get("source_pdf") or "" txt_path = _pdf_text_path(corpus, entry) diff --git a/medbots/vendor_registry.py b/medbots/vendor_registry.py new file mode 100644 index 0000000..b41cdd9 --- /dev/null +++ b/medbots/vendor_registry.py @@ -0,0 +1,78 @@ +"""Unified vendor detection for corpus manifest entries.""" +from __future__ import annotations + +from typing import Any + +VENDORS = ("medsi", "gemotest", "emias", "legacy_flat") + + +def detect_vendor_from_source_pdf(source_pdf: str) -> str: + if not source_pdf: + return "" + normalized = source_pdf.replace("\\", "/") + for vendor in ("medsi", "gemotest", "emias"): + if f"sources/{vendor}" in normalized: + return vendor + if not normalized.startswith("sources/"): + return "legacy_flat" + return "" + + +def detect_vendor_from_entry(entry: dict[str, Any]) -> str: + source_system = (entry.get("source_system") or "").lower() + if source_system in VENDORS: + return source_system + return detect_vendor_from_source_pdf(entry.get("source_pdf") or "") + + +def is_legacy_flat_entry(entry: dict[str, Any]) -> bool: + source_system = (entry.get("source_system") or "").lower() + source_pdf = entry.get("source_pdf") or "" + if source_system == "legacy_flat" and entry.get("extracted_txt"): + return True + return bool(source_pdf and not source_pdf.startswith("sources/") and entry.get("extracted_txt")) + + +def is_lab_source_entry(entry: dict[str, Any]) -> bool: + source_system = (entry.get("source_system") or "").lower() + source_pdf = entry.get("source_pdf") or "" + doc_type = entry.get("doc_type") or "" + if source_system == "legacy_flat": + return doc_type == "lab" + if doc_type == "lab" and ( + source_system == "medsi" or "sources/medsi" in source_pdf + ): + return True + if source_system in ("emias", "gemotest"): + return True + if not source_pdf.startswith("sources/") and doc_type == "lab": + return True + return "sources/emias" in source_pdf or "sources/gemotest" in source_pdf + + +def is_structure_target_entry( + entry: dict[str, Any], + *, + force: bool = False, + sources: set[str] | None = None, +) -> bool: + if entry.get("grok_ingested_at"): + return False + source_pdf = entry.get("source_pdf") or "" + source_system = (entry.get("source_system") or "").lower() + if sources and source_system not in sources: + if not any(s in source_pdf for s in sources): + if not (is_legacy_flat_entry(entry) and "legacy_flat" in sources): + return False + if entry.get("structured_locally_at") and not force: + return False + if is_legacy_flat_entry(entry): + return True + if source_system in ("emias", "gemotest"): + return True + if "sources/emias" in source_pdf or "sources/gemotest" in source_pdf: + return True + if source_system == "medsi" or "sources/medsi" in source_pdf: + if entry.get("user_drop_batch") or entry.get("ingest_note") == "ALL-NEW-FILES drop": + return True + return False diff --git a/tests/test_vendor_registry.py b/tests/test_vendor_registry.py new file mode 100644 index 0000000..10d384c --- /dev/null +++ b/tests/test_vendor_registry.py @@ -0,0 +1,155 @@ +"""Tests for medbots.vendor_registry.""" +from __future__ import annotations + +import pytest + +from medbots.vendor_registry import ( + VENDORS, + detect_vendor_from_entry, + detect_vendor_from_source_pdf, + is_lab_source_entry, + is_legacy_flat_entry, + is_structure_target_entry, +) + + +@pytest.mark.parametrize( + ("source_pdf", "expected"), + [ + ("sources/medsi/2024-01-10/drop/blood.pdf", "medsi"), + ("sources/gemotest/2021-02-14/report.pdf", "gemotest"), + ("sources/emias/2021-01-18/covid.pdf", "emias"), + ("2021-01-18_covid_antibodies.pdf", "legacy_flat"), + ("sources/unknown/vendor.pdf", ""), + ("", ""), + ], +) +def test_detect_vendor_from_source_pdf(source_pdf: str, expected: str) -> None: + assert detect_vendor_from_source_pdf(source_pdf) == expected + + +@pytest.mark.parametrize( + ("entry", "expected"), + [ + ({"source_system": "gemotest", "source_pdf": "sources/gemotest/x.pdf"}, "gemotest"), + ({"source_system": "EMIAS", "source_pdf": "sources/emias/x.pdf"}, "emias"), + ({"source_pdf": "sources/medsi/x.pdf"}, "medsi"), + ({"source_system": "legacy_flat", "source_pdf": "flat.pdf"}, "legacy_flat"), + ({"source_pdf": "flat.pdf", "extracted_txt": "pdf_text/flat.pdf.txt"}, "legacy_flat"), + ], +) +def test_detect_vendor_from_entry(entry: dict, expected: str) -> None: + assert detect_vendor_from_entry(entry) == expected + + +def test_vendors_tuple() -> None: + assert VENDORS == ("medsi", "gemotest", "emias", "legacy_flat") + + +@pytest.mark.parametrize( + ("entry", "expected"), + [ + ({"source_system": "legacy_flat", "doc_type": "lab"}, True), + ({"source_system": "legacy_flat", "doc_type": "consult"}, False), + ({"source_system": "gemotest", "source_pdf": "sources/gemotest/x.pdf"}, True), + ({"source_system": "emias", "source_pdf": "sources/emias/x.pdf"}, True), + ( + { + "source_system": "medsi", + "source_pdf": "sources/medsi/x.pdf", + "doc_type": "lab", + }, + True, + ), + ( + { + "source_pdf": "sources/medsi/x.pdf", + "doc_type": "lab", + }, + True, + ), + ( + { + "source_pdf": "2021-01-18_covid.pdf", + "doc_type": "lab", + }, + True, + ), + ( + { + "source_pdf": "sources/medsi/x.pdf", + "doc_type": "imaging", + }, + False, + ), + ( + { + "source_pdf": "sources/emias/x.pdf", + }, + True, + ), + ], +) +def test_is_lab_source_entry(entry: dict, expected: bool) -> None: + assert is_lab_source_entry(entry) is expected + + +def test_is_legacy_flat_entry() -> None: + assert is_legacy_flat_entry( + { + "source_system": "legacy_flat", + "source_pdf": "flat.pdf", + "extracted_txt": "pdf_text/flat.pdf.txt", + } + ) + assert not is_legacy_flat_entry( + { + "source_system": "gemotest", + "source_pdf": "sources/gemotest/x.pdf", + } + ) + + +def test_is_structure_target_entry_skips_grok_ingested() -> None: + entry = { + "source_system": "gemotest", + "source_pdf": "sources/gemotest/x.pdf", + "grok_ingested_at": "2024-01-01T00:00:00Z", + } + assert not is_structure_target_entry(entry) + + +def test_is_structure_target_entry_skips_already_structured() -> None: + entry = { + "source_system": "gemotest", + "source_pdf": "sources/gemotest/x.pdf", + "structured_locally_at": "2024-01-01T00:00:00Z", + } + assert not is_structure_target_entry(entry) + assert is_structure_target_entry(entry, force=True) + + +def test_is_structure_target_entry_medsi_requires_drop_batch() -> None: + medsi = { + "source_system": "medsi", + "source_pdf": "sources/medsi/x.pdf", + } + assert not is_structure_target_entry(medsi) + assert is_structure_target_entry({**medsi, "user_drop_batch": "batch-1"}) + assert is_structure_target_entry({**medsi, "ingest_note": "ALL-NEW-FILES drop"}) + + +def test_is_structure_target_entry_source_filter() -> None: + entry = { + "source_system": "gemotest", + "source_pdf": "sources/gemotest/x.pdf", + } + assert is_structure_target_entry(entry, sources={"gemotest"}) + assert not is_structure_target_entry(entry, sources={"medsi"}) + legacy = { + "source_system": "legacy_flat", + "source_pdf": "flat.pdf", + "extracted_txt": "pdf_text/flat.pdf.txt", + } + assert is_structure_target_entry(legacy, sources={"legacy_flat"}) + assert not is_structure_target_entry(legacy, sources={"medsi"}) From 366658b05ab727cadca89dcb5d217c3fa1b3366b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 14:22:05 +0000 Subject: [PATCH 07/11] Group lab dedup by LOINC when available, fallback to canonical_key Add _dedup_key() that builds grouping keys as (loinc:code|key:canonical_key, specimen_date). Rows with the same LOINC on the same date are deduplicated even when canonical_key differs. Vendor priority (medsi > gemotest > emias) still applies within each group. Co-authored-by: apodobe --- medbots/dedup_labs.py | 15 ++++++++++-- tests/test_dedup_vendor_priority.py | 36 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/medbots/dedup_labs.py b/medbots/dedup_labs.py index 572bef5..5aad48a 100644 --- a/medbots/dedup_labs.py +++ b/medbots/dedup_labs.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Deduplicate LABS_NORMALIZED rows by canonical_key + specimen_date.""" +"""Deduplicate LABS_NORMALIZED rows by LOINC (when present) or canonical_key + specimen_date.""" + from __future__ import annotations import argparse @@ -43,6 +44,16 @@ def _detect_vendor(row: dict[str, Any], manifest_index: dict[str, str]) -> str: return "other" +def _dedup_key(row: dict[str, Any]) -> tuple[str, str | None]: + specimen_date = row.get("specimen_date") + loinc = row.get("loinc") + if loinc not in (None, ""): + first_part = f"loinc:{loinc}" + else: + first_part = f"key:{row.get('canonical_key') or ''}" + return (first_part, specimen_date) + + def _row_score(row: dict[str, Any], manifest_index: dict[str, str]) -> tuple[int, int, int]: vendor = _detect_vendor(row, manifest_index) tier = _SOURCE_TIER.get(vendor, 9) @@ -66,7 +77,7 @@ def dedup_labs(corpus: Path, *, apply: bool = True) -> dict[str, int]: groups: dict[tuple[str, str | None], list[dict[str, Any]]] = {} for row in rows: - key = (str(row.get("canonical_key") or ""), row.get("specimen_date")) + key = _dedup_key(row) groups.setdefault(key, []).append(row) kept: list[dict[str, Any]] = [] diff --git a/tests/test_dedup_vendor_priority.py b/tests/test_dedup_vendor_priority.py index f0ec8d4..03a0503 100644 --- a/tests/test_dedup_vendor_priority.py +++ b/tests/test_dedup_vendor_priority.py @@ -49,3 +49,39 @@ def test_dedup_medsi_wins_over_emias(tmp_corpus: Path) -> None: assert len(kept) == 1 assert "medsi" in kept[0]["source_path"] assert kept[0]["value"] == "5.2" + + +def test_dedup_same_loinc_different_canonical_key_medsi_wins(tmp_corpus: Path) -> None: + rows = [ + { + "canonical_key": "glucose_emias", + "loinc": "2345-7", + "specimen_date": "2024-03-15", + "source_path": "sources/emias/2024-03-15__blood__abc123.pdf", + "value": "5.0", + "unit": "mmol/L", + }, + { + "canonical_key": "glucose_medsi", + "loinc": "2345-7", + "specimen_date": "2024-03-15", + "source_path": "sources/medsi/2024-03-15__blood__def456.pdf", + "value": "5.2", + "unit": "mmol/L", + "ref_low": 3.9, + "ref_high": 6.1, + }, + ] + _write_labs(tmp_corpus, rows) + + stats = dedup_labs(tmp_corpus, apply=True) + + assert stats["rows_before"] == 2 + assert stats["rows_after"] == 1 + assert stats["removed"] == 1 + assert stats["dup_groups"] == 1 + + kept = _read_labs(tmp_corpus) + assert len(kept) == 1 + assert "medsi" in kept[0]["source_path"] + assert kept[0]["value"] == "5.2" From d600563ce96f51512c7eed2229a4e92e59155252 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 14:23:07 +0000 Subject: [PATCH 08/11] Replace print() with logging in orchestration layer Add medbots/log_config.py with setup_logging() and get_logger(). Use logger.info/error in pipeline/run.py for step progress and failures. Keep user-facing print in cli.py; route structure warnings to logger. Use logger in dedup_labs and merge_labs_corpus main() entry points. Co-authored-by: apodobe --- medbots/cli.py | 23 ++++++++++++++++++----- medbots/dedup_labs.py | 18 +++++++++++++----- medbots/log_config.py | 17 +++++++++++++++++ medbots/merge_labs_corpus.py | 20 +++++++++++++++----- medbots/pipeline/run.py | 25 +++++++++++++++++-------- 5 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 medbots/log_config.py diff --git a/medbots/cli.py b/medbots/cli.py index 403f36d..d5e2b92 100644 --- a/medbots/cli.py +++ b/medbots/cli.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """CLI entry points for medbots-core.""" + from __future__ import annotations import argparse @@ -15,10 +16,13 @@ from medbots.import_apple_health import run_import from medbots.init_instance import init as run_init from medbots.local_structure_pdfs import run as run_structure +from medbots.log_config import get_logger, setup_logging from medbots.pipeline.run import run_pipeline from medbots.pipeline.validate_corpus import main as validate_corpus_main from medbots.scan_sources import scan as run_scan +logger = get_logger(__name__) + def _cmd_init(args: argparse.Namespace) -> int: run_init(args.path, force=args.force) @@ -57,12 +61,16 @@ def _cmd_import_apple_health(args: argparse.Namespace) -> int: def _cmd_validate_apple_health(args: argparse.Namespace) -> int: corpus = resolve_corpus(args.corpus) - return subprocess.call([sys.executable, "-m", "medbots.validate_apple_health", "--corpus", str(corpus)]) + return subprocess.call( + [sys.executable, "-m", "medbots.validate_apple_health", "--corpus", str(corpus)] + ) def _cmd_structure(args: argparse.Namespace) -> int: root = Path(args.bot_root).resolve() if args.bot_root else find_bot_root() - corpus = resolve_corpus(args.corpus) if args.corpus else resolve_corpus(root / "structured_database") + corpus = ( + resolve_corpus(args.corpus) if args.corpus else resolve_corpus(root / "structured_database") + ) source_filter = {s.strip().lower() for s in args.source if s.strip()} or None stats = run_structure( corpus, @@ -73,7 +81,7 @@ def _cmd_structure(args: argparse.Namespace) -> int: print(json.dumps(stats, ensure_ascii=False, indent=2)) if stats.get("errors"): for err in stats["errors"]: - print(f"WARN: {err}", file=sys.stderr) + logger.warning("%s", err) return 0 @@ -101,6 +109,7 @@ def _cmd_patient_dob(args: argparse.Namespace) -> int: def main(argv: list[str] | None = None) -> int: + setup_logging() parser = argparse.ArgumentParser( prog="medbots", description="Medical corpus tools: scan PDFs, extract text, parse labs, run pipeline", @@ -130,10 +139,14 @@ def main(argv: list[str] | None = None) -> int: p_ah.add_argument("--zip", type=Path, required=True, help="Apple Health export.zip from iPhone") p_ah.add_argument("--bot-root", type=Path) p_ah.add_argument("--corpus", type=Path) - p_ah.add_argument("--copy-zip", action="store_true", help="Archive zip under sources/apple_health/") + p_ah.add_argument( + "--copy-zip", action="store_true", help="Archive zip under sources/apple_health/" + ) p_ah.set_defaults(func=_cmd_import_apple_health) - p_ah_val = sub.add_parser("validate-apple-health", help="Check fitness/ after Apple Health import") + p_ah_val = sub.add_parser( + "validate-apple-health", help="Check fitness/ after Apple Health import" + ) p_ah_val.add_argument("--corpus", type=Path) p_ah_val.set_defaults(func=_cmd_validate_apple_health) diff --git a/medbots/dedup_labs.py b/medbots/dedup_labs.py index 5aad48a..cce2401 100644 --- a/medbots/dedup_labs.py +++ b/medbots/dedup_labs.py @@ -5,12 +5,12 @@ import argparse import re -import sys from pathlib import Path from typing import Any from medbots.cli_args import add_corpus_argument, resolve_corpus_from_args from medbots.corpus_io import load_labs, manifest_vendor_index, write_labs +from medbots.log_config import get_logger, setup_logging _SOURCE_TIER = { "medsi": 0, @@ -19,6 +19,9 @@ } +logger = get_logger(__name__) + + def _detect_vendor(row: dict[str, Any], manifest_index: dict[str, str]) -> str: source_path = str(row.get("source_path") or "").lower() for vendor in _SOURCE_TIER: @@ -106,6 +109,7 @@ def dedup_labs(corpus: Path, *, apply: bool = True) -> dict[str, int]: def main() -> int: + setup_logging() ap = argparse.ArgumentParser(description="Deduplicate LABS_NORMALIZED.json") add_corpus_argument(ap) ap.add_argument("--dry-run", action="store_true") @@ -114,12 +118,16 @@ def main() -> int: try: stats = dedup_labs(corpus, apply=not args.dry_run) except FileNotFoundError as exc: - print(f"ERROR: {exc}", file=sys.stderr) + logger.error("%s", exc) return 1 mode = "dry-run" if args.dry_run else "applied" - print( - f"{mode}: rows {stats['rows_before']}->{stats['rows_after']} " - f"removed={stats['removed']} dup_groups={stats['dup_groups']}" + logger.info( + "%s: rows %s->%s removed=%s dup_groups=%s", + mode, + stats["rows_before"], + stats["rows_after"], + stats["removed"], + stats["dup_groups"], ) return 0 diff --git a/medbots/log_config.py b/medbots/log_config.py new file mode 100644 index 0000000..e8c0551 --- /dev/null +++ b/medbots/log_config.py @@ -0,0 +1,17 @@ +"""Centralized logging configuration for medbots.""" + +from __future__ import annotations + +import logging + +_LOG_FORMAT = "%(levelname)s: %(message)s" + + +def setup_logging(level: str = "INFO") -> None: + """Configure root logger with a simple format.""" + numeric_level = getattr(logging, level.upper(), logging.INFO) + logging.basicConfig(level=numeric_level, format=_LOG_FORMAT) + + +def get_logger(name: str) -> logging.Logger: + return logging.getLogger(name) diff --git a/medbots/merge_labs_corpus.py b/medbots/merge_labs_corpus.py index acd79e0..4bb0650 100644 --- a/medbots/merge_labs_corpus.py +++ b/medbots/merge_labs_corpus.py @@ -1,18 +1,27 @@ #!/usr/bin/env python3 """Re-extract lab rows from EMIAS/Gemotest pdf_text into LABS_NORMALIZED.json.""" + from __future__ import annotations import argparse import json -import sys from pathlib import Path from typing import Any -from medbots.corpus_io import default_corpus_root, load_labs, load_manifest, load_patient_dob, write_labs +from medbots.corpus_io import ( + default_corpus_root, + load_labs, + load_manifest, + load_patient_dob, + write_labs, +) from medbots.corpus_writers import _append_lab_rows from medbots.local_structure_pdfs import _entry_title, _parse_entry, _pdf_text_path +from medbots.log_config import get_logger, setup_logging from medbots.vendor_registry import is_lab_source_entry +logger = get_logger(__name__) + def _doc_text_rel(entry: dict[str, Any]) -> str: doc_text = entry.get("doc_text") @@ -124,6 +133,7 @@ def run(corpus: Path, *, dry_run: bool = False, replace: bool = True) -> dict[st def main() -> int: + setup_logging() ap = argparse.ArgumentParser(description="Merge Gemotest/EMIAS labs into LABS_NORMALIZED") ap.add_argument("--corpus", type=Path, default=None) ap.add_argument("--dry-run", action="store_true") @@ -137,14 +147,14 @@ def main() -> int: args.corpus = default_corpus_root() corpus = args.corpus.expanduser().resolve() if not corpus.is_dir(): - print(f"ERROR: corpus not found: {corpus}", file=sys.stderr) + logger.error("corpus not found: %s", corpus) return 1 stats = run(corpus, dry_run=args.dry_run, replace=not args.no_replace) - print(json.dumps(stats, ensure_ascii=False, indent=2)) + logger.info("%s", json.dumps(stats, ensure_ascii=False, indent=2)) if stats.get("errors"): for err in stats["errors"]: - print(f"WARN: {err}", file=sys.stderr) + logger.warning("%s", err) return 0 diff --git a/medbots/pipeline/run.py b/medbots/pipeline/run.py index fbd741c..78b9ddb 100644 --- a/medbots/pipeline/run.py +++ b/medbots/pipeline/run.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Run post-ingest corpus pipeline with bot_config feature flags.""" + from __future__ import annotations import json @@ -11,6 +12,7 @@ from medbots.config import BotConfig, corpus_from_config, feature_enabled, load_config from medbots.corpus_io import resolve_corpus from medbots.dedup_labs import dedup_labs +from medbots.log_config import get_logger, setup_logging from medbots.merge_labs_corpus import run as merge_labs_corpus from medbots.pipeline.apply_loinc_map import apply_loinc_map from medbots.pipeline.extract_goals_from_doc_text import extract_goals @@ -22,17 +24,19 @@ from medbots.pipeline.validate_corpus import main as validate_corpus_main from medbots.pipeline.write_corpus_index import build_index +logger = get_logger(__name__) + def _exit_on_failure(step: str, exc: BaseException | None = None, code: int = 1) -> None: if exc is not None: - print(f"ERROR: {step} failed: {exc}", file=sys.stderr) + logger.error("%s failed: %s", step, exc) raise SystemExit(code) from exc - print(f"ERROR: {step} failed with exit code {code}", file=sys.stderr) + logger.error("%s failed with exit code %s", step, code) raise SystemExit(code) def _run_step(step: str, fn) -> None: - print(f"==> {step}") + logger.info("==> %s", step) try: result = fn() except Exception as exc: @@ -45,7 +49,7 @@ def _run_script(script: Path, corpus: Path, extra: list[str] | None = None) -> N cmd = [sys.executable, str(script), "--corpus", str(corpus)] if extra: cmd.extend(extra) - print(f"==> {script.name}") + logger.info("==> %s", script.name) try: subprocess.run(cmd, check=True) except subprocess.CalledProcessError as exc: @@ -61,6 +65,7 @@ def run_pipeline( corpus: Path | str | None = None, config: BotConfig | None = None, ) -> None: + setup_logging() cfg = config or load_config(bot_root) root = cfg.bot_root if bot_root is None else bot_root.resolve() corp = resolve_corpus(corpus) if corpus is not None else corpus_from_config(cfg) @@ -77,7 +82,9 @@ def run_pipeline( if bridge.is_file(): _run_script(bridge, corp, ["--apply"]) - _run_step("medbots.merge_labs_corpus", lambda: merge_labs_corpus(corp, dry_run=False, replace=True)) + _run_step( + "medbots.merge_labs_corpus", lambda: merge_labs_corpus(corp, dry_run=False, replace=True) + ) _run_step("medbots.pipeline.apply_loinc_map", lambda: apply_loinc_map(corp, apply=True)) _run_step("medbots.dedup_labs", lambda: dedup_labs(corp, apply=True)) @@ -97,7 +104,9 @@ def run_pipeline( lambda: extract_protocols(corp, apply=True), ) - _run_step("medbots.pipeline.generate_discrepancies", lambda: generate_discrepancies(corp, apply=True)) + _run_step( + "medbots.pipeline.generate_discrepancies", lambda: generate_discrepancies(corp, apply=True) + ) _run_step("medbots.pipeline.generate_lhm", lambda: generate_lhm(corp)) if feature_enabled("weekly_pending", cfg): @@ -105,7 +114,7 @@ def run_pipeline( if weekly.is_file(): _run_script(weekly, corp, ["--apply"]) else: - print("==> reconcile_weekly_pending (skipped: script not found)") + logger.info("==> reconcile_weekly_pending (skipped: script not found)") if feature_enabled("goals_reminders", cfg): _run_step( @@ -122,4 +131,4 @@ def _write_index() -> None: _run_step("write_corpus_index", _write_index) - print("Pipeline OK.") + logger.info("Pipeline OK.") From 80d07e0998df09791ba717b3de58428dd04c9cdd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 14:25:08 +0000 Subject: [PATCH 09/11] Add ruff and mypy to dev deps and CI with minimal strictness Configure ruff (E/F/I/UP, line-length 100) and mypy (packages medbots, ignore_missing_imports) in pyproject.toml. Run both in GitHub Actions before pytest. Auto-fix import order and style issues across medbots; fix mypy narrowing in import_apple_health. Temporarily ignore_errors for local_structure_pdfs during incremental parser split. Co-authored-by: apodobe --- .github/workflows/test.yml | 6 ++ medbots/cli_args.py | 1 + medbots/config.py | 1 + medbots/corpus_io.py | 1 + medbots/corpus_writers.py | 8 +- medbots/extract_pdf_text.py | 8 +- medbots/import_apple_health.py | 35 +++--- medbots/init_instance.py | 14 ++- medbots/lab_source_lib.py | 8 +- medbots/local_structure_pdfs.py | 102 +++++++++--------- medbots/pipeline/apply_loinc_map.py | 2 +- .../pipeline/extract_goals_from_doc_text.py | 7 +- .../pipeline/extract_protocols_from_corpus.py | 12 +-- .../extract_supplements_from_corpus.py | 15 ++- medbots/pipeline/generate_discrepancies.py | 21 ++-- medbots/pipeline/generate_lhm.py | 6 +- medbots/pipeline/reconcile_goals.py | 10 +- medbots/pipeline/validate_corpus.py | 7 +- medbots/pipeline/write_corpus_index.py | 17 ++- medbots/scan_sources.py | 5 +- medbots/time_util.py | 5 +- medbots/validate_apple_health.py | 6 +- medbots/vendor_registry.py | 5 +- medbots/zip_safety.py | 5 +- pyproject.toml | 23 +++- tests/test_check_safe_to_push.py | 10 +- tests/test_import_apple_health.py | 4 +- tests/test_lab_source_lib.py | 2 +- tests/test_merge_labs_corpus.py | 2 - tests/test_parse_emias.py | 3 +- tests/test_parse_gemotest.py | 3 +- tests/test_parse_medsi.py | 6 +- tests/test_validate_corpus.py | 4 +- 33 files changed, 222 insertions(+), 142 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 33bdc73..1756314 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,5 +25,11 @@ jobs: python -m pip install --upgrade pip pip install -e ".[dev]" + - name: Ruff + run: ruff check medbots tests + + - name: Mypy + run: mypy medbots + - name: Run pytest run: pytest tests/ -q diff --git a/medbots/cli_args.py b/medbots/cli_args.py index c124420..a8b17c9 100644 --- a/medbots/cli_args.py +++ b/medbots/cli_args.py @@ -1,4 +1,5 @@ """Shared argparse helpers for corpus CLI entrypoints.""" + from __future__ import annotations import argparse diff --git a/medbots/config.py b/medbots/config.py index af0fc64..abfca9a 100644 --- a/medbots/config.py +++ b/medbots/config.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Load bot_config.json and resolve feature flags (with env overrides).""" + from __future__ import annotations import json diff --git a/medbots/corpus_io.py b/medbots/corpus_io.py index af70ba2..9ad151a 100644 --- a/medbots/corpus_io.py +++ b/medbots/corpus_io.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Shared corpus I/O: manifest, labs, patient profile, vendor index.""" + from __future__ import annotations import json diff --git a/medbots/corpus_writers.py b/medbots/corpus_writers.py index 4212717..c28685a 100644 --- a/medbots/corpus_writers.py +++ b/medbots/corpus_writers.py @@ -1,9 +1,10 @@ """Shared corpus writers (doc_text, labs, supplements) — no Grok/LLM.""" + from __future__ import annotations import json import re -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -11,7 +12,7 @@ def _utc_date_slug() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%d") + return datetime.now(UTC).strftime("%Y-%m-%d") def _load_patient_profile(corpus: Path) -> dict[str, Any]: @@ -52,8 +53,7 @@ def _write_to_extracted_images_md(corpus: Path, section_id: str, extracted: dict else: md_path.write_text( "# Извлечение из Telegram ingest\n\n" - f"Пациент: **{patient}**, д.р. **{dob}**.\n" - + entry.lstrip("\n"), + f"Пациент: **{patient}**, д.р. **{dob}**.\n" + entry.lstrip("\n"), encoding="utf-8", ) diff --git a/medbots/extract_pdf_text.py b/medbots/extract_pdf_text.py index 72f1ee5..7338769 100644 --- a/medbots/extract_pdf_text.py +++ b/medbots/extract_pdf_text.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Extract text from manifest PDFs into structured_database/pdf_text/.""" + from __future__ import annotations import json @@ -94,7 +95,12 @@ def main() -> int: import argparse ap = argparse.ArgumentParser(description="Extract PDF text layers into pdf_text/") - ap.add_argument("--bot-root", type=Path, default=Path.cwd(), help="Instance root (sources/ + structured_database/)") + ap.add_argument( + "--bot-root", + type=Path, + default=Path.cwd(), + help="Instance root (sources/ + structured_database/)", + ) ap.add_argument("--corpus", type=Path, default=None, help="structured_database path override") args = ap.parse_args() stats = run(args.bot_root, args.corpus) diff --git a/medbots/import_apple_health.py b/medbots/import_apple_health.py index 6ea464c..4d66dcc 100644 --- a/medbots/import_apple_health.py +++ b/medbots/import_apple_health.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Import Apple Health export.zip into fitness corpus (stream XML, no full extract).""" + from __future__ import annotations import argparse @@ -9,13 +10,13 @@ import zipfile from collections import defaultdict from dataclasses import dataclass, field -from datetime import date, datetime, timezone +from datetime import UTC, date, datetime from pathlib import Path from typing import Any from xml.etree.ElementTree import iterparse from medbots.corpus_io import load_patient_dob, resolve_corpus -from medbots.zip_safety import UnsafeZipError, validate_zip_archive +from medbots.zip_safety import validate_zip_archive VALID_DATE_MIN = "2015-01-01" @@ -80,11 +81,11 @@ def parse_apple_datetime(s: str) -> datetime: def is_valid_day(day: str | None) -> bool: - return bool(day) and day >= VALID_DATE_MIN + return day is not None and day >= VALID_DATE_MIN def track_date(stats: ImportStats, day: str | None) -> None: - if not is_valid_day(day): + if day is None or not is_valid_day(day): return stats.date_min = day if stats.date_min is None else min(stats.date_min, day) stats.date_max = day if stats.date_max is None else max(stats.date_max, day) @@ -96,8 +97,8 @@ def local_date(s: str) -> str: def to_utc_iso(dt: datetime) -> str: if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") def find_main_xml(zf: zipfile.ZipFile) -> str: @@ -213,7 +214,9 @@ def match_route_file(workout_start: str, routes: list[str]) -> str | None: return best -def stream_import(zip_path: Path) -> tuple[dict[str, DailyAgg], list[dict], ImportStats, str | None]: +def stream_import( + zip_path: Path, +) -> tuple[dict[str, DailyAgg], list[dict], ImportStats, str | None]: daily: dict[str, DailyAgg] = defaultdict(DailyAgg) sleep_buf: list[tuple[str, str, str]] = [] workouts: list[dict[str, Any]] = [] @@ -488,7 +491,9 @@ def quality_report( } -def write_summary_md(meta: dict[str, Any], entries: list[dict], workouts: list[dict], ecg: list[dict]) -> str: +def write_summary_md( + meta: dict[str, Any], entries: list[dict], workouts: list[dict], ecg: list[dict] +) -> str: recent_metrics = entries[-7:] recent_workouts = workouts[-10:] lines = [ @@ -514,7 +519,8 @@ def write_summary_md(meta: dict[str, Any], entries: list[dict], workouts: list[d for e in recent_metrics: lines.append( f"| {e['date']} | {e.get('weight_kg', '')} | {e.get('steps', '')} | " - f"{e.get('sleep_hours', '')} | {e.get('exercise_min', '')} | {e.get('hr_bpm', '')} |" + f"{e.get('sleep_hours', '')} | {e.get('exercise_min', '')} | " + f"{e.get('hr_bpm', '')} |" ) else: lines.append("_no data_") @@ -525,7 +531,10 @@ def write_summary_md(meta: dict[str, Any], entries: list[dict], workouts: list[d f"{w.get('duration_min')} min" ) lines.append("") - lines.append("Full data: `BODY_METRICS.json`, `WORKOUTS.json`, `ECG_RECORDS.json`, `APPLE_HEALTH_META.json`.") + lines.append( + "Full data: `BODY_METRICS.json`, `WORKOUTS.json`, `ECG_RECORDS.json`, " + "`APPLE_HEALTH_META.json`." + ) return "\n".join(lines) @@ -576,7 +585,7 @@ def run_import( patient_dob = profile.get("dob") or load_patient_dob(corp) or None quality = quality_report(entries, workouts, stats, export_dob, patient_dob) - imported_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + imported_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") meta = { "version": 1, "imported_at": imported_at, @@ -605,7 +614,9 @@ def run_import( } _save_json(fitness / "BODY_METRICS.json", bm) - wo = _load_json(fitness / "WORKOUTS.json", {"version": 1, "plan": [], "sessions": [], "meta": {}}) + wo = _load_json( + fitness / "WORKOUTS.json", {"version": 1, "plan": [], "sessions": [], "meta": {}} + ) wo["sessions"] = merge_workouts(wo.get("sessions") or [], workouts) wo["meta"] = { "apple_health_last_import": imported_at, diff --git a/medbots/init_instance.py b/medbots/init_instance.py index 63bb3d6..bad11e9 100644 --- a/medbots/init_instance.py +++ b/medbots/init_instance.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Scaffold a new private health-data instance directory.""" + from __future__ import annotations import json @@ -46,7 +47,9 @@ def init(path: Path, *, force: bool = False) -> None: labs = corpus / "LABS_NORMALIZED.json" if not labs.exists() or force: - labs.write_text(json.dumps({"rows": []}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + labs.write_text( + json.dumps({"rows": []}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) loinc = corpus / "labs" / "LOINC_MAP.tsv" if not loinc.exists() or force: @@ -69,9 +72,12 @@ def init(path: Path, *, force: bool = False) -> None: print(f" 4. medbots extract-text --bot-root {root}") print(f" 5. medbots structure --bot-root {root}") print(f" 6. medbots pipeline --bot-root {root}") - print(f" Optional Apple Health: medbots import-apple-health --zip ~/Downloads/export.zip --bot-root {root}") - print(f" Demo without PDFs: medbots structure --bot-root examples/demo-instance") - print(f" Docs: docs/PARSERS.md, docs/ARCHITECTURE.md") + print( + f" Optional Apple Health: medbots import-apple-health " + f"--zip ~/Downloads/export.zip --bot-root {root}" + ) + print(" Demo without PDFs: medbots structure --bot-root examples/demo-instance") + print(" Docs: docs/PARSERS.md, docs/ARCHITECTURE.md") def main() -> int: diff --git a/medbots/lab_source_lib.py b/medbots/lab_source_lib.py index 0eca876..9f8b25f 100644 --- a/medbots/lab_source_lib.py +++ b/medbots/lab_source_lib.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """Shared helpers for lab PDF source import scripts (EMIAS, Gemotest, …).""" + from __future__ import annotations import hashlib import re import zipfile from pathlib import Path -from typing import Optional from xml.etree import ElementTree as ET import fitz @@ -51,7 +51,7 @@ def docx_text(path: Path) -> str: return "\n".join(paras).strip() -def docx_core_date(path: Path, field: str = "modified") -> Optional[str]: +def docx_core_date(path: Path, field: str = "modified") -> str | None: """Return ISO date YYYY-MM-DD from docProps/core.xml (created|modified).""" with zipfile.ZipFile(path) as zf: if "docProps/core.xml" not in zf.namelist(): @@ -111,9 +111,7 @@ def load_owner_dob(repo_or_corpus: Path) -> str: return resolve_owner_dob(repo_or_corpus) -def is_owner_patient_text( - text: str, owner_dob: str | None = None -) -> tuple[bool, str]: +def is_owner_patient_text(text: str, owner_dob: str | None = None) -> tuple[bool, str]: """False when PDF clearly belongs to another patient (child / foreign DOB).""" if owner_dob is None: import os diff --git a/medbots/local_structure_pdfs.py b/medbots/local_structure_pdfs.py index 26b567b..0fd05b9 100644 --- a/medbots/local_structure_pdfs.py +++ b/medbots/local_structure_pdfs.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Structure EMIAS, Gemotest, and Medsi PDFs from pdf_text without external LLM API.""" + from __future__ import annotations import argparse @@ -7,13 +8,16 @@ import json import re import sys -from datetime import datetime, timezone from pathlib import Path -from typing import Any, Optional - -from medbots.corpus_io import bot_root, default_corpus_root, load_manifest, load_patient_dob, write_manifest -from medbots.time_util import utc_now_iso -from medbots.vendor_registry import is_legacy_flat_entry, is_structure_target_entry +from typing import Any + +from medbots.corpus_io import ( + bot_root, + default_corpus_root, + load_manifest, + load_patient_dob, + write_manifest, +) from medbots.corpus_writers import ( _append_lab_rows, _safe_txt_name, @@ -21,22 +25,16 @@ _write_to_extracted_images_md, sha256_bytes, ) +from medbots.time_util import utc_now_iso +from medbots.vendor_registry import is_legacy_flat_entry, is_structure_target_entry _DATE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})") -_DATE_RESEARCH = re.compile( - r"Дата исследования:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE -) +_DATE_RESEARCH = re.compile(r"Дата исследования:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) _EMIAS_DATE = re.compile(r"Дата:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) _PRINT_DATE = re.compile(r"ПЕЧАТЬ:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) -_ORDER_DATE = re.compile( - r"Дата регистрации заказа\s*\n\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE -) -_SLASH_ORDER_DATE = re.compile( - r"дата:\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE -) -_GEMOTEST_DATE_MULTILINE = re.compile( - r"дата:\s*\n\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE -) +_ORDER_DATE = re.compile(r"Дата регистрации заказа\s*\n\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) +_SLASH_ORDER_DATE = re.compile(r"дата:\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE) +_GEMOTEST_DATE_MULTILINE = re.compile(r"дата:\s*\n\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE) _FILENAME_ISO_DATE = re.compile(r"(?:^|/)(20\d{2}-\d{2}-\d{2})__") _SECTION_HEADERS = frozenset( { @@ -111,7 +109,7 @@ def _dmy_to_iso(d: str, m: str, y: str) -> str: return f"{y}-{m}-{d}" -def _date_from_source_path(source_pdf: str) -> Optional[str]: +def _date_from_source_path(source_pdf: str) -> str | None: m = _FILENAME_ISO_DATE.search(source_pdf.replace("\\", "/")) if not m: return None @@ -125,9 +123,9 @@ def _first_iso_date( text: str, *, source_pdf: str = "", - fallback_iso: Optional[str] = None, + fallback_iso: str | None = None, patient_dob: str = "", -) -> Optional[str]: +) -> str | None: path_date = _date_from_source_path(source_pdf) if path_date: return path_date @@ -169,7 +167,7 @@ def _canonical_key(name_ru: str) -> str: return slug[:80] or "analyte" -def _parse_float(value: str) -> Optional[float]: +def _parse_float(value: str) -> float | None: v = value.strip().replace(",", ".") if not v or not _NUMERIC.match(v): return None @@ -179,7 +177,7 @@ def _parse_float(value: str) -> Optional[float]: return None -def _parse_ref_range(ref_text: str) -> tuple[Optional[float], Optional[float], Optional[str]]: +def _parse_ref_range(ref_text: str) -> tuple[float | None, float | None, str | None]: ref = " ".join(ref_text.split()) if not ref or ref.lower().startswith("смотри"): return None, None, ref or None @@ -208,9 +206,9 @@ def _lab_row( name_ru: str, value: Any, unit: str = "", - ref_low: Optional[float] = None, - ref_high: Optional[float] = None, - ref_note: Optional[str] = None, + ref_low: float | None = None, + ref_high: float | None = None, + ref_note: str | None = None, specimen_date: str, facility: str = "ЕМИАС", ) -> dict[str, Any]: @@ -234,7 +232,7 @@ def _lab_row( def _markdown_header( *, doc_type: str, - doc_date: Optional[str], + doc_date: str | None, title: str, institution: str = "—", ) -> str: @@ -280,7 +278,11 @@ def _extract_conclusion(text: str) -> str: ) if m: return " ".join(m.group(1).split())[:500] - m = re.search(r"Основной\s*\n\s*диагноз\s*\n+(.+?)(?:\nРекомендации|\nДата:|\Z)", text, re.DOTALL | re.IGNORECASE) + m = re.search( + r"Основной\s*\n\s*диагноз\s*\n+(.+?)(?:\nРекомендации|\nДата:|\Z)", + text, + re.DOTALL | re.IGNORECASE, + ) if m: return " ".join(m.group(1).split())[:500] return "—" @@ -291,7 +293,7 @@ def parse_emias_lab( title: str, *, source_pdf: str = "", - fallback_iso: Optional[str] = None, + fallback_iso: str | None = None, patient_dob: str = "", ) -> dict[str, Any]: doc_date = _first_iso_date( @@ -359,16 +361,18 @@ def parse_emias_lab( parts.append(f"{row['name_ru']}: {note}") result_summary = "; ".join(parts) - md_lines = [_markdown_header(doc_type="lab", doc_date=doc_date, title=title, institution=facility)] + md_lines = [ + _markdown_header(doc_type="lab", doc_date=doc_date, title=title, institution=facility) + ] if lab_rows: md_lines.append("| Показатель | Результат | Ед. | Референс |") md_lines.append("|------------|-----------|-----|----------|") for row in lab_rows: - val_s = str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") - ref = row.get("ref_note") if row.get("value") is not None else "—" - md_lines.append( - f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |" + val_s = ( + str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") ) + ref = row.get("ref_note") if row.get("value") is not None else "—" + md_lines.append(f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |") else: md_lines.append(text.strip()) @@ -389,7 +393,7 @@ def parse_emias_consult_or_imaging( doc_type: str, *, source_pdf: str = "", - fallback_iso: Optional[str] = None, + fallback_iso: str | None = None, facility_override: str | None = None, patient_dob: str = "", ) -> dict[str, Any]: @@ -481,9 +485,9 @@ def _append_gemotest_row( facility: str, value: Any = None, unit: str = "", - ref_low: Optional[float] = None, - ref_high: Optional[float] = None, - ref_note: Optional[str] = None, + ref_low: float | None = None, + ref_high: float | None = None, + ref_note: str | None = None, ) -> None: if not name_ru or len(name_ru) < 2 or not specimen_date: return @@ -747,7 +751,7 @@ def parse_gemotest( title: str, source_pdf: str, *, - fallback_iso: Optional[str] = None, + fallback_iso: str | None = None, patient_dob: str = "", ) -> dict[str, Any]: subtype = _gemotest_subtype(source_pdf) @@ -809,7 +813,9 @@ def parse_gemotest( md += "| Показатель | Значение | Ед. | Референс |\n" md += "|------------|----------|-----|----------|\n" for row in lab_rows: - val_s = str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") + val_s = ( + str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") + ) ref = ( f"{row['ref_low']}–{row['ref_high']}" if row.get("ref_low") is not None and row.get("ref_high") is not None @@ -869,7 +875,7 @@ def _is_medsi_value_line(line: str) -> bool: return _parse_float(line.strip()) is not None -def _medsi_iso_date(text: str, *, patient_dob: str = "") -> Optional[str]: +def _medsi_iso_date(text: str, *, patient_dob: str = "") -> str | None: m = _EMIAS_DATE.search(text) if m: return _dmy_to_iso(m.group(1), m.group(2), m.group(3)) @@ -929,10 +935,10 @@ def _parse_medsi_lab_rows(text: str, doc_date: str, facility: str) -> list[dict[ continue unit = "" - ref_low: Optional[float] = None - ref_high: Optional[float] = None - ref_note: Optional[str] = None - value: Optional[float] = None + ref_low: float | None = None + ref_high: float | None = None + ref_note: str | None = None + value: float | None = None if i < len(lines) and _is_medsi_unit_line(lines[i].strip()): unit = lines[i].strip() @@ -970,7 +976,7 @@ def parse_medsi_lab( title: str, *, source_pdf: str = "", - fallback_iso: Optional[str] = None, + fallback_iso: str | None = None, patient_dob: str = "", ) -> dict[str, Any]: doc_date = _medsi_iso_date(text, patient_dob=patient_dob) or fallback_iso @@ -1230,9 +1236,7 @@ def run( sha256=sha, ingest_ts=now, ) - labs_added = _append_lab_rows( - corpus, extracted.get("lab_rows") or [], doc_rel - ) + labs_added = _append_lab_rows(corpus, extracted.get("lab_rows") or [], doc_rel) labs_added_total += labs_added updated = dict(entry) diff --git a/medbots/pipeline/apply_loinc_map.py b/medbots/pipeline/apply_loinc_map.py index 9949a0a..62c431b 100644 --- a/medbots/pipeline/apply_loinc_map.py +++ b/medbots/pipeline/apply_loinc_map.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Apply labs/LOINC_MAP.tsv to LABS_NORMALIZED rows where loinc is null.""" + from __future__ import annotations import argparse @@ -9,7 +10,6 @@ from typing import Any from medbots.cli_args import add_corpus_argument, resolve_corpus_from_args - from medbots.corpus_io import load_labs, write_labs diff --git a/medbots/pipeline/extract_goals_from_doc_text.py b/medbots/pipeline/extract_goals_from_doc_text.py index 002b2ff..5db0f84 100644 --- a/medbots/pipeline/extract_goals_from_doc_text.py +++ b/medbots/pipeline/extract_goals_from_doc_text.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Extract exam/supplement reminders from consultation doc_text/*.md (no API).""" + from __future__ import annotations import argparse @@ -7,7 +8,6 @@ import json import re import sys -import sys from pathlib import Path from typing import Any @@ -112,9 +112,7 @@ def _recommendation_lines(body: str) -> list[str]: return [ln.strip() for ln in body.splitlines() if ln.strip()] -def _extract_from_text( - text: str, path: Path, rel_path: str -) -> list[dict[str, Any]]: +def _extract_from_text(text: str, path: Path, rel_path: str) -> list[dict[str, Any]]: doc_date, title = _doc_meta(text, path) items: list[dict[str, Any]] = [] seen: set[str] = set() @@ -236,6 +234,7 @@ def main() -> int: ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() from medbots.corpus_io import default_corpus_root + if getattr(args, "corpus", None) is None: args.corpus = default_corpus_root() corpus = args.corpus.expanduser().resolve() diff --git a/medbots/pipeline/extract_protocols_from_corpus.py b/medbots/pipeline/extract_protocols_from_corpus.py index d89431e..085b0eb 100644 --- a/medbots/pipeline/extract_protocols_from_corpus.py +++ b/medbots/pipeline/extract_protocols_from_corpus.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 """Composer v0: enrich PROTOCOLS.json from doc_text (lifestyle + exam cadence).""" + from __future__ import annotations import argparse import hashlib import json import re -import sys -import sys from pathlib import Path from typing import Any @@ -82,9 +81,7 @@ def extract_protocols(corpus: Path, *, apply: bool = True) -> dict[str, int]: data = {"version": 1, "protocols": [], "meta": {}} existing_ids = {p.get("id") for p in data.get("protocols") or []} - existing_steps = { - tuple(p.get("steps_ru") or []) for p in data.get("protocols") or [] - } + existing_steps = {tuple(p.get("steps_ru") or []) for p in data.get("protocols") or []} new_count = 0 doc_dir = corpus / "doc_text" @@ -131,7 +128,9 @@ def extract_protocols(corpus: Path, *, apply: bool = True) -> dict[str, int]: stats = {"total": len(data.get("protocols") or []), "new": new_count} if apply: proto_path.parent.mkdir(parents=True, exist_ok=True) - proto_path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + proto_path.write_text( + json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) return stats @@ -145,6 +144,7 @@ def main() -> int: ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() from medbots.corpus_io import default_corpus_root + if getattr(args, "corpus", None) is None: args.corpus = default_corpus_root() corpus = args.corpus.expanduser().resolve() diff --git a/medbots/pipeline/extract_supplements_from_corpus.py b/medbots/pipeline/extract_supplements_from_corpus.py index 7309542..06043d8 100644 --- a/medbots/pipeline/extract_supplements_from_corpus.py +++ b/medbots/pipeline/extract_supplements_from_corpus.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Composer v0: extract supplement/medication mentions from doc_text + GOALS_REMINDERS.""" + from __future__ import annotations import argparse @@ -7,7 +8,6 @@ import json import re import sys -import sys from pathlib import Path from typing import Any @@ -86,7 +86,9 @@ def _extract_lines(text: str) -> list[str]: return lines -def _parse_mention(line: str, *, doc_date: str | None, title: str, rel_path: str) -> dict[str, Any] | None: +def _parse_mention( + line: str, *, doc_date: str | None, title: str, rel_path: str +) -> dict[str, Any] | None: known = _KNOWN.search(line) if not known and not re.search(r"\d+\s*мг", line, re.I): return None @@ -205,9 +207,13 @@ def extract_supplements(corpus: Path, *, apply: bool = True) -> dict[str, int]: if apply: primary.parent.mkdir(parents=True, exist_ok=True) - primary.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + primary.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) mirror.parent.mkdir(parents=True, exist_ok=True) - mirror.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + mirror.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) return stats @@ -222,6 +228,7 @@ def main() -> int: ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() from medbots.corpus_io import default_corpus_root + if getattr(args, "corpus", None) is None: args.corpus = default_corpus_root() corpus = args.corpus.expanduser().resolve() diff --git a/medbots/pipeline/generate_discrepancies.py b/medbots/pipeline/generate_discrepancies.py index 37563cc..f49d2ba 100644 --- a/medbots/pipeline/generate_discrepancies.py +++ b/medbots/pipeline/generate_discrepancies.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 """Rule-based DISCREPANCIES.json: out-of-range labs, stale imaging, missing fields.""" + from __future__ import annotations import argparse import hashlib import json import sys -from datetime import date, datetime, timezone +from datetime import UTC, date, datetime from pathlib import Path from typing import Any @@ -53,9 +54,7 @@ def _lab_discrepancies(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: ref_lo = row.get("ref_low") ref_hi = row.get("ref_high") source = row.get("source_path") or "" - detail = ( - f"{name}: {val} {unit} (реф. {ref_lo}–{ref_hi}), дата {spec_date}" - ) + detail = f"{name}: {val} {unit} (реф. {ref_lo}–{ref_hi}), дата {spec_date}" items.append( { "id": _item_id("lab", f"{key}|{spec_date}|{val}"), @@ -76,7 +75,7 @@ def _stale_imaging(corpus: Path, *, years: float = 2.0) -> list[dict[str, Any]]: if not timeline_path.exists(): return [] data = json.loads(timeline_path.read_text(encoding="utf-8")) - today = datetime.now(timezone.utc).date() + today = datetime.now(UTC).date() cutoff_days = int(years * 365.25) items: list[dict[str, Any]] = [] @@ -109,9 +108,7 @@ def _stale_imaging(corpus: Path, *, years: float = 2.0) -> list[dict[str, Any]]: return items -def _missing_field_discrepancies( - rows: list[dict[str, Any]], corpus: Path -) -> list[dict[str, Any]]: +def _missing_field_discrepancies(rows: list[dict[str, Any]], corpus: Path) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] missing_loinc = sum(1 for r in rows if not r.get("loinc")) if missing_loinc: @@ -122,16 +119,13 @@ def _missing_field_discrepancies( "category": "missing_data", "title_ru": "Лаборатория без LOINC", "detail_ru": ( - f"{missing_loinc} из {len(rows)} строк LABS_NORMALIZED " - f"без кода LOINC" + f"{missing_loinc} из {len(rows)} строк LABS_NORMALIZED без кода LOINC" ), "sources": ["LABS_NORMALIZED.json"], } ) - missing_value = [ - r for r in rows if r.get("value") in (None, "") and r.get("canonical_key") - ] + missing_value = [r for r in rows if r.get("value") in (None, "") and r.get("canonical_key")] for row in missing_value[:50]: key = row.get("canonical_key") or "?" spec_date = row.get("specimen_date") or "?" @@ -226,6 +220,7 @@ def main() -> int: ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() from medbots.corpus_io import default_corpus_root + if getattr(args, "corpus", None) is None: args.corpus = default_corpus_root() corpus = args.corpus.expanduser().resolve() diff --git a/medbots/pipeline/generate_lhm.py b/medbots/pipeline/generate_lhm.py index 9118cdd..d6b3f4b 100644 --- a/medbots/pipeline/generate_lhm.py +++ b/medbots/pipeline/generate_lhm.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Generate structured_database/LIVING_HEALTH_SUMMARY.md for LLM consumption.""" + from __future__ import annotations import argparse @@ -115,7 +116,10 @@ def generate_lhm(corpus: Path) -> Path: _read_excerpt(fitness_path, 80), "", "---", - "Полные данные: `LABS_NORMALIZED.json`, `TIMELINE_EVENTS.json`, `genomics/`, `fitness/`.", + ( + "Полные данные: `LABS_NORMALIZED.json`, `TIMELINE_EVENTS.json`, " + "`genomics/`, `fitness/`." + ), "", ] ) diff --git a/medbots/pipeline/reconcile_goals.py b/medbots/pipeline/reconcile_goals.py index 7149454..57a8316 100644 --- a/medbots/pipeline/reconcile_goals.py +++ b/medbots/pipeline/reconcile_goals.py @@ -2,13 +2,14 @@ """ Mark GOALS_REMINDERS items inactive when corpus text matches auto_complete_patterns. """ + from __future__ import annotations import argparse import json import re import sys -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -22,7 +23,7 @@ def _file_doc_date(path: Path) -> float | None: if not m: return None try: - dt = datetime(int(m[1]), int(m[2]), int(m[3]), tzinfo=timezone.utc) + dt = datetime(int(m[1]), int(m[2]), int(m[3]), tzinfo=UTC) return dt.timestamp() except ValueError: return None @@ -48,7 +49,7 @@ def _corpus_haystack( parts.append(timeline.read_text(encoding="utf-8", errors="replace")) cutoff = 0.0 if recent_days is not None: - cutoff = datetime.now(timezone.utc).timestamp() - recent_days * 86400 + cutoff = datetime.now(UTC).timestamp() - recent_days * 86400 for sub in ("pdf_text", "doc_text"): d = corpus / sub if not d.is_dir(): @@ -95,7 +96,7 @@ def reconcile( continue if _matches(patterns, haystack): item["active"] = False - stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d") + stamp = datetime.now(UTC).strftime("%Y-%m-%d") note = f" (авто-снято {stamp}: найдено в корпусе)" item["why_ru"] = (item.get("why_ru") or "").rstrip() + note changed.append(str(item.get("id", "?"))) @@ -131,6 +132,7 @@ def main() -> int: ap.add_argument("--recent-days", type=int, default=0) args = ap.parse_args() from medbots.corpus_io import default_corpus_root + if getattr(args, "corpus", None) is None: args.corpus = default_corpus_root() corpus = args.corpus.expanduser().resolve() diff --git a/medbots/pipeline/validate_corpus.py b/medbots/pipeline/validate_corpus.py index 160e87e..0387078 100755 --- a/medbots/pipeline/validate_corpus.py +++ b/medbots/pipeline/validate_corpus.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import os import sys from pathlib import Path @@ -67,7 +66,8 @@ def main() -> int: warnings.append("manifest.meta.pdf_count missing") elif expected_count != actual_count: errors.append( - f"manifest pdf count mismatch: meta.pdf_count={expected_count}, len(pdfs)={actual_count}" + "manifest pdf count mismatch: " + f"meta.pdf_count={expected_count}, len(pdfs)={actual_count}" ) else: stats.append(f"manifest_meta_pdf_count={expected_count} (matches)") @@ -104,7 +104,8 @@ def main() -> int: if missing_ingest: errors.append( - f"{len(missing_ingest)} manifest PDF(s) without grok_ingested_at or structured_locally_at" + f"{len(missing_ingest)} manifest PDF(s) without " + "grok_ingested_at or structured_locally_at" ) for name in missing_ingest[:5]: errors.append(f" not ingested: {name}") diff --git a/medbots/pipeline/write_corpus_index.py b/medbots/pipeline/write_corpus_index.py index dd201fd..fd5b74e 100644 --- a/medbots/pipeline/write_corpus_index.py +++ b/medbots/pipeline/write_corpus_index.py @@ -1,17 +1,18 @@ #!/usr/bin/env python3 """Write CORPUS_INDEX.json — navigation summary for the corpus.""" + from __future__ import annotations import argparse import json import re from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any from medbots.config import find_bot_root, load_config -from medbots.corpus_io import default_corpus_root, load_patient_dob, resolve_corpus +from medbots.corpus_io import load_patient_dob, resolve_corpus DATE_IN_PDF = re.compile(r"(20\d{2})[-_](\d{2})[-_](\d{2})") @@ -29,7 +30,11 @@ def _entry_date(entry: dict[str, Any]) -> str | None: def build_index(corpus: Path, bot_id: str, vps_path: str) -> dict[str, Any]: manifest_path = corpus / "manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else {"pdfs": [], "meta": {}} + manifest = ( + json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest_path.exists() + else {"pdfs": [], "meta": {}} + ) pdfs = manifest.get("pdfs") or [] meta = manifest.get("meta") or {} @@ -76,7 +81,7 @@ def build_index(corpus: Path, bot_id: str, vps_path: str) -> dict[str, Any]: ] return { - "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "generated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "bot_id": bot_id, "patient": patient, "patient_dob": dob, @@ -108,7 +113,9 @@ def main() -> int: root = (args.bot_root or find_bot_root()).resolve() cfg = load_config(root) - corpus = resolve_corpus(args.corpus) if args.corpus else resolve_corpus(root / "structured_database") + corpus = ( + resolve_corpus(args.corpus) if args.corpus else resolve_corpus(root / "structured_database") + ) index = build_index(corpus, cfg.bot_id, cfg.vps_corpus_path) out = corpus / "CORPUS_INDEX.json" diff --git a/medbots/scan_sources.py b/medbots/scan_sources.py index 7151a17..7fab553 100644 --- a/medbots/scan_sources.py +++ b/medbots/scan_sources.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Scan sources/{emias,medsi,gemotest}/ and register new PDFs in manifest.json.""" + from __future__ import annotations import hashlib @@ -91,7 +92,9 @@ def main() -> int: ap = argparse.ArgumentParser(description="Register PDFs from sources/ into manifest.json") ap.add_argument("--bot-root", type=Path, default=Path.cwd()) ap.add_argument("--corpus", type=Path, default=None) - ap.add_argument("--source", action="append", default=[], help="Vendor filter: emias, medsi, gemotest") + ap.add_argument( + "--source", action="append", default=[], help="Vendor filter: emias, medsi, gemotest" + ) args = ap.parse_args() vendors = {s.strip().lower() for s in args.source if s.strip()} or None stats = scan(args.bot_root, args.corpus, vendors) diff --git a/medbots/time_util.py b/medbots/time_util.py index b511c4b..6f41360 100644 --- a/medbots/time_util.py +++ b/medbots/time_util.py @@ -1,8 +1,9 @@ """Shared UTC timestamp helpers.""" + from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime def utc_now_iso() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/medbots/validate_apple_health.py b/medbots/validate_apple_health.py index 8b9d5c8..2a23dae 100644 --- a/medbots/validate_apple_health.py +++ b/medbots/validate_apple_health.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 """Validate Apple Health fitness files after import.""" + from __future__ import annotations import argparse import json -import sys from pathlib import Path from medbots.corpus_io import resolve_corpus @@ -17,7 +17,9 @@ def validate(corpus: Path) -> tuple[list[str], list[str]]: meta_path = fitness / "APPLE_HEALTH_META.json" if not meta_path.exists(): - errors.append("APPLE_HEALTH_META.json missing — run: medbots import-apple-health --zip export.zip") + errors.append( + "APPLE_HEALTH_META.json missing — run: medbots import-apple-health --zip export.zip" + ) return errors, warnings meta = json.loads(meta_path.read_text(encoding="utf-8")) diff --git a/medbots/vendor_registry.py b/medbots/vendor_registry.py index b41cdd9..811e325 100644 --- a/medbots/vendor_registry.py +++ b/medbots/vendor_registry.py @@ -1,4 +1,5 @@ """Unified vendor detection for corpus manifest entries.""" + from __future__ import annotations from typing import Any @@ -39,9 +40,7 @@ def is_lab_source_entry(entry: dict[str, Any]) -> bool: doc_type = entry.get("doc_type") or "" if source_system == "legacy_flat": return doc_type == "lab" - if doc_type == "lab" and ( - source_system == "medsi" or "sources/medsi" in source_pdf - ): + if doc_type == "lab" and (source_system == "medsi" or "sources/medsi" in source_pdf): return True if source_system in ("emias", "gemotest"): return True diff --git a/medbots/zip_safety.py b/medbots/zip_safety.py index a9ae5ce..d81072b 100644 --- a/medbots/zip_safety.py +++ b/medbots/zip_safety.py @@ -1,4 +1,5 @@ """Zip archive safety checks (zip-slip, zip bombs).""" + from __future__ import annotations import zipfile @@ -40,6 +41,4 @@ def validate_zip_archive(zf: zipfile.ZipFile, *, zip_size: int | None = None) -> ) if total_uncompressed > MAX_UNCOMPRESSED_BYTES: - raise UnsafeZipError( - f"uncompressed zip payload too large: {total_uncompressed} bytes" - ) + raise UnsafeZipError(f"uncompressed zip payload too large: {total_uncompressed} bytes") diff --git a/pyproject.toml b/pyproject.toml index 19600ed..8a982d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0"] +dev = ["pytest>=8.0", "ruff>=0.8", "mypy>=1.13"] [project.scripts] medbots = "medbots.cli:main" @@ -26,3 +26,24 @@ packages = ["medbots"] [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["."] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["S101"] +"medbots/local_structure_pdfs.py" = ["E501"] + +[tool.mypy] +python_version = "3.11" +packages = ["medbots"] +ignore_missing_imports = true + +# Large legacy module; type errors addressed incrementally during parser split. +[[tool.mypy.overrides]] +module = "medbots.local_structure_pdfs" +ignore_errors = true diff --git a/tests/test_check_safe_to_push.py b/tests/test_check_safe_to_push.py index fda2f18..95fc627 100644 --- a/tests/test_check_safe_to_push.py +++ b/tests/test_check_safe_to_push.py @@ -5,8 +5,6 @@ import sys from pathlib import Path -import pytest - ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "scripts")) @@ -74,7 +72,13 @@ def test_pre_push_script_exits_zero_on_current_branch() -> None: text=True, ).strip() proc = subprocess.run( - [sys.executable, str(ROOT / "scripts" / "check_safe_to_push.py"), "--public", "--range", f"{out}..HEAD"], + [ + sys.executable, + str(ROOT / "scripts" / "check_safe_to_push.py"), + "--public", + "--range", + f"{out}..HEAD", + ], cwd=ROOT, capture_output=True, text=True, diff --git a/tests/test_import_apple_health.py b/tests/test_import_apple_health.py index 43b705a..3755f32 100644 --- a/tests/test_import_apple_health.py +++ b/tests/test_import_apple_health.py @@ -23,7 +23,9 @@ def test_normalize_body_fat_fraction() -> None: def test_build_body_metrics_minimal_day() -> None: daily = { - "2024-06-01": DailyAgg(steps=8000, sleep_seconds=7 * 3600, hr_resting_sum=60, hr_resting_n=1), + "2024-06-01": DailyAgg( + steps=8000, sleep_seconds=7 * 3600, hr_resting_sum=60, hr_resting_n=1 + ), } entries = build_body_metrics(daily) assert len(entries) == 1 diff --git a/tests/test_lab_source_lib.py b/tests/test_lab_source_lib.py index 0673f7c..288b5ca 100644 --- a/tests/test_lab_source_lib.py +++ b/tests/test_lab_source_lib.py @@ -7,8 +7,8 @@ content_hash, extract_dates, ingest_priority, - iso_to_created_at, is_owner_patient_text, + iso_to_created_at, safe_slug, ) diff --git a/tests/test_merge_labs_corpus.py b/tests/test_merge_labs_corpus.py index 809243f..b1d33a5 100644 --- a/tests/test_merge_labs_corpus.py +++ b/tests/test_merge_labs_corpus.py @@ -5,8 +5,6 @@ from pathlib import Path from unittest.mock import patch -import pytest - from medbots.local_structure_pdfs import _parse_entry as real_parse_entry from medbots.merge_labs_corpus import run diff --git a/tests/test_parse_emias.py b/tests/test_parse_emias.py index c971a43..22003ff 100644 --- a/tests/test_parse_emias.py +++ b/tests/test_parse_emias.py @@ -5,6 +5,7 @@ from typing import Any import pytest +from conftest import load_pdf_text from medbots.local_structure_pdfs import ( _parse_entry, @@ -12,8 +13,6 @@ parse_emias_lab, ) -from conftest import load_pdf_text - _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") LAB_FIXTURE = ( diff --git a/tests/test_parse_gemotest.py b/tests/test_parse_gemotest.py index 6ac2349..19981f9 100644 --- a/tests/test_parse_gemotest.py +++ b/tests/test_parse_gemotest.py @@ -5,11 +5,10 @@ from typing import Any import pytest +from conftest import load_pdf_text from medbots.local_structure_pdfs import _gemotest_subtype, parse_gemotest -from conftest import load_pdf_text - _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") GEMOTEST_FIXTURES: list[tuple[str, str, str, str]] = [ diff --git a/tests/test_parse_medsi.py b/tests/test_parse_medsi.py index cdd1cb7..e765a7b 100644 --- a/tests/test_parse_medsi.py +++ b/tests/test_parse_medsi.py @@ -5,13 +5,13 @@ from typing import Any import pytest +from conftest import load_pdf_text from medbots.local_structure_pdfs import ( _extract_medsi_facility, _parse_entry, parse_medsi_lab, ) -from conftest import load_pdf_text BIOCHEM_FIXTURE = ( "sources__medsi__2026-06-16__drop_Биохимический_анализ_крови__2e563e81.pdf.txt" @@ -73,7 +73,9 @@ def test_lab_rows_have_required_fields(self, biochem_parsed: dict[str, Any]) -> def test_biochem_glucose_row(self, biochem_parsed: dict[str, Any]) -> None: glucose = next( - r for r in biochem_parsed["lab_rows"] if r["canonical_key"] == "glyukoza_venoznoy_krovi_natoschak" + r + for r in biochem_parsed["lab_rows"] + if r["canonical_key"] == "glyukoza_venoznoy_krovi_natoschak" ) assert glucose["value"] == pytest.approx(4.82) assert glucose["unit"] == "ммоль/л" diff --git a/tests/test_validate_corpus.py b/tests/test_validate_corpus.py index 6ff312c..2faf6c0 100644 --- a/tests/test_validate_corpus.py +++ b/tests/test_validate_corpus.py @@ -18,7 +18,9 @@ def corpus_env(monkeypatch: pytest.MonkeyPatch, tmp_corpus: Path) -> Path: return tmp_corpus -def test_empty_corpus_fails_key_checks(corpus_env: Path, capsys: pytest.CaptureFixture[str]) -> None: +def test_empty_corpus_fails_key_checks( + corpus_env: Path, capsys: pytest.CaptureFixture[str] +) -> None: exit_code = validate_main() captured = capsys.readouterr().out From 00a3db960d2c2a1cff1c19ff1292593219c1c42b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 14:26:56 +0000 Subject: [PATCH 10/11] refactor: split local_structure_pdfs into medbots/parsers package Extract vendor-specific parse_* functions and shared utilities into medbots/parsers/ (common, emias, gemotest, medsi). Keep local_structure_pdfs.py as a thin orchestrator for manifest routing, run(), and backward-compatible re-exports for tests. Pure refactor with no behavior changes; all 100 tests pass. Co-authored-by: apodobe --- medbots/local_structure_pdfs.py | 1025 +------------------------------ medbots/parsers/__init__.py | 11 + medbots/parsers/common.py | 196 ++++++ medbots/parsers/emias.py | 182 ++++++ medbots/parsers/gemotest.py | 448 ++++++++++++++ medbots/parsers/medsi.py | 198 ++++++ 6 files changed, 1060 insertions(+), 1000 deletions(-) create mode 100644 medbots/parsers/__init__.py create mode 100644 medbots/parsers/common.py create mode 100644 medbots/parsers/emias.py create mode 100644 medbots/parsers/gemotest.py create mode 100644 medbots/parsers/medsi.py diff --git a/medbots/local_structure_pdfs.py b/medbots/local_structure_pdfs.py index 0fd05b9..3da35a1 100644 --- a/medbots/local_structure_pdfs.py +++ b/medbots/local_structure_pdfs.py @@ -1,23 +1,15 @@ #!/usr/bin/env python3 """Structure EMIAS, Gemotest, and Medsi PDFs from pdf_text without external LLM API.""" - from __future__ import annotations import argparse import hashlib import json -import re import sys from pathlib import Path from typing import Any -from medbots.corpus_io import ( - bot_root, - default_corpus_root, - load_manifest, - load_patient_dob, - write_manifest, -) +from medbots.corpus_io import bot_root, default_corpus_root, load_manifest, load_patient_dob, write_manifest from medbots.corpus_writers import ( _append_lab_rows, _safe_txt_name, @@ -25,998 +17,29 @@ _write_to_extracted_images_md, sha256_bytes, ) -from medbots.time_util import utc_now_iso -from medbots.vendor_registry import is_legacy_flat_entry, is_structure_target_entry - -_DATE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})") -_DATE_RESEARCH = re.compile(r"Дата исследования:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) -_EMIAS_DATE = re.compile(r"Дата:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) -_PRINT_DATE = re.compile(r"ПЕЧАТЬ:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) -_ORDER_DATE = re.compile(r"Дата регистрации заказа\s*\n\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) -_SLASH_ORDER_DATE = re.compile(r"дата:\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE) -_GEMOTEST_DATE_MULTILINE = re.compile(r"дата:\s*\n\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE) -_FILENAME_ISO_DATE = re.compile(r"(?:^|/)(20\d{2}-\d{2}-\d{2})__") -_SECTION_HEADERS = frozenset( - { - "Биохимия 19 показателей (расширенная)", - "ОБЩЕКЛИНИЧЕСКИЕ ИССЛЕДОВАНИЯ КАЛА", - "Копрограмма", - "БИОХИМИЯ (Капиллярная кровь)", - } -) -_NUMERIC = re.compile(r"^[\d]+(?:[.,]\d+)?$") -_SKIP_LINES = frozenset( - { - "Исследование", - "Значение", - "Ед. изм.", - "Нормальные значения", - "Нормальные \nзначения", - "Диагноз", - "Тест", - "Результат", - "Норма", - "Отклонение", - "Критичность отклонения", - "Критичность", - "отклонения", - "Ед. изм.", - } +from medbots.parsers import ( + parse_emias_consult_or_imaging, + parse_emias_lab, + parse_gemotest, + parse_medsi_lab, ) +from medbots.parsers.common import _utc_ts +from medbots.parsers.gemotest import _gemotest_subtype +from medbots.parsers.medsi import _extract_medsi_facility +from medbots.vendor_registry import is_legacy_flat_entry, is_structure_target_entry -_CYRILLIC = { - "а": "a", - "б": "b", - "в": "v", - "г": "g", - "д": "d", - "е": "e", - "ё": "e", - "ж": "zh", - "з": "z", - "и": "i", - "й": "y", - "к": "k", - "л": "l", - "м": "m", - "н": "n", - "о": "o", - "п": "p", - "р": "r", - "с": "s", - "т": "t", - "у": "u", - "ф": "f", - "х": "kh", - "ц": "ts", - "ч": "ch", - "ш": "sh", - "щ": "sch", - "ъ": "", - "ы": "y", - "ь": "", - "э": "e", - "ю": "yu", - "я": "ya", -} - - -def _utc_ts() -> str: - return utc_now_iso() - - -def _dmy_to_iso(d: str, m: str, y: str) -> str: - return f"{y}-{m}-{d}" - - -def _date_from_source_path(source_pdf: str) -> str | None: - m = _FILENAME_ISO_DATE.search(source_pdf.replace("\\", "/")) - if not m: - return None - iso = m.group(1) - if iso.endswith("-unknown") or "unknown" in iso: - return None - return iso - - -def _first_iso_date( - text: str, - *, - source_pdf: str = "", - fallback_iso: str | None = None, - patient_dob: str = "", -) -> str | None: - path_date = _date_from_source_path(source_pdf) - if path_date: - return path_date - m = _GEMOTEST_DATE_MULTILINE.search(text) - if m: - return _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - for pat in (_EMIAS_DATE, _DATE_RESEARCH, _PRINT_DATE, _ORDER_DATE, _SLASH_ORDER_DATE): - m = pat.search(text) - if m: - iso = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - if iso != patient_dob: - return iso - m = _DATE_DMY.search(text) - if m: - iso = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - if iso != patient_dob: - return iso - if fallback_iso and fallback_iso != patient_dob: - return fallback_iso - return None - - -def _transliterate_ru(text: str) -> str: - out: list[str] = [] - for ch in text.lower(): - if ch in _CYRILLIC: - out.append(_CYRILLIC[ch]) - elif ch.isascii() and (ch.isalnum() or ch in "-_"): - out.append(ch) - else: - out.append("_") - return "".join(out) - - -def _canonical_key(name_ru: str) -> str: - slug = _transliterate_ru(name_ru) - slug = re.sub(r"[^\w]+", "_", slug) - slug = re.sub(r"_+", "_", slug).strip("_") - return slug[:80] or "analyte" - - -def _parse_float(value: str) -> float | None: - v = value.strip().replace(",", ".") - if not v or not _NUMERIC.match(v): - return None - try: - return float(v) - except ValueError: - return None - - -def _parse_ref_range(ref_text: str) -> tuple[float | None, float | None, str | None]: - ref = " ".join(ref_text.split()) - if not ref or ref.lower().startswith("смотри"): - return None, None, ref or None - m = re.search(r"([\d.,]+)\s*[-–]\s*([\d.,]+)", ref) - if m: - lo = _parse_float(m.group(1)) - hi = _parse_float(m.group(2)) - return lo, hi, None - m = re.match(r"^[<≤]\s*=?([\d.,]+)\s*$", ref) - if m: - hi = _parse_float(m.group(1)) - return None, hi, ref if hi is None else None - m = re.match(r"^[>≥]\s*=?([\d.,]+)\s*$", ref) - if m: - lo = _parse_float(m.group(1)) - return lo, None, ref if lo is None else None - if ref.startswith("<") or ref.startswith(">") or ref.startswith("<="): - return None, None, ref - if _NUMERIC.match(ref.replace(",", ".")): - return None, None, ref - return None, None, ref - - -def _lab_row( - *, - name_ru: str, - value: Any, - unit: str = "", - ref_low: float | None = None, - ref_high: float | None = None, - ref_note: str | None = None, - specimen_date: str, - facility: str = "ЕМИАС", -) -> dict[str, Any]: - return { - "canonical_key": _canonical_key(name_ru), - "name_ru": name_ru.strip(), - "name_en": None, - "loinc": None, - "value": value, - "unit": unit or None, - "ref_low": ref_low, - "ref_high": ref_high, - "ref_note": ref_note, - "specimen_date": specimen_date, - "report_date": specimen_date, - "facility": facility, - "source_kind": "pdf_text", - } - - -def _markdown_header( - *, - doc_type: str, - doc_date: str | None, - title: str, - institution: str = "—", -) -> str: - return ( - f"**Тип:** {doc_type}\n" - f"**Дата:** {doc_date or 'дата неизвестна'}\n" - f"**Учреждение:** {institution}\n" - f"**Врач:** —\n\n" - f"**{title}**\n\n" - ) - - -def _extract_emias_facility(text: str) -> str: - for line in text.splitlines(): - line = line.strip() - if line.startswith("ГБУЗ") or line.startswith("МНПЦ") or "поликлиник" in line.lower(): - return line[:120] - return "ЕМИАС" - - -def _extract_medsi_facility(text: str) -> str: - for line in text.splitlines(): - line = line.strip() - if "медси" in line.lower() or "мичуринск" in line.lower(): - return line[:120] - if "медси" in text.lower(): - return "Медси" - return "Медси" - - -def _extract_conclusion(text: str) -> str: - m = re.search( - r"Заключение:?\s*\n+(.+?)(?:\n-{3,}|\nРекомендаци|\nПОДПИСИ|\nЗаключение протокола|\nДата:|\Z)", - text, - re.DOTALL | re.IGNORECASE, - ) - if m: - return " ".join(m.group(1).split())[:500] - m = re.search( - r"Заключение:\s*(.+?)(?:\nЗаключение протокола|\nРекомендаци|\nПОДПИСИ|\Z)", - text, - re.DOTALL | re.IGNORECASE, - ) - if m: - return " ".join(m.group(1).split())[:500] - m = re.search( - r"Основной\s*\n\s*диагноз\s*\n+(.+?)(?:\nРекомендации|\nДата:|\Z)", - text, - re.DOTALL | re.IGNORECASE, - ) - if m: - return " ".join(m.group(1).split())[:500] - return "—" - - -def parse_emias_lab( - text: str, - title: str, - *, - source_pdf: str = "", - fallback_iso: str | None = None, - patient_dob: str = "", -) -> dict[str, Any]: - doc_date = _first_iso_date( - text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob - ) - facility = _extract_emias_facility(text) - lab_rows: list[dict[str, Any]] = [] - title_l = title.lower() - text_l = text.lower() - - if "антител" in title_l or "igg" in text_l or "igm" in text_l: - blocks = re.findall( - r"Определение антител (Ig[MG]) к\s+Coronavirus \(SARS-[\s\n]*CoV-2\)\s*\n([\d.,]+)\s*\n(<[\d.,]+)", - text, - re.IGNORECASE, - ) - for ig_type, val_s, ref_s in blocks: - name_clean = f"Определение антител {ig_type.upper()} к Coronavirus (SARS-CoV-2)" - val = _parse_float(val_s) - ref_note = ref_s.strip() if ref_s else None - if doc_date: - lab_rows.append( - _lab_row( - name_ru=name_clean, - value=val, - unit="Ед/мл", - ref_note=ref_note, - specimen_date=doc_date, - facility=facility, - ) - ) - elif re.search(r"Исследование - \(L", text): - lab_date = doc_date or _medsi_iso_date(text, patient_dob=patient_dob) - if lab_date: - lab_rows = _parse_medsi_lab_rows(text, lab_date, facility) - else: - m = re.search( - r"(РНК\s+Coronavirus[^\n]+|Исследование на коронавирусы[^\n]*)\s*\n\s*(Не обнаружено|обнаружено|Обнаружено)", - text, - re.IGNORECASE, - ) - if m: - test_name = " ".join(m.group(1).split()) - result = m.group(2).strip() - if doc_date: - lab_rows.append( - _lab_row( - name_ru=test_name, - value=None, - ref_note=result, - specimen_date=doc_date, - facility=facility, - ) - ) - - result_summary = "—" - if lab_rows: - parts = [] - for row in lab_rows: - val = row.get("value") - note = row.get("ref_note") - if val is not None: - parts.append(f"{row['name_ru']}: {val}") - elif note: - parts.append(f"{row['name_ru']}: {note}") - result_summary = "; ".join(parts) - - md_lines = [ - _markdown_header(doc_type="lab", doc_date=doc_date, title=title, institution=facility) - ] - if lab_rows: - md_lines.append("| Показатель | Результат | Ед. | Референс |") - md_lines.append("|------------|-----------|-----|----------|") - for row in lab_rows: - val_s = ( - str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") - ) - ref = row.get("ref_note") if row.get("value") is not None else "—" - md_lines.append(f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |") - else: - md_lines.append(text.strip()) - - return { - "doc_date": doc_date, - "doc_type": "lab", - "title_ru": title, - "institution": facility, - "conclusion_ru": result_summary, - "markdown_block": "\n".join(md_lines), - "lab_rows": lab_rows, - } - - -def parse_emias_consult_or_imaging( - text: str, - title: str, - doc_type: str, - *, - source_pdf: str = "", - fallback_iso: str | None = None, - facility_override: str | None = None, - patient_dob: str = "", -) -> dict[str, Any]: - doc_date = _first_iso_date( - text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob - ) - facility = facility_override or _extract_emias_facility(text) - conclusion = _extract_conclusion(text) - if doc_type == "imaging": - mapped_type = "imaging" - elif doc_type == "functional": - mapped_type = "functional" - else: - mapped_type = "consult" - body = text.strip() - md = _markdown_header( - doc_type=mapped_type, - doc_date=doc_date, - title=title, - institution=facility, - ) - md += f"```\n{body}\n```\n\n**Заключение:** {conclusion}" - return { - "doc_date": doc_date, - "doc_type": mapped_type, - "title_ru": title, - "institution": facility, - "conclusion_ru": conclusion, - "markdown_block": md, - "lab_rows": [], - } - - -def _gemotest_subtype(source_pdf: str) -> str: - name = Path(source_pdf).name.lower() - if any(k in name for k in ("справка", "сертификат", "lo-50")): - return "certificate" - if "_e_a_m_" in name or "кала" in name or "копрограмм" in name: - return "microbiome" - if "_e_a_s_" in name: - return "certificate" - if "_e_a_l_" in name or "e_a_l" in name: - return "lab_results" - return "other" - - -def _gemotest_facility(text: str) -> str: - m = re.search(r'(\d+\.\s*"[^"]+")', text) - if m: - return m.group(1) - if "Гемотест" in text: - return 'ООО "Лаборатория Гемотест"' - return "Гемотест" - - -def _is_section_header(line: str) -> bool: - if line in _SECTION_HEADERS: - return True - return line.startswith("Биохимия ") and "показател" in line.lower() - - -def _collect_analyte_name(lines: list[str], start_idx: int) -> str: - name_parts: list[str] = [] - k = start_idx - while k >= 0: - prev = lines[k] - if _DATE_RESEARCH.match(prev) or prev.startswith("ПЕЧАТЬ:"): - break - if prev in _SKIP_LINES or _is_section_header(prev): - k -= 1 - continue - if _NUMERIC.match(prev.replace(",", ".")): - break - if prev.startswith("Нормальный уровень"): - break - if len(prev) > 120: - break - name_parts.insert(0, prev) - k -= 1 - return " ".join(name_parts).strip() - - -def _append_gemotest_row( - rows: list[dict[str, Any]], - seen: set[tuple[str, str]], - *, - name_ru: str, - specimen_date: str, - facility: str, - value: Any = None, - unit: str = "", - ref_low: float | None = None, - ref_high: float | None = None, - ref_note: str | None = None, -) -> None: - if not name_ru or len(name_ru) < 2 or not specimen_date: - return - key = (_canonical_key(name_ru), specimen_date) - if key in seen: - return - seen.add(key) - rows.append( - _lab_row( - name_ru=name_ru, - value=value, - unit=unit, - ref_low=ref_low, - ref_high=ref_high, - ref_note=ref_note, - specimen_date=specimen_date, - facility=facility, - ) - ) - - -def _parse_gemotest_numeric_blocks(text: str, facility: str) -> list[dict[str, Any]]: - """Parse Gemotest blocks: name / value / unit / ref / Дата исследования.""" - lines = [ln.strip() for ln in text.splitlines()] - rows: list[dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() - - for i, line in enumerate(lines): - m = _DATE_RESEARCH.match(line) - if not m or i < 3: - continue - specimen_date = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - ref_line = lines[i - 1] - unit_line = lines[i - 2] - value_line = lines[i - 3] - - if ref_line.startswith("Нормальный уровень"): - continue - - val = _parse_float(value_line) - if val is None: - name_ru = _collect_analyte_name(lines, i - 4) - _append_gemotest_row( - rows, - seen, - name_ru=name_ru, - value=None, - ref_note=value_line, - specimen_date=specimen_date, - facility=facility, - ) - continue - - unit = "" - if unit_line not in _SKIP_LINES and not _NUMERIC.match(unit_line.replace(",", ".")): - unit = unit_line - name_idx = i - 4 - else: - name_idx = i - 3 - - ref_low, ref_high, ref_note = _parse_ref_range(ref_line) - name_ru = _collect_analyte_name(lines, name_idx) - _append_gemotest_row( - rows, - seen, - name_ru=name_ru, - value=val, - unit=unit, - ref_low=ref_low, - ref_high=ref_high, - ref_note=ref_note, - specimen_date=specimen_date, - facility=facility, - ) - return rows - - -def _parse_gemotest_quad_table( - text: str, facility: str, specimen_date: str -) -> list[dict[str, Any]]: - """Parse 4-line Gemotest tables without per-analyte dates (capillary biochemistry).""" - lines = [ln.strip() for ln in text.splitlines()] - rows: list[dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() - stop_markers = ( - "Результат лабораторных", - "Получая данный", - "Электронная подпись", - "ПЕЧАТЬ:", - "Качество исследований", - ) - - start = 0 - for idx, ln in enumerate(lines): - if "Нормальные значения" in ln or ln.startswith("БИОХИМИЯ"): - start = idx + 1 - break - - i = start - while i < len(lines) - 3: - line = lines[i] - if any(marker in line for marker in stop_markers): - break - if line in _SKIP_LINES or _is_section_header(line) or not line: - i += 1 - continue - if ":" in line or line.startswith("№"): - i += 1 - continue - - name = line - val_s = lines[i + 1] - unit = lines[i + 2] - ref = lines[i + 3] - - if _DATE_RESEARCH.match(val_s) or _DATE_RESEARCH.match(unit): - i += 1 - continue - if _NUMERIC.match(unit.replace(",", ".")): - i += 1 - continue - if not re.search(r"[а-яa-z]", name, re.IGNORECASE): - i += 1 - continue - - val = _parse_float(val_s) - ref_low, ref_high, ref_note = _parse_ref_range(ref) - if val is not None: - _append_gemotest_row( - rows, - seen, - name_ru=name, - value=val, - unit=unit if unit not in _SKIP_LINES else "", - ref_low=ref_low, - ref_high=ref_high, - ref_note=ref_note, - specimen_date=specimen_date, - facility=facility, - ) - i += 4 - continue - i += 1 - return rows - - -def _coprogram_norm_continues(norm: str) -> bool: - n = norm.rstrip().lower() - return n.endswith("или") or n.endswith("или,") or "или" in n and not n.endswith("немного") - - -def _read_coprogram_triplet(lines: list[str], i: int) -> tuple[str, str, str, int] | None: - if i + 2 >= len(lines): - return None - name = lines[i] - if lines[i + 1].rstrip().endswith(","): - if i + 3 >= len(lines): - return None - result = f"{lines[i + 1]} {lines[i + 2]}".strip() - norm = lines[i + 3] - next_i = i + 4 - else: - result = lines[i + 1] - norm = lines[i + 2] - next_i = i + 3 - if next_i < len(lines) and _coprogram_norm_continues(norm): - norm = f"{norm} {lines[next_i]}".strip() - next_i += 1 - return name, result, norm, next_i - - -def _parse_gemotest_coprogram_rows( - text: str, specimen_date: str, facility: str -) -> list[dict[str, Any]]: - """Parse coprogram section: name / result [/ multiline] / norm.""" - m = re.search( - r"Копрограмма\s*\n(.+?)(?:\nКачество исследований|\Z)", - text, - re.DOTALL | re.IGNORECASE, - ) - if not m: - return [] - - lines = [ln.strip() for ln in m.group(1).splitlines() if ln.strip()] - rows: list[dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() - i = 0 - while i < len(lines): - name = lines[i] - if not name or name in _SKIP_LINES or _is_section_header(name): - i += 1 - continue - triplet = _read_coprogram_triplet(lines, i) - if not triplet: - break - name, result, norm, i = triplet - if not result or _DATE_RESEARCH.match(result): - continue - - val = _parse_float(result) - ref_low, ref_high, ref_note = _parse_ref_range(norm) - if val is not None: - _append_gemotest_row( - rows, - seen, - name_ru=name, - value=val, - ref_low=ref_low, - ref_high=ref_high, - ref_note=ref_note, - specimen_date=specimen_date, - facility=facility, - ) - else: - _append_gemotest_row( - rows, - seen, - name_ru=name, - value=None, - ref_note=f"{result} (норма: {norm})" if norm else result, - specimen_date=specimen_date, - facility=facility, - ) - return rows - - -def _parse_gemotest_qualitative_table(text: str) -> str: - m = re.search( - r"Копрограмма\s*\n(.+?)(?:\nКачество исследований|\Z)", - text, - re.DOTALL | re.IGNORECASE, - ) - if not m: - return text.strip() - - lines = [ln.strip() for ln in m.group(1).splitlines() if ln.strip()] - rows: list[tuple[str, str, str]] = [] - i = 0 - while i < len(lines): - name = lines[i] - if not name or name in _SKIP_LINES or _is_section_header(name): - i += 1 - continue - triplet = _read_coprogram_triplet(lines, i) - if not triplet: - break - name, val, norm, i = triplet - if val and norm and not _DATE_RESEARCH.match(val): - rows.append((name, val, norm)) - - if not rows: - return text.strip() - out = ["| Показатель | Результат | Норма |", "|------------|-----------|-------|"] - for name, val, norm in rows: - out.append(f"| {name} | {val} | {norm} |") - return "\n".join(out) - - -def parse_gemotest( - text: str, - title: str, - source_pdf: str, - *, - fallback_iso: str | None = None, - patient_dob: str = "", -) -> dict[str, Any]: - subtype = _gemotest_subtype(source_pdf) - facility = _gemotest_facility(text) - doc_date = _first_iso_date( - text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob - ) - - if subtype == "certificate": - md = _markdown_header( - doc_type="consult", - doc_date=doc_date, - title=title, - institution=facility, - ) - md += f"```\n{text.strip()}\n```" - return { - "doc_date": doc_date, - "doc_type": "consult", - "title_ru": title, - "institution": facility, - "conclusion_ru": "Справка/сертификат", - "markdown_block": md, - "lab_rows": [], - } - - if subtype == "microbiome" or "копрограмма" in text.lower(): - table = _parse_gemotest_qualitative_table(text) - lab_rows: list[dict[str, Any]] = [] - if doc_date: - lab_rows = _parse_gemotest_coprogram_rows(text, doc_date, facility) - md = _markdown_header( - doc_type="lab", - doc_date=doc_date, - title=title, - institution=facility, - ) - md += table - return { - "doc_date": doc_date, - "doc_type": "lab", - "title_ru": title, - "institution": facility, - "conclusion_ru": "Качественное исследование", - "markdown_block": md, - "lab_rows": lab_rows, - } - - lab_rows = _parse_gemotest_numeric_blocks(text, facility) - if not lab_rows and doc_date: - lab_rows = _parse_gemotest_quad_table(text, facility, doc_date) - md = _markdown_header( - doc_type="lab", - doc_date=doc_date, - title=title, - institution=facility, - ) - if lab_rows: - md += "| Показатель | Значение | Ед. | Референс |\n" - md += "|------------|----------|-----|----------|\n" - for row in lab_rows: - val_s = ( - str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") - ) - ref = ( - f"{row['ref_low']}–{row['ref_high']}" - if row.get("ref_low") is not None and row.get("ref_high") is not None - else (row.get("ref_note") or "—") - ) - md += f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |\n" - else: - md += text.strip() - - conclusion = "—" - if lab_rows: - conclusion = f"Извлечено показателей: {len(lab_rows)}" - - return { - "doc_date": doc_date, - "doc_type": "lab", - "title_ru": title, - "institution": facility, - "conclusion_ru": conclusion, - "markdown_block": md, - "lab_rows": lab_rows, - } - - -_MEDSI_UNIT = re.compile( - r"^(?:ммоль/л|мг/л|г/л|ед/л|мкмоль/л|%|фл|пг|мм/час|10\*9/л|10\*12/л|клеток/мкл)$", - re.IGNORECASE, -) -_MEDSI_SKIP = frozenset( - { - "венозная", - "Наименование исследования", - "Результат", - "Ед. изм.", - "Нормальные значения", - "Флаг", - "Врач КДЛ:", - } -) - - -def _is_medsi_unit_line(line: str) -> bool: - s = line.strip() - return bool(_MEDSI_UNIT.match(s)) - - -def _is_medsi_ref_line(line: str) -> bool: - s = line.strip() - if not s: - return False - if s.startswith("<") or s.startswith(">") or s.startswith("≤") or s.startswith("≥"): - return True - return bool(re.match(r"^[\d.,]+\s*[-–]\s*[\d.,]+$", s)) - - -def _is_medsi_value_line(line: str) -> bool: - return _parse_float(line.strip()) is not None - - -def _medsi_iso_date(text: str, *, patient_dob: str = "") -> str | None: - m = _EMIAS_DATE.search(text) - if m: - return _dmy_to_iso(m.group(1), m.group(2), m.group(3)) - return _first_iso_date(text, patient_dob=patient_dob) - - -def _parse_medsi_lab_rows(text: str, doc_date: str, facility: str) -> list[dict[str, Any]]: - lines = [ln.rstrip() for ln in text.splitlines()] - rows: list[dict[str, Any]] = [] - i = 0 - in_table = False - - while i < len(lines): - raw = lines[i].strip() - i += 1 - if not raw: - continue - if raw in _MEDSI_SKIP: - continue - if raw.startswith("Исследование - (L"): - in_table = True - continue - if not in_table: - continue - if raw.startswith("Согласно ") or raw.startswith("Диагностические"): - continue - if raw.startswith("Нормальный уровень") or raw.startswith("уровень глюкозы"): - continue - if raw.startswith("Выполнено по методу"): - continue - if re.match(r"^\d{2}\.\d{2}\.\d{4}", raw): - continue - if "Врач" in raw and ":" not in raw[:20]: - continue - - name_parts = [raw] - while i < len(lines): - nxt = lines[i].strip() - if not nxt: - i += 1 - continue - if ( - _is_medsi_unit_line(nxt) - or _is_medsi_ref_line(nxt) - or _is_medsi_value_line(nxt) - or nxt.startswith("Исследование - (L") - ): - break - if nxt in _MEDSI_SKIP: - i += 1 - break - name_parts.append(nxt) - i += 1 - - name = " ".join(name_parts).strip() - if not name or name in _MEDSI_SKIP: - continue - - unit = "" - ref_low: float | None = None - ref_high: float | None = None - ref_note: str | None = None - value: float | None = None - - if i < len(lines) and _is_medsi_unit_line(lines[i].strip()): - unit = lines[i].strip() - i += 1 - - if i < len(lines) and _is_medsi_ref_line(lines[i].strip()): - ref_low, ref_high, ref_note = _parse_ref_range(lines[i].strip()) - i += 1 - - if i < len(lines) and _is_medsi_value_line(lines[i].strip()): - value = _parse_float(lines[i].strip()) - i += 1 - - if value is None and ref_note is None: - continue - - rows.append( - _lab_row( - name_ru=name, - value=value, - unit=unit, - ref_low=ref_low, - ref_high=ref_high, - ref_note=ref_note, - specimen_date=doc_date, - facility=facility, - ) - ) - - return rows - - -def parse_medsi_lab( - text: str, - title: str, - *, - source_pdf: str = "", - fallback_iso: str | None = None, - patient_dob: str = "", -) -> dict[str, Any]: - doc_date = _medsi_iso_date(text, patient_dob=patient_dob) or fallback_iso - facility = "Медси" - if "мичуринск" in text.lower(): - facility = 'Медси "Мичуринский"' - lab_rows: list[dict[str, Any]] = [] - if doc_date: - lab_rows = _parse_medsi_lab_rows(text, doc_date, facility) - - md = _markdown_header( - doc_type="lab", - doc_date=doc_date, - title=title, - institution=facility, - ) - if lab_rows: - md += "| Показатель | Значение | Ед. | Референс |\n" - md += "|------------|----------|-----|----------|\n" - for row in lab_rows: - val_s = str(row["value"]) if row.get("value") is not None else "—" - ref = ( - f"{row['ref_low']}–{row['ref_high']}" - if row.get("ref_low") is not None and row.get("ref_high") is not None - else (row.get("ref_note") or "—") - ) - md += f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |\n" - else: - md += text.strip() - - conclusion = f"Извлечено показателей: {len(lab_rows)}" if lab_rows else "—" - return { - "doc_date": doc_date, - "doc_type": "lab", - "title_ru": title, - "institution": facility, - "conclusion_ru": conclusion, - "markdown_block": md, - "lab_rows": lab_rows, - } +__all__ = [ + "_entry_title", + "_extract_medsi_facility", + "_gemotest_subtype", + "_parse_entry", + "_pdf_text_path", + "parse_emias_consult_or_imaging", + "parse_emias_lab", + "parse_gemotest", + "parse_medsi_lab", + "run", +] def _append_extracted_section_if_new( @@ -1236,7 +259,9 @@ def run( sha256=sha, ingest_ts=now, ) - labs_added = _append_lab_rows(corpus, extracted.get("lab_rows") or [], doc_rel) + labs_added = _append_lab_rows( + corpus, extracted.get("lab_rows") or [], doc_rel + ) labs_added_total += labs_added updated = dict(entry) diff --git a/medbots/parsers/__init__.py b/medbots/parsers/__init__.py new file mode 100644 index 0000000..c3acb80 --- /dev/null +++ b/medbots/parsers/__init__.py @@ -0,0 +1,11 @@ +"""Vendor-specific PDF text parsers.""" +from medbots.parsers.emias import parse_emias_consult_or_imaging, parse_emias_lab +from medbots.parsers.gemotest import parse_gemotest +from medbots.parsers.medsi import parse_medsi_lab + +__all__ = [ + "parse_emias_consult_or_imaging", + "parse_emias_lab", + "parse_gemotest", + "parse_medsi_lab", +] diff --git a/medbots/parsers/common.py b/medbots/parsers/common.py new file mode 100644 index 0000000..47f54cf --- /dev/null +++ b/medbots/parsers/common.py @@ -0,0 +1,196 @@ +"""Shared PDF parsing utilities for EMIAS, Gemotest, and Medsi.""" +from __future__ import annotations + +import re +from typing import Any, Optional + +from medbots.time_util import utc_now_iso + +_DATE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})") + +_DATE_RESEARCH = re.compile(r"Дата исследования:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) + +_EMIAS_DATE = re.compile(r"Дата:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) + +_PRINT_DATE = re.compile(r"ПЕЧАТЬ:\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) + +_ORDER_DATE = re.compile(r"Дата регистрации заказа\s*\n\s*(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE) + +_SLASH_ORDER_DATE = re.compile(r"дата:\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE) + +_GEMOTEST_DATE_MULTILINE = re.compile(r"дата:\s*\n\s*(\d{2})/(\d{2})/(\d{4})", re.IGNORECASE) + +_FILENAME_ISO_DATE = re.compile(r"(?:^|/)(20\d{2}-\d{2}-\d{2})__") + +_NUMERIC = re.compile(r"^[\d]+(?:[.,]\d+)?$") + +_CYRILLIC = { + "а": "a", + "б": "b", + "в": "v", + "г": "g", + "д": "d", + "е": "e", + "ё": "e", + "ж": "zh", + "з": "z", + "и": "i", + "й": "y", + "к": "k", + "л": "l", + "м": "m", + "н": "n", + "о": "o", + "п": "p", + "р": "r", + "с": "s", + "т": "t", + "у": "u", + "ф": "f", + "х": "kh", + "ц": "ts", + "ч": "ch", + "ш": "sh", + "щ": "sch", + "ъ": "", + "ы": "y", + "ь": "", + "э": "e", + "ю": "yu", + "я": "ya", +} + +def _utc_ts() -> str: + return utc_now_iso() + +def _dmy_to_iso(d: str, m: str, y: str) -> str: + return f"{y}-{m}-{d}" + +def _date_from_source_path(source_pdf: str) -> str | None: + m = _FILENAME_ISO_DATE.search(source_pdf.replace("\\", "/")) + if not m: + return None + iso = m.group(1) + if iso.endswith("-unknown") or "unknown" in iso: + return None + return iso + +def _first_iso_date( + text: str, + *, + source_pdf: str = "", + fallback_iso: str | None = None, + patient_dob: str = "", +) -> str | None: + path_date = _date_from_source_path(source_pdf) + if path_date: + return path_date + m = _GEMOTEST_DATE_MULTILINE.search(text) + if m: + return _dmy_to_iso(m.group(1), m.group(2), m.group(3)) + for pat in (_EMIAS_DATE, _DATE_RESEARCH, _PRINT_DATE, _ORDER_DATE, _SLASH_ORDER_DATE): + m = pat.search(text) + if m: + iso = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) + if iso != patient_dob: + return iso + m = _DATE_DMY.search(text) + if m: + iso = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) + if iso != patient_dob: + return iso + if fallback_iso and fallback_iso != patient_dob: + return fallback_iso + return None + +def _transliterate_ru(text: str) -> str: + out: list[str] = [] + for ch in text.lower(): + if ch in _CYRILLIC: + out.append(_CYRILLIC[ch]) + elif ch.isascii() and (ch.isalnum() or ch in "-_"): + out.append(ch) + else: + out.append("_") + return "".join(out) + +def _canonical_key(name_ru: str) -> str: + slug = _transliterate_ru(name_ru) + slug = re.sub(r"[^\w]+", "_", slug) + slug = re.sub(r"_+", "_", slug).strip("_") + return slug[:80] or "analyte" + +def _parse_float(value: str) -> float | None: + v = value.strip().replace(",", ".") + if not v or not _NUMERIC.match(v): + return None + try: + return float(v) + except ValueError: + return None + +def _parse_ref_range(ref_text: str) -> tuple[float | None, float | None, str | None]: + ref = " ".join(ref_text.split()) + if not ref or ref.lower().startswith("смотри"): + return None, None, ref or None + m = re.search(r"([\d.,]+)\s*[-–]\s*([\d.,]+)", ref) + if m: + lo = _parse_float(m.group(1)) + hi = _parse_float(m.group(2)) + return lo, hi, None + m = re.match(r"^[<≤]\s*=?([\d.,]+)\s*$", ref) + if m: + hi = _parse_float(m.group(1)) + return None, hi, ref if hi is None else None + m = re.match(r"^[>≥]\s*=?([\d.,]+)\s*$", ref) + if m: + lo = _parse_float(m.group(1)) + return lo, None, ref if lo is None else None + if ref.startswith("<") or ref.startswith(">") or ref.startswith("<="): + return None, None, ref + if _NUMERIC.match(ref.replace(",", ".")): + return None, None, ref + return None, None, ref + +def _lab_row( + *, + name_ru: str, + value: Any, + unit: str = "", + ref_low: float | None = None, + ref_high: float | None = None, + ref_note: str | None = None, + specimen_date: str, + facility: str = "ЕМИАС", +) -> dict[str, Any]: + return { + "canonical_key": _canonical_key(name_ru), + "name_ru": name_ru.strip(), + "name_en": None, + "loinc": None, + "value": value, + "unit": unit or None, + "ref_low": ref_low, + "ref_high": ref_high, + "ref_note": ref_note, + "specimen_date": specimen_date, + "report_date": specimen_date, + "facility": facility, + "source_kind": "pdf_text", + } + +def _markdown_header( + *, + doc_type: str, + doc_date: str | None, + title: str, + institution: str = "—", +) -> str: + return ( + f"**Тип:** {doc_type}\n" + f"**Дата:** {doc_date or 'дата неизвестна'}\n" + f"**Учреждение:** {institution}\n" + f"**Врач:** —\n\n" + f"**{title}**\n\n" + ) + diff --git a/medbots/parsers/emias.py b/medbots/parsers/emias.py new file mode 100644 index 0000000..ca24bf4 --- /dev/null +++ b/medbots/parsers/emias.py @@ -0,0 +1,182 @@ +"""EMIAS PDF parsers.""" +from __future__ import annotations + +import re +from typing import Any, Optional + +from medbots.parsers.common import ( + _first_iso_date, + _lab_row, + _markdown_header, + _parse_float, +) +from medbots.parsers.medsi import _medsi_iso_date, _parse_medsi_lab_rows + +def _extract_emias_facility(text: str) -> str: + for line in text.splitlines(): + line = line.strip() + if line.startswith("ГБУЗ") or line.startswith("МНПЦ") or "поликлиник" in line.lower(): + return line[:120] + return "ЕМИАС" + +def _extract_conclusion(text: str) -> str: + m = re.search( + r"Заключение:?\s*\n+(.+?)(?:\n-{3,}|\nРекомендаци|\nПОДПИСИ|\nЗаключение протокола|\nДата:|\Z)", + text, + re.DOTALL | re.IGNORECASE, + ) + if m: + return " ".join(m.group(1).split())[:500] + m = re.search( + r"Заключение:\s*(.+?)(?:\nЗаключение протокола|\nРекомендаци|\nПОДПИСИ|\Z)", + text, + re.DOTALL | re.IGNORECASE, + ) + if m: + return " ".join(m.group(1).split())[:500] + m = re.search( + r"Основной\s*\n\s*диагноз\s*\n+(.+?)(?:\nРекомендации|\nДата:|\Z)", + text, + re.DOTALL | re.IGNORECASE, + ) + if m: + return " ".join(m.group(1).split())[:500] + return "—" + +def parse_emias_lab( + text: str, + title: str, + *, + source_pdf: str = "", + fallback_iso: str | None = None, + patient_dob: str = "", +) -> dict[str, Any]: + doc_date = _first_iso_date( + text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) + facility = _extract_emias_facility(text) + lab_rows: list[dict[str, Any]] = [] + title_l = title.lower() + text_l = text.lower() + + if "антител" in title_l or "igg" in text_l or "igm" in text_l: + blocks = re.findall( + r"Определение антител (Ig[MG]) к\s+Coronavirus \(SARS-[\s\n]*CoV-2\)\s*\n([\d.,]+)\s*\n(<[\d.,]+)", + text, + re.IGNORECASE, + ) + for ig_type, val_s, ref_s in blocks: + name_clean = f"Определение антител {ig_type.upper()} к Coronavirus (SARS-CoV-2)" + val = _parse_float(val_s) + ref_note = ref_s.strip() if ref_s else None + if doc_date: + lab_rows.append( + _lab_row( + name_ru=name_clean, + value=val, + unit="Ед/мл", + ref_note=ref_note, + specimen_date=doc_date, + facility=facility, + ) + ) + elif re.search(r"Исследование - \(L", text): + lab_date = doc_date or _medsi_iso_date(text, patient_dob=patient_dob) + if lab_date: + lab_rows = _parse_medsi_lab_rows(text, lab_date, facility) + else: + m = re.search( + r"(РНК\s+Coronavirus[^\n]+|Исследование на коронавирусы[^\n]*)\s*\n\s*(Не обнаружено|обнаружено|Обнаружено)", + text, + re.IGNORECASE, + ) + if m: + test_name = " ".join(m.group(1).split()) + result = m.group(2).strip() + if doc_date: + lab_rows.append( + _lab_row( + name_ru=test_name, + value=None, + ref_note=result, + specimen_date=doc_date, + facility=facility, + ) + ) + + result_summary = "—" + if lab_rows: + parts = [] + for row in lab_rows: + val = row.get("value") + note = row.get("ref_note") + if val is not None: + parts.append(f"{row['name_ru']}: {val}") + elif note: + parts.append(f"{row['name_ru']}: {note}") + result_summary = "; ".join(parts) + + md_lines = [ + _markdown_header(doc_type="lab", doc_date=doc_date, title=title, institution=facility) + ] + if lab_rows: + md_lines.append("| Показатель | Результат | Ед. | Референс |") + md_lines.append("|------------|-----------|-----|----------|") + for row in lab_rows: + val_s = ( + str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") + ) + ref = row.get("ref_note") if row.get("value") is not None else "—" + md_lines.append(f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |") + else: + md_lines.append(text.strip()) + + return { + "doc_date": doc_date, + "doc_type": "lab", + "title_ru": title, + "institution": facility, + "conclusion_ru": result_summary, + "markdown_block": "\n".join(md_lines), + "lab_rows": lab_rows, + } + +def parse_emias_consult_or_imaging( + text: str, + title: str, + doc_type: str, + *, + source_pdf: str = "", + fallback_iso: str | None = None, + facility_override: str | None = None, + patient_dob: str = "", +) -> dict[str, Any]: + doc_date = _first_iso_date( + text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) + facility = facility_override or _extract_emias_facility(text) + conclusion = _extract_conclusion(text) + if doc_type == "imaging": + mapped_type = "imaging" + elif doc_type == "functional": + mapped_type = "functional" + else: + mapped_type = "consult" + body = text.strip() + md = _markdown_header( + doc_type=mapped_type, + doc_date=doc_date, + title=title, + institution=facility, + ) + md += f"```\n{body}\n```\n\n**Заключение:** {conclusion}" + return { + "doc_date": doc_date, + "doc_type": mapped_type, + "title_ru": title, + "institution": facility, + "conclusion_ru": conclusion, + "markdown_block": md, + "lab_rows": [], + } + diff --git a/medbots/parsers/gemotest.py b/medbots/parsers/gemotest.py new file mode 100644 index 0000000..c4f8265 --- /dev/null +++ b/medbots/parsers/gemotest.py @@ -0,0 +1,448 @@ +"""Gemotest PDF parser.""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Optional + +from medbots.parsers.common import ( + _DATE_RESEARCH, + _NUMERIC, + _canonical_key, + _dmy_to_iso, + _first_iso_date, + _lab_row, + _markdown_header, + _parse_float, + _parse_ref_range, +) + +_SECTION_HEADERS = frozenset( + { + "Биохимия 19 показателей (расширенная)", + "ОБЩЕКЛИНИЧЕСКИЕ ИССЛЕДОВАНИЯ КАЛА", + "Копрограмма", + "БИОХИМИЯ (Капиллярная кровь)", + } +) + +_SKIP_LINES = frozenset( + { + "Исследование", + "Значение", + "Ед. изм.", + "Нормальные значения", + "Нормальные \nзначения", + "Диагноз", + "Тест", + "Результат", + "Норма", + "Отклонение", + "Критичность отклонения", + "Критичность", + "отклонения", + "Ед. изм.", + } +) + +def _gemotest_subtype(source_pdf: str) -> str: + name = Path(source_pdf).name.lower() + if any(k in name for k in ("справка", "сертификат", "lo-50")): + return "certificate" + if "_e_a_m_" in name or "кала" in name or "копрограмм" in name: + return "microbiome" + if "_e_a_s_" in name: + return "certificate" + if "_e_a_l_" in name or "e_a_l" in name: + return "lab_results" + return "other" + +def _gemotest_facility(text: str) -> str: + m = re.search(r'(\d+\.\s*"[^"]+")', text) + if m: + return m.group(1) + if "Гемотест" in text: + return 'ООО "Лаборатория Гемотест"' + return "Гемотест" + +def _is_section_header(line: str) -> bool: + if line in _SECTION_HEADERS: + return True + return line.startswith("Биохимия ") and "показател" in line.lower() + +def _collect_analyte_name(lines: list[str], start_idx: int) -> str: + name_parts: list[str] = [] + k = start_idx + while k >= 0: + prev = lines[k] + if _DATE_RESEARCH.match(prev) or prev.startswith("ПЕЧАТЬ:"): + break + if prev in _SKIP_LINES or _is_section_header(prev): + k -= 1 + continue + if _NUMERIC.match(prev.replace(",", ".")): + break + if prev.startswith("Нормальный уровень"): + break + if len(prev) > 120: + break + name_parts.insert(0, prev) + k -= 1 + return " ".join(name_parts).strip() + +def _append_gemotest_row( + rows: list[dict[str, Any]], + seen: set[tuple[str, str]], + *, + name_ru: str, + specimen_date: str, + facility: str, + value: Any = None, + unit: str = "", + ref_low: float | None = None, + ref_high: float | None = None, + ref_note: str | None = None, +) -> None: + if not name_ru or len(name_ru) < 2 or not specimen_date: + return + key = (_canonical_key(name_ru), specimen_date) + if key in seen: + return + seen.add(key) + rows.append( + _lab_row( + name_ru=name_ru, + value=value, + unit=unit, + ref_low=ref_low, + ref_high=ref_high, + ref_note=ref_note, + specimen_date=specimen_date, + facility=facility, + ) + ) + +def _parse_gemotest_numeric_blocks(text: str, facility: str) -> list[dict[str, Any]]: + """Parse Gemotest blocks: name / value / unit / ref / Дата исследования.""" + lines = [ln.strip() for ln in text.splitlines()] + rows: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + + for i, line in enumerate(lines): + m = _DATE_RESEARCH.match(line) + if not m or i < 3: + continue + specimen_date = _dmy_to_iso(m.group(1), m.group(2), m.group(3)) + ref_line = lines[i - 1] + unit_line = lines[i - 2] + value_line = lines[i - 3] + + if ref_line.startswith("Нормальный уровень"): + continue + + val = _parse_float(value_line) + if val is None: + name_ru = _collect_analyte_name(lines, i - 4) + _append_gemotest_row( + rows, + seen, + name_ru=name_ru, + value=None, + ref_note=value_line, + specimen_date=specimen_date, + facility=facility, + ) + continue + + unit = "" + if unit_line not in _SKIP_LINES and not _NUMERIC.match(unit_line.replace(",", ".")): + unit = unit_line + name_idx = i - 4 + else: + name_idx = i - 3 + + ref_low, ref_high, ref_note = _parse_ref_range(ref_line) + name_ru = _collect_analyte_name(lines, name_idx) + _append_gemotest_row( + rows, + seen, + name_ru=name_ru, + value=val, + unit=unit, + ref_low=ref_low, + ref_high=ref_high, + ref_note=ref_note, + specimen_date=specimen_date, + facility=facility, + ) + return rows + +def _parse_gemotest_quad_table( + text: str, facility: str, specimen_date: str +) -> list[dict[str, Any]]: + """Parse 4-line Gemotest tables without per-analyte dates (capillary biochemistry).""" + lines = [ln.strip() for ln in text.splitlines()] + rows: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + stop_markers = ( + "Результат лабораторных", + "Получая данный", + "Электронная подпись", + "ПЕЧАТЬ:", + "Качество исследований", + ) + + start = 0 + for idx, ln in enumerate(lines): + if "Нормальные значения" in ln or ln.startswith("БИОХИМИЯ"): + start = idx + 1 + break + + i = start + while i < len(lines) - 3: + line = lines[i] + if any(marker in line for marker in stop_markers): + break + if line in _SKIP_LINES or _is_section_header(line) or not line: + i += 1 + continue + if ":" in line or line.startswith("№"): + i += 1 + continue + + name = line + val_s = lines[i + 1] + unit = lines[i + 2] + ref = lines[i + 3] + + if _DATE_RESEARCH.match(val_s) or _DATE_RESEARCH.match(unit): + i += 1 + continue + if _NUMERIC.match(unit.replace(",", ".")): + i += 1 + continue + if not re.search(r"[а-яa-z]", name, re.IGNORECASE): + i += 1 + continue + + val = _parse_float(val_s) + ref_low, ref_high, ref_note = _parse_ref_range(ref) + if val is not None: + _append_gemotest_row( + rows, + seen, + name_ru=name, + value=val, + unit=unit if unit not in _SKIP_LINES else "", + ref_low=ref_low, + ref_high=ref_high, + ref_note=ref_note, + specimen_date=specimen_date, + facility=facility, + ) + i += 4 + continue + i += 1 + return rows + +def _coprogram_norm_continues(norm: str) -> bool: + n = norm.rstrip().lower() + return n.endswith("или") or n.endswith("или,") or "или" in n and not n.endswith("немного") + +def _read_coprogram_triplet(lines: list[str], i: int) -> tuple[str, str, str, int] | None: + if i + 2 >= len(lines): + return None + name = lines[i] + if lines[i + 1].rstrip().endswith(","): + if i + 3 >= len(lines): + return None + result = f"{lines[i + 1]} {lines[i + 2]}".strip() + norm = lines[i + 3] + next_i = i + 4 + else: + result = lines[i + 1] + norm = lines[i + 2] + next_i = i + 3 + if next_i < len(lines) and _coprogram_norm_continues(norm): + norm = f"{norm} {lines[next_i]}".strip() + next_i += 1 + return name, result, norm, next_i + +def _parse_gemotest_coprogram_rows( + text: str, specimen_date: str, facility: str +) -> list[dict[str, Any]]: + """Parse coprogram section: name / result [/ multiline] / norm.""" + m = re.search( + r"Копрограмма\s*\n(.+?)(?:\nКачество исследований|\Z)", + text, + re.DOTALL | re.IGNORECASE, + ) + if not m: + return [] + + lines = [ln.strip() for ln in m.group(1).splitlines() if ln.strip()] + rows: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + i = 0 + while i < len(lines): + name = lines[i] + if not name or name in _SKIP_LINES or _is_section_header(name): + i += 1 + continue + triplet = _read_coprogram_triplet(lines, i) + if not triplet: + break + name, result, norm, i = triplet + if not result or _DATE_RESEARCH.match(result): + continue + + val = _parse_float(result) + ref_low, ref_high, ref_note = _parse_ref_range(norm) + if val is not None: + _append_gemotest_row( + rows, + seen, + name_ru=name, + value=val, + ref_low=ref_low, + ref_high=ref_high, + ref_note=ref_note, + specimen_date=specimen_date, + facility=facility, + ) + else: + _append_gemotest_row( + rows, + seen, + name_ru=name, + value=None, + ref_note=f"{result} (норма: {norm})" if norm else result, + specimen_date=specimen_date, + facility=facility, + ) + return rows + +def _parse_gemotest_qualitative_table(text: str) -> str: + m = re.search( + r"Копрограмма\s*\n(.+?)(?:\nКачество исследований|\Z)", + text, + re.DOTALL | re.IGNORECASE, + ) + if not m: + return text.strip() + + lines = [ln.strip() for ln in m.group(1).splitlines() if ln.strip()] + rows: list[tuple[str, str, str]] = [] + i = 0 + while i < len(lines): + name = lines[i] + if not name or name in _SKIP_LINES or _is_section_header(name): + i += 1 + continue + triplet = _read_coprogram_triplet(lines, i) + if not triplet: + break + name, val, norm, i = triplet + if val and norm and not _DATE_RESEARCH.match(val): + rows.append((name, val, norm)) + + if not rows: + return text.strip() + out = ["| Показатель | Результат | Норма |", "|------------|-----------|-------|"] + for name, val, norm in rows: + out.append(f"| {name} | {val} | {norm} |") + return "\n".join(out) + +def parse_gemotest( + text: str, + title: str, + source_pdf: str, + *, + fallback_iso: str | None = None, + patient_dob: str = "", +) -> dict[str, Any]: + subtype = _gemotest_subtype(source_pdf) + facility = _gemotest_facility(text) + doc_date = _first_iso_date( + text, source_pdf=source_pdf, fallback_iso=fallback_iso, patient_dob=patient_dob + ) + + if subtype == "certificate": + md = _markdown_header( + doc_type="consult", + doc_date=doc_date, + title=title, + institution=facility, + ) + md += f"```\n{text.strip()}\n```" + return { + "doc_date": doc_date, + "doc_type": "consult", + "title_ru": title, + "institution": facility, + "conclusion_ru": "Справка/сертификат", + "markdown_block": md, + "lab_rows": [], + } + + if subtype == "microbiome" or "копрограмма" in text.lower(): + table = _parse_gemotest_qualitative_table(text) + lab_rows: list[dict[str, Any]] = [] + if doc_date: + lab_rows = _parse_gemotest_coprogram_rows(text, doc_date, facility) + md = _markdown_header( + doc_type="lab", + doc_date=doc_date, + title=title, + institution=facility, + ) + md += table + return { + "doc_date": doc_date, + "doc_type": "lab", + "title_ru": title, + "institution": facility, + "conclusion_ru": "Качественное исследование", + "markdown_block": md, + "lab_rows": lab_rows, + } + + lab_rows = _parse_gemotest_numeric_blocks(text, facility) + if not lab_rows and doc_date: + lab_rows = _parse_gemotest_quad_table(text, facility, doc_date) + md = _markdown_header( + doc_type="lab", + doc_date=doc_date, + title=title, + institution=facility, + ) + if lab_rows: + md += "| Показатель | Значение | Ед. | Референс |\n" + md += "|------------|----------|-----|----------|\n" + for row in lab_rows: + val_s = ( + str(row["value"]) if row.get("value") is not None else (row.get("ref_note") or "—") + ) + ref = ( + f"{row['ref_low']}–{row['ref_high']}" + if row.get("ref_low") is not None and row.get("ref_high") is not None + else (row.get("ref_note") or "—") + ) + md += f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |\n" + else: + md += text.strip() + + conclusion = "—" + if lab_rows: + conclusion = f"Извлечено показателей: {len(lab_rows)}" + + return { + "doc_date": doc_date, + "doc_type": "lab", + "title_ru": title, + "institution": facility, + "conclusion_ru": conclusion, + "markdown_block": md, + "lab_rows": lab_rows, + } + diff --git a/medbots/parsers/medsi.py b/medbots/parsers/medsi.py new file mode 100644 index 0000000..7d76e78 --- /dev/null +++ b/medbots/parsers/medsi.py @@ -0,0 +1,198 @@ +"""Medsi PDF parser.""" +from __future__ import annotations + +import re +from typing import Any, Optional + +from medbots.parsers.common import ( + _EMIAS_DATE, + _dmy_to_iso, + _first_iso_date, + _lab_row, + _markdown_header, + _parse_float, + _parse_ref_range, +) + +_MEDSI_UNIT = re.compile( + r"^(?:ммоль/л|мг/л|г/л|ед/л|мкмоль/л|%|фл|пг|мм/час|10\*9/л|10\*12/л|клеток/мкл)$", + re.IGNORECASE, +) + +_MEDSI_SKIP = frozenset( + { + "венозная", + "Наименование исследования", + "Результат", + "Ед. изм.", + "Нормальные значения", + "Флаг", + "Врач КДЛ:", + } +) + +def _extract_medsi_facility(text: str) -> str: + for line in text.splitlines(): + line = line.strip() + if "медси" in line.lower() or "мичуринск" in line.lower(): + return line[:120] + if "медси" in text.lower(): + return "Медси" + return "Медси" + +def _is_medsi_unit_line(line: str) -> bool: + s = line.strip() + return bool(_MEDSI_UNIT.match(s)) + +def _is_medsi_ref_line(line: str) -> bool: + s = line.strip() + if not s: + return False + if s.startswith("<") or s.startswith(">") or s.startswith("≤") or s.startswith("≥"): + return True + return bool(re.match(r"^[\d.,]+\s*[-–]\s*[\d.,]+$", s)) + +def _is_medsi_value_line(line: str) -> bool: + return _parse_float(line.strip()) is not None + +def _medsi_iso_date(text: str, *, patient_dob: str = "") -> str | None: + m = _EMIAS_DATE.search(text) + if m: + return _dmy_to_iso(m.group(1), m.group(2), m.group(3)) + return _first_iso_date(text, patient_dob=patient_dob) + +def _parse_medsi_lab_rows(text: str, doc_date: str, facility: str) -> list[dict[str, Any]]: + lines = [ln.rstrip() for ln in text.splitlines()] + rows: list[dict[str, Any]] = [] + i = 0 + in_table = False + + while i < len(lines): + raw = lines[i].strip() + i += 1 + if not raw: + continue + if raw in _MEDSI_SKIP: + continue + if raw.startswith("Исследование - (L"): + in_table = True + continue + if not in_table: + continue + if raw.startswith("Согласно ") or raw.startswith("Диагностические"): + continue + if raw.startswith("Нормальный уровень") or raw.startswith("уровень глюкозы"): + continue + if raw.startswith("Выполнено по методу"): + continue + if re.match(r"^\d{2}\.\d{2}\.\d{4}", raw): + continue + if "Врач" in raw and ":" not in raw[:20]: + continue + + name_parts = [raw] + while i < len(lines): + nxt = lines[i].strip() + if not nxt: + i += 1 + continue + if ( + _is_medsi_unit_line(nxt) + or _is_medsi_ref_line(nxt) + or _is_medsi_value_line(nxt) + or nxt.startswith("Исследование - (L") + ): + break + if nxt in _MEDSI_SKIP: + i += 1 + break + name_parts.append(nxt) + i += 1 + + name = " ".join(name_parts).strip() + if not name or name in _MEDSI_SKIP: + continue + + unit = "" + ref_low: float | None = None + ref_high: float | None = None + ref_note: str | None = None + value: float | None = None + + if i < len(lines) and _is_medsi_unit_line(lines[i].strip()): + unit = lines[i].strip() + i += 1 + + if i < len(lines) and _is_medsi_ref_line(lines[i].strip()): + ref_low, ref_high, ref_note = _parse_ref_range(lines[i].strip()) + i += 1 + + if i < len(lines) and _is_medsi_value_line(lines[i].strip()): + value = _parse_float(lines[i].strip()) + i += 1 + + if value is None and ref_note is None: + continue + + rows.append( + _lab_row( + name_ru=name, + value=value, + unit=unit, + ref_low=ref_low, + ref_high=ref_high, + ref_note=ref_note, + specimen_date=doc_date, + facility=facility, + ) + ) + + return rows + +def parse_medsi_lab( + text: str, + title: str, + *, + source_pdf: str = "", + fallback_iso: str | None = None, + patient_dob: str = "", +) -> dict[str, Any]: + doc_date = _medsi_iso_date(text, patient_dob=patient_dob) or fallback_iso + facility = "Медси" + if "мичуринск" in text.lower(): + facility = 'Медси "Мичуринский"' + lab_rows: list[dict[str, Any]] = [] + if doc_date: + lab_rows = _parse_medsi_lab_rows(text, doc_date, facility) + + md = _markdown_header( + doc_type="lab", + doc_date=doc_date, + title=title, + institution=facility, + ) + if lab_rows: + md += "| Показатель | Значение | Ед. | Референс |\n" + md += "|------------|----------|-----|----------|\n" + for row in lab_rows: + val_s = str(row["value"]) if row.get("value") is not None else "—" + ref = ( + f"{row['ref_low']}–{row['ref_high']}" + if row.get("ref_low") is not None and row.get("ref_high") is not None + else (row.get("ref_note") or "—") + ) + md += f"| {row['name_ru']} | {val_s} | {row.get('unit') or '—'} | {ref} |\n" + else: + md += text.strip() + + conclusion = f"Извлечено показателей: {len(lab_rows)}" if lab_rows else "—" + return { + "doc_date": doc_date, + "doc_type": "lab", + "title_ru": title, + "institution": facility, + "conclusion_ru": conclusion, + "markdown_block": md, + "lab_rows": lab_rows, + } + From adc703452434984c45436d13fd0b47a10016b27d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 14:28:04 +0000 Subject: [PATCH 11/11] Fix ruff/mypy after parser split: pyproject ignores and meta typing Co-authored-by: apodobe --- medbots/local_structure_pdfs.py | 11 +++++++++-- medbots/parsers/common.py | 2 +- medbots/parsers/emias.py | 3 ++- medbots/parsers/gemotest.py | 2 +- medbots/parsers/medsi.py | 2 +- pyproject.toml | 7 +------ 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/medbots/local_structure_pdfs.py b/medbots/local_structure_pdfs.py index 3da35a1..6d9f16b 100644 --- a/medbots/local_structure_pdfs.py +++ b/medbots/local_structure_pdfs.py @@ -9,7 +9,13 @@ from pathlib import Path from typing import Any -from medbots.corpus_io import bot_root, default_corpus_root, load_manifest, load_patient_dob, write_manifest +from medbots.corpus_io import ( + bot_root, + default_corpus_root, + load_manifest, + load_patient_dob, + write_manifest, +) from medbots.corpus_writers import ( _append_lab_rows, _safe_txt_name, @@ -275,7 +281,8 @@ def run( if not dry_run: manifest["pdfs"] = pdfs - meta = manifest.get("meta") if isinstance(manifest.get("meta"), dict) else {} + raw_meta = manifest.get("meta") + meta: dict[str, Any] = raw_meta if isinstance(raw_meta, dict) else {} meta["structured_locally_at"] = now manifest["meta"] = meta write_manifest(corpus, manifest) diff --git a/medbots/parsers/common.py b/medbots/parsers/common.py index 47f54cf..2e7a5b8 100644 --- a/medbots/parsers/common.py +++ b/medbots/parsers/common.py @@ -2,7 +2,7 @@ from __future__ import annotations import re -from typing import Any, Optional +from typing import Any from medbots.time_util import utc_now_iso diff --git a/medbots/parsers/emias.py b/medbots/parsers/emias.py index ca24bf4..e9009af 100644 --- a/medbots/parsers/emias.py +++ b/medbots/parsers/emias.py @@ -2,7 +2,7 @@ from __future__ import annotations import re -from typing import Any, Optional +from typing import Any from medbots.parsers.common import ( _first_iso_date, @@ -12,6 +12,7 @@ ) from medbots.parsers.medsi import _medsi_iso_date, _parse_medsi_lab_rows + def _extract_emias_facility(text: str) -> str: for line in text.splitlines(): line = line.strip() diff --git a/medbots/parsers/gemotest.py b/medbots/parsers/gemotest.py index c4f8265..d9216b3 100644 --- a/medbots/parsers/gemotest.py +++ b/medbots/parsers/gemotest.py @@ -3,7 +3,7 @@ import re from pathlib import Path -from typing import Any, Optional +from typing import Any from medbots.parsers.common import ( _DATE_RESEARCH, diff --git a/medbots/parsers/medsi.py b/medbots/parsers/medsi.py index 7d76e78..df5b68e 100644 --- a/medbots/parsers/medsi.py +++ b/medbots/parsers/medsi.py @@ -2,7 +2,7 @@ from __future__ import annotations import re -from typing import Any, Optional +from typing import Any from medbots.parsers.common import ( _EMIAS_DATE, diff --git a/pyproject.toml b/pyproject.toml index 8a982d5..3374878 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,14 +36,9 @@ select = ["E", "F", "I", "UP"] [tool.ruff.lint.per-file-ignores] "tests/*" = ["S101"] -"medbots/local_structure_pdfs.py" = ["E501"] +"medbots/parsers/*.py" = ["E501"] [tool.mypy] python_version = "3.11" packages = ["medbots"] ignore_missing_imports = true - -# Large legacy module; type errors addressed incrementally during parser split. -[[tool.mypy.overrides]] -module = "medbots.local_structure_pdfs" -ignore_errors = true