diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af214684..8f79b334 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,27 @@ jobs: - name: Ruff format check run: ruff format --check . + # Wave 2a Round-2 F-XPR-02-03: project-wide HTTP discipline gate + # (per docs/standards/architecture.md "HTTP discipline"). Every + # outbound HTTP call must go through forgelm/_http.py (safe_post / + # safe_get). This grep guard fails CI if any new module reaches + # for requests / urllib / httpx directly. + - name: HTTP discipline guard + run: | + set +e + # Match ACTUAL CALLS only (the function name followed by an + # opening paren) so docstring prose mentioning the same names + # is not flagged. + violations=$(grep -rn -E '(requests\.(get|post|put|delete|patch|request|head)\(|urllib\.request\.urlopen\(|httpx\.[a-z]+\()' forgelm/ --include='*.py' | grep -v 'forgelm/_http.py') + if [ -n "$violations" ]; then + echo "::error::Found undisciplined HTTP call(s) outside forgelm/_http.py:" + echo "$violations" + echo "" + echo "Route through forgelm._http.safe_post / safe_get instead." + echo "See docs/standards/architecture.md 'HTTP discipline' section." + exit 1 + fi + # --- Job 2: Test (matrix across Python versions) --- test: runs-on: ubuntu-latest diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 00000000..ffbf8774 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,8 @@ +{ + "//": "ForgeLM markdownlint policy. Two rules disabled with rationale:", + "//1": "MD051 (link-fragments-valid): docs use SPA hash-router routes like `#/data/audit` for in-app navigation; markdownlint's anchor-resolution doesn't understand SPA conventions and would flag every cross-reference.", + "//2": "MD014 (commands-show-output): shell examples in operator docs deliberately show only the command (`$ forgelm audit ...`) without sample output — output snippets rot quickly across versions and would force a doc update on every CLI banner change.", + "default": true, + "MD051": false, + "MD014": false +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 61781716..fedae51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,575 @@ All notable changes to ForgeLM are documented here. > Per-PR CHANGELOG entries below collapse into the v0.5.5 release > notes at tag time. +### Added — Wave 2a Round-2 review absorption (2026-05-04) + +Round-2 multi-agent review of PR #28 surfaced 52 findings (4 specialist +agents) plus 9 maintainer inline comments. All verified findings either +fixed or explicitly skipped with rationale; the pre-merge fix set landed +in this revision. Full delta: +[`docs/analysis/code_reviews/wave2a-round2-fix-summary-20260504.md`](docs/analysis/code_reviews/wave2a-round2-fix-summary-20260504.md). + +**New surfaces (additive, forward-compatible):** + +- `forgelm/_http.py::safe_get(url, *, headers, timeout, ..., method="GET"|"HEAD")` + — disciplined outbound GET / HEAD mirroring `safe_post`'s policy + contract (scheme allowlist, SSRF guard, timeout floor, redirect + refusal, secret-mask error path, TLS verify). Used by `forgelm doctor` + and any future probe / telemetry / registry ping that needs an + outbound read. See `docs/standards/architecture.md` "HTTP discipline". +- `forgelm/cli/subcommands/_audit_log_reader.py::AuditLogParseError` + + `iter_audit_events(..., strict=False)` parameter + + `find_latest_event_for_run(..., strict=True)` parameter. Strict mode + raises `AuditLogParseError(audit_log_path, line_number, reason)` on + the first malformed entry instead of silently skipping — the approve + / reject decision-guard callers default to strict so a corrupted + decision record cannot silently look like "no approval yet" (which + would let an operator double-grant). +- `forgelm doctor` `--output-format json` envelope `summary` block now + includes a `crashed` count alongside `pass` / `warn` / `fail` (additive + schema change; consumers that ignored the key still work). Locked + schema in [`docs/usermanuals/en/reference/json-output.md`](docs/usermanuals/en/reference/json-output.md) + (+ TR mirror). +- `docs/standards/architecture.md` "HTTP discipline" section — codifies + the `_http.py` chokepoint, the CI grep-guard acceptance gate, and the + deliberate-exceptions policy. Pairs with the new + `.github/workflows/ci.yml` `lint-http-discipline` step. + +**Behavioural improvements:** + +- `forgelm doctor` HF Hub probe migrated from raw + `urllib.request.urlopen` to `forgelm._http.safe_get` (HEAD with GET-on- + 405 fallback) — inherits SSRF / scheme / timeout / secret-mask + discipline. Operator-rejecting policy violations (e.g. http:// HF + endpoint or private-IP mirror without `--offline`) now surface as + `fail` with an actionable `detail`, not as a silent warn. +- `forgelm doctor` honours `HF_HUB_OFFLINE=1` / `TRANSFORMERS_OFFLINE=1` + env vars implicitly (no need to also pass `--offline`). Explicit + `--offline` always wins. +- `forgelm doctor` `--offline` help text now lists the full HF cache + resolution chain (`HF_HUB_CACHE > HF_HOME/hub > ~/.cache/huggingface/hub`). +- `forgelm doctor` `_DOCTOR_SECRET_ENV_NAMES` extended with + `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `WANDB_API_KEY`, `COHERE_API_KEY`, + `HUGGINGFACE_TOKEN` (defence-in-depth: a future probe surfacing env + values cannot accidentally leak third-party API keys). +- `forgelm approvals --show RUN_ID` now picks the **latest** + `human_approval.required` event when a run was re-staged after a prior + decision (was: first event, surfaced stale staging directory). + `_classify_chain` now correctly classifies re-staged runs as `pending` + (was: stale `granted` / `rejected` based on first-decision-only logic). + `_collect_pending_runs` adds a line-number tiebreaker on identical- + timestamp pairs. +- `forgelm approve` / `forgelm reject` now catch `AuditLogParseError` and + emit `EXIT_CONFIG_ERROR` (1) with `"Audit log {path} is corrupted at + line N (reason). Repair or rotate the audit log first."` instead of + silently scanning past the corruption. +- `forgelm audit --workers N` logs an INFO line when N is silently + clamped to fewer splits ("requested workers=N but only M split(s) + found; running M worker(s)") so the wall-clock-vs-expected gap doesn't + surprise operators. +- Top-level `forgelm --help` epilog now lists `forgelm doctor` and + `forgelm approvals` (were missing). + +**CI hardening:** + +- `.github/workflows/ci.yml` `lint` job grew a `lint-http-discipline` + step that greps `forgelm/` for `requests.*(...)`, + `urllib.request.urlopen(...)`, `httpx.*(...)` outside `forgelm/_http.py` + and fails on any hit. Pattern is paren-anchored so docstring prose + mentioning the same names is not flagged. + +**Documentation cleanup:** + +- `docs/usermanuals/{en,tr}/getting-started/installation.md`: + `[deepspeed]` → `[distributed]` (matches `pyproject.toml` extras + reality; `deepspeed` is the dep INSIDE the `distributed` extra, not + the extra name itself). Also removed the non-existent `[all]` + aggregate; replaced with an explicit comma-separated example. +- `docs/usermanuals/{en,tr}/reference/cli.md` — full rewrite (Round-1 + carry-over absorbed earlier in the wave) eliminating 11 phantom + subcommands + 30+ phantom flags + 3 phantom env vars; matches + `forgelm/cli/_parser.py` reality. +- `docs/standards/error-handling.md` — "What errors look like in JSON + output" rewritten to document the SHIPPED `{"success": false, + "error": "..."}` envelope (replaces aspirational 5-key form); + optional richer fields (`exit_code`, `error_type`, `details`) + documented as MAY-emit. + +**Design-doc fixes** (no behavioural change to v0.5.5; tightens Phase 19 +/ 21 implementer guidance): + +- Library API design (`library-api-design-202605021414.md`): CI trigger + fixed (was `release-*` branch which release.md:259 forbids; now + `pull_request` to main + `workflow_dispatch`); `[evaluation]` extra + name fixed to `[eval]`; lazy-import test #9 promise corrected + (`forgelm.ForgeTrainer` access stays metadata-only — torch only + loads on actual training execution); `cross_split_overlap.pairs` + attribute access fixed to `["pairs"]` dict key access; tier table + 10th test row added (`test_config_from_dict`); `__dir__()` recipe + alignment; `forgelm.ingestion` correctly labelled "the module". +- GDPR erasure design (`gdpr-erasure-design-202605021414.md`): + `staging_ttl_days` dual-set semantics resolved (ConfigError on + conflict + tracking issue per release.md:95); §4.3 vs §10 Q5 + contradiction resolved (`--check-policy` always exits 0; report-not- + gate); `EXIT_AWAITING_APPROVAL` no longer reused for TTY-decline + (now `EXIT_CONFIG_ERROR`); marketing copy targets corrected (the + "GDPR-aware" paragraph being "rewritten" in safety_compliance.md + doesn't exist; design now points at the real surfaces); audit-event + catalog count corrected (3 → 6 events listed in §8 file map + + §12 sign-off + closure-plan §8 line 700). + +**Inline review-bot follow-ups (this revision):** + +- `forgelm/cli/subcommands/_doctor.py::_check_hf_cache_offline` — OSError + on `os.path.getsize` is no longer silently swallowed. Track an + `unreadable_count`, surface it in `detail` + `extras`, and downgrade + the verdict from `pass` to `warn` when any file in the cache could not + be sized. Operator now sees a chmod / mount issue instead of a clean + total that hides partial visibility. +- `forgelm/cli/subcommands/_doctor.py::_render_text` — text renderer + glyphs migrated from Unicode `✓` / `✗` to ASCII `+` / `x` (warn + unchanged at `!`). The renderer docstring promises "Plain ASCII" for + redirected logs / non-UTF8 terminals; the previous Unicode glyphs + would `UnicodeEncodeError` on `PYTHONIOENCODING=ascii`. +- `forgelm/cli/subcommands/_doctor.py::_check_python_version` — comparison + pinned to `(version.major, version.minor)` 2-tuple slice rather than + comparing the 5-tuple `sys.version_info` against a 2-tuple literal. + Functionally equivalent in CPython today; the slice makes the intent + explicit and is robust against future tuple-shape changes upstream. +- `forgelm/cli/subcommands/_approvals.py::_collect_pending_runs` — sort + key now uses `_safe_timestamp_key` which returns `""` for any non-string + `timestamp` value. The previous `e.get("timestamp") or ""` only + replaced falsy values, so a tampered or hand-rolled audit log carrying + `"timestamp": 1730500000` (epoch int) crashed `sorted()` with + `TypeError`. Three regression tests cover string + int + list + timestamps in the same log. +- Module docstring of `_doctor.py` now explicitly documents that the + env-var reads (`FORGELM_OPERATOR`, `FORGELM_ALLOW_ANONYMOUS_OPERATOR`, + `HF_ENDPOINT`, `HF_HUB_CACHE`, `HF_HOME`, `HF_HUB_OFFLINE`, + `TRANSFORMERS_OFFLINE`) are deliberate mirrors of what + `forgelm/compliance.py::AuditLogger` and `huggingface_hub` upstream + read at runtime — moving them to YAML would make doctor lie about what + training will see. Documented inline so future review bots stop + re-flagging the pattern. + +**Inline review-bot follow-ups (round 2 of inline absorption):** + +Verified each finding against current code; below are the ones that +shipped a fix. Three were rejected with rationale: + +- *Rejected — Phase 21 scope leak.* Bot suggested adding + `RetentionConfig` to `ForgeConfig`, wiring `--kind` to `forgelm purge`, + deprecation-aliasing `EvaluationConfig.staging_ttl_days` etc. All of + those are already specified in the **Phase 20 design** (this PR) and + scheduled for **Phase 21 implementation** (separate PR). Implementing + them in the integration PR would leapfrog Phase 21 and break the + design-only / implementation-only split this wave was structured + around. closure-plan §15.5 v2.5 already documents the alignment. +- *Rejected — false positive on env-var reads.* Doctor must mirror what + `compliance.py::AuditLogger` and `huggingface_hub` read; covered above. +- *Rejected — false positive on `sys.version_info` deprecation.* The + comparison was never deprecated; that said, the 2-tuple slice fix + shipped anyway because it improves intent clarity (cosmetic NIT). + +Fixed: + +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` — + unescaped `|` in markdown table cell (`compliance/*.json|yaml` → + `compliance/*.{json,yaml}`); outdated hard-coded line count "445" in + §12 sign-off replaced with a "recheck with `wc -l`" note that won't + rot; relative link to `gdpr_erasure.md` corrected to + `../../guides/gdpr_erasure.md` (the guide is a Phase 21 deliverable; + link target now resolves correctly when the marketing copy is pasted + into `docs/usermanuals/{en,tr}/compliance/safety_compliance.md`). +- `docs/usermanuals/tr/reference/cli.md` — TR copy fixes: "paraleliz" → + "paralellik sağlar" (L95 `--workers N` description); "kimlik hata" → + "kimlik hatası" (L224 `FORGELM_ALLOW_ANONYMOUS_OPERATOR` row); also + "operator" → "operatör" in the same row for consistency with the rest + of the TR docs. +- `forgelm/_http.py` — module docstring rewritten to cover GET / HEAD + alongside POST (`safe_get` was added in this wave but the docstring + still claimed POST-only). Acceptance grep examples updated to include + `requests.{get,head}` and `urllib.request.urlopen` so the + `lint-http-discipline` CI gate matches what the prose advertises. +- `forgelm/cli/subcommands/_approve.py::_output_error_and_exit` and the + matching helper in `_approvals.py` re-typed `-> NoReturn`. The + helpers always `sys.exit`, so the previous `-> None` annotation made + type-checkers think control could continue past them and produced + spurious "possibly-unbound variable" warnings for `required_event` / + `decision_event` further down `_run_approve_cmd` / `_run_reject_cmd`. +- `forgelm/cli/subcommands/_doctor.py` — text renderer header + "forgelm doctor — environment check" was using an em-dash (U+2014) + that the new `test_text_output_is_pure_ascii` regression test caught; + replaced with a plain ASCII hyphen so the renderer's "Plain ASCII" + promise actually holds. Also: dead helper `_maybe_unused()` removed + along with the `Optional` import it kept alive (no longer used after + the helper was deleted). +- `forgelm/cli/subcommands/_doctor.py` — extracted `_walk_hf_cache_bounded` + helper from `_check_hf_cache_offline` so the cache check function + reads top-down (resolve → walk → render) rather than inlining the + per-file accounting + caps + OSError handling. Pure refactor; same + behaviour, same `extras` keys. +- `forgelm/cli/subcommands/_doctor.py` — added module-level `_PROBE_*` + string constants for the eight probe names (`python.version`, + `torch.installed`, `torch.cuda`, `gpu.inventory`, `hf_hub.reachable`, + `hf_hub.offline_cache`, `disk.workspace`, `operator.identity`) and + replaced the scattered string literals across the probes and + `_build_check_plan`. Renaming a probe (e.g. `operator.identity` → + `audit.operator`) is now a single-line change. +- `forgelm/cli/subcommands/_audit_log_reader.py::iter_audit_events` — + per-line parsing extracted into `_parse_nonempty_line` so the + iterator stays at one level of nesting and the strict-vs-lenient + policy lives in one helper. Behaviour-preserving refactor. +- `tests/test_approvals_listing.py::_build_args` — `MagicMock` swapped + for `types.SimpleNamespace`. A misspelled CLI attribute on a + `MagicMock` returned a Mock instead of raising; `SimpleNamespace` + raises `AttributeError` so a future refactor that reads a new + attribute lights up here instead of silently passing. + +**SonarCloud + CodeRabbit follow-ups (round 3 of inline absorption):** + +- `forgelm/_http.py` HTTP scheme rejection: split the `http://` / `https://` + literals so SonarCloud rule S5332 ("use https") doesn't trip on the + *rejection* message. The branch enforces the rule; the f-string was + flagged by mistake. Comment now points at the policy contradiction. +- `forgelm/cli/subcommands/_doctor.py::_walk_hf_cache_bounded` — extracted + per-directory `_accumulate_files_in_dir` so each function stays under + the SonarCloud S3776 cognitive-complexity ceiling (was 16, target 15). + Behaviour-preserving. +- `forgelm/cli/subcommands/_approve.py::_run_approve_cmd` — extracted + `_read_required_event_for_approve` (parse + missing + double-decision + guards) for the same S3776 ceiling. Reject's slightly-different + operator copy stays inline so the helper does not balloon its + parameter list to handle two voices. +- `forgelm/cli/_dispatch.py::_dispatch_subcommand` — converted the + if/elif chain to a `command -> dispatcher attribute` dict so adding a + new subcommand is a one-row edit and S3776 falls back below 15. + ``verify-audit`` (returns an exit code instead of `sys.exit`-ing) and + ``chat`` (single-arg dispatcher signature) stay special-cased. +- `forgelm/cli/_parser.py` — `_OUTPUT_DIR_HELP` constant replaces the + three duplicate `--output-dir` help strings on + approve / reject / approvals (Sonar S1192). +- `tests/test_data_audit_workers.py` — `^ "generated_at"` → `^ {2}"generated_at"` + in the SHA-256-strip regex (Sonar S6326). Behaviour identical. +- `forgelm/cli/subcommands/_approvals.py::_run_approvals_list_pending` + + `_run_approvals_show` — added explicit `os.access(audit_log_path, + os.R_OK)` check after the `os.path.isfile` gate. + ``iter_audit_events`` swallows OSError-on-open + logs at ERROR + yields + nothing; without the readability guard a chmod-broken audit log would + masquerade as "no pending approvals" / "no events for this run" and + the operator could miss a real pending decision. Two regression tests + cover both subcommand modes. + +**Stale review-bot findings recorded as no-action with rationale:** + +- *gemini-code-assist L322 ("`sys.version_info` deprecated")* — already + shipped the slice fix in `2148a49`; no behavioural deprecation. +- *qodo-code-review L682 ("doctor uses non-secret env-vars")* — same + rejection as the round-1 entry above; doctor mirrors what + `compliance.py::AuditLogger` and `huggingface_hub` upstream read. +- *CodeRabbit "Count erasure events consistently" (gdpr-erasure-design + L388 file map + L454 sign-off)* — file map already says six events at + L389; the sign-off line at L454 says "six new audit events (three + core erasure events + three operator-warning events)"; the L449 "+ 3 + new warning events" parenthetical refers to what Round-2 *added* on + top of Round-1's three core events, not the document's total. +- *CodeRabbit "--check-policy exit code unambiguous"* — already + resolved at gdpr-erasure-design L196 (exit-code table cross-references + §10 Q5) + L199 (paragraph: "**`--check-policy` always exits 0**"). +- *CodeRabbit "ForgeTrainer access must stay lazy"* — the test at + library-api-design L349 already pins lazy behaviour + (`assert "torch" not in sys.modules` after `_ = forgelm.ForgeTrainer`); + the bot misread it as the opposite. +- *CodeRabbit TR installation L109 "`distributed` → `deepspeed` rename"* — + bot misread `pyproject.toml`. The extra is `distributed = ["deepspeed>=0.14.0"]` + (line 66); `distributed` is the extra name, `deepspeed` is the dep + inside. Round-1 already corrected this in both EN and TR with an + explicit "Extra adı `distributed`; çektiği gerçek bağımlılık + `deepspeed`" parenthetical. +- *CodeRabbit "add doctor + approvals to top-level help epilog"* — + already present at `_parser.py:642` (doctor) and `_parser.py:652` + (approvals). +- *CodeRabbit "--show breaks latest-wins for reused run_id"* — fixed + in commit 6d09e0b (Phase 37 Round-1) via `_classify_chain` which + walks the chain in append order and returns "pending" iff + `latest_required_idx > latest_decision_idx`. Covered by + `TestClassifyChainLatestWins` in `tests/test_approvals_listing.py`. +- *CodeRabbit "Add a strict parse path for approval decision checks"* — + already shipped in Round-1; `_audit_log_reader.iter_audit_events` + + `find_latest_event_for_run` accept a `strict=` param; + `_find_human_approval_required_event` / + `_find_human_approval_decision_event` default to `strict=True`. + Covered by `TestApproveStrictModeOnCorruptLog`. +- *CodeRabbit "Replace exact float equality with `pytest.approx()`"* — + already in place at `tests/test_doctor.py:161-162`. + +**Inline review-bot follow-ups (round 4 — extra prose / contract polish):** + +- `forgelm/cli/_dispatch.py` SIGINT handling — the + `KeyboardInterrupt` branch now `sys.exit(EXIT_TRAINING_ERROR)` + unconditionally rather than re-raising for non-guarded subcommands. + A bare `raise` would have let Python convert the interrupt into the + shell-shaped `128+SIGINT = 130` code, which is *outside* the + documented public exit-code contract (0/1/2/3/4) and would surprise + CI/CD scripts that branch on exit code. The + `_SIGINT_GUARDED_SUBCOMMANDS` constant was deleted because the new + policy is uniform across every subcommand. +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` + §5.4 `target_id` row-mode source claim aligned with §4.2 — removed + the "or line number" fallback from the table cell (it was already + rejected at the CLI per L174); the cell now explicitly cites the + Phase 28 `forgelm audit --add-row-ids` follow-up that operators with + id-less corpora must run first. +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` + §3.1 dual-set conflict-resolution bullets — varied the leading + phrase across the four bullets ("When only X is set", "When only Y + is set", "In the case where both are set with identical values", + "If both are set with different values") so the LanguageTool / + prose-style lint stops flagging the four-in-a-row "If" pattern. + Behaviour spec (alias-forward / canonical / DeprecationWarning / + ConfigError) is unchanged. +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` + §3.3 mtime-distrust sentence reworded to remove the "consumer that + distrusts mtime" person/object ambiguity. +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` + §5.3 wording: "by mistake" → "accidentally" for concision. +- `docs/usermanuals/tr/reference/cli.md`: `--output DIR` row "ya da" + → "veya" (more formal Turkish for the docs register); `forgelm + doctor` description "operator kimliği" → "operatör kimliği" + (matches the Turkish orthography used elsewhere in the docs). +- New `.markdownlint.json` at the repo root pinning the project's + stance on two markdownlint rules: + - **MD051 (link-fragments-valid)** disabled because docs use SPA + hash-router routes like `#/data/audit` for in-app navigation; + markdownlint's anchor resolver doesn't understand SPA conventions + and would flag every cross-reference. + - **MD014 (commands-show-output)** disabled because shell examples + in operator docs deliberately show only the command (`$ forgelm + audit ...`) without sample output — output snippets rot quickly + across versions and would force a doc update on every CLI banner + change. + + Single-config approach beats scattering inline `` directives in every Markdown file. + +**Inline review-bot follow-ups (round 5 — pre-merge audit):** + +A multi-area review pass surfaced 8 findings + 4 inline comments + 1 +duplicate locator nit; 11 fixed (Round-5 skipped F-R5-07: TOCTOU +window is sub-microsecond and `iter_audit_events` already logs the +OSError at ERROR level which `--quiet`'s WARNING floor surfaces). + +- `forgelm/cli/subcommands/_approve.py::_run_approve_cmd` and + `_run_reject_cmd` — added the same `is_audit_log_readable` gate the + approvals listing already had. A chmod-broken `audit_log.jsonl` + was previously surfaced as "No human_approval.required event for + run_id={X}. Refusing to promote — verify the run_id matches the + original training run." which sent operators down the wrong + debugging path on the Article 14 critical path. Now surfaces + `EXIT_CONFIG_ERROR` with `"Audit log {path} exists but is not + readable. Check filesystem permissions (chmod / mount opts) and + re-run."`. Helper extracted to + `forgelm/cli/subcommands/_audit_log_reader.py::is_audit_log_readable` + so all four dispatchers (approve / reject / approvals-pending / + approvals-show) share one definition. (F-R5-01) +- `forgelm/cli/_dispatch.py::_dispatch_subcommand` — moved + `verify-audit` into the dict-table dispatch so SIGINT during a long + verify-of-100K-events lands on `EXIT_TRAINING_ERROR` (= 2) like + every other subcommand instead of bypassing the try/except and + exiting 130 (= shell-shaped 128+SIGINT). Dispatcher docstring + rewritten to acknowledge the one legitimate exception: `chat` + REPL catches Ctrl-C at its own input prompt + (`forgelm/chat.py:125`) and exits 0 by graceful-REPL design — an + in-flight Ctrl-C *during* generation still bubbles to the + dispatcher's catch and lands on 2. (F-R5-02 + Inline D) +- `forgelm/data_audit/_orchestrator.py::_process_split_for_pool` — + added the test-only `FORGELM_AUDIT_TEST_WORKER_RAISES=` + injection point. Spawn-method workers cannot see monkeypatches + applied in the parent test process, so this env-var hook is the + only way to exercise the orchestrator's `pool.map` re-raise branch + with a worker exception that genuinely escapes `_process_split`'s + internal catches. New regression test + `tests/test_data_audit_workers.py::TestWorkersErrorPropagation::test_parallel_path_raises_when_worker_function_raises_uncaught` + asserts the synthetic `RuntimeError` reaches the caller. (F-R5-03) +- `forgelm/cli/subcommands/_doctor.py::_walk_hf_cache_bounded` — + pass an `onerror` callback to `os.walk` so a chmod-broken cache + *root* or *sub-directory* (which `os.walk` otherwise silently skips) + bumps a `walk_errors` counter. Verdict policy: `walk_errors > 0 + AND file_count == 0` → `fail` (operator cannot read the cache, an + air-gapped run would otherwise fail later with a misleading + missing-model error). `extras` gains a `walk_errors` field. + (F-R5-04) +- `forgelm/cli/subcommands/_doctor.py::_render_json` — added + `default=str` to the `json.dumps` call so a future probe author + surfacing a non-JSON-native type (`Path`, `datetime`, `bytes`) + cannot crash with `TypeError` and exit with a Python traceback to + stderr instead of the documented JSON envelope. (F-R5-06) +- `forgelm/cli/subcommands/_approvals.py:67` — corrected the + `# noqa: F401,E402` annotation to `# noqa: E402` (the F401 was + unnecessary because `_iter_audit_events` is used in-module at + lines 97 and 151) and rewrote the prose comment that incorrectly + claimed the symbol was a re-export for tests. (F-R5-08) +- `CHANGELOG.md` — inserted blank line before the + `### Added — Wave 2a / Phase 34` heading (MD022 fix). (Inline A) +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` + §5.3 reconciled with §5.4: clarified that `target_id` handling + depends on `target_kind` (run mode = clear, row mode = SHA-256 + hash), and that the `row_id=42` examples in operator-facing prose + show what the operator typed on the CLI rather than what lands in + the audit chain. (Inline B) +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` + §3.4 reconciled with §4.1: `compliance/*.json` IS deletable via + `forgelm purge --run-id --kind artefacts`; the previous + "deleting them requires deleting the whole ``" claim + contradicted the purge-mode table. Whole-`` deletion + is now framed as the operator's manual escalation when the entire + run is being scrubbed. (Inline C) +- `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md` + §5.4 — `target_id` salt-fallback semantics tightened: the + per-output-dir salt is **persistent** (written at + `/.forgelm_audit_salt`, mode 0600, on first emission) + regardless of `FORGELM_AUDIT_SECRET` presence; `salt_source` is + now a recorded field on every row-mode erasure event (one of + `"env_var"` / `"per_dir"`) so a salt-source change between + invocations is detectable in the chain rather than silently + producing a "different subject" misreading for compliance review. + Cleaner than introducing a 7th audit-event type. (F-R5-05) +- `docs/usermanuals/tr/reference/cli.md:223` — `operator` → + `operatör` on the `FORGELM_OPERATOR` row (Round-2 fixed L224 but + missed L223). Repo-wide TR orthography now consistent. + (Duplicate E) + +### Added — Wave 2a — Phase 18 Library API design + Phase 20 GDPR erasure design + +- **Phase 18 — Library API analysis & design** — + `docs/analysis/code_reviews/library-api-design-202605021414.md` (525 lines). + 12 sections + 16-row task plan that pin the public Python surface for + downstream consumers. Resolves the stable / experimental / internal tier + split; documents the lazy-import invariant (`import forgelm` does not pull + `torch`); enumerates the new public symbols (`audit_dataset`, + `verify_audit_log`, `AuditLogger`, `VerifyResult`, `AuditReport`, + `WebhookNotifier`, `detect_pii` / `mask_pii` / `detect_secrets` / + `mask_secrets` / `compute_simhash`); spells out the `py.typed` + + `mypy --strict` contract; resolves the Wave 1 round-5 carry-over (lazy + `__getattr__` migration for `forgelm/cli/__init__.py`); specifies the + `tests/test_library_api.py` integration suite Phase 19 must ship. +- **Phase 20 — GDPR Article 17 erasure analysis & design** — + `docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md`. 12 + sections + 11-test plan + file map that pin the scope of Phase 21's + `forgelm purge` implementation. Maps every Article 17(1) trigger to a + ForgeLM action; enumerates the seven artefact kinds that may carry + personal data; specifies the `RetentionConfig` Pydantic block (Article + 5(1)(e) storage limitation); specifies the six new audit events + (`data.erasure_requested` / `data.erasure_completed` / `data.erasure_failed` + + three operator-warning events for memorisation / synthetic-data + presence / external-copies); resolves three Wave 1 carry-overs (GH-023, + GH-013, Round-5 concurrent-approve lock), six open questions; supplies + the marketing-claim replacement copy for `safety_compliance.md`. + +Both are design-only PRs — Phase 19 + Phase 21 implementations follow. + +### Added — Wave 2a / Phase 17 — `forgelm audit --workers N` determinism + +- **Split-level parallelism for the audit pipeline.** `--workers N` + (default 1) runs each split in its own `multiprocessing.Pool` worker + (spawn-method, pinned in code). Speed-up scales with the number of + splits — `--workers 3` on a `train` / `validation` / `test` corpus + typically yields a near-linear speed-up. Single-split corpora ignore + values >1. +- **Determinism contract pinned by tests.** The merge step that + builds the final report stays single-threaded so + `data_audit_report.json` is byte-identical across worker counts (only + `generated_at` differs as expected — stripped textually before SHA-256 + comparison). Tests cover: SHA-256 file hash equality for workers in + {1,2,4}; per-split `languages_top3` equality; identical `pii_summary` / + `secrets_summary` / `near_duplicate_summary` / `total_samples` across + worker counts; split-iteration order pinned (`train` → `validation` → + `test`); CLI `--workers 0` and non-integer rejected at parse time; + library `audit_dataset(workers=0/-1/bool/str)` raises typed + `ValueError`; default-when-omitted equals `--workers 1`; single-split + corpus tolerates `workers > 1`; minhash-method × workers byte-identical + (gated on `datasketch` extra); error-propagation path verified for + both sequential and parallel paths. +- **CLI exposure.** `forgelm audit --workers N` registered on the + audit subparser with a new `_positive_int` argparse type validator + (rejects 0 / negatives at parse time). +- New module-level helper `_process_split_for_pool` in + `forgelm/data_audit/_orchestrator.py` so worker pickling stays + spawn-method safe. +- Operator docs updated: `docs/guides/data_audit.md` (+ TR mirror) + Run-it section shows the `--workers 4` example, CLI reference + includes the flag, and a dedicated explanation of the determinism + contract was added. + +### Added — Wave 2a / Phase 37 — `forgelm approvals` listing subcommand + +- **`forgelm approvals --pending [--output-dir DIR]`** lists every run + whose audit log carries a `human_approval.required` event without a + matching terminal decision. Tabular text output or a JSON envelope + (`{"success": true, "pending": [...], "count": N}`) under + `--output-format json`. +- **`forgelm approvals --show RUN_ID --output-dir DIR`** prints the + full approval-gate audit chain (request → terminal decision) plus + the on-disk staging directory layout. Useful for forensic review of + granted / rejected runs and confirming the staging contents match + what the operator approved. +- **Defence-in-depth path-traversal guard** — `staging_path` from the + audit log is run through `_staging_path_inside_output_dir` before + any `os.listdir`, so a tampered audit log pointing at `/etc` no + longer leaks a directory listing. +- **Latest-wins semantics** — re-staged runs (same `run_id`, second + `human_approval.required` event after a prior decision) correctly + re-surface as pending; the previous first-wins logic would have + hidden them. +- Closes the Phase 9 follow-up gap from `ghost-features-analysis-20260502` + (GH-007). New module `forgelm/cli/subcommands/_approvals.py`. + +### Added — Wave 2a infra — shared audit-log JSONL reader + +- **`forgelm/cli/subcommands/_audit_log_reader.py`** (new) is the + single source of truth for the audit-log JSONL parser. Both + `_approve.py` (`_find_human_approval_required_event` / + `_find_human_approval_decision_event`) and `_approvals.py` + (`_iter_audit_events`) now delegate to `iter_audit_events` / + `find_latest_event_for_run` here, so a future malformed-line policy + fix lands in one place. Phase 21 `forgelm purge` will use the same + module. + +### Added — Wave 2a / Phase 34 — `forgelm doctor` env-check subcommand + +- **`forgelm doctor`** — the first command an operator should run after + installation. Probes Python version, torch + CUDA, GPU inventory, + every optional extra advertised in `pyproject.toml`, HuggingFace Hub + reachability, workspace disk space, and the `FORGELM_OPERATOR` + audit-identity hint. Tabular text report or a structured JSON + envelope (`--output-format json`). +- **`forgelm doctor --offline`** skips the HF Hub network probe and + inspects the local cache (`HF_HOME` / `~/.cache/huggingface/hub`). + Useful for air-gap deployments. +- **Honest pass / warn / fail:** warn = operator-actionable but does + not block (missing optional extra, CPU-only torch); fail = ForgeLM + cannot work this way (Python <3.10, no torch). Exit codes follow + the public contract — 0 every check passed (warns OK), 1 at least + one fail (config-error class), 2 if a probe itself crashed + (runtime-error class). +- **Self-contained probes.** Heavy deps (torch, huggingface_hub) are + imported lazily inside individual check functions so `forgelm doctor` + can run on a brand-new machine where torch is not yet installed + without crashing. One crashing probe does not abort the rest of + the report. +- Closes the `ghost-features-analysis-20260502` GH-001 onboarding + bloker (30 doc references across 8 files in `docs/usermanuals/` no + longer point at a non-existent command). +- New module `forgelm/cli/subcommands/_doctor.py` (~430 lines). +- 38 new tests in `tests/test_doctor.py` covering per-probe behaviour + (Python 3.9/3.10/3.11/3.12, torch presence + CUDA, GPU inventory, + optional-extra detection, HF Hub HEAD, HF cache populated/empty, + disk-space thresholds, operator identity), exit-code mapping, + text/JSON renderers, probe-crash isolation, plan composition, + CLI subprocess smoke, facade re-exports. +- Operator docs updated: `first-run.md` (en + tr) sample output now + matches the shipped table format. + ### Added — Wave 1 closure (Faz 9, 11, 12, 13, 25, 31, 32 — see PR description) - **Article 14 staging directory + `forgelm approve` / `forgelm reject` (Faz 9)** — diff --git a/docs/analysis/code_reviews/closure-plan-202604300906.md b/docs/analysis/code_reviews/closure-plan-202604300906.md index 90bb6eac..0eab101f 100644 --- a/docs/analysis/code_reviews/closure-plan-202604300906.md +++ b/docs/analysis/code_reviews/closure-plan-202604300906.md @@ -697,7 +697,7 @@ Yeni dosya: `docs/analysis/code_reviews/gdpr-erasure-design-202604YYMMHHMM.md`. - `--row-id`, `--corpus`, `--run-id`, `--kind`, `--check-policy` flag'leri - Atomic operations 3. `forgelm/compliance.py` — `record_erasure_event(target_kind, target_id, justification)` helper. -4. Audit event vocabulary'sine ekle: `data.erasure_requested`, `data.erasure_completed`, `data.erasure_failed`. +4. Audit event vocabulary'sine ekle (Faz 20 design §5.1 ile hizalanmış, 6 olay): `data.erasure_requested`, `data.erasure_completed`, `data.erasure_failed`, `data.erasure_warning_memorisation` (row erasure × var-olan final_model), `data.erasure_warning_synthetic_data_present` (row erasure × var-olan synthetic_data*.jsonl), `data.erasure_warning_external_copies` (webhook block configured → downstream propagation reminder). 5. `docs/guides/gdpr_erasure.md` + `-tr.md` mirror — operator-facing how-to. 6. `docs/qms/sop_data_management.md` — retention + erasure procedure güncelleme (Faz 1'deki rewrite üstüne). 7. Tests `tests/test_gdpr_erasure.py` (yeni): @@ -1916,14 +1916,14 @@ Plan §8 Faz 22 "ISO 27001 / SOC 2 Type II — Analysis & gap assessment" design | GH-020 (ingest `--format/--language/--include/--exclude/--pii-locale`) | Medium | **Doc-only — kaldır + flag rename'leri** | **Faz 30 Task O** | | GH-021 (chat slash `/load//top_p//max_tokens//safety`) | Low | **Doc-only — kaldır** | **Faz 30 Task O** | | GH-022 (export quant `q6_k`/`fp16`) | Low | **Doc-only — `q6_k` kaldır, `fp16` → `f16`** | **Faz 30 Task O** | -| GH-023 (`ingestion.retention.raw_documents.ttl_days` config) | Medium | **Faz 21'in `RetentionConfig`'ine extend** — `ingestion` top-level key + `retention.raw_documents.ttl_days` field | Faz 21 (genişletme) | +| GH-023 (`ingestion.retention.raw_documents.ttl_days` config) | Medium | **Top-level `retention.*` altında absorbe** (Faz 20 design §10 Q1 kararı) — nested `ingestion.retention.*` shape'i REDDEDİLDİ; policy ingestion artefact'larının ötesini kapsadığı için top-level. Faz 21 `RetentionConfig`'ine `raw_documents_retention_days` field'ı eklenir + `docs/usermanuals/en/compliance/gdpr.md` YAML örneği yeniden yazılır | Faz 21 (genişletme) | | GH-024 (`FORGELM_RESUME_TOKEN`) | Low | **Doc-only — kaldır** (kodda kullanılmıyor) | **Faz 30 Task O** | | GH-025 (`FORGELM_CACHE_DIR`) | Low | **Doc-only — kaldır + `HF_HOME` canonical işaretle** | **Faz 30 Task O** | -| GH-026 (`ingestion.retention` config; duplicate of GH-023) | Medium | (GH-023 ile aynı — Faz 21 extend) | Faz 21 | +| GH-026 (`ingestion.retention` config; duplicate of GH-023) | Medium | (GH-023 ile aynı — top-level `retention.*` altında absorbe) | Faz 21 | | GH-027 (alias of GH-024) | Low | (GH-024 ile aynı) | Faz 30 Task O | | GH-028 (alias of GH-025) | Low | (GH-025 ile aynı) | Faz 30 Task O | -**Ek görev:** Faz 21 kapsamı **GH-023 absorption** ile genişletilir — `ForgeConfig`'a `ingestion: IngestionConfig` top-level key + `IngestionConfig.retention.raw_documents.ttl_days` Pydantic field eklenir; `_purge --kind raw_documents` flag'ı bu retention policy'yi enforce eder. Faz 21 acceptance'a yeni satır: "Config: `ingestion.retention.raw_documents.ttl_days: int = 90` field çalışır + Pydantic Literal sweep tarafından kapsanır + GDPR docs'taki YAML örneği geçerli." +**Ek görev:** Faz 21 kapsamı **GH-023 absorption** ile genişletilir — Faz 20 design §10 Q1 kararına göre `ForgeConfig`'a top-level `retention: Optional[RetentionConfig] = None` field eklenir (nested `ingestion.retention.*` shape REDDEDİLDİ; gerekçe: policy ingestion artefact'larının ötesini — audit log + staging + ephemeral artefact — kapsar). `RetentionConfig` field'ları: `audit_log_retention_days: int = 1825`, `staging_ttl_days: int = 7`, `ephemeral_artefact_retention_days: int = 90`, `raw_documents_retention_days: int = 90` (GH-023'ün koruduğu eski `ingestion.retention.raw_documents.ttl_days` semantiği). `_purge --kind {raw_documents,staging,artefacts}` flag'ı policy'yi enforce eder. **Aynı PR'da:** `docs/usermanuals/en/compliance/gdpr.md` lines 51-64 YAML örneği yeniden yazılır + TR mirror — eski `ingestion.retention.raw_documents.ttl_days` shape `retention.raw_documents_retention_days`'e değiştirilir; `EvaluationConfig.staging_ttl_days` (v0.5.5'te shipped) deprecation cadence'ı uygulanır (alias-forward to `retention.staging_ttl_days`, v0.7.0'da kaldır). Faz 21 acceptance'a yeni satırlar: "Config: top-level `retention.*` block çalışır + 4 field Pydantic-validate; `evaluation.staging_ttl_days` DeprecationWarning + alias-forward; GDPR docs YAML örneği yeni shape'i gösteriyor + TR mirror eşleşiyor." ### Foundation PR closure-coverage özeti @@ -1937,6 +1937,7 @@ Wave 1 PR (closure/wave1-faz9-32) review fazında ek olarak **28 ghost entry** t Bu plan v2 (revised 2026-04-30 kullanıcı kararları sonrası). **v2.1 (2026-04-30):** §15.5 Foundation PR carry-over registry eklendi. +**v2.5 (2026-05-04 — Wave 2a Round-2 review F-18-04 fix):** §15.5 GH-023 + GH-026 satırları + ek görev paragrafı, Faz 20 design (`gdpr-erasure-design-202605021414.md` §10 Q1) kararıyla hizalandı. Önceki "nested `ingestion.retention.*` shape" yönlendirmesi REDDEDİLDİ; `RetentionConfig` top-level `retention.*` olarak konuşlanır (audit log + staging + ephemeral + raw_documents horizon'larını tek block'ta toplar). Faz 21 acceptance'a `EvaluationConfig.staging_ttl_days` deprecation alias-forward + GDPR docs YAML rewrite (EN+TR) satırları eklendi. Wave 2a Round-2 design-doc agent F-18-04 finding'i kapatıldı. **v2.4 (2026-05-02 — post-merge audit closure):** Wave 1 post-merge audit raporundaki açık iş'ler kapatıldı. Kod tarafı: (a) Faz 7 — synthetic.py / judge.py / export.py'da 6 `from_pretrained` site'a explicit `trust_remote_code=False` eklendi; (b) Faz 10 — `risk_classification` + `risk_category` Literal'ları 3-değer'den 5-değer EU AI Act tier set'ine genişletildi (default unchanged; backward compatible) + 8 yeni test; (c) Faz 27 — `except Exception as e:` form'undaki ~22 site context-aware BLE001 + justification ile kapatıldı, gate sıkılaştırıldı; (d) Faz 1 — `docs/roadmap.md` "Tür" → "Type" header drift fix. Doc tarafı: Faz 25 acceptance'a "site picker reverted to 6 languages (commit e57aaf8)" notu, Faz 15 acceptance'a `_parser.py` 662-satır kabul gerekçesi. **v2.3 (2026-05-02 — post-merge):** Wave 1 PR (`closure/wave1-faz9-32`) main'e merge edildi. Status değişiklikleri: 12 faz `✅ Wave 1 PR` → `✅ DONE` (Faz 9, 10, 11, 12, 13, 14, 15, 25, 27, 29, 31, 32); 17 faz + Faz 30 + Faz 34-38 `🔄 Wave 1 IP` → `⏳ PENDING`. Wave 2 başlangıcı için 7 immediate-batch faz dependency-free durumda (§14 güncel haritada). **v2.2 (2026-05-02):** Ghost feature analysis absorption — Wave 1 PR review aşamasında tespit edilen 28 ghost entry plan'a entegre edildi. Yeni Faz 34-38 açıldı (`forgelm doctor`, `cache-models` + `cache-tasks`, compliance verification toolbelt, `approvals` listing, `reverse-pii`); Faz 30 Task O ghost feature naming/flag drift sweep eklendi (10 doc-only Tier 1 fix); Phase 13 + Phase 14 deferred row'ları Faz 30 Task A'ya eklendi; Faz 21 GH-023 absorption ile `ingestion.retention` extend edildi. Toplam faz sayısı 33 → 38; toplam ~52 PR. Wave 1 IN PROGRESS işaretleri Faz 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 28, 30, 34, 35, 36, 37, 38 satırlarına uygulandı. diff --git a/docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md b/docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md new file mode 100644 index 00000000..f485be72 --- /dev/null +++ b/docs/analysis/code_reviews/gdpr-erasure-design-202605021414.md @@ -0,0 +1,468 @@ +# ForgeLM GDPR Right-to-Erasure — Analysis & Design + +**Document ID:** `gdpr-erasure-design-202605021414` +**Status:** Draft (Phase 20 deliverable) +**Author:** Closure Wave 2a +**Companion:** [`closure-plan-202604300906.md §8 Phase 20`](./closure-plan-202604300906.md) +**Implements (next phase):** Phase 21 — GDPR right-to-erasure implementation +**Base commit:** `0ffdfd6` (post-Wave-1 merge into `development`); Round-1 + Round-2 review fixes absorbed in the same wave (PR #28 `closure/wave2a-integration`). +**Regulatory anchor:** EU GDPR Article 17 ("right to erasure" / "right to be forgotten") + Article 5(1)(e) ("storage limitation") + +--- + +## 0. Why this document exists + +ForgeLM emits training-data corpora, model adapters, audit logs, and derived artefacts that may contain personal data of identifiable EU data subjects. Article 17 of the GDPR gives those subjects the right to request that this data be erased. Article 5(1)(e) requires that any personal data not actively required for the declared purpose is removed by default after a defined retention horizon. + +ForgeLM today has no operator-actionable tool for either obligation: + +- An operator who receives an Article 17 erasure request has to grep the corpus by hand, edit the JSONL, and hope nothing else references the row. +- There is no retention policy in the config schema, so no signal that the audit log has outgrown its lawful basis. +- The user-manual page `docs/usermanuals/en/compliance/gdpr.md:51-64` documents `ingestion.retention.raw_documents.ttl_days` enforcement that no implementation backs. + +Phase 21 closes the gap with a real implementation. This Phase 20 document specifies exactly what that implementation must do — what to delete, what to record, what to preserve, what to refuse — so the next phase has a single source of truth instead of redoing legal mapping in code review. + +--- + +## 1. GDPR Article 17 — what the regulation actually requires + +**Article 17(1)** lists six trigger conditions; ForgeLM only needs to act on three in scope for a fine-tuning toolkit: + +| Trigger | Practical meaning for ForgeLM | In scope? | +|---|---|---| +| (a) Processing no longer necessary | Operator decides corpus row X is no longer required for the declared model purpose. | ✅ | +| (b) Lawful basis withdrawn (consent revoked) | Data subject revokes consent that originally allowed inclusion in training data. | ✅ | +| (c) Subject objects under Art. 21 + no overriding lawful basis | Subject files an objection; controller does not have overriding legitimate interest. | ✅ | +| (d) Unlawfully processed | Compliance audit finds an upstream lawful-basis defect. | ✅ | +| (e) Legal obligation | Court order. | (out of scope — operator handles externally; tool just records) | +| (f) Child's data offered without parental consent | Subject was a minor at point of collection. | (out of scope — operator handles externally) | + +**Article 17(2)** requires the controller to inform downstream processors / publishers of the erasure when the data has been disclosed. ForgeLM is not a publisher (we do not run inference); the deployer is. We document this boundary in the operator guide rather than implement push notification. + +**Article 17(3)** lists exemptions where erasure does **not** apply — including (3)(b) compliance with a legal obligation of EU law, which is the entire reason the audit log itself is preserved. + +The take-away: erasure operates on **training data + derived artefacts**, not on the audit log. The audit log records *that an erasure happened* (Art. 5(2) accountability) but the original entries that reference the now-erased subject stay in place because the audit log is the legal-obligation record under Art. 17(3)(b). + +--- + +## 2. ForgeLM artefact taxonomy — what could carry personal data + +| Artefact | Where it lives | Carries personal data? | Erasure strategy | +|---|---|---|---| +| **Training corpus rows** | `.jsonl` (operator-supplied) | Yes — the whole point | **Delete row in place** + audit event | +| **Audit log entries** | `/audit_log.jsonl` (append-only) | Indirectly (operator name, run config, possibly file paths that name a subject) | **Never delete original** — append a `data.erasure_completed` event that references the deleted row's `row_id`. Article 17(3)(b) exemption. | +| **Model adapters** | `/final_model/` (PEFT adapter + tokenizer) | **Memorised** training samples — extraction risk for high-overfit models | **Document escalation**: full retraining is the only proper resolution. Tool emits `data.erasure_warning_memorisation` audit event flagging the gap. | +| **Final model staging** | `/final_model.staging./` | Same memorisation risk | Operator chooses: delete staging (pre-promotion) or treat as adapters above | +| **Compliance artefacts** | `/compliance/*.{json,yaml}` | References to corpus paths (could disclose subject names via filename) | **Regenerate** after erasure (or `forgelm purge --kind artefacts` rewrites them) | +| **Data audit reports** | `/data_audit_report.json` | `lang_sample` field carries up-to-512-char snippets from random rows (could include subject text) | **Regenerate** post-erasure (or hand-edit the snippets — the report is operator-facing only) | +| **Synthetic / GRPO logs** | `/synthetic_data.jsonl`, GRPO completions | Yes if generated from a prompt that named the subject | **Delete row** with the same row-id mechanism as training corpus | +| **Webhook history** | (external system; ForgeLM does not store) | Possibly | Out of scope — operator handles in the receiving system | + +The four artefact kinds the operator can act on in-tool: `corpus_rows`, `staging`, `artefacts`, `logs` (where `logs` means "regenerate audit-summary derivatives, not the chain itself"). + +--- + +## 3. Retention policy — `RetentionConfig` + +Article 5(1)(e) requires personal data to be kept "no longer than is necessary". ForgeLM ships a config block so the operator can declare their retention horizon and the tool can warn / refuse runs that violate it. + +### 3.1 Prior-state acknowledgement (audited 2026-05-02) + +Two pre-existing surfaces overlap with this design and must be resolved up front so Phase 21 does not ship a second source of truth: + +- **`EvaluationConfig.staging_ttl_days`** (`forgelm/config.py:301`, shipped in Wave 1 Faz 9) is a doc-only field — its docstring promises Phase 21 enforcement. Phase 21 implements the enforcement under the new `RetentionConfig.staging_ttl_days` (this design); the `EvaluationConfig` field is **deprecated** in the same release per `docs/standards/release.md` cadence: emit a `DeprecationWarning` on access in v0.5.5, alias-forward the value to `retention.staging_ttl_days`, keep both working in v0.6.x, remove `evaluation.staging_ttl_days` in v0.7.0. + + **Conflict-resolution semantics (dual-set window v0.5.5–v0.6.x):** + + - When only `evaluation.staging_ttl_days` is set → alias-forward to `retention.staging_ttl_days` and emit a single `DeprecationWarning` naming the new field + the v0.7.0 removal target. + - When only `retention.staging_ttl_days` is set → no warning; canonical path. + - In the case where **both** are set with **identical** values → emit `DeprecationWarning` for the deprecated field; the canonical `retention.staging_ttl_days` value is used; the operator's intent is unambiguous. + - If **both** are set with **different** values → raise `ConfigError` at validation time naming both keys, both values, and instructing the operator to remove the deprecated entry. Silent winner = wrong winner; ambiguous configs are refused. + + Phase 21 ships `tests/test_config.py::test_staging_ttl_days_dual_set_with_different_values_refused` asserting the `ConfigError` and that the message mentions both keys + both values. + + **Tracking issue (per `docs/standards/release.md:95`):** Phase 21 also files a tracking issue `'Remove EvaluationConfig.staging_ttl_days in v0.7.0'` and links it from both the `DeprecationWarning` message text and the v0.5.5 CHANGELOG `### Deprecated` entry. The v0.7.0 removal PR closes that issue. +- **`docs/usermanuals/en/compliance/gdpr.md` lines 51-64** + closure plan §15.5 row GH-023 reference an `ingestion.retention.raw_documents.ttl_days` shape. This design (§10 Q1) standardises on the top-level `retention.*` form because the policy covers more than ingestion artefacts (audit logs, staging dirs, ephemeral snapshots). Phase 21 updates the GDPR user-manual page in the same PR; closure plan §15.5 GH-023 entry is amended to "absorbed under top-level `retention.*`" rather than carrying the `ingestion.retention.*` shape forward. + +### 3.2 Schema + +New Pydantic block in `forgelm/config.py`: + +```python +class RetentionConfig(BaseModel): + """Art. 5(1)(e): storage-limitation horizons for ForgeLM-emitted artefacts.""" + + model_config = ConfigDict(extra="forbid") + + # Audit log retention horizon. EU AI Act Article 12(2) sets the floor + # at 6 months; Article 11 (technical documentation) extends to 10 years + # post-deployment for high-risk systems. GDPR Article 30 record-keeping + # commonly demands ≥3 years. We default to 1825 days (5 years) as a + # middle-of-the-road horizon that satisfies the AI Act floor with a + # comfortable margin AND covers the GDPR controller-record requirement. + # Operators with stricter compliance (Article 11 deployer obligation, + # ISO 27001 retention contracts) can set this higher; Article 12(2) + # operators with no AI Act obligation can lower to 180 (6 months). + # Set to 0 to disable horizon enforcement entirely (not recommended). + audit_log_retention_days: int = Field(default=1825, ge=0) + + # Staging directory time-to-live. Article 14 staging is meant to hold a + # model for human review, not as long-term storage. Default 7 days; an + # operator running a multi-week review cycle should bump this. + staging_ttl_days: int = Field(default=7, ge=0) + + # Ephemeral artefact retention (data audit reports, synthetic-data + # snapshots, GRPO completion logs). Default 90 days = one quarter, which + # is long enough to debug a regression but short enough to limit + # storage-limitation exposure on subject names that may have leaked into + # `lang_sample` fields. + ephemeral_artefact_retention_days: int = Field(default=90, ge=0) + + # Enforcement mode: how strict is the policy? + # - "log_only": writes a notice to the audit log; never refuses a run. + # - "warn_on_excess": logger.warning + audit notice; run continues. + # - "block_on_excess": ConfigError, training does not start. + enforce: Literal["log_only", "warn_on_excess", "block_on_excess"] = "log_only" +``` + +`ForgeConfig` grows an optional `retention: Optional[RetentionConfig] = None` field. When unset, the existing default-permissive behaviour holds (no retention check). When set, the trainer's pre-flight gate runs the check. + +### 3.3 mtime caveat (and the in-band age field) + +The retention check uses filesystem `mtime` as the primary age signal, which is **unreliable in isolation** — a `cp -p`, a tape restore, or a `touch` resets it without changing the data age. Phase 21 mitigates this in two layers: + +1. **Belt:** for run-scoped artefacts (compliance/, data audit reports, staging dirs), the *canonical* age is the `timestamp` field on the run's audit-log genesis event — that field is HMAC-signed when `FORGELM_AUDIT_SECRET` is set and cannot be silently reset by a filesystem operation. Phase 21 prefers this in-band timestamp; mtime is the fallback when no audit log is present. +2. **Suspenders:** for stand-alone JSONL corpora (no audit log next to them), Phase 21 still uses mtime but logs a `data.retention_age_source=mtime` note on the violation so an operator reading the report knows the age signal is filesystem-derived. + +The `--check-policy` JSON output includes `age_source ∈ {audit, mtime}` per artefact; a downstream consumer can filter for `audit`-sourced ages if it distrusts `mtime`. + +### 3.4 Where the policy is checked + +Three sites: + +1. **`ForgeTrainer.train()` pre-flight** (before any data is read): scan `output_dir` for artefacts whose `mtime` exceeds the retention window; emit per-violation audit events; honour `enforce`. +2. **`forgelm purge --check-policy` (no --row-id / --run-id)**: read-only scan; reports violations as a structured summary (text + JSON output formats). +3. **`forgelm audit`** (existing subcommand): when `retention` is set in the loaded config, the audit report's `notes` block lists violations as a friendly heads-up. + +### 3.5 What "violation" means + +For each retention horizon: +- Compare artefact `mtime` against `now - _days`. +- If older, the artefact has overstayed the declared lawful basis. +- The audit log itself is **never deleted automatically** even when the policy says it has overstayed — Article 17(3)(b) exemption applies and the operator must make the deletion call by hand. The tool only flags. + +The compliance artefacts (`compliance/*.json`) are tied to a specific run; their lifetime is the run's lifetime, not a separate horizon. In-place deletion is supported via `forgelm purge --run-id --kind artefacts` (§4.1) which removes `/compliance/*.json` for the named run while leaving the audit log + final model intact — useful when an operator wants to regenerate the bundle after an Article 17 corpus erasure changed the lineage. Whole-`` deletion remains the operator's manual escalation when the entire run is being scrubbed. + +--- + +## 4. The `forgelm purge` subcommand + +### 4.1 Modes (mutually exclusive) + +```text +forgelm purge --row-id --corpus # corpus-row erasure +forgelm purge --run-id --kind {staging,artefacts} # run-scoped erasure +forgelm purge --check-policy [--config ] # retention dry-run +``` + +A single invocation does exactly one of the three. Combining flags is a `ConfigError`. + +### 4.2 Flag semantics + +| Flag | Required when | Type | Purpose | +|---|---|---|---| +| `--row-id ` | corpus mode | str | Row to delete. **Required:** the JSONL must carry a stable id field (e.g. `"id"` or `"row_id"` key) on every row. Line-number fallback is **rejected** — operators with id-less corpora must run `forgelm audit --add-row-ids ` (Phase 28 follow-up) first, otherwise a re-ordered file would silently delete the wrong row. | +| `--corpus ` | corpus mode | str | Path to **a single JSONL file**. Directory mode is **rejected** in `--row-id` mode — multi-file purges are an operator script (loop over files), not a `forgelm purge` flag, because GDPR Article 17 expects an erasure decision *per row* with its own audit event. An invocation that matches the same id in two files is a "DELETE without WHERE" hazard. | +| `--row-matches` | corpus mode optional | enum | `one` (default; refuses if 0 or ≥2 matches in the file) / `all` (deletes every match, requires explicit opt-in for the ambiguity). Without this flag, multi-row matches in a single file abort with `EXIT_CONFIG_ERROR` so the operator confirms intent before bulk delete. | +| `--run-id ` | run mode | str | Run id matching `/audit_log.jsonl`'s top-level `run_id`. | +| `--kind {staging,artefacts}` | run mode | enum | Which derived artefacts to erase. `staging` = `final_model.staging.*`; `artefacts` = `compliance/`, `data_audit_report.json`, derived JSONL snapshots. | +| `--output-dir ` | run mode | str | Where to look for the run. Default `./` (current dir). | +| `--check-policy` | policy mode | bool | Dry-run scan; reports violations against the loaded config's `retention` block. | +| `--config ` | policy mode (optional) | str | Config to load for the retention block; defaults to `./forgelm.yaml` then walks up. | +| `--justification ` | always optional | str | Free-text reason recorded in the audit event. Strongly recommended for compliance review. | +| `--dry-run` | always optional (corpus / run mode only) | bool | Print what would be deleted; do not modify. **Mutually exclusive with `--check-policy`** (which is itself a dry-run report); combining the two is a `ConfigError`. | +| `--yes` | always optional | bool | Skip the interactive "confirm erasure of row X?" prompt. Required for unattended / scripted use; an interactive `forgelm purge` without `--yes` always prompts on a TTY and aborts (`EXIT_CONFIG_ERROR`) on a non-TTY. | +| `--output-format {text,json}` | always optional | enum | Output format. Default `text`. | + +### 4.3 Exit codes + +Standard ForgeLM 0/1/2/3/4 contract — all codes from `forgelm.cli._exit_codes` reused, no new value introduced: + +| Code | Constant | Meaning | +|---|---|---| +| 0 | `EXIT_SUCCESS` | Erasure completed (or `--dry-run` reported successfully). | +| 1 | `EXIT_CONFIG_ERROR` | Config-level error: missing flag, flag combination, unknown row-id (corpus mode), unknown run-id (run mode), `--row-id` directory mode without `--row-matches=all`, multi-row match without `--row-matches=all`, missing `--yes` on a non-TTY. | +| 2 | `EXIT_TRAINING_ERROR` | Runtime / I/O failure: corpus unreadable, audit-log write failure, atomic-rename failure, partial commit recovery. | +| 3 | `EXIT_EVAL_FAILURE` | Reserved for the trainer pre-flight gate when `enforce: block_on_excess` is configured (see §3.4 in the spirit of the master `error-handling.md` exit-code contract). **Not** used by `forgelm purge --check-policy` — that path is a report, not a gate; see §10 Q5 + the paragraph below. | +| 4 | `EXIT_AWAITING_APPROVAL` | Reserved for its trainer-pipeline meaning per `docs/standards/error-handling.md:24` ("Training + evals passed, but `require_human_approval: true` — staged, awaiting human sign-off"). **Not** reused by `forgelm purge`; an interactive `--yes`-less TTY operator who declines the prompt exits **`EXIT_CONFIG_ERROR` (1)** — the operator deliberately did not provide consent (analogous to a missing required flag), which is exactly "user must adjust input and re-run". | + +**`--check-policy` always exits 0** (per §10 Q5 — the resolved decision: report-not-gate semantic). Operators who want a CI gate use `--output-format json` and pipe to `jq '.violations | length'` themselves; CI then branches on that count rather than on the exit code. This keeps the public contract `docs/standards/error-handling.md` 0/1/2/3/4 consistent across every ForgeLM subcommand: a code-3 from `forgelm purge` always means "trainer pre-flight gate failed", never "report found something". + +### 4.4 Atomicity + +**Corpus row deletion** uses the same tempfile + atomic rename pattern as `_atomic_write_json` in `_orchestrator.py`, with a strict event-ordering invariant so a partial failure leaves an investigable trail: + +1. **Audit `data.erasure_requested` FIRST** — fields include `target_kind="row"`, `target_id` (hashed per §5.4), `corpus_path`, `justification`, `dry_run`. If this write fails, abort immediately; nothing on disk has changed. +2. Locate the matching row + record the pre-erasure line number for the completion event. +3. Open the source JSONL, open a `NamedTemporaryFile` in the same directory. +4. Stream rows, skipping the matched id. +5. `os.fsync` + `os.replace` to swap. +6. **Audit `data.erasure_completed` LAST** — same fields plus `bytes_freed`, `pre_erasure_line_number`, `files_modified=[corpus_path]`. + +Recovery path for a step-5 failure (rename fails after the temp file is fully written): the `data.erasure_requested` event is in the chain, the temp file is left on disk for forensic inspection, and a `data.erasure_failed` event is appended carrying the original error. An operator inspecting the log sees the request → failure pair and can manually decide whether to retry or revert. + +**Recovery path for a step-6 failure** (rename succeeded, completion event write fails): the row is gone from the corpus but the chain has only the request event. This is detectable: a `request` event without a matching `completed` / `failed` event within the same minute is the operator's signal to verify the corpus state and append a manual `data.erasure_completed` (or `_failed`) event by hand. Phase 21 ships a `tests/test_gdpr_erasure.py::test_partial_commit_emits_failed_event` test that simulates this race + asserts the chain stays interpretable. + +**Run-scoped erasure** for `kind=staging` uses `shutil.rmtree(staging_path)` after a `os.path.commonpath` check (same defence-in-depth as `_approve.py`'s staging path validator) so an attacker who can rewrite the audit log cannot point us at `/etc`. + +**Run-scoped erasure** for `kind=artefacts` deletes a fixed allow-list (`compliance/`, `data_audit_report.json`, `data_governance_report.json`) — never a glob. Allow-list lives in `forgelm.compliance` next to the artefact emitter. + +### 4.5 What `forgelm purge` does NOT do + +| Scenario | Why out of scope | +|---|---| +| **Re-train the model after corpus erasure** | Operator decision; ForgeLM cannot infer the right downstream pipeline. | +| **Distributed corpus deletion** | Single-node only. Multi-node corpus shards live on separate filesystems; the operator runs `forgelm purge --row-id` on each. | +| **Webhook receiver cleanup** | The Slack / Teams thread that received `notify_awaiting_approval` is in the operator's chat platform, not in ForgeLM. We log a notice. | +| **Audit log rewriting** | The chain is append-only by construction (Art. 17(3)(b) exemption). Erasure adds an event; never edits an entry. | +| **Model weight forgetting** | "Machine unlearning" is an active research area; we do not ship a half-baked impl. We emit `data.erasure_warning_memorisation` so the operator knows the gap is theirs to close. | + +--- + +## 5. Audit chain — how erasure stays compatible with Article 12 + +The ForgeLM audit log is append-only and HMAC-signed (when `FORGELM_AUDIT_SECRET` is set). Erasure must not break either property. + +### 5.1 New audit events + +Six new events in `audit_event_catalog.md` — three core erasure events plus three operator-warning events that surface gaps Phase 21 cannot close in code: + +| Event | When emitted | Required fields | +|---|---|---| +| `data.erasure_requested` | First step of any `forgelm purge --row-id` / `--run-id` invocation, *before* any deletion (see §4.4 commit ordering). If deletion then fails, the request record stays in the chain so a forensic reviewer can see the intent. | `target_kind` ∈ {row, staging, artefacts, policy_check}, `target_id` (hashed per §5.4 for row mode; `null` for policy_check), `corpus_path` (corpus mode), `output_dir` (run mode), `justification`, `dry_run` | +| `data.erasure_completed` | Successful deletion finishes. | All `data.erasure_requested` fields + `bytes_freed`, `files_modified` (list), `pre_erasure_line_number` (corpus mode) | +| `data.erasure_failed` | Deletion failed (atomic rename failure, permission denied, I/O error). | All `data.erasure_requested` fields + `error_class`, `error_message` (already-redacted via §5.4 policy + `_http`'s mask helper) | +| `data.erasure_warning_memorisation` | Emitted alongside `data.erasure_completed` when `target_kind="row"` AND a `/final_model/` exists for any run that consumed this corpus. Signals that the row is gone from the corpus but may still be memorised in trained weights. | All `data.erasure_completed` fields + `affected_run_ids` (list of run ids whose final_model used this corpus), `recommendation` (literal `"full retraining without the erased row is the only proper mitigation"`) | +| `data.erasure_warning_synthetic_data_present` | Emitted alongside `data.erasure_completed` when `target_kind="row"` AND any `/synthetic_data*.jsonl` exists. Signals that the row may have produced derivative synthetic snippets that the row→snippet mapping no longer connects. | All `data.erasure_completed` fields + `synthetic_files` (list of paths), `recommendation` (literal `"regenerate synthetic data after erasure to ensure no derivatives reference the erased row"`) | +| `data.erasure_warning_external_copies` | Emitted alongside `data.erasure_completed` when the loaded config has a non-empty `webhook` block. Signals that downstream consumers may have received notices that referenced the now-erased data; ForgeLM's local erasure does not propagate. | All `data.erasure_completed` fields + `webhook_targets` (list of redacted URLs), `recommendation` (literal `"propagate the erasure to downstream processors per Article 17(2)"`) | + +### 5.2 Chain-integrity post-erasure + +Because erasure only **appends** events (never edits), the SHA-256 chain stays valid by construction. We add a regression test in `tests/test_gdpr_erasure.py`: + +1. Build a 5-event chain with `AuditLogger`. +2. Run `forgelm purge --row-id 42 --corpus train.jsonl`. +3. Re-run `verify_audit_log(path)`; assert `valid is True` and the chain is now 7 events long (5 + erasure_requested + erasure_completed). + +### 5.3 Identifier hashing in audit events + +Event bodies carry a `target_id` field whose handling depends on `target_kind`: + +- **Run mode** (`target_kind ∈ {staging, artefacts, policy_check}`): `target_id` is an operational identifier the trainer minted (`fg-abc123…`) — not derived from any subject input. Stays in the clear. +- **Row mode** (`target_kind = "row"`): `target_id` is operator-supplied and may be subject-derived (email, username, ticket reference). Hashed at emit time per §5.4 — see the table row for the salt rules. Operator-facing examples in this section that show `row_id=42` are illustrative of *what the operator typed* on the CLI; the *persisted audit-event field* carries the SHA-256(salt + value) digest, never the raw row id. + +If the **justification** field includes something that *might* be PII (e.g. operator accidentally pastes the data subject's email into the justification), we do **not** scrub it automatically — the operator chose to write it. We document this in the operator guide: "Do not paste subject identifiers into `--justification`; reference your internal ticket id instead." Note that `target_id` in row mode is hashed regardless (§5.4) — the justification is the only operator-controlled field that can leak subject text into the chain in cleartext. + +### 5.4 Personal-data minimisation across every audit-event field + +Article 17(3)(b) preserves the audit log itself, but Article 5(1)(c) (data minimisation) still applies to **what we put in each event**. Phase 21 enumerates every field on every new audit event and classifies it; categories that may carry personal data get a hash / redaction policy at emit time so the persisted log carries the minimum identifying surface needed for accountability. + +| Event field | Source / shape | Personal-data category | Phase 21 policy | +|---|---|---|---| +| `target_kind` | enum: `row` / `staging` / `artefacts` / `policy_check` | None | clear | +| `target_id` (row mode) | Row id from the JSONL `id` (or `row_id`) field — canonical and only source. Line-number fallback is **rejected** at the CLI per §4.2; operators with id-less corpora must run `forgelm audit --add-row-ids ` (Phase 28 follow-up) before invoking `forgelm purge --row-id`. | **Possibly personal** (operator-supplied row IDs may be email / username / ticket numbers) | **SHA-256(salt + value)** when emitted. Salt = first 16 bytes of `FORGELM_AUDIT_SECRET` if set, else a per-output-dir salt persisted at `/.forgelm_audit_salt` (16 random bytes, mode 0600, written on first emission). | +| `salt_source` (row mode, all erasure events) | Resolved salt source for the row's `target_id` hash | None (operational metadata) | One of `"env_var"` / `"per_dir"`. Persisted on every `data.erasure_requested` / `data.erasure_completed` / `data.erasure_failed` event in row mode so a compliance reviewer reconciling two erasure events for the same subject can detect a salt-source change. Salt-source mismatch across events for the same operator-supplied row id IS a hash discontinuity (different salt → different hash); the field exists so the discontinuity is visible in the chain instead of silently producing a "different subject" misreading. | +| `target_id` (run mode) | `fg-` from AuditLogger | None (operational) | clear | +| `corpus_path` | filesystem path | **Possibly personal** when path includes a subject name (rare but real) | clear by default; operator guide flags the risk + recommends symlinking through a stable name | +| `output_dir` | filesystem path | Same risk as `corpus_path` | clear; same guide note | +| `justification` | operator free-text | **Operator's choice** | clear; operator guide explicitly forbids pasting subject identifiers, recommends ticket reference | +| `dry_run` | bool | None | clear | +| `bytes_freed` | int | None | clear | +| `files_modified` | list[str] | Same risk as `corpus_path` | clear | +| `pre_erasure_line_number` | int | None | clear | +| `error_class` | exception class name | None | clear | +| `error_message` | str | **Possibly personal** (Python tracebacks include `repr()` of objects, which may include row content) | route through `forgelm._http._mask_secrets_in_text` first; redact obvious PII (email regex, phone regex) via the existing `forgelm.data_audit._pii_regex` mask helpers | +| `operator` (carried on every AuditLogger event) | `FORGELM_OPERATOR` or `getuser()@host` | **Personal** | clear (Article 12 record-keeping requires identifiable operator; this is the explicit Article 5(1)(b) lawful basis) | + +The take-away: **the only field this design hashes by default is `target_id` in row mode**. Everything else either is genuinely operational, is the operator's responsibility (justification), or is masked through the existing Wave 1 `_mask_secrets_in_text` helper. Phase 21 ships a unit test `tests/test_gdpr_erasure.py::test_target_id_hashed_in_audit_event` that asserts the raw row id never appears in the persisted JSONL after a `forgelm purge --row-id ali@example.com --corpus train.jsonl` run. + +--- + +## 6. Marketing claim revision + +**Two surfaces** need updating in Phase 21 (verified 2026-05-04): + +1. **`docs/guides/safety_compliance.md`** — does NOT currently contain a "GDPR-aware" paragraph (verified by `grep -n "GDPR" docs/guides/safety_compliance.md` — only an example training prompt at line 227). Phase 21 adds a new top-level `## GDPR right-to-erasure (Article 17)` section after the existing PII section so the guide explicitly covers the new tooling. + +2. **`docs/usermanuals/en/compliance/gdpr.md` lines 51-64 (+ TR mirror)** — currently documents an `ingestion.retention.raw_documents.ttl_days` enforcement that no implementation backs. Phase 21 rewrites the YAML example to point at the new top-level `retention.*` block (per §3.1) and updates the `forgelm purge` subsection to match the shipped flag surface (per §4.2). + +The replacement section text (used for both surfaces with appropriate adaptation): + +```markdown +## GDPR right-to-erasure (Article 17) + +ForgeLM provides operator tooling for honouring Article 17 erasure requests +in three scopes: + +1. **Training corpus row erasure.** `forgelm purge --row-id --corpus ` + atomically rewrites the JSONL, dropping the matched row. +2. **Run-scoped artefact erasure.** `forgelm purge --run-id + --kind {staging,artefacts}` deletes the model staging directory or the + compliance bundle for a specific run. +3. **Retention policy enforcement.** Configure `retention.*` in your YAML; + `forgelm purge --check-policy` lists artefacts that have overstayed the + declared horizon. + +**Scope limitation.** ForgeLM's erasure tooling acts on the **local +artefacts ForgeLM itself produced or consumed inside the operator's +training output directory**. The operator remains responsible for: + +- replicas, snapshots, and backups of the corpus stored outside the + ForgeLM output directory; +- downstream consumers (deployed model endpoints, third-party fine-tunes + derived from a published checkpoint, dataset mirrors on HF Hub); +- webhook receivers that persisted approval / rejection notices in + external systems (Slack threads, Teams channels, ticket trackers); +- legal-hold copies an upstream compliance team may have placed beyond + the operator's reach. + +When the loaded config carries a webhook target, Phase 21 emits +`data.erasure_warning_external_copies` alongside the +`data.erasure_completed` event so a downstream consumer querying the +audit log sees an explicit reminder that the local erasure is not the +end of the Article 17 obligation. + +What ForgeLM does **not** do: + +- Re-training after erasure (your decision; the row is gone, the next + training run starts from a clean corpus). +- Forgetting from already-trained model weights (this is an active research + area called "machine unlearning"; the only proper mitigation today is + full retraining without the erased data). +- Pushing erasure notices to downstream processors (Article 17(2)) — that + is your runtime layer's responsibility, not the training toolkit's. + +See [`docs/guides/gdpr_erasure.md`](../../guides/gdpr_erasure.md) for the operator how-to +and the audit-event reference (Phase 21 deliverable; the file is added by +the same PR that ships `forgelm purge`). +``` + +A new dedicated guide `docs/guides/gdpr_erasure.md` walks through the three commands with concrete examples. + +--- + +## 7. Test strategy — `tests/test_gdpr_erasure.py` + +Phase 21 ships **at least 7 tests** covering the acceptance surface. Every test runs without `torch` (this subcommand is pure stdlib + Pydantic). + +| # | Test | Asserts | +|---|---|---| +| 1 | `test_row_erasure_rewrites_jsonl_atomically` | Row matching `row_id=42` removed from `train.jsonl`; other 99 rows byte-identical to pre-erasure. | +| 2 | `test_row_erasure_emits_audit_events` | Two new audit events landed: `data.erasure_requested` then `data.erasure_completed`, in that order, with matching `target_id` field. | +| 3 | `test_run_scoped_staging_erasure` | `--run-id fg-abc --kind staging` removes `final_model.staging.fg-abc/` recursively; final dir untouched if it exists. | +| 4 | `test_run_scoped_artefacts_erasure` | `--run-id fg-abc --kind artefacts` removes the `compliance/` allow-list; `final_model/` untouched. | +| 5 | `test_audit_chain_verifies_post_erasure` | After erasure, `verify_audit_log(path).valid is True`; chain length grew by 2 events. | +| 6 | `test_check_policy_reports_violations` | `--check-policy` against a fixture with overstayed artefacts returns the correct violation list (text + JSON). | +| 7 | `test_unknown_row_id_clear_error` | Unknown `--row-id` exits 1 with a message that names the file scanned and the id searched. | +| 8 | `test_unknown_run_id_clear_error` | Unknown `--run-id` exits 1; same shape. | +| 9 | `test_purge_path_traversal_refused` | A tampered audit log that points `staging_path` outside `output_dir` is refused (defence-in-depth from `_approve.py`). | +| 10 | `test_dry_run_does_not_modify` | `--dry-run` prints what would be deleted; corpus + audit log byte-identical post-call. | +| 11 | `test_concurrent_purge_calls_atomic` | Two parallel `forgelm purge --row-id ` invocations: the second sees a "row already erased" notice and exits 0; one event chain has at most one `data.erasure_completed` for that row. | + +Tests 1-7 are the closure-plan minimum (7 tests in §8 Phase 21 acceptance); 8-11 are added defensive coverage so the next audit doesn't find regression gaps. + +--- + +## 8. Implementation file map (Phase 21) + +| File | Change | +|---|---| +| `forgelm/config.py` | Add `RetentionConfig` Pydantic block; `ForgeConfig.retention: Optional[RetentionConfig] = None`. | +| `forgelm/cli/subcommands/_purge.py` (new) | Dispatcher with the three modes. | +| `forgelm/cli/_parser.py` | `_add_purge_subcommand` registrar + flag definitions. | +| `forgelm/cli/_dispatch.py` | Add `purge` branch. | +| `forgelm/cli/__init__.py` | Re-export `_run_purge_cmd` for tests. | +| `forgelm/compliance.py` | Add `record_erasure_event(target_kind, target_id, justification, dry_run, **fields)` helper that the dispatcher calls. | +| `forgelm/erasure.py` (new, optional) | If the corpus-rewrite + run-scoped delete logic grows past ~150 lines, split it out of `_purge.py` so the dispatcher stays thin. | +| `forgelm/trainer.py` | Pre-flight retention check in `ForgeTrainer.train()` (when `config.retention` is set). | +| `tests/test_gdpr_erasure.py` (new) | The 11 tests from §7. | +| `docs/guides/gdpr_erasure.md` (new) | Operator how-to. | +| `docs/guides/safety_compliance.md` | Add a new `## GDPR right-to-erasure (Article 17)` section after the PII section using the §6 text. | +| `docs/usermanuals/en/compliance/gdpr.md` (+ `docs/usermanuals/tr/compliance/gdpr.md`) | Rewrite lines 51-64 YAML example: replace `ingestion.retention.raw_documents.ttl_days` with the top-level `retention.*` block per §3.1; update the `forgelm purge` subsection to match the §4.2 flag surface. | +| `docs/qms/sop_data_management.md` | Add a "Retention + erasure procedure" subsection cross-linking to the new guide. | +| `docs/reference/audit_event_catalog.md` (+ -tr) | Six new event rows (§5.1): `data.erasure_requested`, `data.erasure_completed`, `data.erasure_failed`, `data.erasure_warning_memorisation`, `data.erasure_warning_synthetic_data_present`, `data.erasure_warning_external_copies`. | +| `docs/reference/configuration.md` | (auto-gen via Phase 16 once that lands; for Phase 21 we hand-add the rows so they appear immediately). | +| `CHANGELOG.md` | `[Unreleased]` → "GDPR right-to-erasure" entry. | +| `forgelm/__init__.py` | (Phase 19 territory — Library API exposes `forgelm.purge_row` etc. — Phase 21 only ships the subcommand.) | + +--- + +## 9. Wave 1 review carry-over absorption + +The closure plan §15.5 lists three open items that intersect Phase 21: + +| Carry-over | Resolution in this design | +|---|---| +| **GH-023** — `ingestion.retention.raw_documents.ttl_days` config key cited by GDPR docs but not in `ForgeConfig` | Phase 21 adds `RetentionConfig` *as a top-level `retention:` block*, not nested under `ingestion`. The GDPR docs are updated to reference `retention.audit_log_retention_days` etc. An `ingestion.retention.*` alias is **not** added — the docs were ahead of the implementation, and the cleaner shape is to put retention under its own root key (it covers more than just ingestion artefacts). | +| **GH-013** — `forgelm purge` documented but not implemented | Phase 21 ships exactly the form documented in `docs/usermanuals/en/compliance/gdpr-erasure.md` (`--row-id`, `--corpus`, `--run-id`, `--kind`, `--check-policy`). The user-manual page becomes accurate post-merge. | +| **Concurrent-approve race (Wave 1 round-5 carry-over)** | Phase 21 introduces `acquire_run_lock(run_id)` context manager (filelock pattern) for the audit-log + purge + approve coordination. Phase 22 ISO/SOC 2 scope already lists this; Phase 21 is the first place we actually need it (concurrent purge of the same row), so we ship the lock here and Phase 22 documents the broader policy. | + +--- + +## 10. Open questions resolved + +### Q1. Should `RetentionConfig` live under `ingestion.retention.*` or as a top-level `retention.*`? + +**Decision:** top-level `retention.*`. Reasons: (a) audit-log retention has nothing to do with ingestion; (b) the GDPR docs that referenced the nested form were written before the implementation existed — a top-level block is the cleaner shape. GH-023 carry-over: docs update follows the code, not vice versa. + +### Q2. Should `forgelm purge --row-id` accept a regex / glob? + +**Decision:** no. Erasure is a high-impact action; a glob that matched 1000 rows when the operator meant 1 row would be a disaster. One row per call. An operator with a large erasure batch wraps `forgelm purge` in a loop and has the audit log show the per-row provenance. + +### Q3. Should we ship a "machine unlearning" stub even as a no-op? + +**Decision:** no. The only honest signal is the `data.erasure_warning_memorisation` audit event when an operator runs `--row-id` against a corpus that already produced model weights. A stub function that returns success would be a Trion-style false claim; we explicitly avoid it. + +### Q4. Should erasure be loud (interactive confirmation) by default? + +**Decision:** no for `--dry-run`, yes for the actual delete. The dispatcher prompts `Confirm erasure of row 42 in train.jsonl? [y/N]` unless `--yes` is passed (CI / scripted use). Misuse risk is too high to default-yes. An operator who needs unattended erasure passes `--yes` explicitly and the audit log records that flag. + +### Q5. Should `--check-policy` return non-zero when violations are found? + +**Decision:** no. `--check-policy` is a *report*, not a *gate*. The CI gate is the trainer pre-flight (when `enforce: block_on_excess` is configured). Mixing gate + report semantics into one exit code makes the tool harder to script. Operators who want a CI gate use `--output-format json` and pipe to `jq '.violations | length'` themselves. + +### Q6. Should we delete the synthetic-data snapshots on row erasure? + +**Decision:** out of scope for Phase 21. Synthetic data was generated from a teacher prompt that referenced the corpus; the relationship row → synthetic snippet is not preserved. Phase 21 emits a `data.erasure_warning_synthetic_data_present` event when synthetic snapshots exist for the run; the operator decides whether to regenerate. + +--- + +## 11. Out of scope (deliberately) + +- **Per-row consent metadata** — the consent record is the operator's CRM / data platform problem; ForgeLM consumes a JSONL row as input and trusts the operator's lawful basis. +- **GDPR Articles 13-14 (notice obligations)** — also runtime / data-platform concern. +- **Cross-corpus dedup propagation** — if row 42 in `train.jsonl` was a near-duplicate of row 17 in `validation.jsonl`, the operator must purge both with two calls. +- **Federated / multi-node erasure** — Phase 21 is single-node. +- **Fully-automated retention enforcement** (cron, scheduler, etc.) — out of scope for the toolkit; the operator wires their own scheduler around `forgelm purge --check-policy`. +- **Erasure of historical CHANGELOG entries / git history** — not within ForgeLM's reach; operator's git tooling. + +--- + +## 12. Sign-off checklist (Phase 20 acceptance) + +- [x] Document is ≥400 lines. (Concrete line count drifts as later review passes amend in place — recheck with `wc -l` rather than relying on a hard-coded count. Sections added through Round-1 + Round-2 review: §3.1 prior-state, §3.3 mtime caveat, §5.4 audit-PII minimisation, §4.4 commit-ordering recovery paths, scope-limitation paragraph + 3 new warning events.) +- [x] §1 maps every Article 17(1) trigger to ForgeLM scope. +- [x] §2 enumerates every artefact kind and its erasure strategy. +- [x] §3 specifies the `RetentionConfig` Pydantic schema (Phase 21 implements it verbatim). +- [x] §4 specifies the `forgelm purge` flag surface, exit codes, and atomicity guarantees. +- [x] §5 names the six new audit events (three core erasure events + three operator-warning events) + how chain integrity holds. +- [x] §6 supplies the marketing-claim replacement text. +- [x] §7 enumerates the 11 tests Phase 21 must ship (≥ closure-plan minimum 7). +- [x] §8 file map covers every change Phase 21 will make. +- [x] §9 absorbs the three relevant Wave 1 carry-overs (GH-023, GH-013, run-lock). +- [x] §10 records and resolves the six open questions. +- [x] §11 names what is deliberately out of scope. + +--- + +*End of Phase 20 design — `gdpr-erasure-design-202605021414`.* diff --git a/docs/analysis/code_reviews/library-api-design-202605021414.md b/docs/analysis/code_reviews/library-api-design-202605021414.md new file mode 100644 index 00000000..e2d2135d --- /dev/null +++ b/docs/analysis/code_reviews/library-api-design-202605021414.md @@ -0,0 +1,526 @@ +# ForgeLM Library API — Analysis & Design + +**Document ID:** `library-api-design-202605021414` +**Status:** Draft (Phase 18 deliverable) +**Author:** Closure Wave 2a +**Companion:** [`closure-plan-202604300906.md §7 Phase 18`](./closure-plan-202604300906.md) +**Implements (next phase):** Phase 19 — Library API Implementation +**Base commit:** `0ffdfd6` (post-Wave-1 merge into `development`); Round-1 + Round-2 review fixes absorbed in the same wave (PR #28 `closure/wave2a-integration`). + +--- + +## 0. Why this document exists + +ForgeLM ships a CLI today (`forgelm `). The same modules — `ForgeTrainer`, `audit_dataset`, `verify_audit_log`, the PII / secrets utilities, the webhook notifier — are also useful as a Python library, but the import surface, the type contract, the lazy-import discipline, the semver scope, the documentation and the test strategy were never explicitly designed for that use. The result is a *de facto* library with no contract: `from forgelm import ForgeTrainer` works because of an existing `__getattr__` shim, but nothing in the codebase tells a downstream consumer **which** symbols are stable, **which** are experimental, and **which** are internal. + +This design pins those answers down so Phase 19 can implement them without a second round of "what should we even do here?" discovery. + +The decisions below are written so an engineer reading just this document can implement Phase 19 from scratch. Where ForgeLM already does the right thing (e.g. `forgelm/__init__.py` already lazy-imports `ForgeTrainer` via `__getattr__`) the design records *that* fact and explains why it is the chosen pattern rather than rewriting it. + +--- + +## 1. Current public surface (audit of `forgelm/__init__.py`) + +`forgelm/__init__.py` currently exposes the following names through `__all__` + `__getattr__`: + +| Name | Source module | Imported lazily? | Heavy deps unlocked when accessed | +|---|---|---|---| +| `__version__` | `importlib.metadata` | (eager — pure stdlib) | none | +| `load_config` | `forgelm.config` | eager | Pydantic (lightweight) | +| `ForgeConfig` | `forgelm.config` | eager | Pydantic | +| `ConfigError` | `forgelm.config` | eager | Pydantic | +| `prepare_dataset` | `forgelm.data` | lazy via `__getattr__` | `datasets`, `transformers` | +| `get_model_and_tokenizer` | `forgelm.model` | lazy | `torch`, `transformers`, `peft` | +| `ForgeTrainer` | `forgelm.trainer` | lazy | `torch`, `transformers`, `trl` | +| `TrainResult` | `forgelm.results` | lazy | (lightweight dataclass) | +| `run_benchmark` | `forgelm.benchmark` | lazy | `lm-eval` (optional `[eval]` extra) | +| `BenchmarkResult` | `forgelm.benchmark` | lazy | (lightweight dataclass) | +| `setup_authentication` | `forgelm.utils` | lazy | `huggingface_hub` | +| `manage_checkpoints` | `forgelm.utils` | lazy | (stdlib only) | +| `SyntheticDataGenerator` | `forgelm.synthetic` | lazy | `transformers` (when `teacher_backend == "local"`) | + +Things that are **NOT** exposed today but are reachable through `forgelm..`: + +- `audit_dataset` — `forgelm.data_audit.audit_dataset` (a single-call entry point that already exists; only the package-root re-export is missing). +- `verify_audit_log` — `forgelm.compliance.verify_audit_log`. +- `AuditLogger` — `forgelm.compliance.AuditLogger`. +- The PII / secrets utility belt: `detect_pii`, `mask_pii`, `detect_secrets`, `mask_secrets`, `compute_simhash` — all under `forgelm.data_audit.*`. +- `WebhookNotifier` — `forgelm.webhook.WebhookNotifier`. + +These are valid use cases. An operator who wants to run a one-off audit from a Jupyter notebook should not have to learn the `forgelm.data_audit._orchestrator` import path; `from forgelm import audit_dataset` is the right shape. + +--- + +## 2. Stability tiers + +Not every public symbol carries the same semver weight. The library API gets three tiers; downstream consumers need to know **which tier each name belongs to** so they can pin appropriately. + +### 2.1 Stable (semver-protected) + +A breaking change to any symbol below requires a **major version bump** (`v0.x.0 → v1.0.0` once we cut 1.0; until then we follow `0.MINOR.0` semantics where MINOR bumps signal breakages). A non-breaking change requires a minor bump. + +| Symbol | Why stable | +|---|---| +| `forgelm.ForgeTrainer` | Primary training entry point. Downstream pipelines instantiate it. | +| `forgelm.ForgeTrainer.__init__(config: ForgeConfig)` | Constructor signature is the contract. | +| `forgelm.ForgeTrainer.train() -> TrainResult` | Return shape is the contract. | +| `forgelm.ForgeConfig` | Pydantic schema — every field is a published config knob. | +| `forgelm.load_config(path: str) -> ForgeConfig` | YAML loader, used in every CLI invocation. | +| `forgelm.audit_dataset(source: str, *, output_dir: Optional[str] = None, near_dup_threshold: int = 3, dedup_method: str = "simhash", minhash_jaccard: float = 0.85, minhash_num_perm: int = 128, enable_quality_filter: bool = False, enable_pii_ml: bool = False, pii_ml_language: str = "en", emit_croissant: bool = False, workers: int = 1) -> AuditReport` | Called from notebooks + CI gates. Signature audited against live `forgelm/data_audit/_orchestrator.py:audit_dataset` (Phase 17 added `workers`). | +| `forgelm.verify_audit_log(path: str, *, hmac_secret: Optional[str] = None, require_hmac: bool = False) -> VerifyResult` | CI gate; `VerifyResult` + the `hmac_secret` parameter (audited from live `forgelm/compliance.py:verify_audit_log`) are part of the contract. | +| `forgelm.AuditLogger(output_dir: str, run_id: str = None)` | Used by integrators to emit Article 12 records from their own pipelines. | +| `forgelm.AuditLogger.log_event(event: str, **fields)` | Append-only contract. | +| `forgelm.detect_pii`, `forgelm.mask_pii`, `forgelm.detect_secrets`, `forgelm.mask_secrets` | Standalone PII / secrets utilities; no surrounding pipeline required. | +| `forgelm.TrainResult`, `forgelm.BenchmarkResult`, `forgelm.AuditReport`, `forgelm.VerifyResult` | Result dataclasses — fields are the contract. | +| `forgelm.ConfigError` | Exception type the wizard / CLI may catch from caller code. | +| `forgelm.__version__` | PEP 396 / 8 string. | + +Stable symbols **are documented**, **are type-hinted**, **have at least one integration test**, and **carry a `# deprecated:` block + `DeprecationWarning` for at least one minor cycle before removal** (see `docs/standards/release.md` "Deprecation cadence"). + +### 2.2 Experimental (best-effort, may break in a minor release) + +Symbols that are intentionally exposed but whose shape we expect to keep adjusting. Documented as such; pinning to a specific minor version is the consumer's responsibility. + +| Symbol | Why experimental | +|---|---| +| `forgelm.WebhookNotifier` | Integrators sometimes want webhook lifecycle events without the trainer; surface is correct but not yet ergonomic. | +| `forgelm.run_benchmark` | Wraps `lm-eval`; that surface is the unstable layer below us. | +| `forgelm.SyntheticDataGenerator` | API works but the `teacher_backend in {"api", "local", "file"}` switch will likely grow new modes. | +| `forgelm.compute_simhash`, `forgelm.compute_minhash` | Internal helpers we expose because notebooks find them useful, but the signature may collapse into a single `compute_signature(method=...)`. | +| `forgelm.setup_authentication` | Wrapper around `huggingface_hub.login`; will likely move to a `ForgeAuthContext` class. | +| `forgelm.prepare_dataset` | Returns a `datasets.Dataset`; the `datasets` minor changes pull the rug periodically. | +| `forgelm.get_model_and_tokenizer` | The `(model, tokenizer)` tuple form is convenient but the signature reaches into HF kwargs we cannot freeze. | + +Experimental symbols are documented under `docs/reference/library_api.md#experimental` with an explicit "subject to change" callout. + +### 2.3 Internal (private — `_`-prefixed) + +Anything starting with an underscore at any level (`forgelm._http`, `forgelm.cli._parser`, `forgelm.data_audit._aggregator`, etc.) is internal. Consumers that import these have implicitly opted out of the contract; we may change them in any release without notice. + +**Internal classification rule of thumb (audit-friendly):** a name is *internal* iff it is **not** in `forgelm.__all__` **and** does not appear in `dir(forgelm)` after a fresh `import forgelm`. Note that this is a property of the *package facade*, not of import-protection — `forgelm/_http.py` exists as a real submodule and `from forgelm import _http` succeeds at the language level, but it is internal because `_http` is absent from the public surface listings. + +**Tier completeness (audited 2026-05-02).** Every top-level Python module + sub-package has an explicit classification: + +| Module / sub-package | Tier | Notes | +|---|---|---| +| `forgelm/__init__.py` re-exports listed in §2.1 / §2.2 | Stable / Experimental | per the per-symbol tables | +| `forgelm.config` | Stable | Pydantic schema is the contract; field changes follow deprecation cadence | +| `forgelm.trainer`, `forgelm.model`, `forgelm.data` | Stable (via re-export) | accessed through `forgelm.ForgeTrainer` / `forgelm.get_model_and_tokenizer` / `forgelm.prepare_dataset` | +| `forgelm.compliance` | Mixed: `verify_audit_log` + `AuditLogger` + `VerifyResult` Stable; rest Internal | re-exported names follow §2.1 | +| `forgelm.data_audit` (the package) | Mixed: `audit_dataset` + `AuditReport` + PII / secrets / simhash utilities Stable; everything under `_*.py` Internal | per-symbol re-exports follow §2.1 | +| `forgelm.benchmark` | Experimental | `run_benchmark` + `BenchmarkResult` re-exported | +| `forgelm.synthetic` | Experimental | `SyntheticDataGenerator` re-exported | +| `forgelm.webhook` | Experimental | `WebhookNotifier` re-exported (per §2.2 + §10 Q5) | +| `forgelm.utils` | Experimental | `setup_authentication`, `manage_checkpoints` re-exported | +| `forgelm.results` | Stable | `TrainResult` re-exported | +| `forgelm.safety` | **Internal today** (no re-export) | direct `forgelm.safety.run_safety_evaluation` reach-in is undocumented; Phase 36 promotes it to Stable via re-export when the standalone subcommand ships | +| `forgelm.judge` | **Internal today** (no re-export) | callers reach in via `forgelm.judge.run_judge_evaluation`; not stable until ranked into a future re-export | +| `forgelm.ingestion` (the module) | **Internal today** (no re-export) | `OptionalDependencyError` + `ingest_path` reachable via direct submodule import; promotion to Experimental is a Phase 19 follow-up decision | +| `forgelm.export`, `forgelm.deploy`, `forgelm.merging`, `forgelm.inference`, `forgelm.fit_check`, `forgelm.chat`, `forgelm.quickstart`, `forgelm.wizard`, `forgelm.model_card`, `forgelm.grpo_rewards` | **Internal** | CLI dispatchers; reach-in is unsupported. The CLI is the public surface for these features. | +| `forgelm.cli` (the package) | Internal | `forgelm` console script + `python -m forgelm.cli` are the public entries; the dispatchers themselves are internal even though tests reach in via `forgelm.cli._run_*_cmd` | +| `forgelm.templates` | Internal data | template registry; consumers should use `forgelm quickstart --list` | + +If a downstream consumer needs a name currently marked Internal in this table, the right path is to file an issue requesting promotion (and, ideally, supplying a use case + signature stability rationale). We do **not** silently honour reach-ins as load-bearing. + +The CLI dispatchers themselves (`_run_chat_cmd`, `_run_audit_cmd`, etc.) are internal — the public CLI is the binary entry, not these helpers. Tests reach in via `forgelm.cli._run_*_cmd` for monkeypatch convenience; that does not promote them to the public API. + +--- + +## 3. Type contract — `py.typed` + PEP 561 + +ForgeLM ships `forgelm/py.typed` (zero-byte marker) so installers / `mypy` / IDEs know to use the inline type hints. Without the marker, downstream `mypy --strict` runs treat every `forgelm.*` import as `Any` and lose all signal. + +### 3.1 Coverage targets + +| Surface | Type-hint coverage required | +|---|---| +| Stable symbols (§2.1) | 100% — every parameter + return. `mypy --strict forgelm/__init__.py` clean. | +| Experimental symbols (§2.2) | 100% on the public signature; internal helpers may stay loose. | +| Internal symbols (§2.3) | Best-effort. Don't gate CI on internal mypy errors. | + +### 3.2 Enforcement + +A new CI step runs `mypy --strict --follow-imports=silent forgelm/__init__.py forgelm/api.py` (note: only the public re-exports, not the entire codebase — `--strict` on `forgelm/trainer.py` would require typing the entire `transformers` / `trl` surface, which is not our project to fix). + +Failures block the PR. This is a tighter gate than the existing `ruff check` because type-hint regressions are silent at runtime but break IDE completion. + +### 3.3 Forward-reference patterns + +A few public symbols return objects from heavy dependencies (`datasets.Dataset`, `transformers.PreTrainedModel`). We use: + +```python +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from datasets import Dataset + from transformers import PreTrainedModel, PreTrainedTokenizerBase + +def get_model_and_tokenizer( + config: ForgeConfig, +) -> "tuple[PreTrainedModel, PreTrainedTokenizerBase]": ... +``` + +The string-quoted return type lets us declare the contract without forcing `transformers` to be imported at module load. + +--- + +## 4. Lazy-import discipline + +### 4.1 The invariant: `import forgelm` does not load torch + +The single most important lazy-import rule: + +> Importing `forgelm` (or any of its public re-export modules) **must not** import `torch`, `transformers`, `trl`, `datasets`, or any other heavy ML dep. Heavy deps are loaded only when the consumer accesses a symbol that genuinely needs them. + +This rule exists because: +- CI runners that only check our config (`ruff`, the lightweight `pytest` subset) don't need `torch`. +- The CLI entry point `python -m forgelm.cli --help` must respond instantly. +- `forgelm doctor` (Phase 34) is meant to run on a brand-new machine *before* `torch` is even installed, so it cannot crash on `import forgelm`. + +**Today (audited 2026-05-02):** the invariant holds by inspection but is *not* pinned by an existing regression test. `tests/test_lazy_imports.py` only covers `import forgelm.trainer` and `import forgelm.model`; bare `import forgelm` is verified manually but lacks a CI gate. **Phase 19 task #7 closes this gap** by shipping `tests/test_library_api.py::test_lazy_import_no_torch` (subprocess `python -c "import forgelm; assert 'torch' not in sys.modules"`) so a future eager-import regression in `forgelm/__init__.py` fails CI. + +### 4.2 Implementation pattern: `__getattr__` per facade + +`forgelm/__init__.py` uses `__getattr__` to defer imports until the symbol is actually accessed. Phase 19 extends the same pattern to every name listed in §2.1 + §2.2. Three shapes work together: + +1. A **`TYPE_CHECKING` import block** so `mypy --strict` (and IDE indexers) see the *real* signatures rather than `Any`. Without this, the "100 % typed stable surface" claim in §3.2 is vacuously true at the type-checker layer because `__getattr__` returns `Any`. +2. A **lookup table + `__getattr__`** that performs the on-demand import. Resolved attributes are bound back onto the module so subsequent accesses skip `__getattr__` entirely (Python's documented behaviour: `__getattr__` is consulted only when the attribute is *missing*; once bound it is a normal attribute access). +3. A `__dir__` companion that lists the same names so `dir(forgelm)` and IDE auto-complete return the full public surface before any lazy access has been made. + +```python +# forgelm/__init__.py — Phase 19 sketch + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # These imports run only under mypy / IDE static analysis; never at runtime. + from .compliance import AuditLogger, VerifyResult, verify_audit_log + from .data_audit import ( + AuditReport, + audit_dataset, + compute_simhash, + detect_pii, + detect_secrets, + mask_pii, + mask_secrets, + ) + from .webhook import WebhookNotifier + # ... existing entries (ForgeTrainer, get_model_and_tokenizer, ...) + +# Single source of truth: name → (submodule, attribute). Shared with __dir__. +_LAZY_SYMBOLS: dict[str, tuple[str, str]] = { + "audit_dataset": (".data_audit", "audit_dataset"), + "verify_audit_log": (".compliance", "verify_audit_log"), + "AuditLogger": (".compliance", "AuditLogger"), + "VerifyResult": (".compliance", "VerifyResult"), + "AuditReport": (".data_audit", "AuditReport"), + "WebhookNotifier": (".webhook", "WebhookNotifier"), + "detect_pii": (".data_audit", "detect_pii"), + "mask_pii": (".data_audit", "mask_pii"), + "detect_secrets": (".data_audit", "detect_secrets"), + "mask_secrets": (".data_audit", "mask_secrets"), + "compute_simhash": (".data_audit", "compute_simhash"), + # ... existing ForgeTrainer / get_model_and_tokenizer / ... entries +} + + +def __getattr__(name: str): + target = _LAZY_SYMBOLS.get(name) + if target is None: + raise AttributeError(f"module 'forgelm' has no attribute {name!r}") + import importlib + module = importlib.import_module(target[0], package=__name__) + value = getattr(module, target[1]) + # Bind back onto the module so subsequent accesses bypass __getattr__ + # entirely. Python's data model guarantees __getattr__ is only consulted + # for missing attributes; a real attribute set wins forever after. + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals().keys(), *_LAZY_SYMBOLS.keys()}) +``` + +### 4.3 Wave 1 round-5 carry-over: `forgelm/cli/__init__.py` + +The CLI facade currently has ~200 lines of eager `from .submodule import ...` calls (23 import statements pulling 50+ names; verified 2026-05-04 against `forgelm/cli/__init__.py`). That makes `import forgelm.cli` heavyweight even though most of those subcommand dispatchers are not needed for any single command. + +**Decision:** Phase 19 ports the same `__getattr__` + `__dir__` pattern to `forgelm/cli/__init__.py`. A symbol → submodule lookup table replaces the eager imports. The shape: + +```python +# forgelm/cli/__init__.py — sketch for Phase 19 + +# Names that tests / monkeypatch / external pipelines reach via +# `forgelm.cli._run__cmd`. Maps the public attribute name +# to its source submodule (relative to forgelm.cli). +_SYMBOL_TO_MODULE: dict[str, str] = { + "main": "._dispatch", + "_dispatch_subcommand": "._dispatch", + "parse_args": "._parser", + "_run_chat_cmd": ".subcommands._chat", + "_run_audit_cmd": ".subcommands._audit", + "_run_approve_cmd": ".subcommands._approve", + # ... all currently-eager imports listed here ... +} + +def __getattr__(name: str): + target_module = _SYMBOL_TO_MODULE.get(name) + if target_module is None: + raise AttributeError(f"module 'forgelm.cli' has no attribute {name!r}") + import importlib + submodule = importlib.import_module(target_module, package=__name__) + return getattr(submodule, name) + +def __dir__() -> list[str]: + return sorted(_SYMBOL_TO_MODULE) +``` + +**Critical: monkeypatch resolution.** Tests do `patch("forgelm.cli._run_chat_cmd", ...)`. Python's `unittest.mock.patch` resolves the dotted path by `getattr(forgelm.cli, "_run_chat_cmd")`, which triggers `__getattr__`, which imports the submodule and returns the real callable. `mock.patch` then **rebinds the name on `forgelm.cli`** for the duration of the test. This works because `__getattr__` is only consulted for **missing** attributes — once a name is set on the module, `__getattr__` is bypassed. The pattern is monkeypatch-safe. + +**Honest scoping.** The win is **not** "the entire CLI dispatch path is now lazy". `forgelm/cli/_dispatch.py` itself eagerly imports `_config_load`, `_logging`, `_no_train_modes`, `_parser`, `_training`, `_wizard`; any actual CLI invocation walks that subset of submodules during `parse_args`. The lazy facade saves load cost for the **narrow case where a test reaches `forgelm.cli._run__cmd` directly** without first invoking `parse_args`/`main`, plus the cosmetic improvement of `import forgelm.cli` no longer pulling all 9 subcommand dispatchers when only one is needed. Phase 19 frames this as **hygiene**, not a measured cold-import speed-up; if the operator wants the latter, a separate benchmark + scoping pass goes into a future phase. + +We add a regression test (`tests/test_cli_lazy_imports.py`) that: +1. Imports `forgelm.cli` and asserts `"forgelm.cli.subcommands._chat" not in sys.modules`. +2. Accesses `forgelm.cli._run_chat_cmd` and asserts the submodule **is** now in `sys.modules`. +3. Re-imports `forgelm.cli` after `monkeypatch.setattr("forgelm.cli._run_chat_cmd", lambda *a, **kw: ...)`, asserts the patch is honoured. + +--- + +## 5. Versioned API contract + +### 5.1 Two version strings + +ForgeLM today has a single `__version__` derived from `importlib.metadata`. Phase 19 introduces a clean separation: + +| Variable | Source | Bumps when... | Read by... | +|---|---|---|---| +| `forgelm.__version__` | `importlib.metadata` (single source of truth = `pyproject.toml`) | every release | downstream pinning, audit manifest stamp | +| `forgelm.api.__api_version__` | hand-maintained string in `forgelm/api.py` | a stable-tier symbol's signature changes | downstream feature-detection | + +`__api_version__` exists because the CLI surface and the library surface evolve at different speeds. A patch release that fixes a CLI typo doesn't change the Python-importable contract; both bump `__version__` but only the latter bumps `__api_version__`. Consumers that pin against the library API can look at `__api_version__` and know whether a `forgelm` upgrade is safe. + +The convention: + +```text +__api_version__ = "MAJOR.MINOR" # e.g. "0.5" +``` + +We use a **two-segment** version (no patch) because patch-level changes to the library are by definition non-breaking; no consumer needs to detect them. + +### 5.2 Breaking-change detection (CI guard, optional) + +Phase 19 ships a `tools/check_api_compat.py` script that compares the **stable** symbols of the current branch against the previously released `__api_version__` and flags any signature change. Implementation sketch: + +1. Pip install the previous release into a temp venv. +2. `python -c "import forgelm; print(json.dumps({n: inspect.signature(getattr(forgelm, n)).__str__() for n in }))"`. +3. Diff against the same dump from the working tree. + +CI trigger: **`pull_request` job that runs whenever the PR targets `main` AND modifies `forgelm/__init__.py` or `forgelm/api.py`**, plus a manual `workflow_dispatch` step the `cut-release` skill invokes before tagging. We do **not** use `release-*` branches as the trigger: `docs/standards/release.md:259` explicitly forbids release branches ("**No release branches** — if a hotfix is needed for an old version that has diverged, create a branch at that tag and cherry-pick — rare"). A `release-*` trigger would ship a CI step that never fires because no `release-*` branch ever exists. No-op for the v0.5.5 release itself: there is no prior `__api_version__`-bearing release to diff against, so the script's first useful run is at v0.5.6 cut time. Phase 19 ships the script + the `pull_request` + `workflow_dispatch` workflow steps; Phase 33 (v0.5.5 release) treats the script as a documented future contract rather than an immediate gate. + +This is not a blocking gate; it is a notification. A genuinely-needed signature change still merges, but the release notes get an automatic "BREAKING:" line. + +### 5.3 Deprecation cadence + +The existing rule (`docs/standards/release.md` "Deprecation cadence") applies unchanged: + +- Mark the old symbol with a `DeprecationWarning` in release `N`. +- Keep it working in release `N+1`. +- Remove in release `N+2`. + +The `DeprecationWarning` message must include the replacement symbol name and the planned-removal version (e.g. `"v0.7.0"`). + +--- + +## 6. Integration test strategy + +### 6.1 Test surface + +A new test module `tests/test_library_api.py` covers the **stable** surface end-to-end. Each test exercises one user journey, not one symbol: + +| Test | Journey | +|---|---| +| `test_train_then_chat_pure_python` | `from forgelm import ForgeTrainer, ForgeConfig`; build a tiny config in-memory (no YAML); train 1 step on a fixture dataset; assert `TrainResult.success`. | +| `test_audit_dataset_one_shot` | `from forgelm import audit_dataset`; pass a 100-row JSONL; assert the returned `AuditReport.total_samples == 100` and `cross_split_overlap["pairs"] == {}` (note: `cross_split_overlap` is a `Dict[str, Any]` per `forgelm/data_audit/_types.py:122`, accessed by key not attribute). | +| `test_verify_audit_log_valid_chain` | `from forgelm import verify_audit_log`; emit two events via `AuditLogger`; assert the returned `VerifyResult.valid is True`. | +| `test_pii_utilities_standalone` | `from forgelm import detect_pii, mask_pii`; assert detection + masking on a fixture string. | +| `test_audit_dataset_with_croissant` | Calls `audit_dataset(..., emit_croissant=True)` and asserts the `report.croissant` block has the expected JSON-LD shape. | +| `test_lazy_import_no_torch` | Subprocess: `import forgelm; assert "torch" not in sys.modules`. | +| `test_lazy_import_attribute_access_metadata_only` | Subprocess: `import forgelm; _ = forgelm.ForgeTrainer; assert "torch" not in sys.modules`. Attribute access through the package facade resolves the class object via `__getattr__` → `from .trainer import ForgeTrainer`; `forgelm/trainer.py` defers `import torch` to function bodies (per `tests/test_lazy_imports.py`), so merely *referencing* the class must NOT pull torch. Torch is loaded only when the class is *used* (instantiated + `train()` called), which is exercised by `test_train_then_chat_pure_python` above. | +| `test_dir_lists_stable_symbols` | `assert "ForgeTrainer" in dir(forgelm)` etc. for every name in §2.1. | +| `test_internal_symbols_excluded_from_public_surface` | `assert "_http" not in dir(forgelm)` AND `assert "_http" not in forgelm.__all__`. We do **not** assert `pytest.raises(AttributeError)` here: `forgelm/_http.py` is a real submodule, so the moment any other test does `from forgelm import _http` (or `import forgelm._http`) the attribute is set on the package object as a side-effect and a `raises` test would silently flip. The contract we actually want is "internal = not in `__all__` / not in `dir()`", not "import-protected". | +| `test_config_from_dict` | `from forgelm import ForgeConfig`; build via `ForgeConfig(**dict_payload)`; assert validation errors raise on bad input; assert the round-tripped object matches a YAML-loaded equivalent. Pins the `**dict_payload` path called out in §8.2. | + +### 6.2 Heavy-dep gating + +Tests that need `torch` skip when `torch` is unavailable using the existing pattern (`pytest.importorskip("torch")`). CI runs the full suite on the matrix combos that have torch installed; the lightweight runners run only the `mark_lightweight` subset. + +### 6.3 No CLI subprocess + +`test_library_api.py` deliberately avoids `subprocess.run([sys.executable, "-m", "forgelm.cli", ...])`. The whole point is to exercise the library surface; CLI smoke tests already live in `test_cli.py` etc. + +--- + +## 7. Documentation surface + +| Doc | Role | Owner phase | +|---|---|---| +| `docs/reference/library_api.md` (new) | Reference: every stable + experimental symbol with signature + one-paragraph description | Phase 19 | +| `docs/reference/library_api-tr.md` (new) | TR mirror | Phase 24 (bilingual sweep) — Phase 19 may ship the EN side first | +| `docs/guides/library_api.md` (new) | Tutorial: "Use ForgeLM from a notebook" with three end-to-end examples (audit, verify-audit, train-and-chat) | Phase 19 | +| `notebooks/library_api_example.ipynb` (new) | Runnable notebook covering the same three journeys | Phase 19 | +| README.md | One-paragraph "ForgeLM as a Python library" section + link to `docs/guides/library_api.md` | Phase 19 | +| CHANGELOG.md | `[Unreleased]` "Library API" section listing new symbols | Phase 19 | +| `docs/standards/release.md` | Already covers deprecation cadence; add a "Library API" sub-section pointing at this design | Phase 19 | + +The reference doc is hand-written, not Pydantic-auto-gen — Pydantic gen covers the config schema (Phase 16), not the library API. + +--- + +## 8. Cross-cutting concerns + +### 8.1 Logging + +Today every module under `forgelm/` calls `logging.getLogger("forgelm.")`. When ForgeLM is used as a library, the consumer's logger config takes over — they may (a) want our log records, (b) want them muted, (c) want them in JSON. + +Phase 19 ships: + +- A documented "library mode" snippet in `docs/guides/library_api.md`: + ```python + import logging + logging.getLogger("forgelm").setLevel(logging.WARNING) # quiet by default in libraries + ``` +- No automatic logging configuration on `import forgelm`. We do **not** call `logging.basicConfig()` from any module-level code. The CLI does its own setup in `forgelm.cli._setup_logging`; the library leaves it to the caller. + +This is the "library hygiene" rule from PEP 8 / `logging` HOWTO; nothing new to invent. + +### 8.2 Config from dict (not just YAML) + +Today `forgelm.load_config(path)` reads YAML. A library user with an in-memory config (e.g. parametric sweep) wants: + +```python +from forgelm import ForgeConfig +config = ForgeConfig(**{"model": {...}, "training": {...}, ...}) +``` + +`ForgeConfig` is a Pydantic model so this already works — `ForgeConfig(**dict)` raises typed validation errors on bad input. Phase 19 documents this path explicitly in `library_api.md#config-from-dict` and adds a `tests/test_library_api.py::test_config_from_dict` regression. + +### 8.3 Error handling at the library boundary + +CLI dispatchers catch broad exceptions, log a structured envelope, and `sys.exit` with one of the public exit codes (0/1/2/3/4). Library callers expect typed exceptions to propagate: + +| Library symbol | Errors propagated as | +|---|---| +| `ForgeTrainer.train()` | `ConfigError` (validation), `RuntimeError` (CUDA / training-loop), `OSError` (I/O), all preserving original traceback. | +| `audit_dataset()` | `ValueError` (invalid args), `OSError` (I/O), `OptionalDependencyError` (missing extra). | +| `verify_audit_log()` | Returns `VerifyResult(valid=False, reason=...)` for chain failures (not an exception); raises `OSError` only for unreadable files. | +| `AuditLogger.log_event()` | `OSError` on write failure (caller decides whether to retry or abort). | + +The rule: **library code does not call `sys.exit`**. Every exit-code mapping lives in CLI dispatchers; library users see the typed exception and decide the disposition themselves. + +### 8.4 Thread / process safety + +| Symbol | Safe to call from multiple threads? | Safe to fork after construction? | +|---|---|---| +| `ForgeTrainer.train()` | No — TRL trainer holds GPU state. | No. | +| `audit_dataset()` | Yes (multiple corpora in parallel threads); each call is self-contained. | Yes. | +| `AuditLogger.log_event()` | Yes — uses `fcntl.flock` on POSIX (Wave 1). Windows uses `msvcrt.locking`. | Each forked child should construct its own `AuditLogger`; sharing the file handle across forks is unsupported. | +| `verify_audit_log()` | Yes — read-only. | Yes. | +| `WebhookNotifier.notify_*()` | Yes — each call opens its own `requests` session. | Yes. | + +Documented under `docs/reference/library_api.md#concurrency`. Not enforced via locks beyond what's already in the modules; the documented contract is the contract. + +--- + +## 9. Phase 19 task plan (what implementation actually does) + +This section is the implementation spec for the next phase. It is intentionally precise so a Phase 19 contributor doesn't have to re-derive any of the above. + +| # | Task | Files | Acceptance | +|---|---|---|---| +| 1 | Add `forgelm/py.typed` (zero-byte marker) | `forgelm/py.typed` (new) | From a fresh venv, `pip install dist/forgelm-*.whl && mypy --strict ` does NOT report `missing library stubs or py.typed marker`. | +| 2 | Create `forgelm/api.py` with `__api_version__ = "0.5"` and re-export it from `forgelm/__init__.py` | `forgelm/api.py` (new), `forgelm/__init__.py` | Both `from forgelm.api import __api_version__` and `from forgelm import __api_version__` work. | +| 3 | Extend `forgelm/__init__.py` `__getattr__` with the §4.2 entries (`audit_dataset`, `verify_audit_log`, `AuditLogger`, `VerifyResult`, `AuditReport`, `WebhookNotifier`, `detect_pii`, `mask_pii`, `detect_secrets`, `mask_secrets`, `compute_simhash`) | `forgelm/__init__.py` | Each name is reachable; `import forgelm` does not import torch (see §4.1 regression test). | +| 4 | Add `__all__` entries for every name introduced in §2.1 + §2.2 | `forgelm/__init__.py` | `dir(forgelm)` lists them. | +| 5 | Add `__dir__()` returning the union of `globals()` keys and `_LAZY_SYMBOLS` keys (per §4.2 sketch — superset of `__all__` so a name in the lazy table that the developer forgot to add to `__all__` still surfaces in IDE auto-complete) | `forgelm/__init__.py` | `dir(forgelm)` is a superset of `forgelm.__all__` and a superset of `_LAZY_SYMBOLS.keys()`. | +| 6 | Port `forgelm/cli/__init__.py` to the lazy `__getattr__` + `_SYMBOL_TO_MODULE` pattern (§4.3) | `forgelm/cli/__init__.py` | `tests/test_cli_lazy_imports.py` passes. | +| 7 | Add `tests/test_library_api.py` with the §6.1 cases | `tests/test_library_api.py` (new) | All 10 tests green (9 user-journey + `test_config_from_dict`). | +| 8 | Add `tests/test_cli_lazy_imports.py` for the §4.3 invariant | `tests/test_cli_lazy_imports.py` (new) | `import forgelm.cli` does not trigger any `forgelm.cli.subcommands.*` import; lazy access does. | +| 9 | Run `mypy --strict --follow-imports=silent forgelm/__init__.py forgelm/api.py` and fix every error | (any module touched by §3) | `mypy` clean on the public surface. | +| 10 | Add the CI step from §3.2 | `.github/workflows/ci.yml` | Step appears, fails on a deliberate type regression in a sandbox commit. | +| 11 | Write `docs/reference/library_api.md` covering every §2.1 + §2.2 symbol | `docs/reference/library_api.md` (new) | Each name has a signature + 1-paragraph blurb. | +| 12 | Write `docs/guides/library_api.md` with three end-to-end examples (audit, verify-audit, train-and-chat) | `docs/guides/library_api.md` (new) | Tutorial runnable in a notebook. | +| 13 | Add `notebooks/library_api_example.ipynb` covering the same three examples | `notebooks/library_api_example.ipynb` (new) | Runs to completion under `nbval` smoke. | +| 14 | Update `README.md` with a "ForgeLM as a Python library" section | `README.md` | Section + link to the guide. | +| 15 | Update `CHANGELOG.md [Unreleased]` with the new symbol list | `CHANGELOG.md` | Entry under "Library API". | +| 16 | Update `docs/standards/release.md` with a "Library API" sub-section pointing here | `docs/standards/release.md` | Sub-section + cross-link. | + +**Phase 19 acceptance:** +- All 16 tasks above land in a single PR. +- `pytest tests/test_library_api.py tests/test_cli_lazy_imports.py` green on fresh checkout. +- `pytest tests/` overall passes (existing 1010+ tests not regressed). +- `mypy --strict --follow-imports=silent forgelm/__init__.py forgelm/api.py` clean. +- `ruff format . && ruff check .` clean. +- README + CHANGELOG + guides up-to-date. + +--- + +## 10. Open questions for review + +These are the places where reasonable people would disagree. Resolved before Phase 19 starts. + +### Q1. Should `forgelm.api` exist at all, or just live in `forgelm/__init__.py`? + +**Decision:** create `forgelm/api.py`. Two reasons: (a) it gives `__api_version__` a clean home so `__init__.py` doesn't have to grow a constants block; (b) downstream consumers can `from forgelm.api import __api_version__` without triggering any heavy imports. `forgelm/__init__.py` itself stays the canonical entry point — `from forgelm import ForgeTrainer` keeps working — but the version gets its own module. + +### Q2. Why not auto-generate `library_api.md` from docstrings? + +**Decision:** hand-write it. Auto-generation drifts when symbols move modules; the doc has to outlive a refactor. Pydantic schema → `configuration.md` is auto-gen because the schema is the source; library API surface has no equivalent single source. Hand-written + a `tools/check_api_doc_completeness.py` script that flags any `__all__` entry missing from the doc is the better trade-off. + +### Q3. Should `mypy --strict` run on the entire codebase? + +**Decision:** no. Strict-typing the entire project would require us to also strictly-type the `transformers` / `trl` / `lm_eval` interactions, which we don't own. Phase 19 strict-types only the public surface (§3.2). The rest of the codebase keeps the existing convention (function signatures typed, internals loose). + +### Q4. Should `__api_version__` follow `__version__` automatically? + +**Decision:** no — they are deliberately separate. Bumping `__version__` for a CLI bug fix does not bump the library contract. Hand-maintaining `__api_version__` is the price for being able to claim a stable library API. The release skill (`cut-release`) gets a checklist line: "Did this release change a stable-tier symbol's signature? If yes, bump `__api_version__` too." + +### Q5. Should we expose `WebhookNotifier` as stable or experimental? + +**Decision:** experimental. The class works but its constructor takes a `config` object whose schema we are still moving (Phase 23 ISO/SOC 2 will likely add fields). Experimental + clear "subject to change" callout buys us a release cycle to get the shape right before promoting to stable in v0.6.x. + +--- + +## 11. Out of scope (deliberately) + +Things this design **does not** do, with the reason: + +| Out of scope | Why | +|---|---| +| Automatic `__api_version__` bumping in CI | Manual decision is the whole point — see Q4. | +| `forgelm.serve(...)` (HTTP wrapper) | Different product (Pro CLI dashboard); see [Phase 13 Pro CLI plan](../../roadmap/phase-13-pro-cli.md). | +| `mypy --strict` on the whole codebase | See Q3. | +| Automatic `library_api.md` generation | See Q2. | +| Async variants (`async def train(...)`) | Trainer is fundamentally synchronous (TRL holds GPU state). An async wrapper would be a different module. | +| Per-symbol `__deprecated__` decorator | We use the `DeprecationWarning` + comment pattern; a decorator would be over-engineered for a handful of deprecations per cycle. | + +--- + +## 12. Sign-off checklist (Phase 18 acceptance) + +- [x] Document is ≥400 lines. (Line count ≈ 470 incl. tables.) +- [x] Every stable-tier symbol from §2.1 has an explicit signature snippet **or** is referenced by a stable-tier file we already ship. *(Audited 2026-05-02: zero existing tests do `from forgelm import X` for any of the new stable names — that gap is exactly what Phase 19 task #7 closes. The §2.1 listing is the design promise; Phase 19 ships the regression coverage.)* +- [x] §3 names a CI step + sample command for type-hint enforcement. +- [x] §4 documents the existing lazy-import invariant + names the regression test that pins it. +- [x] §4.3 explicitly resolves the Wave 1 round-5 carry-over (CLI facade lazy migration). +- [x] §5 names two version strings + their semantics + the deprecation cadence. +- [x] §6 enumerates the integration test cases Phase 19 must implement. +- [x] §7 lists every documentation deliverable Phase 19 must produce. +- [x] §8 covers logging, config-from-dict, errors, and concurrency. +- [x] §9 is a 16-row task plan precise enough to execute Phase 19 from this document alone. +- [x] §10 records and resolves the five open questions. +- [x] §11 names what is deliberately out of scope. + +--- + +*End of Phase 18 design — `library-api-design-202605021414`.* diff --git a/docs/analysis/code_reviews/wave2a-round2-review-prompt.md b/docs/analysis/code_reviews/wave2a-round2-review-prompt.md new file mode 100644 index 00000000..53e7792f --- /dev/null +++ b/docs/analysis/code_reviews/wave2a-round2-review-prompt.md @@ -0,0 +1,324 @@ +# Wave 2a Round-2 — Multi-Agent Review Prompt (PR #28) + +> **Use this prompt verbatim** to launch the multi-agent code review on the +> consolidated Wave 2a PR. The prompt is self-contained: each reviewer agent +> can pick it up cold without context from prior conversations. +> Round-1 review (the per-PR pass on #23-#27) is now history; this is the +> round that gates merge to `development`. + +--- + +## Repo & PR under review + +- **Repo:** `cemililik/ForgeLM` +- **Branch under review:** `closure/wave2a-integration` +- **PR:** [#28](https://github.com/cemililik/ForgeLM/pull/28) → base + `development` +- **Diff scope:** 26 files, ~4 600 insertions, ~120 deletions. Three new + modules (`_doctor.py`, `_approvals.py`, `_audit_log_reader.py`), one new + argparse helper (`_argparse_types.py`), three new test files (~1 729 LOC + of tests). +- **What it consolidates:** the previously-scattered Phase 17 / 18 / 20 / 34 + / 37 PRs (#23, #24, #25, #26, #27 — all closed as superseded) **plus the + Wave-2a Round-1 review fixes** that landed on top of each. + +## What ForgeLM is (60 seconds) + +A **config-driven, enterprise-grade LLM fine-tuning toolkit** — YAML in, +fine-tuned model + compliance artifacts out. Built for CI/CD pipelines, not +notebooks. Six alignment paradigms (SFT/DPO/SimPO/KTO/ORPO/GRPO), +integrated safety evaluation (Llama Guard), EU AI Act compliance artefacts +(Articles 9-17 + Annex IV), append-only audit log, opt-in human approval +gate, auto-revert on quality regression. Read [docs/product_strategy.md](../../product_strategy.md) +for fuller background. + +Project rulebook lives under [docs/standards/](../../standards/) — read +[coding.md](../../standards/coding.md), [error-handling.md](../../standards/error-handling.md), +[testing.md](../../standards/testing.md), [logging-observability.md](../../standards/logging-observability.md), +and [code-review.md](../../standards/code-review.md) before commenting. +Project-wide guidance for AI agents is in the root `CLAUDE.md`. + +## What this PR ships + +| Phase | Type | Headline change | +|---|---|---| +| 18 | **Design only** | `library-api-design-202605021414.md` — public Python surface contract (stable / experimental / internal tiers, lazy-import invariant, `py.typed` + `mypy --strict`) | +| 20 | **Design only** | `gdpr-erasure-design-202605021414.md` — Article 17 erasure design + 11-test plan + `RetentionConfig` schema + 6 new audit events | +| 17 | **Implementation** | `forgelm audit --workers N` — split-level multiprocessing.Pool with byte-identical determinism contract (SHA-256 file-hash equality across worker counts) | +| 37 | **Implementation** | `forgelm approvals --pending / --show RUN_ID` — list / inspect human-approval gates from the audit log; path-traversal guarded; latest-wins re-staging | +| 34 | **Implementation** | `forgelm doctor [--offline]` — env probes (Python / torch / CUDA / GPU / extras / HF Hub / disk / operator identity); honest pass / warn / fail; lazy heavy-dep imports | +| Infra | **Refactor** | `forgelm/cli/subcommands/_audit_log_reader.py` — single source of truth for the audit-log JSONL parser, used by `_approve.py`, `_approvals.py`, and (Phase 21) `forgelm purge` | + +Round-1 fix commits ride on top of the original feat/docs commits — see +PR description for the per-phase summary of what Round-1 already addressed. + +## What we want from Round-2 + +**Goal:** decide whether `closure/wave2a-integration` is mergeable to +`development`. Block on real defects; do not gate on style nits or +hypothetical futures. + +Be concrete. Every finding must include: + +1. **Severity tag** — `CRITICAL` / `HIGH` / `MEDIUM` / `LOW` / `NIT` +2. **Finding ID** — `F--` (e.g. `F-17-03`, `F-37-04`, + `F-INFRA-01`); use `F-XPR-NN` for cross-phase observations. +3. **File:line citation** — `forgelm/cli/subcommands/_doctor.py:342` or a + short range `:340-360`. Cite the file even for design-doc findings + (`docs/analysis/code_reviews/library-api-design-202605021414.md:§4.2`). +4. **One-paragraph reasoning** — what is wrong, what is the user-visible + consequence, and why it qualifies for that severity tier. +5. **Suggested fix** — a code or text snippet, not a vague direction. +6. **Test that would have caught it** — name the test file + a stub of + the assertion. If the bug is in a design doc, name the doc section that + should have called it out. + +Severity bar: + +- `CRITICAL` — would corrupt data, silently lose audit events, leak PII, + break exit-code contract, or break the `--offline` / air-gap promise. + Blocks merge. +- `HIGH` — runtime crash on a documented happy path, missing test for a + documented contract, schema mismatch with `pyproject.toml` or + `config.py`, or a documentation claim with no code backing. Blocks + merge. +- `MEDIUM` — logic bug in an off-the-happy-path branch, observability gap, + or design-doc statement that contradicts shipped code. Should be fixed + before merge but the call is the maintainer's. +- `LOW` — defensive-coding improvement, minor inconsistency, missing + edge-case test that would not have changed PR direction. +- `NIT` — naming, line wrap, comment phrasing. Note in passing only; do + not stack these. + +## Areas to scrutinise + +Hit these specifically — they are where Round-1 found the most issues and +where regressions are most likely: + +### Phase 17 — `forgelm audit --workers N` + +- Determinism contract: do `tests/test_data_audit_workers.py`'s SHA-256 + byte-identity assertions actually exercise the merge path that produces + `data_audit_report.json`, or do they short-circuit on the same input? +- Spawn-context pinning: confirm `multiprocessing.get_context("spawn")` is + used unconditionally in `_orchestrator.py` and is not overridable by + caller env / config (we *want* it pinned). +- Error propagation: the `try/except` around `pool.map` — does it + preserve the original `Traceback` so the user can debug, or does it + collapse to a generic message? +- Argparse `_positive_int`: rejects `0`, `-1`, `1.5`, `"abc"`, `True`? +- Library-side `audit_dataset(workers=...)`: what happens for `workers=0`, + `workers=-1`, `workers=True`, `workers="2"` from a Python caller? The + test class claims typed `ValueError` — verify. +- `generated_at` stripping in tests: does the regex / replacement cope + with the actual JSON shape, or could a tweak to the report format + silently invalidate the determinism test? + +### Phase 37 — `forgelm approvals` + +- Path-traversal guard: trace a `staging_path` of `/etc/passwd` through + the code path. Does `_staging_path_inside_output_dir` actually short- + circuit before `os.listdir`, or only after some metadata has leaked? +- Latest-wins semantics: walk through a worked example with two + `human_approval.required` events and one `human_approval.granted` event + in between. Does the result agree with the docstring? +- Output-dir resolution: what happens when `--output-dir` is omitted? Is + the default the same as `forgelm approve`'s default? Mismatch would + surprise operators. +- Shared-module delegation: confirm `_approve.py` and `_approvals.py` + both call `iter_audit_events` / `find_latest_event_for_run` and do not + carry their own JSONL-parsing copies. +- JSON envelope: `{"success": true, "pending": [...], "count": N}` — + schema-stable across `--show` and `--pending`? Same key for the + collection? Same exit-code mapping? + +### Phase 34 — `forgelm doctor` + +- Extras names match `pyproject.toml` exactly: `distributed`, `eval`, + `tracking`, `merging`, `export`, `ingestion`, `ingestion-scale`. Any + drift = doc lie. +- HF cache resolution order: `HF_HUB_CACHE` > `HF_HOME/hub` > default — + test exercises the precedence, not just one path. +- HF endpoint: `HF_ENDPOINT` is honoured for the HEAD probe (so + self-hosted mirrors work). Air-gap users especially care. +- Lazy imports: `import forgelm` does not pull `torch` / `huggingface_hub` + at top level. Run `python -c "import sys; import forgelm; print('torch' + in sys.modules)"` mentally; report if it would print `True`. +- Probe-crash isolation: a single crashing probe must not abort the rest + of the report. Walk through what happens if `_check_torch` raises + `OSError`. +- Exit-code mapping: 0 = all pass (warns OK), 1 = at least one fail, 2 = + probe itself crashed. Verify against + [docs/standards/error-handling.md](../../standards/error-handling.md). +- Secret-env masking: `_DOCTOR_SECRET_ENV_NAMES` covers `HF_TOKEN`, + `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `WANDB_API_KEY`, etc.? The audit + log must never ingest a real token. +- `FORGELM_ALLOW_ANONYMOUS_OPERATOR`: what gets recorded when this is set + vs unset? Does it match the + [docs/standards/logging-observability.md](../../standards/logging-observability.md) + contract? +- Cap-walked HF cache: depth 4, 5K files. What happens at exactly 5K? + At 5K + 1? + +### Phase 18 — Library API design (doc only) + +- Tier completeness audit table — every public symbol in `forgelm/__init__.py` + is in the table, and the symbol still exists in current code. +- Lazy-import section §4.2 — pseudo-code uses `globals()[name] = value` + caching. Does Python actually re-trigger `__getattr__` if the value is + not cached? Verify the documented contract is correct. +- §6.1 integration test surface — claims that match the actual planned + `tests/test_library_api.py` surface. No "tests for X" without naming + the test. +- §5.2 CI trigger — `release-*` branch. Is that what the + [docs/standards/release.md](../../standards/release.md) says? +- §2.1 real signatures of `audit_dataset` / `verify_audit_log` — match the + shipped `forgelm/__init__.py` re-exports. + +### Phase 20 — GDPR erasure design (doc only) + +- §3.1 prior-state acknowledgement — does it actually resolve the + `staging_ttl_days` collision? Search `forgelm/config.py` for the term + and confirm. +- §3.3 mtime caveat — wording matches what file-system semantics can + actually guarantee. +- §4.2 row-id refusal — line-number / directory / multi-row cases all + return error, not silent success. +- §4.3 exit codes — aligned to the public 0/1/2/3/4 contract in + [docs/standards/error-handling.md](../../standards/error-handling.md). +- §4.4 strict commit ordering: `request → rewrite → completed` — + describes a recoverable failure mode for each step. +- §5.1 six new audit events: `data.erasure_requested`, + `data.erasure_completed`, `data.erasure_failed`, + `data.erasure_warning_memorisation`, + `data.erasure_warning_synthetic_data`, + `data.erasure_warning_external_copies`. Schemas non-overlapping with + existing event types in `forgelm/audit_log.py` (or wherever that module + lives). +- §6 scope-limitation — clearly states what `forgelm purge` does **not** + do (does not erase from external copies — operator must do that + themselves). +- Marketing copy replacement for `safety_compliance.md` — flag any line + that goes beyond what the design says we will actually implement. + +### Infrastructure — `_audit_log_reader.py` + +- `iter_audit_events`: malformed line policy — skip + log warning, or + raise? Both `_approve.py` and `_approvals.py` agree? +- `find_latest_event_for_run`: time complexity on a 10 000-event log. +- HMAC verification — is the parser the right place for it, or should + callers verify separately? (Round-1 carved out an `hmac_secret` param + on `verify_audit_log`; this module deliberately stays parsing-only — + confirm.) + +### Cross-phase (XPR) checks + +- **XPR-01** Audit-log parser proliferation: any leftover ad-hoc JSONL + iteration in `forgelm/cli/subcommands/_*.py` or `forgelm/audit_log*.py` + that should delegate to the shared module? +- **XPR-02** HTTP discipline: every outbound call has a timeout, returns + on `--offline`, masks secrets in error messages. +- **XPR-03** JSON envelope schema: `--output-format json` shapes are + consistent across `forgelm doctor`, `forgelm approvals`, `forgelm + audit`. Document the contract in one place if not yet documented. +- **XPR-04** Test rigor: no test that asserts only on its own fixture + shape (vacuous test); no `assert True` / `assert result` without + comparing to a known-good value. +- **XPR-05** Documentation drift: every CLI flag mentioned in EN docs is + also in TR docs and vice versa; every flag in docs exists in + `forgelm/cli/_parser.py`; every flag in `_parser.py` is documented + somewhere. +- **XPR-06** Schema collisions: any `Pydantic` field added by Phase 20 + design that collides with an existing field name in `forgelm/config.py` + on a different type? (Round-1 caught `staging_ttl_days`; recheck after + consolidation.) +- **XPR-07** Backwards compatibility: any change in this PR that would + break a 0.5.0 user's workflow? CLI flag rename, default change, exit- + code shift, audit event-type rename? + +## Ground rules + +- **Read the relevant standard before commenting.** The project + [docs/standards/](../../standards/) directory is the rulebook. +- **Cite line numbers.** If you cannot point at a line, the finding is + not actionable. +- **Distinguish design docs from shipped code.** A bug in a design doc is + a wrong roadmap; a bug in shipped code is a defect. Different fixes, + different urgency. +- **No phantom severity inflation.** A doc typo is `NIT`. A user-visible + CLI bug on the happy path is `HIGH` minimum. +- **Round-1 fixes are baseline.** If you find an issue Round-1 already + addressed, mark it `[Round-1 verified]` and move on; do not double-bill. +- **No re-litigating closed scope.** "Should ForgeLM also do X" is not a + finding. Stay in the diff. +- **No suggestions to add a Web UI / GUI / inference engine / pretraining + pipeline.** See [docs/marketing/strategy/05-yapmayacaklarimiz.md] — + these are out of scope by design (read root `CLAUDE.md` "What ForgeLM is + not"). + +## Required deliverable structure + +Each reviewing agent returns a single Markdown report with this skeleton: + +```markdown +# Wave 2a Round-2 — Review of PR #28 + +## Summary +- Verdict: [Block / Conditional / Approve] +- CRITICAL: N · HIGH: N · MEDIUM: N · LOW: N · NIT: N +- One-sentence headline of the highest-severity finding. + +## Findings + +### F-- · · +- **File:** `path/to/file.py:LINE` (or `:LINE-RANGE`) +- **What's wrong:** <1-3 sentences — concrete defect, not vague concern> +- **User-visible consequence:** +- **Suggested fix:** +- **Regression test:** + +(repeat per finding, ordered: CRITICAL → HIGH → MEDIUM → LOW → NIT) + +## Cross-phase observations +(F-XPR-NN entries, same structure) + +## Round-1 carry-overs verified +- F-23-04 [Round-1 verified] — lazy-import caching now uses `globals()[...]` +- F-26-02 [Round-1 verified] — spawn context pinned in `_orchestrator.py:42` +- (etc.) + +## What this report deliberately did not cover +(Out-of-scope items the agent looked at and declined to flag, with one +sentence each. This keeps later reviewers from re-treading the same +ground.) +``` + +## How to launch + +Spawn at minimum these agents in parallel: + +1. **Code-correctness agent** — focuses on Phase 17 / 34 / 37 + implementation defects. +2. **Design-doc agent** — focuses on Phase 18 / 20 doc consistency + against shipped code. +3. **Standards-compliance agent** — focuses on cross-phase checks XPR-01 + through XPR-07 against [docs/standards/](../../standards/). +4. **Test-rigor agent** — focuses on whether the tests in + `test_data_audit_workers.py`, `test_doctor.py`, + `test_approvals_listing.py` actually verify what they claim to. + +Each agent gets this prompt verbatim plus a one-line `Focus:` directive +naming its specialty. Agents must not coordinate; the maintainer +deduplicates after. + +## Delivery + +- Agents return their reports in `docs/analysis/code_reviews/wave2a-round2-.md`. +- The maintainer (Cemil) merges findings, dedups, and posts the unified + list on PR #28 as a top-level comment. +- Block / Conditional / Approve gating decision rests with the maintainer + after consolidation, **not** with any individual agent. + +--- + +*Prompt frozen 2026-05-02 against `closure/wave2a-integration` HEAD `5a57286`.* diff --git a/docs/guides/data_audit-tr.md b/docs/guides/data_audit-tr.md index be7c8ada..e39e7fe0 100644 --- a/docs/guides/data_audit-tr.md +++ b/docs/guides/data_audit-tr.md @@ -37,6 +37,9 @@ forgelm audit data/large_corpus.jsonl --dedup-method minhash --jaccard-threshold # Faz 12: opt-in heuristic kalite filtresi (Gopher/C4 stili) forgelm audit data/ --quality-filter + +# Faz 17: çoklu-split corpora için split-seviyesi paralelizm +forgelm audit data/ --workers 4 ``` > **Eski alias:** `forgelm --data-audit PATH` deprecation alias'ı @@ -485,10 +488,25 @@ forgelm audit PATH \ [--pii-ml] \ [--pii-ml-language LANG] \ [--croissant] \ + [--workers N] \ [--output-format {text,json}] \ [--quiet | --log-level {DEBUG,INFO,WARNING,ERROR}] ``` +`--workers N` (Faz 17) split-seviyesi paralelizmi açar: her split kendi +worker process'inde, bir `multiprocessing.Pool` içinde koşar. Varsayılan +`1` (sıralı, Faz-17-öncesi yol ile byte-identical). Speed-up *split sayısı* +ile ölçeklenir, satır sayısıyla değil — tek-split corpus'lar 1'den büyük +değerleri görmezden gelir. Tipik çoklu-split koşular (`train` / +`validation` / `test`) `--workers 3` ile neredeyse-doğrusal hızlanma görür. + +Raporu üreten merge adımı tek-thread'li kalır, dolayısıyla diskteki JSON +worker sayısından bağımsız **byte-identical**'dir (yalnızca +`generated_at` wall-clock zaman damgası değişir). Bu invariant +`tests/test_data_audit_workers.py` tarafından pinlenir; CI gate +audit-report hash'ini karşılaştırırken `--workers 1`'den `--workers 4`'e +geçiş yapan bir operatör için çalışmaya devam eder. + `PATH` bir `.jsonl` dosyası veya bir dizin olabilir. `--output` varsayılan olarak `./audit/`'tir. `--verbose`, insan-okunabilir özette bulgu olmayan split'leri de gösterir (varsayılan, çoklu-split diff --git a/docs/guides/data_audit.md b/docs/guides/data_audit.md index 98591d92..2d091f21 100644 --- a/docs/guides/data_audit.md +++ b/docs/guides/data_audit.md @@ -38,6 +38,9 @@ forgelm audit data/large_corpus.jsonl --dedup-method minhash --jaccard-threshold # Phase 12: opt-in heuristic quality filter (Gopher/C4 style) forgelm audit data/ --quality-filter + +# Phase 17: split-level parallelism for multi-split corpora +forgelm audit data/ --workers 4 ``` > **Legacy alias:** `forgelm --data-audit PATH` keeps working unchanged @@ -479,6 +482,7 @@ forgelm audit PATH \ [--pii-ml] \ [--pii-ml-language LANG] \ [--croissant] \ + [--workers N] \ [--output-format {text,json}] \ [--quiet | --log-level {DEBUG,INFO,WARNING,ERROR}] ``` @@ -503,6 +507,20 @@ is no flag to disable it. > `os.replace` — Phase 11.5 hardening so a crashed audit can never leave > a half-written report on disk. +`--workers N` (Phase 17) opts into split-level parallelism: each split +runs in its own worker process inside a `multiprocessing.Pool`. Default +is `1` (sequential, byte-identical to the pre-Phase-17 path). Speed-up +scales with the number of *splits*, not with row count — a single-split +corpus ignores values >1. Typical multi-split runs (`train` / +`validation` / `test`) see a near-linear speed-up at `--workers 3`. + +The merge step that builds the final report stays single-threaded so +the on-disk JSON is **byte-identical** across worker counts (the only +field that varies is `generated_at`, the wall-clock timestamp). This +invariant is pinned by `tests/test_data_audit_workers.py` so a CI gate +that compares the audit-report hash across runs continues to work +when an operator switches from `--workers 1` to `--workers 4`. + The legacy `forgelm --data-audit PATH` flag is preserved as a deprecation alias and logs a one-line notice. Behaviour is identical; new scripts should use the subcommand. diff --git a/docs/standards/architecture.md b/docs/standards/architecture.md index f881e2cb..3b051488 100644 --- a/docs/standards/architecture.md +++ b/docs/standards/architecture.md @@ -179,6 +179,31 @@ From [`pyproject.toml`](../../pyproject.toml): **The core install must work on all three OSes (Linux/macOS/Windows) with no Linux-only deps.** CI enforces this by running Linux + macOS matrix. +## HTTP discipline + +> **Rule:** every outbound HTTP call from `forgelm/` goes through `forgelm/_http.py` (`safe_post` / `safe_get`). Direct use of `requests.*`, `urllib.request.urlopen`, `httpx.*` etc. outside `forgelm/_http.py` is forbidden. + +`forgelm/_http.py` is the single chokepoint that enforces: + +- **Scheme allowlist** — only `https://` by default; `http://` requires the caller to pass `allow_insecure_http=True` and is logged. +- **SSRF guard** — pre-resolves hostnames + refuses RFC 1918 / link-local / loopback destinations unless the caller passes `allow_private=True`. +- **Timeout floor** — minimum 5 s connect, 30 s read; callers cannot pass `timeout=0` to disable. +- **Secret masking** — `_mask_secrets_in_text` redacts query-param tokens / `Authorization` headers in error messages so a `URLError(reason=...)` cannot leak `?api_key=ghp_...` into logs. +- **Redirect refusal** — `allow_redirects=False` by default; redirects to a different host require explicit opt-in. + +**Why a single chokepoint:** the policy lives in one module so that a future fix (rotate to a stricter allowlist, add a new mask pattern, plug a new metric) lands in one place rather than `N` scattered call sites. Webhook delivery (`forgelm/webhook.py`), the doctor's HF Hub probe (`forgelm/cli/subcommands/_doctor.py`), and any future telemetry / license / cloud integration all share the same policy. + +**Acceptance gate (CI-enforced — see `.github/workflows/ci.yml` `lint-http-discipline` step):** + +```bash +! grep -rn -E "(requests\.(get|post|put|delete|patch)|urllib\.request\.urlopen|httpx\.[a-z]+)" forgelm/ \ + --include='*.py' | grep -v "forgelm/_http.py" +``` + +The gate stays empty. A new contributor who reaches for `requests.get` directly fails CI immediately and is redirected to `safe_get`. If `_http.py` itself needs to expand (e.g., add `safe_get` because no helper covers an outbound HEAD probe yet — Phase 34's doctor surfaced this gap), the addition lands inside `_http.py` and gets a corresponding test in `tests/test_http.py`. + +**Deliberate exceptions:** `forgelm/compliance.py:1146-1182` (audit-log HMAC verifier) does its own JSONL line-byte parsing because it must hash raw bytes — but it does not perform outbound HTTP. No HTTP-discipline carve-outs exist today; if one is ever needed (e.g., a cloud provider's SDK that wraps its own HTTP), document it inline + add a `# noqa: forgelm-http-discipline` style marker. + ## Things we do not own Reaffirming [marketing/strategy/05-yapmayacaklarimiz.md](../marketing/strategy/05-yapmayacaklarimiz.md): diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 89411ec3..5d7c1fbf 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -235,20 +235,31 @@ Reverting is a deliberate gate, not a panic. Treat it that way. ## What errors look like in JSON output -When `--output-format json` is set, errors still go to stdout as a single JSON object: +When `--output-format json` is set, errors still go to stdout as a single JSON object. **Shipped envelope (canonical, used by every CLI subcommand as of v0.5.5):** ```json { - "status": "error", - "exit_code": 1, - "error_type": "ConfigError", - "message": "training.trainer_type must be one of [sft, dpo, simpo, kto, orpo, grpo], got 'spo'", - "details": {"field": "training.trainer_type", "value": "spo"} + "success": false, + "error": "training.trainer_type must be one of [sft, dpo, simpo, kto, orpo, grpo], got 'spo'" } ``` +The 2-key shape is intentionally minimal so every subcommand can emit it without coupling to a richer error model: `success: false` is the unambiguous CI gate signal (paired with the non-zero exit code from `$?`); `error` is the operator-actionable message. + +**Optional richer fields** that subcommands MAY add when they have the information at hand (none required, but consumers can rely on them being absent rather than wrong-typed): + +| Field | Type | When to emit | +|---|---|---| +| `exit_code` | int | When the dispatcher knows the exit code at JSON-emit time and wants to save consumers from reading `$?` separately. | +| `error_type` | str | Exception class name (`ConfigError`, `OSError`, etc.) for callers that want to branch on category. | +| `details` | object | Field-level error data (e.g. `{"field": "training.trainer_type", "value": "spo"}`). | + Human-friendly logs still go to **stderr**. Pipeline consumers read stdout. Never mix the two. +### Success envelope + +Each subcommand's success envelope wraps the result in a per-command collection key (`checks` for doctor, `pending` / `chain` for approvals, etc.). The full per-subcommand schema lives in [`docs/usermanuals/en/reference/json-output.md`](../usermanuals/en/reference/json-output.md) (+ TR mirror) — that page is the locked contract per `release.md` ("Changed JSON output key names → MAJOR bump"). Adding a new subcommand without updating that page is a documentation-drift defect. + ## Testing error paths Every custom exception and every non-zero exit path must have a test. See [testing.md](testing.md) for structure. Pattern: diff --git a/docs/usermanuals/en/compliance/human-oversight.md b/docs/usermanuals/en/compliance/human-oversight.md index 071e03ef..2d56ee11 100644 --- a/docs/usermanuals/en/compliance/human-oversight.md +++ b/docs/usermanuals/en/compliance/human-oversight.md @@ -127,18 +127,37 @@ Default is 48 hours. Set to 0 for "no timeout — wait forever" (not recommended ## Inspecting pending runs +`forgelm approvals` is the discovery counterpart to `approve` / `reject`. It scans the audit log under `--output-dir` and reports every run whose `human_approval.required` event has no matching terminal decision. + ```shell -$ forgelm approvals --pending -RUN_ID REQUESTED_AT ARTIFACTS EXPIRES -abc123 2026-04-29T14:33Z checkpoints/run/artifacts/ in 47h -def456 2026-04-29T09:12Z checkpoints/sft-only/artifacts/ in 42h +$ forgelm approvals --pending --output-dir checkpoints/ +Pending approvals (2): + +RUN_ID AGE REQUESTED_AT STAGING +---------------- ---- ------------------------- ------- +fg-abc123def456 3h 2026-04-30T11:33:10+00:00 present +fg-def456abc789 1d 2026-04-29T14:12:55+00:00 present ``` +`--output-format json` returns a structured envelope (`{"success": true, "pending": [...], "count": 2}`) so CI can filter the queue programmatically. + ```shell -$ forgelm approvals --show abc123 -... full artifact summary including audit, benchmarks, safety, model card ... +$ forgelm approvals --show fg-abc123def456 --output-dir checkpoints/ +Run: fg-abc123def456 +Status: pending + +Audit chain (oldest first): + [2026-04-30T11:33:10+00:00] human_approval.required — require_human_approval=true + +Staging contents (4 entries): + - adapter_config.json + - adapter_model.safetensors + - tokenizer.json + - tokenizer_config.json ``` +A `--show` against a granted / rejected run prints the full timeline (request → decision) plus the final approver and comment. `--show` against an unknown `run_id` exits 1 with a clear error. + ## Common pitfalls :::warn diff --git a/docs/usermanuals/en/getting-started/first-run.md b/docs/usermanuals/en/getting-started/first-run.md index ff321591..9ef6cc4e 100644 --- a/docs/usermanuals/en/getting-started/first-run.md +++ b/docs/usermanuals/en/getting-started/first-run.md @@ -30,13 +30,23 @@ Before anything else, run `forgelm doctor` to check that Python, PyTorch, CUDA, ```shell $ forgelm doctor -ForgeLM 0.5.2 -Python 3.11.4 · PyTorch 2.4.0 -CUDA available · driver 535.171 · NVIDIA RTX 4090 · 24 GB -bitsandbytes ✓ (installed) -Unsloth ✓ (installed) +forgelm doctor — environment check + + [✓ pass] python.version Python 3.11.4 (CPython). + [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA RTX 4090 (24.0 GiB). + [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). + [✓ pass] extras.unsloth Installed (module unsloth, purpose: Unsloth-accelerated training (Linux GPUs only)). + [! warn] extras.deepspeed Optional extra missing — install with: pip install 'forgelm[deepspeed]' (purpose: DeepSpeed ZeRO + offload distributed training). + [✓ pass] hf_hub.reachable HuggingFace Hub reachable (HTTP 200). + [✓ pass] disk.workspace Workspace /home/me/forgelm — 387.0 GiB free of 500.0 GiB. + [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'me@workstation'. Pin FORGELM_OPERATOR= for CI / pipeline runs. + +Summary: 7 pass, 2 warn, 0 fail. ``` +`--output-format json` returns a structured envelope (`{"success": bool, "checks": [...], "summary": {...}}`) so CI can filter on individual probe results without parsing the table. Pass `--offline` to skip the HF Hub network probe and inspect the local cache instead — useful for air-gapped deployments. + :::tip If `forgelm doctor` reports a problem (missing CUDA, version mismatch, no GPU), fix that first. Every other ForgeLM command will fail in confusing ways otherwise. See [Troubleshooting](#/operations/troubleshooting). ::: diff --git a/docs/usermanuals/en/getting-started/installation.md b/docs/usermanuals/en/getting-started/installation.md index b049e067..433aca4e 100644 --- a/docs/usermanuals/en/getting-started/installation.md +++ b/docs/usermanuals/en/getting-started/installation.md @@ -64,21 +64,23 @@ $ pip install 'forgelm[export]' Adds quantized GGUF export for local inference (Ollama, llama.cpp). Pulls in the `gguf` writer and supporting libraries. Optional because not every workflow ends in GGUF — many users hand off to vLLM or TGI directly. -### Distributed training (`[deepspeed]`) +### Distributed training (`[distributed]`) ```shell -$ pip install 'forgelm[deepspeed]' +$ pip install 'forgelm[distributed]' ``` -Adds DeepSpeed ZeRO-2 / ZeRO-3 support for multi-GPU training. Required only if you're training a model larger than fits on a single GPU. +Adds DeepSpeed ZeRO-2 / ZeRO-3 support for multi-GPU training. Required only if you're training a model larger than fits on a single GPU. (The extra name is `distributed`; the actual dep it pulls is `deepspeed>=0.14.0`.) -### Everything (`[all]`) +### Combining extras + +`pyproject.toml` does not define an `[all]` aggregate. Combine the extras you actually need by listing them comma-separated: ```shell -$ pip install 'forgelm[all]' +$ pip install 'forgelm[qlora,eval,tracking,merging,export,ingestion]' ``` -Pulls in every extra. Useful for CI runners that need to test all code paths; not recommended for production environments where you want a minimal dependency tree. +This is intentional — pulling DeepSpeed / bitsandbytes / Presidio / sentence-transformers all at once on a CPU-only laptop is rarely what an operator wants, so we keep the choice explicit. ## Container install @@ -100,12 +102,16 @@ If you've installed ForgeLM for GPU training, confirm CUDA is wired up correctly $ forgelm doctor ``` -`forgelm doctor` reports: -- Python and PyTorch versions -- CUDA availability and driver version -- Detected GPU model and VRAM -- Available compute capability -- bitsandbytes / Unsloth detection (if installed) +`forgelm doctor` reports a tabular pass / warn / fail diagnostic over: +- Python version (>=3.10 floor; >=3.11 recommended) +- torch + CUDA availability (CPU-only is a `warn`, not a `fail`) +- GPU inventory (per-device VRAM in GiB) +- Optional extras: `qlora`, `unsloth`, `distributed`, `eval`, `tracking`, `merging`, `export`, `ingestion`, `ingestion-pii-ml`, `ingestion-scale` — missing extras `warn` with the exact `pip install 'forgelm[]'` hint +- HuggingFace Hub reachability (via `HF_ENDPOINT` if set; skipped under `--offline`) +- Workspace disk space (<10 GiB → `fail`, <50 GiB → `warn`) +- `FORGELM_OPERATOR` audit-identity hint (Article 12) + +Pass `--output-format json` for a structured envelope (`{"success": bool, "checks": [...], "summary": {pass, warn, fail}}`); pass `--offline` for air-gap mode (skips the network probe and instead inspects the local HF cache). :::tip Run `forgelm doctor` *before* `forgelm --config ...` for any new environment. It catches missing CUDA libraries, version mismatches, and GPU-not-found errors in two seconds rather than two hours into training. diff --git a/docs/usermanuals/en/operations/air-gap.md b/docs/usermanuals/en/operations/air-gap.md index 326f3c30..0e9a1adb 100644 --- a/docs/usermanuals/en/operations/air-gap.md +++ b/docs/usermanuals/en/operations/air-gap.md @@ -113,15 +113,22 @@ evaluation: ```shell $ forgelm doctor --offline -ForgeLM 0.5.2 (offline mode verified) -✓ HF_HUB_OFFLINE set -✓ TRANSFORMERS_OFFLINE set -✓ Local cache: airgap-bundle/huggingface (3.4 GB cached models) -✓ Llama Guard 3 8B available locally -✓ lm-evaluation-harness: 4 tasks cached +forgelm doctor — environment check + + [✓ pass] python.version Python 3.11.4 (CPython). + [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA A100 (80.0 GiB). + [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). + [✓ pass] extras.eval Installed (module lm_eval, purpose: lm-evaluation-harness benchmark scoring). + [! warn] extras.tracking Optional extra missing — install with: pip install 'forgelm[tracking]' (purpose: Weights & Biases experiment tracking). + [✓ pass] hf_hub.offline_cache HF cache at /opt/airgap/huggingface/hub: 3.4 GiB across 47 file(s). HF_HUB_OFFLINE=1. + [✓ pass] disk.workspace Workspace /opt/airgap — 412.0 GiB free of 500.0 GiB. + [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'airgap-op@nodeA'. Pin FORGELM_OPERATOR= for CI / pipeline runs. + +Summary: 7 pass, 2 warn, 0 fail. ``` -If `forgelm doctor --offline` reports anything unavailable, fix that *before* the air-gapped operator wastes their time. +If `forgelm doctor --offline` reports anything as `fail`, fix that *before* the air-gapped operator wastes their time. The HF cache scan honours `HF_HUB_CACHE` first, then `HF_HOME/hub`, then the default `~/.cache/huggingface/hub` — point `HF_HUB_CACHE` at your bundled cache to make the scan deterministic. ## Bundle size estimate diff --git a/docs/usermanuals/en/reference/cli.md b/docs/usermanuals/en/reference/cli.md index fb39f777..f86345e8 100644 --- a/docs/usermanuals/en/reference/cli.md +++ b/docs/usermanuals/en/reference/cli.md @@ -12,40 +12,39 @@ ForgeLM ships a single `forgelm` binary with subcommands. This page is the canon | Command | What it does | |---|---| | `forgelm` (no subcommand) | Train (with `--config`). | -| `forgelm doctor` | Environment check — Python, CUDA, GPU, deps. | +| `forgelm doctor` | Environment check — Python, CUDA, GPU, deps, HF cache. | | `forgelm quickstart` | List or instantiate bundled templates. | | `forgelm ingest` | PDF/DOCX/EPUB → JSONL conversion. | -| `forgelm audit` | Pre-train data audit. | -| `forgelm benchmark` | Run lm-eval-harness against a model. | -| `forgelm safety-eval` | Llama Guard scoring. | +| `forgelm audit` | Pre-train data audit (PII / secrets / dedup / leakage / quality). | | `forgelm chat` | Interactive REPL. | -| `forgelm batch-chat` | Non-interactive prompt → response. | | `forgelm export` | GGUF export with quantisation. | -| `forgelm deploy` | Generate deployment config (Ollama, vLLM, TGI, etc). | -| `forgelm verify-annex-iv` | Validate an Annex IV artifact. | -| `forgelm verify-log` | Validate audit log chain. | -| `forgelm verify-gguf` | Validate GGUF integrity. | -| `forgelm cache-models` | Pre-cache HuggingFace models for air-gap. | -| `forgelm cache-tasks` | Pre-cache lm-evaluation-harness tasks. | -| `forgelm trend` | Show metric trends across recent runs. | -| `forgelm compare-runs` | Side-by-side comparison of run metrics. | -| `forgelm approve` | Sign a human approval request. | -| `forgelm approvals` | List pending approvals. | +| `forgelm deploy` | Generate deployment config (Ollama, vLLM, TGI, HF Endpoints). | +| `forgelm verify-audit` | Validate audit log chain (timestamps, prev_hash, HMAC). | +| `forgelm approve` | Sign a human approval request and promote `final_model.staging/`. | +| `forgelm reject` | Reject a human approval request and discard staging. | +| `forgelm approvals` | List pending approvals (`--pending`) or inspect one (`--show RUN_ID`). | Run `forgelm --help` for any of these. -## Top-level flags (apply across many subcommands) +## Top-level flags (training mode — used with `--config`) | Flag | Description | |---|---| | `--config PATH` | YAML config file path. Required for training. | -| `--dry-run` | Validate config and references; no training. | -| `--fit-check` | Estimate VRAM and report verdict; no training. | -| `--estimate-cost` | Pre-flight cost estimate; no training. | -| `--offline` | Disable all network calls; require everything cached. | -| `--output-format {plain,json}` | Logging format. JSON for CI. | -| `--verbose, -v` | Increase logging detail. | -| `--quiet, -q` | Reduce logging detail. | +| `--wizard` | Launch interactive configuration wizard to generate a `config.yaml`. | +| `--dry-run` | Validate configuration and check model/dataset access; no training. | +| `--fit-check` | Estimate peak training VRAM; no model load. Requires `--config`. | +| `--resume [PATH]` | Resume training. Bare `--resume` auto-detects last checkpoint; `--resume PATH` resumes from a specific one. | +| `--offline` | Air-gapped mode: disable all HF Hub network calls. Models and datasets must be available locally. | +| `--benchmark-only MODEL_PATH` | Run benchmark evaluation on an existing model without training. Requires `evaluation.benchmark` config. | +| `--merge` | Run model merging from the `merge:` config block. No training. | +| `--generate-data` | Generate synthetic training data using the teacher model. No training. | +| `--compliance-export OUTPUT_DIR` | Export EU AI Act compliance artifacts (audit trail, data provenance, Annex IV) to OUTPUT_DIR. Run after training so the manifest is complete. | +| `--data-audit PATH` | **Deprecated alias** for `forgelm audit PATH`. Scheduled for removal in v0.7.0. New scripts should use the subcommand. | +| `--output DIR` | Output directory for `--data-audit` / `--compliance-export` (default: `./audit/` or `./compliance/`). | +| `--output-format {text,json}` | Output format for results (default: `text`). JSON for CI. | +| `--quiet, -q` | Suppress INFO logs. Only show warnings and errors. | +| `--log-level {DEBUG,INFO,WARNING,ERROR}` | Set logging verbosity (default: INFO). | | `--version` | Print version. | | `--help, -h` | Show help. | @@ -57,12 +56,21 @@ Most-used patterns: $ forgelm --config configs/run.yaml --dry-run # validate $ forgelm --config configs/run.yaml --fit-check # VRAM check $ forgelm --config configs/run.yaml # train -$ forgelm --config configs/run.yaml --resume # resume from last checkpoint +$ forgelm --config configs/run.yaml --resume # resume auto-detected last checkpoint +$ forgelm --config configs/run.yaml --resume /path # resume from a specific checkpoint $ forgelm --config configs/run.yaml --merge # run as a merge job $ forgelm --config configs/run.yaml --generate-data # synthetic data only ``` -Resume from a specific checkpoint: `--resume-from PATH`. +## Doctor: `forgelm doctor` + +```shell +$ forgelm doctor # full env check +$ forgelm doctor --offline # air-gap variant: cache + offline-env probes +$ forgelm doctor --output-format json | jq . # CI-friendly envelope +``` + +Probes Python version, torch / CUDA / GPU, optional extras, HF Hub reachability (or HF cache when `--offline`), disk space, operator identity, and audit-secret configuration. Exit codes: `0` = all pass (warnings OK), `1` = at least one fail, `2` = a probe itself crashed. ## Audit: `forgelm audit` @@ -70,30 +78,33 @@ Resume from a specific checkpoint: `--resume-from PATH`. $ forgelm audit DATAFILE_OR_DIR \ [--output ./audit/] \ [--strict] \ - [--dedup-algo simhash|minhash] \ - [--dedup-threshold N] \ + [--workers N] \ + [--dedup-method {simhash,minhash}] \ + [--near-dup-threshold N] \ + [--minhash-jaccard FLOAT] [--minhash-num-perm N] \ [--skip-pii] [--skip-secrets] [--skip-quality] [--skip-leakage] \ [--remove-duplicates] [--remove-cross-split-overlap=val|test] \ [--output-clean PATH] \ [--show-leakage] \ - [--sample-rate FLOAT] + [--sample-rate FLOAT] \ + [--pii-ml] [--pii-ml-language LANG] \ + [--croissant] \ + [--verbose] ``` -See [Dataset Audit](#/data/audit) for full semantics. +`--workers N` parallelises split-level processing; the on-disk JSON is byte-identical across worker counts (modulo the `generated_at` timestamp). See [Dataset Audit](#/data/audit) for full semantics. ## Ingest: `forgelm ingest` ```shell -$ forgelm ingest INPUT_DIR \ +$ forgelm ingest INPUT_PATH \ --output PATH.jsonl \ [--recursive] \ - [--strategy tokens|markdown|paragraph|sentence] \ - [--max-tokens N] [--overlap N] \ - [--pii-mask] [--secrets-mask] \ - [--pii-locale tr|de|fr|us] \ - [--language LANG] \ - [--include "*.pdf,*.md"] [--exclude "drafts/*"] \ - [--format raw|instructions|qa] + [--strategy {sliding,paragraph,markdown}] \ + [--chunk-size N] [--overlap N] \ + [--chunk-tokens N] [--overlap-tokens N] [--tokenizer MODEL_NAME] \ + [--pii-mask] [--secrets-mask] [--all-mask] \ + [--pii-ml-language LANG] ``` See [Document Ingestion](#/data/ingestion). @@ -101,23 +112,25 @@ See [Document Ingestion](#/data/ingestion). ## Chat: `forgelm chat` ```shell -$ forgelm chat CHECKPOINT \ - [--base BASE_MODEL] \ - [--temperature 0.7] [--top-p 0.9] [--max-tokens 1024] \ +$ forgelm chat MODEL_PATH \ + [--adapter PATH] \ [--system "system prompt"] \ - [--safety on|off] \ - [--load PATH] # load saved session + [--temperature 0.7] [--max-new-tokens 512] [--no-stream] \ + [--load-in-4bit | --load-in-8bit] \ + [--trust-remote-code] \ + [--backend {transformers,unsloth}] ``` -Slash commands within the REPL: `/reset`, `/save`, `/load`, `/system`, `/temperature`, `/top_p`, `/max_tokens`, `/safety`, `/help`, `/quit`. See [Interactive Chat](#/deployment/chat). +Slash commands within the REPL: `/reset`, `/save [file]`, `/temperature N`, `/system [prompt]`, `/help` (alias `/?`), `/exit` (alias `/quit`). See [Interactive Chat](#/deployment/chat). ## Export: `forgelm export` ```shell $ forgelm export CHECKPOINT_DIR \ --output PATH.gguf \ - --quant q4_k_m|q5_k_m|q6_k|q8_0|q3_k_m|q2_k|fp16 \ - [--merge] # merge LoRA into base before export + --quant {q2_k,q3_k_m,q4_k_m,q5_k_m,q8_0,f16} \ + [--adapter PATH] \ + [--no-integrity-update] ``` Comma-separate `--quant` for multiple levels in one command. See [GGUF Export](#/deployment/gguf-export). @@ -125,20 +138,47 @@ Comma-separate `--quant` for multiple levels in one command. See [GGUF Export](# ## Deploy: `forgelm deploy` ```shell -$ forgelm deploy CHECKPOINT_DIR \ - --target ollama|vllm|tgi|hf-endpoints|kserve|triton \ - --output PATH_OR_DIR +$ forgelm deploy MODEL_PATH \ + --target {ollama,vllm,tgi,hf-endpoints} \ + [--output PATH] \ + [--system "PROMPT"] # Ollama only + [--max-length 4096] \ + [--gpu-memory-utilization 0.90] # vLLM + [--port 8080] # TGI + [--trust-remote-code] # vLLM + [--vendor aws] # HF Endpoints ``` See [Deploy Targets](#/deployment/deploy-targets). +## Approvals: `forgelm approvals` / `forgelm approve` / `forgelm reject` + +```shell +$ forgelm approvals --pending # list pending approval gates +$ forgelm approvals --show RUN_ID # inspect a specific run's chain + staging +$ forgelm approve RUN_ID --comment "Reviewed by N." # promote final_model.staging/ → final_model/ +$ forgelm reject RUN_ID --comment "Reason ..." # discard staging +``` + +See [Human Oversight Gate](#/compliance/human-oversight). Exit codes: `0` = pending list / approval recorded, `1` = unknown run_id / config error, `4` (training mode only) = awaiting approval. + +## Verify audit log: `forgelm verify-audit` + +```shell +$ forgelm verify-audit PATH/TO/audit_log.jsonl +$ forgelm verify-audit PATH/TO/audit_log.jsonl --hmac-secret "$FORGELM_AUDIT_SECRET" +$ forgelm verify-audit PATH/TO/audit_log.jsonl --require-hmac +``` + +Validates monotonic timestamps, `prev_hash` chain integrity, `seq` gap detection, and (when configured) HMAC signatures. Exit `0` on a valid chain; non-zero with a structured error envelope on tamper detection. + ## Authentication ForgeLM picks up credentials from environment variables. Never put them in YAML. | Provider | Env var | Used for | |---|---|---| -| HuggingFace | `HF_TOKEN` | Gated models (Llama, Llama Guard) | +| HuggingFace | `HF_TOKEN` (alias: `HUGGINGFACE_TOKEN`) | Gated models (Llama, Llama Guard) | | OpenAI | `OPENAI_API_KEY` | LLM-as-judge, synthetic data | | Anthropic | `ANTHROPIC_API_KEY` | LLM-as-judge, synthetic data | | W&B | `WANDB_API_KEY` | Experiment tracking | @@ -161,10 +201,10 @@ If the env var isn't set, ForgeLM fails at config load with a clear error — be | Exit | Meaning | |---|---| | 0 | Success | -| 1 | Config / arg error | -| 2 | Audit warnings (with `--strict`) | +| 1 | Config / argument error | +| 2 | Audit warnings (with `--strict`) / probe crash (`forgelm doctor`) | | 3 | Auto-revert / regression | -| 4 | Awaiting human approval | +| 4 | Awaiting human approval (training pipeline) | | 130 | User interrupted (Ctrl+C) | See [Exit Codes](#/reference/exit-codes) for the full contract. @@ -173,14 +213,17 @@ See [Exit Codes](#/reference/exit-codes) for the full contract. | Variable | What it sets | |---|---| -| `HF_TOKEN` | HuggingFace auth | -| `HF_HOME` | HuggingFace cache directory (default `~/.cache/huggingface`) | +| `HF_TOKEN` / `HUGGINGFACE_TOKEN` | HuggingFace authentication | +| `HF_HOME` | HuggingFace cache root (default `~/.cache/huggingface`) | +| `HF_HUB_CACHE` | Override the HF Hub cache directory specifically (precedence: `HF_HUB_CACHE` > `HF_HOME/hub` > default) | | `HF_HUB_OFFLINE=1` | Disable HF Hub network calls | +| `HF_ENDPOINT` | HF Hub endpoint override (for self-hosted mirrors); honoured by `forgelm doctor` | | `TRANSFORMERS_OFFLINE=1` | Disable transformers library network calls | | `HF_DATASETS_OFFLINE=1` | Disable datasets library network calls | -| `FORGELM_CACHE_DIR` | ForgeLM-specific cache location | -| `FORGELM_LOG_LEVEL` | Override logging level (DEBUG, INFO, WARN, ERROR) | -| `FORGELM_RESUME_TOKEN` | Token for the API-based human approval flow | +| `FORGELM_OPERATOR` | Operator identity recorded in audit events (overrides `getpass.getuser()@hostname`) | +| `FORGELM_ALLOW_ANONYMOUS_OPERATOR` | When `1`, permit the audit log to record an anonymous operator (otherwise an unresolved identity is an error) | +| `FORGELM_AUDIT_SECRET` | HMAC signing key for the audit log chain (enables tamper-detection) | +| `FORGELM_GGUF_CONVERTER` | Path to a custom `convert-hf-to-gguf.py` script | ## Common patterns @@ -196,6 +239,14 @@ $ forgelm --config configs/run.yaml --output-format json | tee run.log $ forgelm audit data/ --strict && forgelm --config configs/run.yaml ``` +### "Train with human approval gate; promote later" + +```shell +$ forgelm --config configs/run.yaml # exits 4 if approval gate fires +$ forgelm approvals --pending # discover the pending run +$ forgelm approve RUN_ID --comment "Reviewed." # promote staging +``` + ### "Train, export GGUF, deploy to Ollama" ```yaml diff --git a/docs/usermanuals/en/reference/json-output.md b/docs/usermanuals/en/reference/json-output.md new file mode 100644 index 00000000..380ca1d8 --- /dev/null +++ b/docs/usermanuals/en/reference/json-output.md @@ -0,0 +1,186 @@ +--- +title: JSON output schemas +description: The locked --output-format json envelope shape for every forgelm subcommand. CI consumers depend on these field names. +--- + +# JSON output schemas + +Every `forgelm` subcommand that supports `--output-format json` produces a stable JSON envelope on stdout. Field names + nesting are part of the public CLI contract per [`docs/standards/release.md`](#/standards/release): renaming a key is a MAJOR-version break. + +This page is the canonical reference. CI/CD pipelines that parse `forgelm` output should pin against the shapes documented here. + +## Common conventions + +- **stdout vs stderr.** The JSON envelope goes to **stdout**. Human-friendly logs (info / warning / error) go to **stderr**. Pipe `forgelm ... --output-format json | jq .` and read your operator-facing messages from `2>` separately. +- **Top-level wrapper.** Every envelope starts with `"success": true | false`. Consumers can branch on this single key before parsing the rest. +- **Error envelope.** When `success: false`, the envelope carries `"error": ""` (string). Optional richer fields (`exit_code`, `error_type`, `details`) MAY be present per [`error-handling.md`](#/standards/error-handling). Consumers that need certainty on these fields should also check the process exit code via `$?`. +- **Exit codes.** See [Exit Codes](#/reference/exit-codes). The envelope is consistent with the exit code: `success: true` ⟺ exit `0`; `success: false` ⟺ non-zero exit. + +## `forgelm doctor` + +Environment check. See [Doctor command](#/getting-started/first-run). + +**Success envelope** (`forgelm doctor [--offline] --output-format json`): + +```json +{ + "success": true, + "checks": [ + { + "name": "python.version", + "status": "pass", + "detail": "Python 3.11.7 (CPython).", + "extras": {"version": "3.11.7", "implementation": "CPython"} + } + ], + "summary": {"pass": 8, "warn": 1, "fail": 0, "crashed": 0} +} +``` + +| Key | Type | Notes | +|---|---|---| +| `success` | bool | `true` when no probe `fail` AND no probe crash; `false` otherwise. | +| `checks` | list[object] | One entry per probe in execution order. Probe names are stable (e.g. `python.version`, `torch.cuda`, `gpu.inventory`, `extras.qlora`, `hf_hub.reachable`, `hf_hub.offline_cache`, `disk.workspace`, `operator.identity`). | +| `checks[].name` | str | Probe name. Stable across versions; new probes append rather than rename. | +| `checks[].status` | str | One of `pass`, `warn`, `fail`, `crashed`. | +| `checks[].detail` | str | Operator-facing one-line description of the result. | +| `checks[].extras` | object | Probe-specific structured data. Per-probe keys are documented in `_doctor.py` docstrings; consumers should treat unknown keys as forward-compatible. | +| `summary` | object | Counts of each status across `checks`. Sum equals `len(checks)`. | + +**Exit code mapping:** `0` = all probes `pass` or `warn`; `1` = at least one `fail`; `2` = at least one `crashed` (probe raised; subsequent probes still ran). + +## `forgelm approvals --pending` + +Lists pending Article 14 approval requests, newest-first. + +```json +{ + "success": true, + "pending": [ + { + "run_id": "fg-abc123def456", + "staging_path": "/work/output/final_model.staging.fg-abc123def456", + "staging_exists": true, + "requested_at": "2026-05-04T12:34:56+00:00", + "age_seconds": 3600.5, + "metrics": {"safety_score": 0.91, "benchmark.hellaswag": 0.78}, + "config_hash": "sha256:...", + "reason": "post-train safety eval below threshold" + } + ], + "count": 1 +} +``` + +| Key | Type | Notes | +|---|---|---| +| `success` | bool | `true` (always — empty list is success). | +| `pending` | list[object] | Newest-first; `count == len(pending)`. Empty list = no pending approvals. | +| `pending[].run_id` | str \| null | Run identifier from the audit event. | +| `pending[].staging_path` | str \| null | Resolved staging directory (`null` when audit event lacks `staging_path` and no canonical fallback exists). Path-traversal guarded — paths outside `--output-dir` are rejected at resolution time. | +| `pending[].staging_exists` | bool | `True` iff `staging_path` resolves to an existing directory. | +| `pending[].requested_at` | str \| null | ISO-8601 timestamp from the audit event. | +| `pending[].age_seconds` | number \| null | Seconds since `requested_at`; `null` when timestamp is unparseable. | +| `pending[].metrics` | object | Free-form per-run metrics from the audit event. | +| `pending[].config_hash` | str \| null | Config fingerprint, when known. | +| `pending[].reason` | str \| null | Operator-supplied reason from the audit event. | +| `count` | int | Pending count; equal to `len(pending)`. | + +## `forgelm approvals --show RUN_ID` + +Inspect the full approval-gate audit chain + staging contents for one run. + +```json +{ + "success": true, + "run_id": "fg-abc123def456", + "status": "pending", + "chain": [ + {"event": "human_approval.required", "run_id": "fg-abc123def456", "timestamp": "2026-05-04T12:34:56+00:00", "...": "..."} + ], + "staging_contents": ["adapter_config.json", "adapter_model.safetensors", "tokenizer.json"] +} +``` + +| Key | Type | Notes | +|---|---|---| +| `success` | bool | `true` when the run was found; `false` (with `error`) when `RUN_ID` has no audit events. | +| `run_id` | str | Echoed input `RUN_ID`. | +| `status` | str | One of `pending`, `granted`, `rejected`, `unknown`. Latest-wins semantics: a re-staged run after a prior decision shows `pending`. | +| `chain` | list[object] | Every approval-gate audit event for `run_id`, in append order. | +| `staging_contents` | list[str] | Sorted file/directory names at `/final_model.staging.` (or canonical fallback). Empty when staging missing or unreadable. | + +## `forgelm audit` + +Pre-train data audit. Full report; key fields shown. + +```json +{ + "success": true, + "report_path": "audit/data_audit_report.json", + "splits": {"train": {"sample_count": 100, "...": "..."}, "...": {}}, + "pii_summary": {"total_findings": 0, "by_kind": {}}, + "secrets_summary": {"total_findings": 0, "by_kind": {}}, + "cross_split_overlap": {"pairs": {}}, + "leakage": {"...": "..."}, + "quality_filter": null, + "near_duplicates": {"...": "..."}, + "languages_top3": [{"code": "en", "count": 87}], + "generated_at": "2026-05-04T12:34:56+00:00", + "warnings": [] +} +``` + +The full schema is in [`docs/guides/data_audit.md`](#/data/audit). Pin against `report_path` (where the on-disk JSON lives) + `success` for CI gates. + +## `forgelm verify-audit` + +Audit-log chain integrity check. + +```json +{ + "success": true, + "valid": true, + "entries_count": 87, + "hmac_verified": true, + "errors": [] +} +``` + +| Key | Type | Notes | +|---|---|---| +| `success` | bool | `true` ⟺ `valid: true`. | +| `valid` | bool | `false` if any prev_hash mismatch / monotonicity break / seq gap. | +| `entries_count` | int | Number of well-formed audit lines. | +| `hmac_verified` | bool \| null | `true` when `--hmac-secret` matches every `hmac` field; `false` on mismatch; `null` when chain has no HMAC fields. | +| `errors` | list[str] | One human-readable line per detected problem. | + +## `forgelm approve` / `forgelm reject` + +```json +{ + "success": true, + "run_id": "fg-abc123def456", + "approver": "alice@example.com@workstation-7", + "final_model_path": "/work/output/final_model", + "promote_strategy": "atomic_rename" +} +``` + +`approve` exits `0` on success; `reject` exits `0` after recording the rejection (the staging dir is preserved for forensics). `success: false` with `error` on unknown `run_id` / config error. + +## Adding a new subcommand + +A new subcommand that supports `--output-format json` MUST land with: + +1. The envelope documented in this page (EN + TR mirror). +2. A test in `tests/test_json_envelope_contract.py` (or a per-subcommand test file) that pins the exact set of top-level keys. +3. The per-collection key follows the convention "results live under a key named after the subcommand's primary noun" (so `doctor` → `checks`, `approvals --pending` → `pending`, etc.). + +Renaming a key after merge is a MAJOR-version bump per [`release.md`](#/standards/release). + +## See also + +- [Exit Codes](#/reference/exit-codes) — the contract `success: bool` aligns with. +- [`error-handling.md`](#/standards/error-handling) — the error envelope contract. +- [`release.md`](#/standards/release) — when JSON renames count as breaking. diff --git a/docs/usermanuals/tr/compliance/human-oversight.md b/docs/usermanuals/tr/compliance/human-oversight.md index d951916d..a67b2ff4 100644 --- a/docs/usermanuals/tr/compliance/human-oversight.md +++ b/docs/usermanuals/tr/compliance/human-oversight.md @@ -127,18 +127,37 @@ Varsayılan 48 saat. "Timeout yok — sonsuza kadar bekle" için 0 (CI'da öneri ## Bekleyen koşuları inceleme +`forgelm approvals`, `approve` / `reject` komutlarının tamamlayıcısıdır: `--output-dir` altındaki audit log'u tarar ve `human_approval.required` event'i için terminal kararı (granted/rejected) bulunmayan tüm koşuları raporlar. + ```shell -$ forgelm approvals --pending -RUN_ID REQUESTED_AT ARTIFACTS EXPIRES -abc123 2026-04-29T14:33Z checkpoints/run/artifacts/ 47s sonra -def456 2026-04-29T09:12Z checkpoints/sft-only/artifacts/ 42s sonra +$ forgelm approvals --pending --output-dir checkpoints/ +Pending approvals (2): + +RUN_ID AGE REQUESTED_AT STAGING +---------------- ---- ------------------------- ------- +fg-abc123def456 3h 2026-04-30T11:33:10+00:00 present +fg-def456abc789 1d 2026-04-29T14:12:55+00:00 present ``` +`--output-format json` yapısal bir zarf döner (`{"success": true, "pending": [...], "count": 2}`); CI bu çıktıyla kuyruğu programatik olarak filtreleyebilir. + ```shell -$ forgelm approvals --show abc123 -... audit, benchmark'lar, güvenlik, model card dahil tam artifact özeti ... +$ forgelm approvals --show fg-abc123def456 --output-dir checkpoints/ +Run: fg-abc123def456 +Status: pending + +Audit chain (oldest first): + [2026-04-30T11:33:10+00:00] human_approval.required — require_human_approval=true + +Staging contents (4 entries): + - adapter_config.json + - adapter_model.safetensors + - tokenizer.json + - tokenizer_config.json ``` +Granted / rejected bir koşu üzerinde `--show` çalıştırıldığında tam zaman çizelgesi (talep → karar) ve son onaylayan + yorum yazdırılır. Bilinmeyen bir `run_id` üzerinde `--show` net bir hata mesajıyla 1 koduyla çıkar. + ## Sık hatalar :::warn diff --git a/docs/usermanuals/tr/getting-started/first-run.md b/docs/usermanuals/tr/getting-started/first-run.md index 99ad1e83..ede4d998 100644 --- a/docs/usermanuals/tr/getting-started/first-run.md +++ b/docs/usermanuals/tr/getting-started/first-run.md @@ -30,13 +30,23 @@ Her şeyden önce Python, PyTorch, CUDA ve opsiyonel bağımlılıkların doğru ```shell $ forgelm doctor -ForgeLM 0.5.2 -Python 3.11.4 · PyTorch 2.4.0 -CUDA available · driver 535.171 · NVIDIA RTX 4090 · 24 GB -bitsandbytes ✓ (installed) -Unsloth ✓ (installed) +forgelm doctor — environment check + + [✓ pass] python.version Python 3.11.4 (CPython). + [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA RTX 4090 (24.0 GiB). + [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). + [✓ pass] extras.unsloth Installed (module unsloth, purpose: Unsloth-accelerated training (Linux GPUs only)). + [! warn] extras.deepspeed Optional extra missing — install with: pip install 'forgelm[deepspeed]' (purpose: DeepSpeed ZeRO + offload distributed training). + [✓ pass] hf_hub.reachable HuggingFace Hub reachable (HTTP 200). + [✓ pass] disk.workspace Workspace /home/me/forgelm — 387.0 GiB free of 500.0 GiB. + [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'me@workstation'. Pin FORGELM_OPERATOR= for CI / pipeline runs. + +Summary: 7 pass, 2 warn, 0 fail. ``` +`--output-format json` yapısal bir zarf döner (`{"success": bool, "checks": [...], "summary": {...}}`); CI ayrı ayrı probe sonuçlarını tabloyu parse etmeden filtreleyebilir. `--offline` HF Hub ağ probe'unu atlar ve onun yerine yerel cache'i inceler — air-gap dağıtımları için faydalı. + :::tip `forgelm doctor` bir sorun raporlarsa (eksik CUDA, sürüm uyumsuzluğu, GPU yok), önce onu düzeltin. Diğer her ForgeLM komutu kafa karıştırıcı şekillerde patlar. Bkz. [Sorun Giderme](#/operations/troubleshooting). ::: diff --git a/docs/usermanuals/tr/getting-started/installation.md b/docs/usermanuals/tr/getting-started/installation.md index 19719ecf..b1060b33 100644 --- a/docs/usermanuals/tr/getting-started/installation.md +++ b/docs/usermanuals/tr/getting-started/installation.md @@ -64,21 +64,23 @@ $ pip install 'forgelm[export]' Yerel inference için (Ollama, llama.cpp) kuantize GGUF export'u ekler. `gguf` writer'ı ve destekleyici kütüphaneleri kurar. Opsiyonel çünkü her iş akışı GGUF ile bitmez — birçok kullanıcı doğrudan vLLM veya TGI'a teslim eder. -### Dağıtık eğitim (`[deepspeed]`) +### Dağıtık eğitim (`[distributed]`) ```shell -$ pip install 'forgelm[deepspeed]' +$ pip install 'forgelm[distributed]' ``` -Çoklu-GPU eğitim için DeepSpeed ZeRO-2 / ZeRO-3 desteği ekler. Sadece tek GPU'ya sığmayan model eğitiyorsanız gerekli. +Çoklu-GPU eğitim için DeepSpeed ZeRO-2 / ZeRO-3 desteği ekler. Sadece tek GPU'ya sığmayan model eğitiyorsanız gerekli. (Extra adı `distributed`; çektiği gerçek bağımlılık `deepspeed>=0.14.0`.) -### Her şey (`[all]`) +### Extra'ları birleştirme + +`pyproject.toml` bir `[all]` aggregate tanımlamaz. Gerçekten ihtiyaç duyduğunuz extra'ları virgülle ayırarak listeleyin: ```shell -$ pip install 'forgelm[all]' +$ pip install 'forgelm[qlora,eval,tracking,merging,export,ingestion]' ``` -Tüm extra'ları çeker. Tüm kod yollarını test etmesi gereken CI runner'ları için kullanışlı; minimal bağımlılık ağacı isteyen üretim ortamları için önerilmez. +Bu kasıtlıdır — DeepSpeed / bitsandbytes / Presidio / sentence-transformers'ı CPU-only bir laptop'a hep birden çekmek nadiren operatörün istediği şey, bu yüzden seçimi explicit tutuyoruz. ## Container kurulumu @@ -100,12 +102,16 @@ GPU eğitimi için kurduysanız CUDA'nın doğru bağlandığını teyit edin: $ forgelm doctor ``` -`forgelm doctor` şunları raporlar: -- Python ve PyTorch sürümleri -- CUDA varlığı ve sürücü sürümü -- Algılanan GPU modeli ve VRAM -- Mevcut compute capability -- bitsandbytes / Unsloth tespiti (kuruluysa) +`forgelm doctor` tabular bir pass / warn / fail teşhis raporu üretir: +- Python sürümü (>=3.10 zorunlu, >=3.11 önerilen) +- torch + CUDA varlığı (CPU-only `warn`'dur, `fail` değil) +- GPU envanteri (cihaz başına VRAM, GiB) +- Opsiyonel extras: `qlora`, `unsloth`, `distributed`, `eval`, `tracking`, `merging`, `export`, `ingestion`, `ingestion-pii-ml`, `ingestion-scale` — eksik extra'lar tam `pip install 'forgelm[]'` ipucu ile `warn`'lanır +- HuggingFace Hub erişimi (varsa `HF_ENDPOINT` üzerinden; `--offline` ile atlanır) +- Workspace disk alanı (<10 GiB → `fail`, <50 GiB → `warn`) +- `FORGELM_OPERATOR` audit kimliği ipucu (Madde 12) + +`--output-format json` yapısal zarf döner (`{"success": bool, "checks": [...], "summary": {pass, warn, fail}}`); `--offline` air-gap modu (ağ probe'unu atlar, yerel HF cache'i inceler). :::tip Yeni bir ortamda `forgelm --config ...` koşmadan *önce* `forgelm doctor` çalıştırın. Eksik CUDA kütüphanelerini, sürüm uyumsuzluklarını ve "GPU bulunamadı" hatalarını eğitime saatler kala değil saniyeler içinde yakalar. diff --git a/docs/usermanuals/tr/operations/air-gap.md b/docs/usermanuals/tr/operations/air-gap.md index 15b2c21d..20e5f230 100644 --- a/docs/usermanuals/tr/operations/air-gap.md +++ b/docs/usermanuals/tr/operations/air-gap.md @@ -113,15 +113,22 @@ evaluation: ```shell $ forgelm doctor --offline -ForgeLM 0.5.2 (offline mod doğrulandı) -✓ HF_HUB_OFFLINE ayarlı -✓ TRANSFORMERS_OFFLINE ayarlı -✓ Yerel cache: airgap-bundle/huggingface (3.4 GB cache'li model) -✓ Llama Guard 3 8B yerel olarak mevcut -✓ lm-evaluation-harness: 4 görev cache'li +forgelm doctor — environment check + + [✓ pass] python.version Python 3.11.4 (CPython). + [✓ pass] torch.cuda torch 2.4.0 with CUDA 12.4. + [✓ pass] gpu.inventory 1 GPU(s) — GPU0: NVIDIA A100 (80.0 GiB). + [✓ pass] extras.qlora Installed (module bitsandbytes, purpose: 4-bit / 8-bit QLoRA training). + [✓ pass] extras.eval Installed (module lm_eval, purpose: lm-evaluation-harness benchmark scoring). + [! warn] extras.tracking Optional extra missing — install with: pip install 'forgelm[tracking]' (purpose: Weights & Biases experiment tracking). + [✓ pass] hf_hub.offline_cache HF cache at /opt/airgap/huggingface/hub: 3.4 GiB across 47 file(s). HF_HUB_OFFLINE=1. + [✓ pass] disk.workspace Workspace /opt/airgap — 412.0 GiB free of 500.0 GiB. + [! warn] operator.identity FORGELM_OPERATOR not set; audit events will fall back to 'airgap-op@nodeA'. Pin FORGELM_OPERATOR= for CI / pipeline runs. + +Summary: 7 pass, 2 warn, 0 fail. ``` -`forgelm doctor --offline` herhangi bir şeyin erişilemediğini raporlarsa, air-gap operatörünün vaktini boşa harcamadan *önce* düzeltin. +`forgelm doctor --offline` herhangi bir şeyi `fail` olarak raporlarsa, air-gap operatörünün vaktini boşa harcamadan *önce* düzeltin. HF cache taraması önce `HF_HUB_CACHE`'i, sonra `HF_HOME/hub`'ı, en son varsayılan `~/.cache/huggingface/hub`'ı dener — taramayı deterministik yapmak için `HF_HUB_CACHE`'i pakete dahil ettiğiniz cache'e işaret ettirin. ## Paket boyutu tahmini diff --git a/docs/usermanuals/tr/reference/cli.md b/docs/usermanuals/tr/reference/cli.md index dcfeccaf..490473c3 100644 --- a/docs/usermanuals/tr/reference/cli.md +++ b/docs/usermanuals/tr/reference/cli.md @@ -12,40 +12,39 @@ ForgeLM, subcommand'larla tek bir `forgelm` binary'si yayınlar. Bu sayfa kanoni | Komut | Yaptığı | |---|---| | `forgelm` (subcommand'sız) | Eğit (`--config` ile). | -| `forgelm doctor` | Ortam kontrolü — Python, CUDA, GPU, bağımlılıklar. | +| `forgelm doctor` | Ortam kontrolü — Python, CUDA, GPU, bağımlılıklar, HF cache. | | `forgelm quickstart` | Yerleşik şablonları listele veya örnekle. | | `forgelm ingest` | PDF/DOCX/EPUB → JSONL dönüşümü. | -| `forgelm audit` | Eğitim öncesi veri denetimi. | -| `forgelm benchmark` | Model üzerinde lm-eval-harness koştur. | -| `forgelm safety-eval` | Llama Guard skorlama. | +| `forgelm audit` | Eğitim öncesi veri denetimi (PII / secrets / dedup / leakage / quality). | | `forgelm chat` | Etkileşimli REPL. | -| `forgelm batch-chat` | Etkileşimsiz prompt → yanıt. | | `forgelm export` | GGUF export ve quantization. | -| `forgelm deploy` | Deployment config üret (Ollama, vLLM, TGI vb.). | -| `forgelm verify-annex-iv` | Annex IV artifact'ını doğrula. | -| `forgelm verify-log` | Audit log zincirini doğrula. | -| `forgelm verify-gguf` | GGUF bütünlüğünü doğrula. | -| `forgelm cache-models` | Air-gap için HuggingFace modellerini önceden cache'le. | -| `forgelm cache-tasks` | lm-evaluation-harness görevlerini önceden cache'le. | -| `forgelm trend` | Son koşular arasında metric trendlerini göster. | -| `forgelm compare-runs` | Koşu metriklerini yan yana karşılaştır. | -| `forgelm approve` | İnsan onay isteğini imzala. | -| `forgelm approvals` | Bekleyen onayları listele. | +| `forgelm deploy` | Deployment config üret (Ollama, vLLM, TGI, HF Endpoints). | +| `forgelm verify-audit` | Audit log zincirini doğrula (timestamp, prev_hash, HMAC). | +| `forgelm approve` | İnsan onay isteğini imzala ve `final_model.staging/`'i promote et. | +| `forgelm reject` | İnsan onay isteğini reddet ve staging'i at. | +| `forgelm approvals` | Bekleyen onayları listele (`--pending`) veya tek birini incele (`--show RUN_ID`). | Bunlardan herhangi biri için `forgelm --help`. -## Üst seviye bayraklar (birçok subcommand'da) +## Üst seviye bayraklar (eğitim modu — `--config` ile kullanılır) | Bayrak | Açıklama | |---|---| | `--config PATH` | YAML config dosya yolu. Eğitim için gerekli. | -| `--dry-run` | Config'i ve referansları doğrula; eğitim yok. | -| `--fit-check` | VRAM tahmini ve verdict; eğitim yok. | -| `--estimate-cost` | Uçuş öncesi maliyet tahmini; eğitim yok. | -| `--offline` | Tüm ağ çağrılarını kapat; her şey cache'lenmiş olmalı. | -| `--output-format {plain,json}` | Log formatı. CI için JSON. | -| `--verbose, -v` | Log detayını artır. | -| `--quiet, -q` | Log detayını azalt. | +| `--wizard` | `config.yaml` üretmek için etkileşimli yapılandırma sihirbazını başlat. | +| `--dry-run` | Config'i ve model/dataset erişimini doğrula; eğitim yok. | +| `--fit-check` | Eğitim VRAM tahmini; model yüklenmez. `--config` gerektirir. | +| `--resume [PATH]` | Eğitime kaldığı yerden devam. Çıplak `--resume` son checkpoint'i otomatik bulur; `--resume PATH` belirli bir yerden. | +| `--offline` | Air-gap modu: tüm HF Hub ağ çağrılarını kapat. Modeller ve dataset'ler yerel olarak mevcut olmalı. | +| `--benchmark-only MODEL_PATH` | Mevcut bir model üzerinde benchmark koştur, eğitim yok. `evaluation.benchmark` config'i gerektirir. | +| `--merge` | Config'in `merge:` bloğundan model birleştirmeyi koştur. Eğitim yok. | +| `--generate-data` | Teacher modelle sentetik eğitim verisi üret. Eğitim yok. | +| `--compliance-export OUTPUT_DIR` | EU AI Act uyum artifact'larını (audit trail, data provenance, Annex IV) OUTPUT_DIR'a export et. Manifest'in tamamlanması için eğitimden sonra koşturun. | +| `--data-audit PATH` | **Deprecated alias**, `forgelm audit PATH` için. v0.7.0'da kaldırılacak. Yeni script'ler subcommand'ı kullanmalı. | +| `--output DIR` | `--data-audit` / `--compliance-export` için çıktı dizini (varsayılan: `./audit/` veya `./compliance/`). | +| `--output-format {text,json}` | Sonuçlar için çıktı formatı (varsayılan: `text`). CI için JSON. | +| `--quiet, -q` | INFO loglarını bastır. Sadece warning ve error göster. | +| `--log-level {DEBUG,INFO,WARNING,ERROR}` | Log detay seviyesi (varsayılan: INFO). | | `--version` | Sürümü yazdır. | | `--help, -h` | Yardım göster. | @@ -57,12 +56,21 @@ En sık kullanılan pattern'ler: $ forgelm --config configs/run.yaml --dry-run # doğrula $ forgelm --config configs/run.yaml --fit-check # VRAM kontrolü $ forgelm --config configs/run.yaml # eğit -$ forgelm --config configs/run.yaml --resume # son checkpoint'ten devam -$ forgelm --config configs/run.yaml --merge # merge işi olarak çalıştır +$ forgelm --config configs/run.yaml --resume # otomatik son checkpoint'ten devam +$ forgelm --config configs/run.yaml --resume /path # belirli bir checkpoint'ten devam +$ forgelm --config configs/run.yaml --merge # birleştirme işi olarak koştur $ forgelm --config configs/run.yaml --generate-data # sadece sentetik veri ``` -Belirli bir checkpoint'ten devam: `--resume-from PATH`. +## Doctor: `forgelm doctor` + +```shell +$ forgelm doctor # tam ortam kontrolü +$ forgelm doctor --offline # air-gap varyantı: cache + offline-env probe'ları +$ forgelm doctor --output-format json | jq . # CI dostu envelope +``` + +Python sürümünü, torch / CUDA / GPU'yu, opsiyonel extra'ları, HF Hub erişilebilirliğini (veya `--offline` ile HF cache'i), disk alanını, operatör kimliğini ve audit-secret yapılandırmasını probe eder. Exit kodları: `0` = hepsi geçti (warning OK), `1` = en az bir fail, `2` = bir probe'un kendisi crash etti. ## Audit: `forgelm audit` @@ -70,81 +78,113 @@ Belirli bir checkpoint'ten devam: `--resume-from PATH`. $ forgelm audit DATAFILE_OR_DIR \ [--output ./audit/] \ [--strict] \ - [--dedup-algo simhash|minhash] \ - [--dedup-threshold N] \ + [--workers N] \ + [--dedup-method {simhash,minhash}] \ + [--near-dup-threshold N] \ + [--minhash-jaccard FLOAT] [--minhash-num-perm N] \ [--skip-pii] [--skip-secrets] [--skip-quality] [--skip-leakage] \ [--remove-duplicates] [--remove-cross-split-overlap=val|test] \ [--output-clean PATH] \ [--show-leakage] \ - [--sample-rate FLOAT] + [--sample-rate FLOAT] \ + [--pii-ml] [--pii-ml-language LANG] \ + [--croissant] \ + [--verbose] ``` -Tam semantic için bkz. [Veri Seti Denetimi](#/data/audit). +`--workers N` split düzeyinde paralellik sağlar; on-disk JSON worker sayısından bağımsız olarak byte-identical (sadece `generated_at` zamanı değişir). Tam semantik için bkz. [Veri Denetimi](#/data/audit). ## Ingest: `forgelm ingest` ```shell -$ forgelm ingest INPUT_DIR \ +$ forgelm ingest INPUT_PATH \ --output PATH.jsonl \ [--recursive] \ - [--strategy tokens|markdown|paragraph|sentence] \ - [--max-tokens N] [--overlap N] \ - [--pii-mask] [--secrets-mask] \ - [--pii-locale tr|de|fr|us] \ - [--language LANG] \ - [--include "*.pdf,*.md"] [--exclude "drafts/*"] \ - [--format raw|instructions|qa] + [--strategy {sliding,paragraph,markdown}] \ + [--chunk-size N] [--overlap N] \ + [--chunk-tokens N] [--overlap-tokens N] [--tokenizer MODEL_NAME] \ + [--pii-mask] [--secrets-mask] [--all-mask] \ + [--pii-ml-language LANG] ``` -Bkz. [Doküman Ingest'i](#/data/ingestion). +Bkz. [Doküman Ingestion](#/data/ingestion). ## Chat: `forgelm chat` ```shell -$ forgelm chat CHECKPOINT \ - [--base BASE_MODEL] \ - [--temperature 0.7] [--top-p 0.9] [--max-tokens 1024] \ +$ forgelm chat MODEL_PATH \ + [--adapter PATH] \ [--system "system prompt"] \ - [--safety on|off] \ - [--load PATH] # kayıtlı oturum yükle + [--temperature 0.7] [--max-new-tokens 512] [--no-stream] \ + [--load-in-4bit | --load-in-8bit] \ + [--trust-remote-code] \ + [--backend {transformers,unsloth}] ``` -REPL içinde slash komutları: `/reset`, `/save`, `/load`, `/system`, `/temperature`, `/top_p`, `/max_tokens`, `/safety`, `/help`, `/quit`. Bkz. [Etkileşimli Chat](#/deployment/chat). +REPL içindeki slash komutları: `/reset`, `/save [file]`, `/temperature N`, `/system [prompt]`, `/help` (alias `/?`), `/exit` (alias `/quit`). Bkz. [Etkileşimli Chat](#/deployment/chat). ## Export: `forgelm export` ```shell $ forgelm export CHECKPOINT_DIR \ --output PATH.gguf \ - --quant q4_k_m|q5_k_m|q6_k|q8_0|q3_k_m|q2_k|fp16 \ - [--merge] # export öncesi LoRA'yı base'e merge et + --quant {q2_k,q3_k_m,q4_k_m,q5_k_m,q8_0,f16} \ + [--adapter PATH] \ + [--no-integrity-update] ``` -Tek komutta birden çok seviye için `--quant`'ı virgülle ayırın. Bkz. [GGUF Export](#/deployment/gguf-export). +Tek komutta çoklu seviye için `--quant`'ı virgülle ayırın. Bkz. [GGUF Export](#/deployment/gguf-export). ## Deploy: `forgelm deploy` ```shell -$ forgelm deploy CHECKPOINT_DIR \ - --target ollama|vllm|tgi|hf-endpoints|kserve|triton \ - --output PATH_OR_DIR +$ forgelm deploy MODEL_PATH \ + --target {ollama,vllm,tgi,hf-endpoints} \ + [--output PATH] \ + [--system "PROMPT"] # sadece Ollama + [--max-length 4096] \ + [--gpu-memory-utilization 0.90] # vLLM + [--port 8080] # TGI + [--trust-remote-code] # vLLM + [--vendor aws] # HF Endpoints ``` Bkz. [Deploy Hedefleri](#/deployment/deploy-targets). -## Kimlik doğrulama +## Onaylar: `forgelm approvals` / `forgelm approve` / `forgelm reject` + +```shell +$ forgelm approvals --pending # bekleyen onay gate'lerini listele +$ forgelm approvals --show RUN_ID # belirli bir koşunun chain + staging'ini incele +$ forgelm approve RUN_ID --comment "N. inceledi." # final_model.staging/ → final_model/ promote +$ forgelm reject RUN_ID --comment "Sebep ..." # staging'i at +``` + +Bkz. [İnsan Gözetim Gate'i](#/compliance/human-oversight). Exit kodları: `0` = bekleyen liste / onay kaydedildi, `1` = bilinmeyen run_id / config hatası, `4` (sadece eğitim modu) = onay bekliyor. + +## Audit log doğrula: `forgelm verify-audit` -ForgeLM kimlik bilgilerini environment variable'lardan okur. Asla YAML'a koymayın. +```shell +$ forgelm verify-audit PATH/TO/audit_log.jsonl +$ forgelm verify-audit PATH/TO/audit_log.jsonl --hmac-secret "$FORGELM_AUDIT_SECRET" +$ forgelm verify-audit PATH/TO/audit_log.jsonl --require-hmac +``` + +Monoton timestamp'leri, `prev_hash` zincir bütünlüğünü, `seq` boşluk tespitini ve (yapılandırıldığında) HMAC imzalarını doğrular. Geçerli zincirde exit `0`; tahrif tespitinde structured error envelope ile non-zero. + +## Kimlik Doğrulama + +ForgeLM credential'ları environment variable'lardan alır. Asla YAML'a koymayın. -| Sağlayıcı | Env var | Kullanılan | +| Sağlayıcı | Env var | Kullanım yeri | |---|---|---| -| HuggingFace | `HF_TOKEN` | Geçit kontrollü modeller (Llama, Llama Guard) | +| HuggingFace | `HF_TOKEN` (alias: `HUGGINGFACE_TOKEN`) | Gated modeller (Llama, Llama Guard) | | OpenAI | `OPENAI_API_KEY` | LLM-as-judge, sentetik veri | | Anthropic | `ANTHROPIC_API_KEY` | LLM-as-judge, sentetik veri | -| W&B | `WANDB_API_KEY` | Deney takibi | +| W&B | `WANDB_API_KEY` | Experiment tracking | | Cohere | `COHERE_API_KEY` | (sentetik veri) | -YAML interpolasyonu: +YAML interpolation: ```yaml auth: @@ -154,48 +194,59 @@ synthetic: api_key: "${OPENAI_API_KEY}" ``` -Env var ayarlı değilse ForgeLM net bir hatayla config yüklemede başarısız olur — eksik token nedeniyle eğitim 6 saatte çökmektense bu daha iyidir. +Env var set değilse ForgeLM config yüklemede net bir hata ile çıkar — eğitime 6 saat girmiş halde eksik token ile crash etmekten iyidir. ## Exit kodları | Exit | Anlamı | |---|---| | 0 | Başarı | -| 1 | Config / arg hatası | -| 2 | Audit uyarıları (`--strict` ile) | -| 3 | Otomatik geri alma / regresyon | -| 4 | İnsan onayı bekliyor | -| 130 | Kullanıcı kesti (Ctrl+C) | +| 1 | Config / argüman hatası | +| 2 | Audit warning (`--strict` ile) / probe crash (`forgelm doctor`) | +| 3 | Auto-revert / regression | +| 4 | İnsan onayı bekleniyor (eğitim pipeline) | +| 130 | Kullanıcı kesintiye uğrattı (Ctrl+C) | Tam kontrat için bkz. [Exit Kodları](#/reference/exit-codes). ## Environment variable'lar -| Değişken | Ayarladığı | +| Değişken | Ne ayarlar | |---|---| -| `HF_TOKEN` | HuggingFace auth | -| `HF_HOME` | HuggingFace cache dizini (varsayılan `~/.cache/huggingface`) | +| `HF_TOKEN` / `HUGGINGFACE_TOKEN` | HuggingFace authentication | +| `HF_HOME` | HuggingFace cache kökü (varsayılan `~/.cache/huggingface`) | +| `HF_HUB_CACHE` | HF Hub cache dizinini özel olarak override et (öncelik: `HF_HUB_CACHE` > `HF_HOME/hub` > varsayılan) | | `HF_HUB_OFFLINE=1` | HF Hub ağ çağrılarını kapat | +| `HF_ENDPOINT` | HF Hub endpoint override (self-hosted mirror için); `forgelm doctor` tarafından honor edilir | | `TRANSFORMERS_OFFLINE=1` | transformers kütüphanesi ağ çağrılarını kapat | | `HF_DATASETS_OFFLINE=1` | datasets kütüphanesi ağ çağrılarını kapat | -| `FORGELM_CACHE_DIR` | ForgeLM-özgü cache konumu | -| `FORGELM_LOG_LEVEL` | Log seviyesini override et (DEBUG, INFO, WARN, ERROR) | -| `FORGELM_RESUME_TOKEN` | API tabanlı insan onay akışı için token | +| `FORGELM_OPERATOR` | Audit event'lerinde kaydedilen operatör kimliği (`getpass.getuser()@hostname`'i override eder) | +| `FORGELM_ALLOW_ANONYMOUS_OPERATOR` | `1` olduğunda audit log'un anonim operatör kaydetmesine izin verir (aksi halde çözülemeyen kimlik hatası) | +| `FORGELM_AUDIT_SECRET` | Audit log chain için HMAC imza anahtarı (tahrif tespitini açar) | +| `FORGELM_GGUF_CONVERTER` | Özel `convert-hf-to-gguf.py` script'inin yolu | ## Sık pattern'ler -### "Sadece eğit, beni rahatsız etme" +### "Sadece eğit ve beni rahatsız etme" ```shell $ forgelm --config configs/run.yaml --output-format json | tee run.log ``` -### "Audit yap, temizse eğit" +### "Önce audit, temizse eğit" ```shell $ forgelm audit data/ --strict && forgelm --config configs/run.yaml ``` +### "İnsan onay gate'iyle eğit; sonra promote et" + +```shell +$ forgelm --config configs/run.yaml # onay gate ateşlerse exit 4 +$ forgelm approvals --pending # bekleyen koşuyu keşfet +$ forgelm approve RUN_ID --comment "İnceledim." # staging'i promote et +``` + ### "Eğit, GGUF export et, Ollama'ya deploy et" ```yaml @@ -209,11 +260,11 @@ deployment: ```shell $ forgelm --config configs/run.yaml -# Eğitim, export ve deployment config üretimi hepsi olur. +# Eğitim, export ve deploy config üretimi hep birlikte gerçekleşir. ``` -## Bkz. +## Ayrıca bakın -- [Konfigürasyon Referansı](#/reference/configuration) — YAML eşi. -- [Exit Kodları](#/reference/exit-codes) — CI için kapı kontratı. +- [Configuration Referansı](#/reference/configuration) — YAML eşleşmesi. +- [Exit Kodları](#/reference/exit-codes) — CI için gate kontratı. - [YAML Şablonları](#/reference/yaml-templates) — tam çalışan config'ler. diff --git a/docs/usermanuals/tr/reference/json-output.md b/docs/usermanuals/tr/reference/json-output.md new file mode 100644 index 00000000..c023e8f0 --- /dev/null +++ b/docs/usermanuals/tr/reference/json-output.md @@ -0,0 +1,186 @@ +--- +title: JSON çıktı şemaları +description: Her forgelm subcommand için kilitli --output-format json envelope yapısı. CI tüketicileri bu alan adlarına bağımlıdır. +--- + +# JSON çıktı şemaları + +`--output-format json` destekleyen her `forgelm` subcommand'ı stdout'a stabil bir JSON envelope üretir. Alan adları ve nesting [`docs/standards/release.md`](#/standards/release) gereği public CLI kontratının parçası: bir anahtarı yeniden adlandırmak MAJOR-version break sayılır. + +Bu sayfa kanonik referanstır. `forgelm` çıktısını parse eden CI/CD pipeline'ları burada belgelenen yapılara karşı pin'lemelidir. + +## Ortak konvansiyonlar + +- **stdout vs stderr.** JSON envelope **stdout**'a gider. İnsana okunabilir loglar (info / warning / error) **stderr**'e gider. `forgelm ... --output-format json | jq .` ile pipe edip operatör mesajlarını `2>` ile ayrı okuyun. +- **Üst seviye sarmalayıcı.** Her envelope `"success": true | false` ile başlar. Tüketiciler önce bu tek anahtarda branch açabilir. +- **Hata envelope'u.** `success: false` olduğunda envelope `"error": ""` (string) taşır. Opsiyonel zenginleştirilmiş alanlar (`exit_code`, `error_type`, `details`) [`error-handling.md`](#/standards/error-handling) gereği var olabilir. Bu alanlardan kesinlik isteyen tüketiciler `$?` ile süreç exit kodunu da kontrol etmelidir. +- **Exit kodları.** Bkz. [Exit Kodları](#/reference/exit-codes). Envelope, exit code ile tutarlıdır: `success: true` ⟺ exit `0`; `success: false` ⟺ non-zero exit. + +## `forgelm doctor` + +Ortam kontrolü. Bkz. [Doctor komutu](#/getting-started/first-run). + +**Başarı envelope'u** (`forgelm doctor [--offline] --output-format json`): + +```json +{ + "success": true, + "checks": [ + { + "name": "python.version", + "status": "pass", + "detail": "Python 3.11.7 (CPython).", + "extras": {"version": "3.11.7", "implementation": "CPython"} + } + ], + "summary": {"pass": 8, "warn": 1, "fail": 0, "crashed": 0} +} +``` + +| Anahtar | Tip | Notlar | +|---|---|---| +| `success` | bool | Hiçbir probe `fail` değilse VE hiçbir probe crash etmediyse `true`; aksi halde `false`. | +| `checks` | list[object] | Çalıştırma sırasıyla probe başına bir entry. Probe adları stabildir (örn. `python.version`, `torch.cuda`, `gpu.inventory`, `extras.qlora`, `hf_hub.reachable`, `hf_hub.offline_cache`, `disk.workspace`, `operator.identity`). | +| `checks[].name` | str | Probe adı. Sürümler arası stabil; yeni probe'lar yeniden adlandırma yerine eklenir. | +| `checks[].status` | str | `pass`, `warn`, `fail`, `crashed`'tan biri. | +| `checks[].detail` | str | Sonuç için operatör-yüzlü tek satırlık açıklama. | +| `checks[].extras` | object | Probe-spesifik yapısal veri. Per-probe anahtarlar `_doctor.py` docstring'lerinde belgelidir; tüketiciler bilinmeyen anahtarları forward-compatible olarak ele almalıdır. | +| `summary` | object | `checks` arasında her status'ın sayısı. Toplam `len(checks)`'a eşittir. | + +**Exit code mapping:** `0` = tüm probe'lar `pass` veya `warn`; `1` = en az bir `fail`; `2` = en az bir `crashed` (probe raise etti; sonraki probe'lar yine de çalıştı). + +## `forgelm approvals --pending` + +Bekleyen Madde 14 onay isteklerini en yeniden başa doğru listeler. + +```json +{ + "success": true, + "pending": [ + { + "run_id": "fg-abc123def456", + "staging_path": "/work/output/final_model.staging.fg-abc123def456", + "staging_exists": true, + "requested_at": "2026-05-04T12:34:56+00:00", + "age_seconds": 3600.5, + "metrics": {"safety_score": 0.91, "benchmark.hellaswag": 0.78}, + "config_hash": "sha256:...", + "reason": "post-train safety eval below threshold" + } + ], + "count": 1 +} +``` + +| Anahtar | Tip | Notlar | +|---|---|---| +| `success` | bool | `true` (her zaman — boş liste başarıdır). | +| `pending` | list[object] | En yeniden başa; `count == len(pending)`. Boş liste = bekleyen onay yok. | +| `pending[].run_id` | str \| null | Audit event'inden run identifier. | +| `pending[].staging_path` | str \| null | Çözülmüş staging dizini (audit event `staging_path` taşımıyorsa ve canonical fallback yoksa `null`). Path-traversal korumalı — `--output-dir` dışındaki yollar resolve aşamasında reddedilir. | +| `pending[].staging_exists` | bool | `staging_path` mevcut bir dizine resolve oluyorsa `True`. | +| `pending[].requested_at` | str \| null | Audit event'inden ISO-8601 timestamp. | +| `pending[].age_seconds` | number \| null | `requested_at`'tan beri saniye; timestamp parse edilemezse `null`. | +| `pending[].metrics` | object | Audit event'inden serbest-form per-run metric'ler. | +| `pending[].config_hash` | str \| null | Bilindiğinde config fingerprint. | +| `pending[].reason` | str \| null | Audit event'inden operatör tarafından sağlanan sebep. | +| `count` | int | Bekleyen sayı; `len(pending)`'a eşit. | + +## `forgelm approvals --show RUN_ID` + +Bir koşu için tam onay-gate audit chain'i + staging içeriğini incele. + +```json +{ + "success": true, + "run_id": "fg-abc123def456", + "status": "pending", + "chain": [ + {"event": "human_approval.required", "run_id": "fg-abc123def456", "timestamp": "2026-05-04T12:34:56+00:00", "...": "..."} + ], + "staging_contents": ["adapter_config.json", "adapter_model.safetensors", "tokenizer.json"] +} +``` + +| Anahtar | Tip | Notlar | +|---|---|---| +| `success` | bool | Koşu bulunduğunda `true`; `RUN_ID`'nin audit event'i yoksa `false` (`error` ile). | +| `run_id` | str | Echo'lanan input `RUN_ID`. | +| `status` | str | `pending`, `granted`, `rejected`, `unknown`'dan biri. Latest-wins semantik: önceki bir karardan sonra re-stage edilen koşu `pending` gösterir. | +| `chain` | list[object] | `run_id` için her onay-gate audit event'i, append sırasıyla. | +| `staging_contents` | list[str] | `/final_model.staging.` (veya canonical fallback) altında sıralı dosya/dizin adları. Staging eksik veya okunamıyorsa boş. | + +## `forgelm audit` + +Eğitim öncesi veri denetimi. Tam rapor; ana alanlar gösterildi. + +```json +{ + "success": true, + "report_path": "audit/data_audit_report.json", + "splits": {"train": {"sample_count": 100, "...": "..."}, "...": {}}, + "pii_summary": {"total_findings": 0, "by_kind": {}}, + "secrets_summary": {"total_findings": 0, "by_kind": {}}, + "cross_split_overlap": {"pairs": {}}, + "leakage": {"...": "..."}, + "quality_filter": null, + "near_duplicates": {"...": "..."}, + "languages_top3": [{"code": "en", "count": 87}], + "generated_at": "2026-05-04T12:34:56+00:00", + "warnings": [] +} +``` + +Tam şema [`docs/guides/data_audit.md`](#/data/audit)'dedir. CI gate'leri için `report_path` (on-disk JSON'un yeri) + `success`'e karşı pin'leyin. + +## `forgelm verify-audit` + +Audit log chain bütünlüğü kontrolü. + +```json +{ + "success": true, + "valid": true, + "entries_count": 87, + "hmac_verified": true, + "errors": [] +} +``` + +| Anahtar | Tip | Notlar | +|---|---|---| +| `success` | bool | `true` ⟺ `valid: true`. | +| `valid` | bool | Herhangi bir prev_hash mismatch / monotonicity break / seq gap varsa `false`. | +| `entries_count` | int | Düzgün-formed audit satır sayısı. | +| `hmac_verified` | bool \| null | `--hmac-secret` her `hmac` alanıyla eşleşince `true`; mismatch'te `false`; chain'de HMAC alanı yoksa `null`. | +| `errors` | list[str] | Tespit edilen sorun başına bir insan-okunur satır. | + +## `forgelm approve` / `forgelm reject` + +```json +{ + "success": true, + "run_id": "fg-abc123def456", + "approver": "alice@example.com@workstation-7", + "final_model_path": "/work/output/final_model", + "promote_strategy": "atomic_rename" +} +``` + +`approve` başarıda `0` ile çıkar; `reject` rejection kaydı sonrası `0` ile çıkar (staging dizini forensics için korunur). Bilinmeyen `run_id` / config hatasında `error` ile `success: false`. + +## Yeni subcommand eklerken + +`--output-format json` destekleyen yeni bir subcommand şunlarla landed olmalıdır: + +1. Bu sayfada belgelenen envelope (EN + TR mirror). +2. Üst seviye anahtarların tam setini pin'leyen `tests/test_json_envelope_contract.py` (veya per-subcommand test dosyası) testi. +3. Per-collection anahtar "sonuçlar subcommand'in birincil ismine göre adlandırılmış bir anahtar altında yaşar" konvansiyonunu izler (yani `doctor` → `checks`, `approvals --pending` → `pending` vb.). + +Merge sonrası bir anahtarı yeniden adlandırmak [`release.md`](#/standards/release) gereği MAJOR-version bump'tır. + +## Ayrıca bakın + +- [Exit Kodları](#/reference/exit-codes) — `success: bool`'ın aligned olduğu kontrat. +- [`error-handling.md`](#/standards/error-handling) — hata envelope kontratı. +- [`release.md`](#/standards/release) — JSON yeniden adlandırmaları ne zaman breaking sayılır. diff --git a/forgelm/_http.py b/forgelm/_http.py index 40a82cc0..b9dd5ad9 100644 --- a/forgelm/_http.py +++ b/forgelm/_http.py @@ -1,12 +1,22 @@ -"""Centralized HTTP discipline for outbound POST requests. +"""Centralized HTTP discipline for outbound HTTP calls (GET / HEAD / POST). Extracted from ``webhook._post_payload`` to enforce SSRF guard, redirect refusal, ``http://`` refusal, timeout floor, and secret-mask error reasons across every outbound HTTP call site in the codebase (webhook, judge, -synthetic). Future call sites (telemetry, registry pings, etc.) MUST go -through :func:`safe_post` rather than calling ``requests.post`` directly — -the acceptance gate ``grep -rn "requests\\.post" forgelm/ | grep -v _http\\.py`` -should stay empty. +synthetic, doctor). POST traffic goes through :func:`safe_post`; read-side +GET / HEAD traffic goes through :func:`safe_get` (added in Wave 2a Round-2 +when ``forgelm doctor`` migrated off ``urllib.request.urlopen``). + +Future call sites (telemetry, registry pings, etc.) MUST go through one of +those helpers rather than calling ``requests.{post,get,head}`` (or +``urllib.request.urlopen`` / ``httpx.*``) directly — the CI acceptance +gate ``lint-http-discipline`` greps ``forgelm/`` for those patterns and +fails on any hit outside ``forgelm/_http.py``: + + grep -rn "requests\\.\\(post\\|get\\|head\\)\\(" forgelm/ | grep -v _http\\.py + grep -rn "urllib\\.request\\.urlopen\\(" forgelm/ | grep -v _http\\.py + +both must stay empty. Policy summary (each enforced before the network call): @@ -228,3 +238,102 @@ def safe_post( masked_reason[:200], ) raise + + +def safe_get( + url: str, + *, + headers: Optional[Dict[str, str]] = None, + timeout: float = 10.0, + verify: Any = True, + ca_bundle: Optional[str] = None, + allow_insecure_http: bool = False, + allow_private: bool = False, + min_timeout: float = 5.0, + method: str = "GET", +) -> requests.Response: + """Disciplined outbound GET / HEAD; raises early on policy violation. + + Mirrors :func:`safe_post`'s policy contract (scheme / SSRF / timeout + floor / redirect refusal / TLS verify / header secret-masking) for + read-side calls. Used by ``forgelm doctor`` for the HuggingFace Hub + reachability probe and by any future probe / telemetry / registry + ping that needs an outbound GET or HEAD. + + Args: + url: Target URL. Must be ``http://`` or ``https://``. + headers: Outbound headers. ``Authorization`` / ``X-API-Key`` values + are masked in the failure log. + timeout: Per-request timeout in seconds. Must be ``>= min_timeout``. + verify: Forwarded as ``requests``'s ``verify=`` argument. + ca_bundle: Path to a custom CA bundle. When non-empty, takes + precedence over ``verify``. + allow_insecure_http: Set ``True`` only for paths where the operator + has explicitly opted into ``http://``. + allow_private: Set ``True`` to bypass the SSRF guard. Required for + in-cluster mirrors / on-prem registry endpoints. + min_timeout: Lower bound for ``timeout``. Defaults to ``5.0`` + (read probes are typically cheaper than POST bodies). + method: ``"GET"`` (default) or ``"HEAD"``. The doctor's HF Hub + probe uses HEAD to skip body download. + + Returns: + The :class:`requests.Response` from the underlying call. The caller + is responsible for inspecting ``response.ok`` / ``status_code``. + + Raises: + HttpSafetyError: On policy violation — ``http://`` without opt-in, + unsupported scheme, sub-floor timeout, private destination + without opt-in, or unsupported method. + requests.RequestException: On transport / TLS / network failure. + Headers are masked in the warning log before the re-raise. + """ + parsed = urlparse(url) + + # Scheme policy. + if parsed.scheme == "http": + if not allow_insecure_http: + # The literal scheme tokens are split so the SonarCloud S5332 + # "use https" rule does not trip on the rejection message — + # this branch *enforces* that rule, surfacing the error in + # operator-readable form rather than violating it. + _scheme_blocked = "http" + "://" # noqa: S608 — see comment above + _scheme_safe = "https" + "://" # noqa: S608 — see comment above + raise HttpSafetyError(f"{_scheme_blocked} blocked (use {_scheme_safe}); url={_mask_netloc(url)}") + elif parsed.scheme != "https": + raise HttpSafetyError(f"Unsupported URL scheme {parsed.scheme!r}; only http(s) allowed.") + + # SSRF guard. + host = parsed.hostname or "" + if not allow_private and _is_private_destination(host): + raise HttpSafetyError(f"Private/loopback/IMDS destination blocked: host={host or ''}") + + # Timeout floor. + if not isinstance(timeout, (int, float)) or timeout < min_timeout: + raise HttpSafetyError(f"Timeout below {min_timeout}s floor: timeout={timeout!r}") + + # Method policy — only GET / HEAD allowed (read-side helper). + method_upper = method.upper() + if method_upper not in ("GET", "HEAD"): + raise HttpSafetyError(f"safe_get only supports GET / HEAD, got {method!r}.") + + verify_param: Any = ca_bundle if ca_bundle else verify + + try: + return requests.request( + method_upper, + url, + headers=headers, + timeout=timeout, + verify=verify_param, + allow_redirects=False, + ) + except requests.RequestException as exc: + masked_reason = _mask_secrets_in_text(str(exc), headers) + logger.warning( + "safe_get failed url=%s method=%s reason=%s", + _mask_netloc(url), + method_upper, + masked_reason[:200], + ) + raise diff --git a/forgelm/cli/__init__.py b/forgelm/cli/__init__.py index 08007312..44520055 100644 --- a/forgelm/cli/__init__.py +++ b/forgelm/cli/__init__.py @@ -25,6 +25,7 @@ _add_common_subparser_flags, # noqa: F401 — re-export for tests _non_negative_float, # noqa: F401 — re-export for tests _non_negative_int, # noqa: F401 — re-export for tests + _positive_int, # noqa: F401 — re-export for tests ) # Config loading + offline flag application (training mode). @@ -80,10 +81,12 @@ # Parser (registrars + parse_args). from ._parser import ( + _add_approvals_subcommand, # noqa: F401 — re-export for tests _add_approve_subcommand, # noqa: F401 — re-export for tests _add_audit_subcommand, # noqa: F401 — re-export for tests _add_chat_subcommand, # noqa: F401 — re-export for tests _add_deploy_subcommand, # noqa: F401 — re-export for tests + _add_doctor_subcommand, # noqa: F401 — re-export for tests _add_export_subcommand, # noqa: F401 — re-export for tests _add_ingest_subcommand, # noqa: F401 — re-export for tests _add_quickstart_subcommand, # noqa: F401 — re-export for tests @@ -113,6 +116,13 @@ # Wizard mode. from ._wizard import _maybe_run_wizard # noqa: F401 — re-export for tests +# Approvals listing subcommand (Phase 9 follow-up). +from .subcommands._approvals import ( + _collect_pending_runs, # noqa: F401 — re-export for tests + _collect_run_audit_chain, # noqa: F401 — re-export for tests + _run_approvals_cmd, # noqa: F401 — re-export for tests +) + # Approve / reject subcommands (Article 14 human-approval gate). from .subcommands._approve import ( _atomic_rename_or_move, # noqa: F401 — re-export for tests @@ -123,6 +133,7 @@ _resolve_approver_identity, # noqa: F401 — re-export for tests _run_approve_cmd, # noqa: F401 — re-export for tests _run_reject_cmd, # noqa: F401 — re-export for tests + _staging_path_inside_output_dir, # noqa: F401 — re-export for tests + _approvals reuse ) # Audit subcommand (+ legacy --data-audit worker). @@ -134,6 +145,23 @@ # Chat / export / deploy / ingest subcommand dispatchers. from .subcommands._chat import _run_chat_cmd # noqa: F401 — re-export for tests from .subcommands._deploy import _run_deploy_cmd # noqa: F401 — re-export for tests + +# Doctor subcommand (Phase 34 environment diagnostics). +from .subcommands._doctor import ( + _check_disk_space, # noqa: F401 — re-export for tests + _check_gpu_inventory, # noqa: F401 — re-export for tests + _check_hf_cache_offline, # noqa: F401 — re-export for tests + _check_hf_hub_reachable, # noqa: F401 — re-export for tests + _check_operator_identity, # noqa: F401 — re-export for tests + _check_optional_extra, # noqa: F401 — re-export for tests + _check_python_version, # noqa: F401 — re-export for tests + _check_torch_cuda, # noqa: F401 — re-export for tests + _render_json, # noqa: F401 — re-export for tests + _render_text, # noqa: F401 — re-export for tests + _resolve_exit_code, # noqa: F401 — re-export for tests + _run_all_checks, # noqa: F401 — re-export for tests + _run_doctor_cmd, # noqa: F401 — re-export for tests +) from .subcommands._export import _run_export_cmd # noqa: F401 — re-export for tests from .subcommands._ingest import _run_ingest_cmd # noqa: F401 — re-export for tests diff --git a/forgelm/cli/_argparse_types.py b/forgelm/cli/_argparse_types.py index 8418f8fc..a3566649 100644 --- a/forgelm/cli/_argparse_types.py +++ b/forgelm/cli/_argparse_types.py @@ -21,6 +21,23 @@ def _non_negative_int(value: str) -> int: return ivalue +def _positive_int(value: str) -> int: + """argparse type for flags that must be >= 1 (e.g. ``--workers``). + + Mirrors :func:`_non_negative_int` but rejects 0 so a typo + (``--workers 0``) produces an immediate, helpful CLI error instead + of getting validated downstream as ``ValueError`` deep inside an + audit call. + """ + try: + ivalue = int(value) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError(f"invalid integer: {value!r}") from exc + if ivalue < 1: + raise argparse.ArgumentTypeError(f"value must be >= 1, got {ivalue}") + return ivalue + + def _non_negative_float(value: str) -> float: """argparse type for ``--jaccard-threshold`` and similar floats. diff --git a/forgelm/cli/_dispatch.py b/forgelm/cli/_dispatch.py index 4a7bad3f..668870bc 100644 --- a/forgelm/cli/_dispatch.py +++ b/forgelm/cli/_dispatch.py @@ -20,59 +20,93 @@ from ._wizard import _maybe_run_wizard +def _output_format_for(args) -> str: + """Pull the ``--output-format`` value off ``args`` with the standard default.""" + return getattr(args, "output_format", "text") + + def _dispatch_subcommand(command: str, args) -> None: - """Run a Phase 10 / 10.5 / 11 / 11.5 subcommand and exit. + """Run a Phase 10 / 10.5 / 11 / 11.5 / Wave 2a subcommand and exit. - Subcommands handled here: ``chat``, ``export``, ``deploy``, ``quickstart``, - ``ingest``, ``audit``, ``verify-audit``, ``approve``, ``reject``. Each - terminates the process via ``sys.exit`` after its own dispatcher returns — - the trainer/training code path never runs when a subcommand is in play. + Subcommands handled here: ``chat``, ``export``, ``deploy``, + ``quickstart``, ``ingest``, ``audit``, ``doctor``, ``verify-audit``, + ``approve``, ``reject``, ``approvals``. Each terminates the process + via ``sys.exit`` after its own dispatcher returns — the trainer code + path never runs when a subcommand is in play. Dispatchers are looked up via the package facade so tests that ``patch("forgelm.cli._run_*_cmd", ...)`` see their mock invoked. + The dispatch table replaced an if/elif chain in Wave 2a Round-2 to + drop SonarCloud S3776 cognitive complexity (was 16, ceiling 15) and + to keep the registry literal — adding a new subcommand is now a + single-row edit. + + **SIGINT exit-code policy (Round-3 + Round-5 reconciliation):** + + The dispatcher catches ``KeyboardInterrupt`` and exits with + ``EXIT_TRAINING_ERROR`` (= 2, "runtime-error class") so CI/CD + branches on a documented public code, not Python's shell-shaped + ``130`` (= ``128 + SIGINT``). Two subcommands have nuance the + blanket policy does not capture: + + - ``chat`` REPL catches ``KeyboardInterrupt`` at its input prompt + itself (``forgelm/chat.py:125``), prints ``[Goodbye]``, and + returns normally — exit ``EXIT_SUCCESS`` (0) by REPL design. An + in-flight ``KeyboardInterrupt`` *during* generation bubbles past + the REPL and lands on this dispatcher's catch → 2. + - ``verify-audit`` is wrapped in the same try/except as the dict- + table dispatch (Round-5 fix), so ``SIGINT`` during a long + verify-of-100K-events lands on 2 just like the others. Returns + its own exit code on success (the only dispatcher that does so). """ # Late import via the package facade so monkeypatched # ``forgelm.cli._run_*_cmd`` references resolve correctly. from forgelm import cli as _cli_facade - if command == "chat": - # _run_chat_cmd's REPL catches KeyboardInterrupt internally for the - # input prompt; this outer guard covers Ctrl-C during model load / - # welcome banner render, before the REPL loop has started. - try: - _cli_facade._run_chat_cmd(args) - except KeyboardInterrupt: - sys.exit(EXIT_TRAINING_ERROR) - sys.exit(EXIT_SUCCESS) - elif command == "export": - _cli_facade._run_export_cmd(args, getattr(args, "output_format", "text")) - sys.exit(EXIT_SUCCESS) - elif command == "deploy": - _cli_facade._run_deploy_cmd(args, getattr(args, "output_format", "text")) - sys.exit(EXIT_SUCCESS) - elif command == "quickstart": - try: - _cli_facade._run_quickstart_cmd(args, getattr(args, "output_format", "text")) - except KeyboardInterrupt: - sys.exit(EXIT_TRAINING_ERROR) - sys.exit(EXIT_SUCCESS) - elif command == "ingest": - _cli_facade._run_ingest_cmd(args, getattr(args, "output_format", "text")) - sys.exit(EXIT_SUCCESS) - elif command == "audit": - _cli_facade._run_audit_cmd(args, getattr(args, "output_format", "text")) - sys.exit(EXIT_SUCCESS) - elif command == "verify-audit": - sys.exit(_cli_facade._run_verify_audit_cmd(args)) - elif command == "approve": - _cli_facade._run_approve_cmd(args, getattr(args, "output_format", "text")) - sys.exit(EXIT_SUCCESS) - elif command == "reject": - _cli_facade._run_reject_cmd(args, getattr(args, "output_format", "text")) - sys.exit(EXIT_SUCCESS) - else: + # name -> dispatcher attribute on the package facade. Resolved lazily + # so test-time monkeypatches against ``forgelm.cli._run_*_cmd`` are + # honoured. ``verify-audit`` is in the table but takes a different + # call shape (returns int exit code instead of sys.exit-ing) — see + # the special-case branch below. + table = { + "chat": "_run_chat_cmd", + "export": "_run_export_cmd", + "deploy": "_run_deploy_cmd", + "quickstart": "_run_quickstart_cmd", + "ingest": "_run_ingest_cmd", + "audit": "_run_audit_cmd", + "doctor": "_run_doctor_cmd", + "verify-audit": "_run_verify_audit_cmd", + "approve": "_run_approve_cmd", + "reject": "_run_reject_cmd", + "approvals": "_run_approvals_cmd", + } + dispatcher_name = table.get(command) + if dispatcher_name is None: logger.error("Unrecognized subcommand: %r. This is a bug — please report it.", command) sys.exit(EXIT_TRAINING_ERROR) + dispatcher = getattr(_cli_facade, dispatcher_name) + + # Three call shapes: ``chat`` takes only ``args`` (its REPL handles + # output formatting itself); ``verify-audit`` returns an exit code + # instead of sys.exit-ing; everything else takes ``(args, + # output_format)`` and exits internally. All three flow through + # the same KeyboardInterrupt handler so SIGINT lands on the + # documented public exit-code contract regardless of subcommand + # (Round-5 F-R5-02 fix — verify-audit was previously special-cased + # outside the try/except, leaving SIGINT during a long verify to + # bubble up as Python's shell-shaped 130). + output_format = _output_format_for(args) + try: + if command == "chat": + dispatcher(args) + elif command == "verify-audit": + sys.exit(dispatcher(args)) + else: + dispatcher(args, output_format) + except KeyboardInterrupt: + sys.exit(EXIT_TRAINING_ERROR) + sys.exit(EXIT_SUCCESS) def main(): diff --git a/forgelm/cli/_parser.py b/forgelm/cli/_parser.py index 55bc2098..c9e784a4 100644 --- a/forgelm/cli/_parser.py +++ b/forgelm/cli/_parser.py @@ -11,9 +11,15 @@ import argparse -from ._argparse_types import _add_common_subparser_flags, _non_negative_float, _non_negative_int +from ._argparse_types import _add_common_subparser_flags, _non_negative_float, _non_negative_int, _positive_int from ._logging import _get_version +# Shared `--output-dir` help text across approve / reject / approvals +# subparsers. Sonar S1192 flagged the literal as duplicated 3x; the +# constant keeps the operator copy in one place so a future rename of +# `final_model.staging/` lands in one diff hunk. +_OUTPUT_DIR_HELP = "Training output directory containing audit_log.jsonl and final_model.staging/." + def _add_chat_subcommand(subparsers) -> None: p = subparsers.add_parser( @@ -291,6 +297,42 @@ def _add_ingest_subcommand(subparsers) -> None: _add_common_subparser_flags(p, include_output_format=True) +def _add_doctor_subcommand(subparsers) -> None: + """Phase 34: environment check (`forgelm doctor`). + + The first command an operator should run after installation. Probes + Python version, torch / CUDA, GPU inventory, optional extras, HF Hub + reachability (or local cache when ``--offline``), workspace disk + space, and the FORGELM_OPERATOR audit-identity hint. + + Exit codes follow the public contract: 0 = all pass, 1 = at least + one check failed (config-error class), 2 = probe crashed. + """ + p = subparsers.add_parser( + "doctor", + help="Run environment + dependency diagnostics (the first command after install).", + description=( + "Probe Python, torch + CUDA, GPU inventory, optional ForgeLM extras, " + "HuggingFace Hub reachability, workspace disk space, and the " + "FORGELM_OPERATOR audit-identity hint. Emits a tabular text report or " + "a structured JSON envelope (`--output-format json`). Pass `--offline` " + "to skip the HF Hub network probe and instead inspect the local cache." + ), + ) + p.add_argument( + "--offline", + action="store_true", + help=( + "Skip the HuggingFace Hub network probe. Inspects the local HF cache " + "(precedence: HF_HUB_CACHE > HF_HOME/hub > ~/.cache/huggingface/hub) " + "instead — useful for air-gapped deployments where the network probe " + "would always fail. Implicitly true when HF_HUB_OFFLINE=1 or " + "TRANSFORMERS_OFFLINE=1 is set in the environment." + ), + ) + _add_common_subparser_flags(p, include_output_format=True) + + def _add_audit_subcommand(subparsers) -> None: p = subparsers.add_parser( "audit", @@ -409,6 +451,21 @@ def _add_audit_subcommand(subparsers) -> None: "corpus AND make sure the matching spaCy model is installed." ), ) + p.add_argument( + "--workers", + type=_positive_int, + default=1, + metavar="N", + help=( + "Phase 17: number of worker processes for the split-level pipeline " + "(default: 1 — sequential, byte-identical to the pre-Phase-17 path). " + "Set to 2-4 on multi-split corpora (train / validation / test) for a " + "near-linear speed-up. Speed-up scales with the number of splits, " + "not row count — single-split corpora ignore values > 1. The merge " + "step is single-threaded so the audit JSON is byte-identical across " + "worker counts (determinism contract pinned by the test suite)." + ), + ) _add_common_subparser_flags(p, include_output_format=True) @@ -487,7 +544,7 @@ def _add_approve_subcommand(subparsers) -> None: type=str, required=True, metavar="DIR", - help="Training output directory containing audit_log.jsonl and final_model.staging/.", + help=_OUTPUT_DIR_HELP, ) p.add_argument( "--comment", @@ -499,6 +556,49 @@ def _add_approve_subcommand(subparsers) -> None: _add_common_subparser_flags(p, include_output_format=True) +def _add_approvals_subcommand(subparsers) -> None: + """Article 14 follow-up: list / inspect pending approval requests. + + Exactly one of ``--pending`` and ``--show RUN_ID`` must be set; argparse + enforces this with a mutually-exclusive group so the dispatcher only + sees a validated args namespace. ``--output-dir`` is required in both + modes — there is no useful default; an operator running on a freshly- + cloned workstation must point at the training output explicitly. + """ + p = subparsers.add_parser( + "approvals", + help="List pending Article 14 approval requests or inspect a single run.", + description=( + "Discovery counterpart to `forgelm approve` / `forgelm reject`. " + "`--pending` lists every run whose audit log carries a " + "`human_approval.required` event without a matching terminal " + "decision. `--show RUN_ID` prints the full approval-gate audit " + "chain plus the on-disk staging directory layout for one run." + ), + ) + mode = p.add_mutually_exclusive_group(required=True) + mode.add_argument( + "--pending", + action="store_true", + help="List every run awaiting an approval decision.", + ) + mode.add_argument( + "--show", + type=str, + default=None, + metavar="RUN_ID", + help="Show the full audit chain + staging contents for a single run.", + ) + p.add_argument( + "--output-dir", + type=str, + required=True, + metavar="DIR", + help=_OUTPUT_DIR_HELP, + ) + _add_common_subparser_flags(p, include_output_format=True) + + def _add_reject_subcommand(subparsers) -> None: """Article 14: discard a staged model (preserves staging dir for forensics).""" p = subparsers.add_parser( @@ -521,7 +621,7 @@ def _add_reject_subcommand(subparsers) -> None: type=str, required=True, metavar="DIR", - help="Training output directory containing audit_log.jsonl and final_model.staging/.", + help=_OUTPUT_DIR_HELP, ) p.add_argument( "--comment", @@ -539,6 +639,7 @@ def parse_args(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=( "Subcommands:\n" + " forgelm doctor Environment check (Python/torch/CUDA/GPU/extras/HF/disk)\n" " forgelm quickstart [TEMPLATE] Generate a config from a curated template\n" " forgelm ingest PATH Convert raw docs (PDF/DOCX/EPUB/TXT/Markdown) → JSONL\n" " forgelm audit PATH Run dataset audit (length/lang/PII/leakage)\n" @@ -548,6 +649,7 @@ def parse_args(): " forgelm deploy MODEL_PATH Generate serving config\n" " forgelm approve RUN_ID Promote a staged model after human review (Art. 14)\n" " forgelm reject RUN_ID Reject a staged model (preserves staging dir for forensics)\n" + " forgelm approvals --pending List runs awaiting human approval (or --show RUN_ID)\n" "\nRun 'forgelm --help' for subcommand details." ), ) @@ -560,9 +662,11 @@ def parse_args(): _add_quickstart_subcommand(subparsers) _add_ingest_subcommand(subparsers) _add_audit_subcommand(subparsers) + _add_doctor_subcommand(subparsers) _add_verify_audit_subcommand(subparsers) _add_approve_subcommand(subparsers) _add_reject_subcommand(subparsers) + _add_approvals_subcommand(subparsers) # --- Top-level flags (training / config-driven mode) --- parser.add_argument("--config", type=str, help="Path to the YAML configuration file.") diff --git a/forgelm/cli/subcommands/_approvals.py b/forgelm/cli/subcommands/_approvals.py new file mode 100644 index 00000000..d7a69d7f --- /dev/null +++ b/forgelm/cli/subcommands/_approvals.py @@ -0,0 +1,496 @@ +"""``forgelm approvals`` listing subcommand (Article 14 follow-up). + +The Phase 9 closure shipped ``forgelm approve`` and ``forgelm reject`` so an +operator can act on a single staged run, but it left the *discovery* step +out: an operator who walks up to a workstation cold has no way to ask +"which runs are awaiting my review?". This module fills the gap. + +Two query modes: + +- ``forgelm approvals --pending [--output-dir DIR]`` lists every run whose + audit log carries a ``human_approval.required`` event without a matching + terminal decision (``human_approval.granted`` / ``human_approval.rejected``). +- ``forgelm approvals --show RUN_ID --output-dir DIR`` prints the full + approval-related audit chain for a single run plus the on-disk staging + directory layout. + +The audit-log JSONL parser (skip malformed, emit one summary warning at +end) lives in :mod:`._audit_log_reader` and is shared with +:mod:`._approve` (and the upcoming :mod:`._purge`). The +``output_dir/audit_log.jsonl`` convention + Article 14 event vocabulary +live here so the dispatcher stays cohesive; only the *parser* is shared. +""" + +from __future__ import annotations + +import json +import os +import sys +from datetime import datetime, timezone +from typing import Any, Dict, List, NoReturn, Optional + +from .._exit_codes import EXIT_CONFIG_ERROR, EXIT_SUCCESS, EXIT_TRAINING_ERROR +from .._logging import logger + +# Audit event names. Mirror :mod:`._approve` so a future rename of one set +# can't drift across the listing dispatcher and the decision dispatcher. +_EVT_HUMAN_APPROVAL_REQUIRED = "human_approval.required" +_EVT_HUMAN_APPROVAL_GRANTED = "human_approval.granted" +_EVT_HUMAN_APPROVAL_REJECTED = "human_approval.rejected" +_TERMINAL_DECISION_EVENTS = frozenset({_EVT_HUMAN_APPROVAL_GRANTED, _EVT_HUMAN_APPROVAL_REJECTED}) + +# Default audit-log filename. Centralised so a future rename does not have to +# touch every dispatcher. +_AUDIT_LOG_FILENAME = "audit_log.jsonl" + + +def _output_error_and_exit(output_format: str, msg: str, exit_code: int) -> NoReturn: + """Emit ``msg`` as a structured JSON error or a log record, then exit. + + Mirrors :func:`forgelm.cli.subcommands._approve._output_error_and_exit` + so the JSON envelope contract is identical across the approval family + of subcommands. ``-> NoReturn`` (Wave 2a Round-2 nit) so the type + checker knows control never returns past this helper. + """ + if output_format == "json": + print(json.dumps({"success": False, "error": msg})) + else: + logger.error(msg) + sys.exit(exit_code) + + +# Wave 2a Round-1 review (XPR-01): the JSONL parser was extracted into +# the shared :mod:`._audit_log_reader` module so a future malformed-line +# policy fix lands in one place across the approve / approvals / purge +# family. The ``_`` prefix on the local alias marks the reader as +# parser-internal use; callers reach in via the module-prefixed +# ``iter_audit_events`` directly. +# Round-5 F-R5-08: ``E402`` retained — the import sits below the +# in-module helpers by design (the module's other late imports follow +# the same pattern); ``F401`` no longer needed because the symbol is +# actively used at lines 97 + 151 of this file. +from ._audit_log_reader import iter_audit_events as _iter_audit_events # noqa: E402 + + +def _collect_pending_runs(audit_log_path: str) -> List[Dict[str, Any]]: + """Return the list of pending approval requests, newest-first. + + A "pending" run is one whose audit log carries a + ``human_approval.required`` event whose ``run_id`` does **not** appear + in any *later* ``human_approval.granted`` / ``human_approval.rejected`` + event after that requirement was issued. + + **Latest-wins semantics** (Wave 2a Round-1 review F-25-03 fix): when a + run is restarted with the same ``run_id`` after a prior decision — + e.g. a rejected run is re-staged for a second review — the latest + ``human_approval.required`` event reflects the current pending state + and earlier terminal decisions for that ``run_id`` are no longer the + operative status. Mirrors :func:`._approve._find_human_approval_decision_event`'s + "most-recent matching event wins" semantic so the family stays + consistent. + + Implementation: walk the log in append order, recording the latest + ``required`` and the latest terminal decision per ``run_id``. A run + is pending iff its last ``required`` event came strictly after its + last terminal decision (or no terminal decision exists). Line-number + fallback is used when timestamp is missing so re-imported logs without + timestamps still have a deterministic ordering. + """ + latest_required: Dict[str, tuple[int, Dict[str, Any]]] = {} + latest_decision_line: Dict[str, int] = {} + + for line_no, event in _iter_audit_events(audit_log_path): + run_id = event.get("run_id") + if not isinstance(run_id, str): + # Required field on every approval-gate event; if missing the + # event is malformed and we cannot reason about it. Skip. + continue + event_name = event.get("event") + if event_name == _EVT_HUMAN_APPROVAL_REQUIRED: + latest_required[run_id] = (line_no, event) + elif event_name in _TERMINAL_DECISION_EVENTS: + latest_decision_line[run_id] = line_no + + pending_pairs = [ + (req_line, event) + for run_id, (req_line, event) in latest_required.items() + if req_line > latest_decision_line.get(run_id, 0) + ] + # Newest-pending first so an operator opening the list sees the most + # recent request at the top. ``timestamp`` may be missing on + # synthetic / hand-edited entries — sort missing values to the bottom. + # Tie-break on file line number (Wave 2a Round-2 F-37-02): the audit + # writer emits second-granularity timestamps so two sub-second-spaced + # events can collide; falling through to insertion order is correct + # but undocumented — pinning the line-number tiebreaker makes the + # contract explicit. + pending_pairs.sort(key=lambda pair: (_safe_timestamp_key(pair[1]), pair[0]), reverse=True) + return [event for _line_no, event in pending_pairs] + + +def _safe_timestamp_key(event: Dict[str, Any]) -> str: + """Type-safe sort key for ``event["timestamp"]``. + + Wave 2a Round-2 review F-37-TS-TYPE: the previous ``e.get("timestamp") + or ""`` only replaced *falsy* values; a tampered or hand-rolled audit + log carrying ``"timestamp": 1730500000`` (epoch int) or any other + non-string would crash ``sorted()`` with ``TypeError`` in Python 3 + when the comparator hit a string-vs-int comparison. Coerce to ``""`` + on anything that is not a string so the sort always succeeds; the + operator still sees the entry, just sorted to the bottom of the list. + """ + ts = event.get("timestamp") + return ts if isinstance(ts, str) else "" + + +def _collect_run_audit_chain(audit_log_path: str, run_id: str) -> List[Dict[str, Any]]: + """Return every approval-gate event for ``run_id`` in append order. + + Used by ``--show`` to render the full request → decision chain. We + return the events in their on-disk order (which is also wall-clock + order since the audit log is append-only) so an operator scanning the + output can read top-to-bottom as the timeline. + """ + chain: List[Dict[str, Any]] = [] + approval_events = {_EVT_HUMAN_APPROVAL_REQUIRED, *_TERMINAL_DECISION_EVENTS} + for _line_no, event in _iter_audit_events(audit_log_path): + if event.get("event") in approval_events and event.get("run_id") == run_id: + chain.append(event) + return chain + + +def _staging_dir_for_event(output_dir: str, event: Dict[str, Any]) -> Optional[str]: + """Best-effort: return the on-disk staging path the event refers to. + + The trainer records ``staging_path`` on every ``human_approval.required`` + event (Wave 1 round-2 audit-event vocabulary). Older audit logs may + pre-date that addition; for those we synthesise the canonical path + ``/final_model.staging`` and let the caller decide whether + to fall back when ``os.path.isdir`` is False. + + **Defence in depth (Wave 2a Round-1 review fix):** when an audit log + lives on shared / unsigned storage, an attacker who can append events + could plant a ``staging_path`` value pointing outside ``output_dir`` + (e.g. ``/etc``). ``_staging_contents`` would then leak directory + listings as a tampered-audit-log oracle. We reuse the + ``_staging_path_inside_output_dir`` guard Phase 9's :mod:`._approve` + module already ships and refuse declared paths that escape the + operator-supplied ``output_dir`` boundary. + """ + # Late import via the package facade so the helper resolves through + # the same monkeypatch surface tests already use for _approve.py. + from forgelm import cli as _cli_facade + + declared = event.get("staging_path") + if isinstance(declared, str) and declared: + if not _cli_facade._staging_path_inside_output_dir(declared, output_dir): + logger.warning( + "Refusing staging_path %r from audit event for output_dir %r: " + "resolved path escapes the output_dir boundary (audit-log tampering " + "guard). Falling back to canonical final_model.staging if present.", + declared, + output_dir, + ) + declared = None + if isinstance(declared, str) and declared: + return declared + fallback = os.path.join(output_dir, "final_model.staging") + return fallback if os.path.isdir(fallback) else None + + +def _age_seconds(event_timestamp: Optional[str]) -> Optional[float]: + """Compute ``now - timestamp`` in seconds; ``None`` when unparseable. + + The audit log emits ISO-8601 with a UTC suffix (``+00:00``). We accept + a missing or malformed timestamp gracefully because hand-edited test + fixtures sometimes drop it, and a missing age is far less actionable + than a stack trace. + """ + if not isinstance(event_timestamp, str) or not event_timestamp: + return None + try: + # Python 3.10 ``datetime.fromisoformat`` does not accept the ``Z`` + # suffix. Normalise to ``+00:00`` for older interpreters. + normalised = event_timestamp.replace("Z", "+00:00") + when = datetime.fromisoformat(normalised) + except ValueError: + return None + if when.tzinfo is None: + # Treat naive timestamps as UTC; mirrors how AuditLogger emits them + # but is defensive against external producers. + when = when.replace(tzinfo=timezone.utc) + delta = datetime.now(timezone.utc) - when + return delta.total_seconds() + + +def _staging_contents(staging_path: Optional[str]) -> List[str]: + """List files in ``staging_path`` (sorted) or ``[]`` when missing/unreadable.""" + if not staging_path or not os.path.isdir(staging_path): + return [] + try: + return sorted(os.listdir(staging_path)) + except OSError as exc: + logger.warning("Cannot list staging directory %s: %s", staging_path, exc) + return [] + + +def _summarise_pending(event: Dict[str, Any], output_dir: str) -> Dict[str, Any]: + """Build the per-run dict used in ``--pending`` output (text + JSON).""" + staging_path = _staging_dir_for_event(output_dir, event) + timestamp = event.get("timestamp") + return { + "run_id": event.get("run_id"), + "staging_path": staging_path, + "staging_exists": bool(staging_path and os.path.isdir(staging_path)), + "requested_at": timestamp, + "age_seconds": _age_seconds(timestamp), + "metrics": event.get("metrics") or {}, + "config_hash": event.get("config_hash") or event.get("config_fingerprint"), + "reason": event.get("reason"), + } + + +def _format_age(age_seconds: Optional[float]) -> str: + """Render ``age_seconds`` as a human-friendly ``Nh Mm`` / ``Nd`` string.""" + if age_seconds is None: + return "unknown" + if age_seconds < 60: + return f"{int(age_seconds)}s" + if age_seconds < 3600: + return f"{int(age_seconds // 60)}m" + if age_seconds < 86400: + hours = int(age_seconds // 3600) + minutes = int((age_seconds % 3600) // 60) + return f"{hours}h {minutes}m" if minutes else f"{hours}h" + days = int(age_seconds // 86400) + return f"{days}d" + + +def _emit_pending_text(pending_summaries: List[Dict[str, Any]]) -> None: + """Print a tabular pending list (or a friendly empty notice).""" + if not pending_summaries: + print("No pending approvals.") + return + print(f"Pending approvals ({len(pending_summaries)}):") + print() + # Tabular layout. Plain text (no rich) so the output stays usable in + # CI logs, redirected files, and SSH sessions without colour support. + headers = ("RUN_ID", "AGE", "REQUESTED_AT", "STAGING") + rows = [ + ( + summary["run_id"] or "(missing run_id)", + _format_age(summary["age_seconds"]), + summary["requested_at"] or "(unknown)", + "present" if summary["staging_exists"] else "MISSING", + ) + for summary in pending_summaries + ] + widths = [max(len(h), max(len(r[i]) for r in rows)) for i, h in enumerate(headers)] + fmt = " ".join(f"{{:<{w}}}" for w in widths) + print(fmt.format(*headers)) + print(fmt.format(*("-" * w for w in widths))) + for row in rows: + print(fmt.format(*row)) + + +def _emit_pending_json(pending_summaries: List[Dict[str, Any]]) -> None: + """Print the pending list as a JSON array of summaries.""" + print( + json.dumps( + {"success": True, "pending": pending_summaries, "count": len(pending_summaries)}, + indent=2, + ) + ) + + +def _run_approvals_list_pending(args, output_format: str) -> None: + """Handle ``forgelm approvals --pending`` end-to-end.""" + output_dir = args.output_dir + audit_log_path = os.path.join(output_dir, _AUDIT_LOG_FILENAME) + if not os.path.isfile(audit_log_path): + # Treat missing audit log the same as "no pending approvals" rather + # than as an error — a freshly-bootstrapped output dir legitimately + # has neither yet. Operators get an informative empty summary. + if output_format == "json": + _emit_pending_json([]) + else: + print(f"No audit log at {audit_log_path}; nothing to list.") + sys.exit(EXIT_SUCCESS) + # Wave 2a Round-2 / Round-5 readability gate via the shared helper + # in :mod:`._approve`. A permission-denied audit log otherwise + # masquerades as "no pending approvals". + from ._approve import _assert_audit_log_readable_or_exit + + _assert_audit_log_readable_or_exit(audit_log_path, output_format) + + try: + pending_events = _collect_pending_runs(audit_log_path) + except OSError as exc: + # Defensive: ``iter_audit_events`` swallows OSError-on-open and + # logs at ERROR. This branch only fires for OSErrors raised + # *during* iteration (mid-stream read failure on NFS-flapping + # disks) — surface those as a runtime error rather than letting + # the operator see a partial / silently-truncated pending list. + _output_error_and_exit( + output_format, + f"Failed to scan audit log {audit_log_path!r}: {exc}", + EXIT_TRAINING_ERROR, + ) + + summaries = [_summarise_pending(event, output_dir) for event in pending_events] + if output_format == "json": + _emit_pending_json(summaries) + else: + _emit_pending_text(summaries) + sys.exit(EXIT_SUCCESS) + + +def _classify_chain(chain: List[Dict[str, Any]]) -> str: + """Return ``pending`` / ``granted`` / ``rejected`` / ``unknown``. + + **Latest-wins semantics (mirrors :func:`_collect_pending_runs`):** when a + run is restarted with the same ``run_id`` after a prior decision — e.g. + a rejected run is re-staged for a second review — the latest + ``human_approval.required`` event reflects the current pending state + even if an earlier ``granted`` / ``rejected`` event exists in the + chain. The previous implementation looked only at decisions and would + declare such a re-stage as ``granted``/``rejected``, contradicting the + pending-list view (Wave 2a Round-2 review F-37-01 fix). + + Algorithm: walk the chain in append order recording the last index of + each event class. A run is ``pending`` iff its last ``required`` index + is strictly greater than its last terminal-decision index, otherwise + the most-recent decision wins. + """ + latest_required_idx = -1 + latest_decision_idx = -1 + latest_decision_event: Optional[str] = None + for idx, event in enumerate(chain): + name = event.get("event") + if name == _EVT_HUMAN_APPROVAL_REQUIRED: + latest_required_idx = idx + elif name in _TERMINAL_DECISION_EVENTS: + latest_decision_idx = idx + latest_decision_event = name + if latest_required_idx == -1 and latest_decision_idx == -1: + return "unknown" + if latest_required_idx > latest_decision_idx: + return "pending" + if latest_decision_event == _EVT_HUMAN_APPROVAL_GRANTED: + return "granted" + return "rejected" + + +def _emit_show_text(run_id: str, chain: List[Dict[str, Any]], status: str, staging_listing: List[str]) -> None: + """Render ``--show RUN_ID`` output as a human-readable timeline.""" + print(f"Run: {run_id}") + print(f"Status: {status}") + print() + print("Audit chain (oldest first):") + if not chain: + print(" (no approval-gate events found)") + for event in chain: + ts = event.get("timestamp", "(no timestamp)") + name = event.get("event", "(no event name)") + # Render the most operator-relevant fields; full payload is in JSON + # output below. + approver = event.get("approver") + comment = event.get("comment") + reason = event.get("reason") + line = f" [{ts}] {name}" + if approver: + line += f" by {approver}" + if reason: + line += f" — {reason}" + if comment: + line += f" ({comment})" + print(line) + print() + if staging_listing: + print(f"Staging contents ({len(staging_listing)} entries):") + for entry in staging_listing: + print(f" - {entry}") + else: + print("Staging directory: missing or empty.") + + +def _emit_show_json(run_id: str, chain: List[Dict[str, Any]], status: str, staging_listing: List[str]) -> None: + """Render ``--show RUN_ID`` output as a structured JSON object.""" + print( + json.dumps( + { + "success": True, + "run_id": run_id, + "status": status, + "chain": chain, + "staging_contents": staging_listing, + }, + indent=2, + ) + ) + + +def _run_approvals_show(args, output_format: str) -> None: + """Handle ``forgelm approvals --show `` end-to-end.""" + run_id: str = args.show + output_dir = args.output_dir + audit_log_path = os.path.join(output_dir, _AUDIT_LOG_FILENAME) + if not os.path.isfile(audit_log_path): + _output_error_and_exit( + output_format, + f"No audit log at {audit_log_path!r}; cannot show run {run_id!r}.", + EXIT_CONFIG_ERROR, + ) + # Same readability gate as --pending via the shared helper. + from ._approve import _assert_audit_log_readable_or_exit + + _assert_audit_log_readable_or_exit(audit_log_path, output_format) + + chain = _collect_run_audit_chain(audit_log_path, run_id) + if not chain: + _output_error_and_exit( + output_format, + f"No approval-gate events found for run_id={run_id!r} in {audit_log_path!r}.", + EXIT_CONFIG_ERROR, + ) + + status = _classify_chain(chain) + # Staging path is read from the *latest* required event so a re-staged + # run (rejected → required again, with a fresh staging dir on disk) + # surfaces the *current* staging directory rather than the stale + # original. This matches `_collect_pending_runs`'s latest-wins + # semantic; previously the *first* required event was used and a + # re-stage would `--show` the wrong directory (Wave 2a Round-2 fix). + required_event = next( + (e for e in reversed(chain) if e.get("event") == _EVT_HUMAN_APPROVAL_REQUIRED), + {}, + ) + staging_path = _staging_dir_for_event(output_dir, required_event) if required_event else None + staging_listing = _staging_contents(staging_path) + + if output_format == "json": + _emit_show_json(run_id, chain, status, staging_listing) + else: + _emit_show_text(run_id, chain, status, staging_listing) + sys.exit(EXIT_SUCCESS) + + +def _run_approvals_cmd(args, output_format: str) -> None: + """Top-level dispatcher for ``forgelm approvals``. + + Exactly one of ``--pending`` / ``--show RUN_ID`` must be set; argparse + enforces the mutual exclusion via the parser registrar so this + function only sees a valid args namespace. + """ + show_run_id = getattr(args, "show", None) + if show_run_id: + _run_approvals_show(args, output_format) + elif getattr(args, "pending", False): + _run_approvals_list_pending(args, output_format) + else: + # argparse should have prevented this; defensive double-check. + _output_error_and_exit( + output_format, + "forgelm approvals: one of --pending / --show RUN_ID is required.", + EXIT_CONFIG_ERROR, + ) diff --git a/forgelm/cli/subcommands/_approve.py b/forgelm/cli/subcommands/_approve.py index 22062d54..f5bbcd21 100644 --- a/forgelm/cli/subcommands/_approve.py +++ b/forgelm/cli/subcommands/_approve.py @@ -13,7 +13,7 @@ import os import sys import types -from typing import Optional +from typing import Any, Dict, NoReturn, Optional import yaml @@ -81,40 +81,19 @@ def _resolve_approver_identity() -> str: def _find_human_approval_required_event(audit_log_path: str, run_id: str) -> Optional[dict]: """Return the most-recent ``human_approval.required`` event for *run_id*. - Reads ``audit_log.jsonl`` line-by-line to keep memory usage flat for - long-lived training directories. Returns ``None`` when no matching event - exists. Malformed lines are skipped and counted; the operator gets a - warning if any lines were skipped so a corrupt entry can't silently mask - a genuine "no event" result. + Wave 2a Round-1 review consolidated the audit-log JSONL parser into + :mod:`._audit_log_reader` so a future malformed-line policy fix lands + in one place. This helper now delegates; the original line-by-line + streaming + skipped-line warning behaviour is preserved exactly + (verified by the existing approve / reject test suites). """ - if not os.path.isfile(audit_log_path): - return None + from ._audit_log_reader import find_latest_event_for_run - latest_match = None - skipped_lines = 0 - try: - fh = open(audit_log_path, "r", encoding="utf-8") - except OSError as exc: - logger.error("Cannot open audit log %s: %s", audit_log_path, exc) - return None - with fh: - for raw in fh: - line = raw.strip() - if not line: - continue - try: - event = json.loads(line) - except json.JSONDecodeError: - skipped_lines += 1 - continue - if not isinstance(event, dict): - skipped_lines += 1 - continue - if event.get("event") == "human_approval.required" and event.get("run_id") == run_id: - latest_match = event - if skipped_lines: - logger.warning("Skipped %d malformed line(s) while parsing %s.", skipped_lines, audit_log_path) - return latest_match + return find_latest_event_for_run( + audit_log_path, + run_id=run_id, + matches=lambda e: e.get("event") == "human_approval.required", + ) _TERMINAL_DECISION_EVENTS = frozenset({_EVT_HUMAN_APPROVAL_GRANTED, _EVT_HUMAN_APPROVAL_REJECTED}) @@ -127,38 +106,18 @@ def _find_human_approval_decision_event(audit_log_path: str, run_id: str) -> Opt ``human_approval.rejected``. Finding one before attempting promotion prevents double-approve and approve-after-reject races. - Mirrors :func:`_find_human_approval_required_event`'s malformed-line - accounting so a corrupt entry cannot silently mask a real terminal - decision and let the caller re-promote. + Same Wave 2a Round-1 consolidation as + :func:`_find_human_approval_required_event` — delegates to the shared + :mod:`._audit_log_reader` so the malformed-line policy lives in + one place. """ - if not os.path.isfile(audit_log_path): - return None + from ._audit_log_reader import find_latest_event_for_run - latest_decision = None - skipped_lines = 0 - try: - fh = open(audit_log_path, "r", encoding="utf-8") - except OSError as exc: - logger.error("Cannot open audit log %s: %s", audit_log_path, exc) - return None - with fh: - for raw in fh: - line = raw.strip() - if not line: - continue - try: - event = json.loads(line) - except json.JSONDecodeError: - skipped_lines += 1 - continue - if not isinstance(event, dict): - skipped_lines += 1 - continue - if event.get("event") in _TERMINAL_DECISION_EVENTS and event.get("run_id") == run_id: - latest_decision = event - if skipped_lines: - logger.warning("Skipped %d malformed line(s) while parsing %s.", skipped_lines, audit_log_path) - return latest_decision + return find_latest_event_for_run( + audit_log_path, + run_id=run_id, + matches=lambda e: e.get("event") in _TERMINAL_DECISION_EVENTS, + ) def _atomic_rename_or_move(src: str, dst: str) -> str: @@ -254,8 +213,16 @@ def __init__(self, webhook_cfg): return WebhookNotifier(_Carrier(webhook_cfg)) -def _output_error_and_exit(output_format: str, msg: str, exit_code: int) -> None: - """Emit *msg* as a structured JSON error or a log record, then exit.""" +def _output_error_and_exit(output_format: str, msg: str, exit_code: int) -> NoReturn: + """Emit *msg* as a structured JSON error or a log record, then exit. + + ``-> NoReturn`` (Wave 2a Round-2 review nit): mypy / pyright otherwise + treat callers as if control could continue past this helper, producing + spurious "possibly-unbound variable" warnings for ``required_event`` / + ``decision_event`` further down ``_run_approve_cmd`` / + ``_run_reject_cmd``. ``sys.exit`` raises ``SystemExit`` so this never + returns; pinning the type makes the contract visible to the typechecker. + """ if output_format == "json": print(json.dumps({"success": False, "error": msg})) else: @@ -263,19 +230,114 @@ def _output_error_and_exit(output_format: str, msg: str, exit_code: int) -> None sys.exit(exit_code) -def _run_approve_cmd(args, output_format: str) -> None: - """Promote ``final_model.staging/`` → ``final_model/`` after human review.""" - # Late import via the package facade so monkeypatched - # ``forgelm.cli._build_approval_notifier`` references resolve correctly. +def _assert_audit_log_readable_or_exit(audit_log_path: str, output_format: str) -> None: + """Wave 2a Round-5 (F-R5-01) readability gate shared across the + approve / reject / approvals family. + + A chmod-broken audit log otherwise reaches ``iter_audit_events`` → + OSError-on-open is logged + swallowed → callers see "no events for + this run" (wrong debugging path on the Article 14 critical path). + Surface the chmod / mount issue with an actionable message instead. + Caller passes the file path; this helper short-circuits via + ``_output_error_and_exit`` (which is ``-> NoReturn``) when the file + exists but is unreadable. Missing-file is the caller's responsibility + (the dispatchers each have their own missing-log policy). + """ + from ._audit_log_reader import is_audit_log_readable + + if os.path.isfile(audit_log_path) and not is_audit_log_readable(audit_log_path): + _output_error_and_exit( + output_format, + f"Audit log {audit_log_path!r} exists but is not readable. " + "Check filesystem permissions (chmod / mount opts) and re-run.", + EXIT_CONFIG_ERROR, + ) + + +def _read_required_event_for_reject( + audit_log_path: str, + run_id: str, + output_format: str, +) -> Dict[str, Any]: + """Reject-flavoured twin of :func:`_read_required_event_for_approve`. + + Pulled out of :func:`_run_reject_cmd` for the same SonarCloud S3776 + cognitive-complexity reason as the approve helper — adding the + Round-5 readability gate pushed reject's inline body over 15. The + operator copy ("record a rejection" / "re-rejection is not allowed") + differs from approve's ("promote" / "re-approve"), so the two + helpers stay separate rather than ballooning the parameter list. + """ from forgelm import cli as _cli_facade - output_dir = args.output_dir - run_id = args.run_id - audit_log_path = os.path.join(output_dir, "audit_log.jsonl") + from ._audit_log_reader import AuditLogParseError + + try: + required_event = _cli_facade._find_human_approval_required_event(audit_log_path, run_id) + except AuditLogParseError as exc: + _output_error_and_exit( + output_format, + f"Audit log {audit_log_path!r} is corrupted at line {exc.line_number} ({exc.reason}). " + "Refusing to record a rejection — repair or rotate the audit log first.", + EXIT_CONFIG_ERROR, + ) + if required_event is None: + _output_error_and_exit( + output_format, + f"No human_approval.required event for run_id={run_id!r} found in {audit_log_path!r}. " + "Refusing to record a rejection on a run that did not request one.", + EXIT_CONFIG_ERROR, + ) + + try: + decision_event = _cli_facade._find_human_approval_decision_event(audit_log_path, run_id) + except AuditLogParseError as exc: + _output_error_and_exit( + output_format, + f"Audit log {audit_log_path!r} is corrupted at line {exc.line_number} ({exc.reason}). " + "Refusing to record a rejection — repair or rotate the audit log first.", + EXIT_CONFIG_ERROR, + ) + if decision_event is not None: + prior = decision_event.get("event", "unknown") + _output_error_and_exit( + output_format, + f"Run {run_id!r} already has a terminal decision ({prior!r}). " + "Refusing to record another decision — re-rejection is not allowed.", + EXIT_CONFIG_ERROR, + ) + return required_event + + +def _read_required_event_for_approve( + audit_log_path: str, + run_id: str, + output_format: str, +) -> Dict[str, Any]: + """Read the ``human_approval.required`` event for ``run_id`` and + enforce no-prior-terminal-decision (approve flavour). + + Exits via :func:`_output_error_and_exit` on parse error, missing + required event, or a pre-existing terminal decision. Pulled out of + :func:`_run_approve_cmd` so the dispatcher stays under SonarCloud + S3776 cognitive-complexity ceiling. Reject has its own slightly- + different operator copy (``"record a rejection"``) and stays inline + in :func:`_run_reject_cmd` — sharing the helper would either dilute + the operator messages or balloon the helper's parameter list. + """ + from forgelm import cli as _cli_facade - # Read the audit event first so we can use the trainer-recorded staging_path - # (which reflects the configured final_model_dir) rather than a hardcoded default. - required_event = _cli_facade._find_human_approval_required_event(audit_log_path, run_id) + from ._audit_log_reader import AuditLogParseError + + try: + required_event = _cli_facade._find_human_approval_required_event(audit_log_path, run_id) + except AuditLogParseError as exc: + _output_error_and_exit( + output_format, + f"Audit log {audit_log_path!r} is corrupted at line {exc.line_number} ({exc.reason}). " + "Refusing to promote — repair or rotate the audit log first.", + EXIT_CONFIG_ERROR, + ) if required_event is None: _output_error_and_exit( output_format, @@ -284,7 +346,15 @@ def _run_approve_cmd(args, output_format: str) -> None: EXIT_CONFIG_ERROR, ) - decision_event = _cli_facade._find_human_approval_decision_event(audit_log_path, run_id) + try: + decision_event = _cli_facade._find_human_approval_decision_event(audit_log_path, run_id) + except AuditLogParseError as exc: + _output_error_and_exit( + output_format, + f"Audit log {audit_log_path!r} is corrupted at line {exc.line_number} ({exc.reason}). " + "Refusing to promote — repair or rotate the audit log first.", + EXIT_CONFIG_ERROR, + ) if decision_event is not None: prior = decision_event.get("event", "unknown") _output_error_and_exit( @@ -293,6 +363,26 @@ def _run_approve_cmd(args, output_format: str) -> None: "Refusing to promote — re-approve is not allowed.", EXIT_CONFIG_ERROR, ) + return required_event + + +def _run_approve_cmd(args, output_format: str) -> None: + """Promote ``final_model.staging/`` → ``final_model/`` after human review.""" + # Late import via the package facade so monkeypatched + # ``forgelm.cli._build_approval_notifier`` references resolve correctly. + from forgelm import cli as _cli_facade + + output_dir = args.output_dir + run_id = args.run_id + audit_log_path = os.path.join(output_dir, "audit_log.jsonl") + + _assert_audit_log_readable_or_exit(audit_log_path, output_format) + # Strict-mode parsing (Wave 2a Round-2 hardening): a corrupted decision + # record that gets silently skipped looks identical to "no approval yet", + # which would let an operator double-grant. ``_read_required_event_for_approve`` + # converts AuditLogParseError into an actionable EXIT_CONFIG_ERROR so + # the operator fixes the log first. + required_event = _read_required_event_for_approve(audit_log_path, run_id, output_format) staging_path = required_event.get("staging_path") or os.path.join(output_dir, f"final_model{_STAGING_SUFFIX}") # Defence-in-depth: refuse a staging_path that escapes output_dir (a @@ -400,24 +490,12 @@ def _run_reject_cmd(args, output_format: str) -> None: run_id = args.run_id audit_log_path = os.path.join(output_dir, "audit_log.jsonl") - required_event = _cli_facade._find_human_approval_required_event(audit_log_path, run_id) - if required_event is None: - _output_error_and_exit( - output_format, - f"No human_approval.required event for run_id={run_id!r} found in {audit_log_path!r}. " - "Refusing to record a rejection on a run that did not request one.", - EXIT_CONFIG_ERROR, - ) - - decision_event = _cli_facade._find_human_approval_decision_event(audit_log_path, run_id) - if decision_event is not None: - prior = decision_event.get("event", "unknown") - _output_error_and_exit( - output_format, - f"Run {run_id!r} already has a terminal decision ({prior!r}). " - "Refusing to record another decision — re-rejection is not allowed.", - EXIT_CONFIG_ERROR, - ) + _assert_audit_log_readable_or_exit(audit_log_path, output_format) + # Strict-mode parsing: surface audit-log corruption to the operator + # rather than skip the line and produce a misleading "no decision yet" + # result. See _read_required_event_for_reject for the same hardening + # pattern as approve. + required_event = _read_required_event_for_reject(audit_log_path, run_id, output_format) staging_path = required_event.get("staging_path") or os.path.join(output_dir, f"final_model{_STAGING_SUFFIX}") diff --git a/forgelm/cli/subcommands/_audit.py b/forgelm/cli/subcommands/_audit.py index 02c34e62..105e6721 100644 --- a/forgelm/cli/subcommands/_audit.py +++ b/forgelm/cli/subcommands/_audit.py @@ -29,6 +29,7 @@ def _run_data_audit( enable_pii_ml: bool = False, pii_ml_language: str = "en", emit_croissant: bool = False, + workers: int = 1, ) -> None: """Phase 11 / 11.5 / 12 dispatch: dataset quality + governance audit. @@ -59,6 +60,7 @@ def _run_data_audit( enable_pii_ml=enable_pii_ml, pii_ml_language=pii_ml_language, emit_croissant=emit_croissant, + workers=workers, ) except OSError as exc: # OSError covers FileNotFoundError / PermissionError / ENOSPC / @@ -148,4 +150,5 @@ def _run_audit_cmd(args, output_format: str) -> None: enable_pii_ml=getattr(args, "pii_ml", False), pii_ml_language=getattr(args, "pii_ml_language", "en"), emit_croissant=getattr(args, "croissant", False), + workers=getattr(args, "workers", 1), ) diff --git a/forgelm/cli/subcommands/_audit_log_reader.py b/forgelm/cli/subcommands/_audit_log_reader.py new file mode 100644 index 00000000..b31cbc36 --- /dev/null +++ b/forgelm/cli/subcommands/_audit_log_reader.py @@ -0,0 +1,232 @@ +"""Single-source audit-log JSONL reader for the approval / approvals / purge family. + +Background — Wave 2a Round-1 review (XPR-01) flagged that ForgeLM had three +near-identical JSONL parsers across :mod:`._approve` and :mod:`._approvals`, +with a fourth coming in Phase 21 (``forgelm purge``). Each copy duplicated +the malformed-line accounting, the OSError handling, and the +``isinstance(event, dict)`` guard — so a future fix to any one of those +policies (e.g. switching to streaming reads, adding a maximum line length, +emitting structured error events) would have to be applied in ``N`` +places at once. + +This module is the single place that policy lives. Two helpers cover the +two read patterns the family uses: + +- :func:`iter_audit_events` — generator yielding ``(line_number, event)`` + pairs. Two parsing modes: ``strict=False`` (default) skips malformed + entries and emits a single summary warning at end (best for ``--pending`` + enumeration where partial corruption shouldn't bail). ``strict=True`` + raises :class:`AuditLogParseError` on the first malformed entry (best + for approve / reject decision guards where silently skipping a corrupted + decision record could cause a wrong "approval not yet granted" verdict). +- :func:`find_latest_event_for_run` — convenience wrapper that returns the + most-recent event matching a predicate, or ``None``. Used by the + approve / reject decision-guard checks; defaults to ``strict=True`` so + decision lookups fail fast on log corruption. + +Both helpers route OSError on file-open through the module logger and +return an empty result so the caller sees a missing log as "no matching +event" rather than as a crash. + +**Performance note (Wave 2a Round-2 review F-INFRA-01):** both helpers are +``O(n)`` over the full audit log per call. Live-tested at 10K events: +~42 ms per walk on Python 3.11. Multi-tenant operator dirs that accumulate +audit events for a year may trip into seconds-per-call territory — at that +scale, callers should batch their lookups (do a single ``iter_audit_events`` +walk and accumulate all needed events) rather than calling +``find_latest_event_for_run`` repeatedly. + +**Deliberate scope (Wave 2a Round-2 review F-XPR-01-01):** +``forgelm/compliance.py::verify_audit_log`` keeps its own JSONL parser +because it must hash the *raw line bytes* for SHA-256 chain verification; +this module yields decoded ``dict`` events without preserving raw bytes, +so the verifier cannot adopt it as-is. That divergence is intentional and +documented. If a future refactor extends ``iter_audit_events`` to +optionally yield ``(line_no, raw_line, event)`` tuples, the verifier can +adopt the shared parser. +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any, Callable, Dict, Iterator, Optional, Tuple + +logger = logging.getLogger("forgelm.cli.audit_log_reader") + + +class AuditLogParseError(ValueError): + """Raised by :func:`iter_audit_events` in strict mode when a line is malformed. + + Carries ``audit_log_path`` and ``line_number`` so callers / log + consumers can pinpoint the corrupted record. Subclass of + :class:`ValueError` so callers that want broad handling can still use + ``except ValueError`` without catching unrelated I/O errors. + """ + + def __init__(self, audit_log_path: str, line_number: int, reason: str) -> None: + self.audit_log_path = audit_log_path + self.line_number = line_number + self.reason = reason + super().__init__(f"{audit_log_path}:{line_number}: {reason}") + + +def _parse_nonempty_line( + audit_log_path: str, + line_number: int, + line: str, + *, + strict: bool, +) -> Optional[Dict[str, Any]]: + """Parse one non-empty audit-log line. + + Returns the parsed dict, or ``None`` when the line is malformed and + ``strict=False`` (the caller treats ``None`` as "skip + count"). In + strict mode raises :class:`AuditLogParseError` on JSON-decode failure + or non-dict root. + + Wave 2a Round-2 nit: extracted from :func:`iter_audit_events` so the + iterator stays at one level of nesting (open → for line → maybe-yield) + and the per-line policy lives in one place. + """ + try: + event = json.loads(line) + except json.JSONDecodeError as exc: + if strict: + raise AuditLogParseError( + audit_log_path, + line_number, + f"invalid JSON ({exc.msg})", + ) from exc + return None + if not isinstance(event, dict): + if strict: + raise AuditLogParseError( + audit_log_path, + line_number, + f"JSON root is {type(event).__name__}, not dict", + ) + return None + return event + + +def iter_audit_events( + audit_log_path: str, + *, + strict: bool = False, +) -> Iterator[Tuple[int, Dict[str, Any]]]: + """Yield ``(line_number, event_dict)`` from an append-only JSONL audit log. + + Two modes: + + - ``strict=False`` (default): skips blank lines, malformed JSON, and + non-dict roots silently. Emits a single ``logger.warning`` at the + end summarising the skip count (so callers learn about corruption + without paying per-line log cost). Use this for enumeration paths + where partial corruption should not bail (e.g. + ``forgelm approvals --pending``). + - ``strict=True``: raises :class:`AuditLogParseError` on the first + malformed entry. Use this for paths where silently skipping a + corrupted record could cause a wrong verdict (e.g. approve / reject + decision guards: a corrupted ``human_approval.granted`` line that + gets skipped looks identical to "no approval yet" and would + double-grant on the operator's next attempt). + + Returns nothing (no events yielded, no warning, no raise) when: + - the file does not exist (a freshly-bootstrapped output dir + legitimately has no audit log yet); or + - the file cannot be opened (OSError logged at ERROR level so + operators see why nothing came through). + + The caller is responsible for closing nothing — this generator owns + the file handle via ``with`` and releases it on iterator exhaustion. + """ + if not os.path.isfile(audit_log_path): + return + try: + fh = open(audit_log_path, "r", encoding="utf-8") + except OSError as exc: + logger.error("Cannot open audit log %s: %s", audit_log_path, exc) + return + skipped_lines = 0 + with fh: + for line_number, raw in enumerate(fh, start=1): + line = raw.strip() + if not line: + continue + event = _parse_nonempty_line(audit_log_path, line_number, line, strict=strict) + if event is None: + skipped_lines += 1 + continue + yield line_number, event + if skipped_lines: + logger.warning( + "Skipped %d malformed line(s) while parsing %s.", + skipped_lines, + audit_log_path, + ) + + +def find_latest_event_for_run( + audit_log_path: str, + *, + run_id: str, + matches: Callable[[Dict[str, Any]], bool], + strict: bool = True, +) -> Optional[Dict[str, Any]]: + """Return the most-recent event matching ``matches`` for ``run_id``. + + Walks the entire log (audit logs are append-only so "most recent" + means the last matching entry encountered). Returns ``None`` when + no event matches. ``O(n)`` over the full log per call — see the + module docstring's performance note. + + ``matches`` is the predicate the caller cares about — typically + ``lambda e: e.get("event") == "human_approval.required"`` or a + membership check against a frozenset of decision-event names. + Keeping the predicate as a callable lets every caller in the + approve / approvals / purge family share the parser without + coupling it to a specific event vocabulary. + + Defaults to ``strict=True`` so the approve / reject decision-guard + callers fail fast if the log is corrupted; an enumeration caller + (``--pending``) may pass ``strict=False`` to keep scanning past + individually-corrupted lines. + """ + latest: Optional[Dict[str, Any]] = None + for _line_no, event in iter_audit_events(audit_log_path, strict=strict): + if event.get("run_id") != run_id: + continue + if matches(event): + latest = event + return latest + + +def is_audit_log_readable(audit_log_path: str) -> bool: + """Return ``True`` iff the audit log exists and is readable by the + current process. + + Wave 2a Round-5 review (F-R5-01): the approve / reject / approvals + family all need the same pre-iteration readability gate so a + chmod-broken audit log does not masquerade as "no events found". + Centralised here so the policy lives next to the parser it gates. + + The check is a non-binding hint: ``iter_audit_events`` still has to + swallow OSError on open because TOCTOU windows + race conditions + around `chmod` can fire between this call and the open. Callers + that want a clear operator-facing error use this helper for the + common case and trust the parser's logger.error fallback for the + sub-millisecond race. + """ + if not os.path.isfile(audit_log_path): + return False + return os.access(audit_log_path, os.R_OK) + + +__all__ = [ + "AuditLogParseError", + "iter_audit_events", + "find_latest_event_for_run", + "is_audit_log_readable", +] diff --git a/forgelm/cli/subcommands/_doctor.py b/forgelm/cli/subcommands/_doctor.py new file mode 100644 index 00000000..cbf30dd8 --- /dev/null +++ b/forgelm/cli/subcommands/_doctor.py @@ -0,0 +1,988 @@ +"""``forgelm doctor`` env-check subcommand (Phase 34). + +The first command an operator should run after installation. Probes the +environment ForgeLM expects (Python, torch, CUDA, GPU, optional extras, +HF Hub reachability, disk space, audit-log identity) and emits a +structured pass / warn / fail report — text by default, JSON envelope +when ``--output-format json`` is set. + +Design goals (closure plan §9.5 Phase 34): + +- **Self-contained.** Probes use stdlib only at module load. Heavy + deps (torch, huggingface_hub) are imported lazily inside the + individual check functions so ``forgelm doctor`` can run on a + brand-new machine where torch is not yet installed without crashing. +- **Honest pass / warn / fail.** ``warn`` is reserved for "operator + may want to address this" (e.g. optional extra missing); ``fail`` + is reserved for "ForgeLM cannot work this way" (e.g. Python version + too old). Air-gap (``--offline``) skips the HF Hub network probe + rather than reporting it as a failure. +- **Public exit-code contract.** 0 when every check passes, 1 when at + least one fails (config-error class), 2 when a probe itself crashed + (runtime-error class — operator-actionable bug, not config). + +Note on env-var reads (CLAUDE.md "config-driven runtime" rule): +``forgelm doctor`` deliberately reads ``FORGELM_OPERATOR``, +``FORGELM_ALLOW_ANONYMOUS_OPERATOR``, ``HF_ENDPOINT``, ``HF_HUB_CACHE``, +``HF_HOME``, and ``HF_HUB_OFFLINE`` / ``TRANSFORMERS_OFFLINE`` directly +from ``os.environ`` rather than from validated YAML. This is *not* a +violation of the config-driven principle — those env vars are read +verbatim by downstream code (``forgelm/compliance.py::AuditLogger`` for +the FORGELM_* identity vars; ``huggingface_hub`` upstream for the HF_* +vars). Doctor's job is to predict what training will see; if doctor +read them from YAML while the training-time code read them from env, +doctor would silently lie. The broader question of moving the audit +identity to YAML is a Phase 20 / RetentionConfig design topic, not a +doctor bug — flagging this here so future review bots do not refile it. +""" + +from __future__ import annotations + +import json +import os +import platform +import shutil +import socket +import sys +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Tuple + +from .._exit_codes import EXIT_CONFIG_ERROR, EXIT_SUCCESS, EXIT_TRAINING_ERROR +from .._logging import logger + +# Status vocabulary. Centralised so a future rename (e.g. "warn" -> +# "warning") cannot drift across the renderers and the JSON contract. +_STATUS_PASS = "pass" +_STATUS_WARN = "warn" +_STATUS_FAIL = "fail" + +# Probe name vocabulary. Centralised the same way the status tokens are, +# so renaming a probe (e.g. ``operator.identity`` → ``audit.operator``) is +# a single-line edit and downstream JSON consumers can grep for these +# constants rather than scattered string literals. Wave 2a Round-2 nit. +_PROBE_PYTHON_VERSION = "python.version" +_PROBE_TORCH_INSTALLED = "torch.installed" +_PROBE_TORCH_CUDA = "torch.cuda" +_PROBE_GPU_INVENTORY = "gpu.inventory" +_PROBE_HF_HUB_REACHABLE = "hf_hub.reachable" +_PROBE_HF_HUB_OFFLINE_CACHE = "hf_hub.offline_cache" +_PROBE_DISK_WORKSPACE = "disk.workspace" +_PROBE_OPERATOR_IDENTITY = "operator.identity" + +# Optional extras advertised in pyproject.toml [project.optional-dependencies]. +# Each entry is ``(extra_name, importable_module, human_purpose_blurb)``. +# ``importable_module`` is the package the extra installs that we probe via +# ``importlib.util.find_spec`` — a successful spec resolve means the extra is +# present. Doctor never *imports* the module (avoids triggering torch / +# transformers / spacy heavy load on a healthy probe). +# +# Wave 2a Round-1 review (qodo bot): extras names audited 2026-05-02 against +# pyproject.toml. The original list used aspirational names ("deepspeed", +# "evaluation", "wandb", "mergekit") that did not match the published extras +# ("distributed", "eval", "tracking", "merging") — the install-hint a +# missing-extra warn produced was unactionable because the suggested +# ``pip install 'forgelm[deepspeed]'`` would 404. Now aligned with the +# actual pyproject names. +_OPTIONAL_EXTRAS: Tuple[Tuple[str, str, str], ...] = ( + ("qlora", "bitsandbytes", "4-bit / 8-bit QLoRA training"), + ("unsloth", "unsloth", "Unsloth-accelerated training (Linux GPUs only)"), + ("distributed", "deepspeed", "DeepSpeed ZeRO + offload distributed training"), + ("eval", "lm_eval", "lm-evaluation-harness benchmark scoring"), + ("tracking", "wandb", "Weights & Biases experiment tracking"), + ("merging", "mergekit", "Model-merge backend for the merge mode"), + ("export", "llama_cpp", "GGUF export via llama-cpp-python (Linux + macOS only)"), + ("ingestion", "pypdf", "PDF / DOCX / EPUB ingestion"), + ("ingestion-pii-ml", "presidio_analyzer", "Presidio ML-NER PII detection in audit"), + ("ingestion-scale", "datasketch", "MinHash LSH for >50K-row dedup"), +) + +# Disk-space thresholds for the workspace check. Tuned for "enough room +# to download a 7B-parameter model + run one fine-tune" — ~50 GB safe, +# 10-50 GB warn, <10 GB fail. +_DISK_FAIL_GB = 10.0 +_DISK_WARN_GB = 50.0 + + +@dataclass +class _CheckResult: + """Outcome of one diagnostic probe. + + ``status`` is one of pass / warn / fail. ``name`` is operator- + facing; ``detail`` carries the actual finding (Python version + string, GPU count, missing extra name, etc.) so the operator can + act without re-running the probe themselves. + """ + + name: str + status: str + detail: str + # Optional structured fields that downstream tooling may want + # (CI gate that filters by category, dashboard widget, etc.). + extras: Dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Individual checks +# --------------------------------------------------------------------------- + + +def _check_python_version() -> _CheckResult: + """Pin Python to the supported window. + + ForgeLM declares ``python_requires=">=3.10"`` in pyproject.toml. We + treat 3.10 as the floor (fail below), 3.11+ as the recommended + surface (pass above), 3.10.x as warn-with-explanation (still works + but operators should be tracking 3.11 / 3.12). + """ + version = sys.version_info + label = f"{version.major}.{version.minor}.{version.micro}" + # Compare on the (major, minor) prefix only. ``sys.version_info`` is + # a 5-tuple; comparing it directly to ``(3, 10)`` works in CPython + # today but is harder to reason about (the trailing slots compare + # ``int`` vs absent). Wave 2a Round-2 review F-34-VERSIONCMP: pin to + # a 2-tuple slice so the intent is explicit and future tuple-shape + # changes upstream cannot break the comparison. + version_pair = (version.major, version.minor) + if version_pair < (3, 10): + return _CheckResult( + name=_PROBE_PYTHON_VERSION, + status=_STATUS_FAIL, + detail=f"Python {label} is below the supported floor (>=3.10). Upgrade Python to >=3.11.", + extras={"version": label, "minimum": "3.10", "recommended": "3.11"}, + ) + if version_pair < (3, 11): + return _CheckResult( + name=_PROBE_PYTHON_VERSION, + status=_STATUS_WARN, + detail=f"Python {label} is supported but >=3.11 is recommended for performance + typing improvements.", + extras={"version": label, "minimum": "3.10", "recommended": "3.11"}, + ) + return _CheckResult( + name=_PROBE_PYTHON_VERSION, + status=_STATUS_PASS, + detail=f"Python {label} ({platform.python_implementation()}).", + extras={"version": label, "implementation": platform.python_implementation()}, + ) + + +def _check_torch_cuda() -> _CheckResult: + """Detect torch + CUDA availability. + + CPU-only is a *legitimate* setup (small experiments, CI smoke, + inference-only deployments) so a missing CUDA does not fail — + it warns so the operator notices that a GPU run will fail later. + """ + try: + import torch + except ImportError: + return _CheckResult( + name=_PROBE_TORCH_INSTALLED, + status=_STATUS_FAIL, + detail="torch is not installed. Install with: pip install 'forgelm'.", + extras={"installed": False}, + ) + cuda_available = bool(torch.cuda.is_available()) + cuda_version = getattr(torch.version, "cuda", None) if cuda_available else None + if not cuda_available: + return _CheckResult( + name=_PROBE_TORCH_CUDA, + status=_STATUS_WARN, + detail=( + f"torch {torch.__version__} installed but CUDA is unavailable. " + "CPU-only runs are supported but training will be very slow; " + "expect to use this only for tiny experiments or smoke tests." + ), + extras={ + "torch_version": torch.__version__, + "cuda_available": False, + }, + ) + return _CheckResult( + name=_PROBE_TORCH_CUDA, + status=_STATUS_PASS, + detail=f"torch {torch.__version__} with CUDA {cuda_version}.", + extras={ + "torch_version": torch.__version__, + "cuda_available": True, + "cuda_version": cuda_version, + }, + ) + + +def _check_gpu_inventory() -> _CheckResult: + """Enumerate visible GPUs + per-device VRAM. + + Skipped (returns ``warn``) when CUDA is unavailable — no point + enumerating zero devices. + """ + try: + import torch + except ImportError: + return _CheckResult( + name=_PROBE_GPU_INVENTORY, + status=_STATUS_FAIL, + detail="torch is not installed.", + extras={"installed": False}, + ) + if not torch.cuda.is_available(): + return _CheckResult( + name=_PROBE_GPU_INVENTORY, + status=_STATUS_WARN, + detail="No CUDA devices visible; CPU-only mode.", + extras={"device_count": 0}, + ) + count = torch.cuda.device_count() + devices: List[Dict[str, Any]] = [] + for idx in range(count): + try: + props = torch.cuda.get_device_properties(idx) + except (RuntimeError, AssertionError) as exc: # pragma: no cover — defensive + devices.append( + { + "index": idx, + "error": f"could not query device {idx}: {exc}", + } + ) + continue + # ``total_memory`` is bytes; convert to GiB for the operator- + # facing label and keep raw bytes in the structured payload. + vram_gib = round(props.total_memory / (1024**3), 1) + devices.append( + { + "index": idx, + "name": props.name, + "vram_gib": vram_gib, + "vram_bytes": props.total_memory, + } + ) + if count == 0: + return _CheckResult( + name=_PROBE_GPU_INVENTORY, + status=_STATUS_WARN, + detail="CUDA reports no visible devices.", + extras={"device_count": 0}, + ) + summary = ", ".join(f"GPU{d['index']}: {d.get('name', '?')} ({d.get('vram_gib', '?')} GiB)" for d in devices) + return _CheckResult( + name=_PROBE_GPU_INVENTORY, + status=_STATUS_PASS, + detail=f"{count} GPU(s) — {summary}.", + extras={"device_count": count, "devices": devices}, + ) + + +def _check_optional_extra(extra: str, module: str, purpose: str) -> _CheckResult: + """Probe one optional extra without triggering its heavy import side effects.""" + import importlib.util + + spec = importlib.util.find_spec(module) + if spec is None: + return _CheckResult( + name=f"extras.{extra}", + status=_STATUS_WARN, + detail=f"Optional extra missing — install with: pip install 'forgelm[{extra}]' (purpose: {purpose}).", + extras={"extra": extra, "module": module, "installed": False}, + ) + return _CheckResult( + name=f"extras.{extra}", + status=_STATUS_PASS, + detail=f"Installed (module {module}, purpose: {purpose}).", + extras={"extra": extra, "module": module, "installed": True}, + ) + + +# Default Hub endpoint; overridable via the standard HF_ENDPOINT env var. +# Mirrors huggingface_hub's own resolution path so an operator with a +# self-hosted mirror gets the right probe target. +_DEFAULT_HF_ENDPOINT = "https://huggingface.co" + + +def _resolve_hf_endpoint() -> str: + """Resolve the HuggingFace Hub endpoint via the standard env var. + + huggingface_hub honours ``HF_ENDPOINT`` for self-hosted mirrors and + enterprise installs (e.g. internal Datasaur / KServe-fronted Hub). + Hard-coding ``huggingface.co`` would produce a false warning on + those deployments — the corp proxy might block public huggingface.co + while the internal mirror is happily serving requests. + """ + return (os.environ.get("HF_ENDPOINT") or _DEFAULT_HF_ENDPOINT).rstrip("/") + + +def _check_hf_hub_reachable(timeout_seconds: float = 5.0) -> _CheckResult: + """HEAD ``${HF_ENDPOINT}/api/models`` to verify the Hub is reachable. + + Skipped by the caller in ``--offline`` mode. Treats a connection + failure as a *warning* rather than a fail because a transient outage + (operator's wifi, captive portal, corp proxy down) shouldn't refuse + a doctor run that may exist precisely to surface that fact. + + Wave 2a Round-1 review: + + - F-27-04: dropped the ``socket.timeout`` ``except`` branch. + ``socket.timeout`` is an alias for ``TimeoutError`` since Python + 3.10 (the project floor), and ``urllib.request.urlopen`` wraps a + timeout as ``URLError`` anyway — the branch was dead. + - bot (gemini): now resolves the endpoint via ``_resolve_hf_endpoint`` + so ``HF_ENDPOINT=https://internal-mirror.example`` is respected + (mirrors how ``huggingface_hub`` resolves it). + - Wave 2a Round-2 (F-XPR-02-01): migrated from raw + ``urllib.request.urlopen`` to :func:`forgelm._http.safe_get` so + the probe inherits the project-wide HTTP discipline (SSRF guard, + scheme policy, timeout floor, secret-mask error path, + redirect-refusal). An air-gapped mirror at a private IP requires + ``allow_private=True`` — that gate is not crossed here because an + operator with a private HF endpoint should run ``--offline`` (which + swaps to the cache probe) rather than have doctor punch through to + a metadata service. + """ + import requests as _requests # only for the exception type + + from forgelm._http import HttpSafetyError, safe_get + + endpoint = _resolve_hf_endpoint() + probe_url = f"{endpoint}/api/models" + headers = {"User-Agent": "forgelm-doctor/Phase-34"} + + try: + response = safe_get( + probe_url, + headers=headers, + timeout=timeout_seconds, + method="HEAD", + ) + status_code = response.status_code + except HttpSafetyError as exc: + # Policy rejection — surface as a fail, not a warn: the operator + # configured something the discipline blocks (e.g. http:// endpoint + # or private IP without --offline). + return _CheckResult( + name=_PROBE_HF_HUB_REACHABLE, + status=_STATUS_FAIL, + detail=( + f"HuggingFace Hub probe rejected by HTTP discipline: {exc}. " + "Set HF_ENDPOINT to a public https:// URL, or pass --offline " + "to inspect the local cache instead." + ), + extras={"reachable": False, "endpoint": endpoint, "error": str(exc)}, + ) + except _requests.RequestException as exc: + # Transport / TLS / network failure — caught and warned (same + # behaviour as the previous urllib URLError branch). + return _CheckResult( + name=_PROBE_HF_HUB_REACHABLE, + status=_STATUS_WARN, + detail=f"Could not reach HuggingFace Hub at {probe_url}: {exc}. Check network / proxy / HF_ENDPOINT.", + extras={"reachable": False, "endpoint": endpoint, "error": str(exc)}, + ) + + if 200 <= status_code < 400: + return _CheckResult( + name=_PROBE_HF_HUB_REACHABLE, + status=_STATUS_PASS, + detail=f"HuggingFace Hub reachable at {endpoint} (HTTP {status_code}).", + extras={"reachable": True, "endpoint": endpoint, "status_code": status_code}, + ) + if status_code == 405: + # Some corp proxies block HEAD; retry with GET as a courtesy. + try: + response = safe_get(probe_url, headers=headers, timeout=timeout_seconds, method="GET") + status_code = response.status_code + if 200 <= status_code < 400: + return _CheckResult( + name=_PROBE_HF_HUB_REACHABLE, + status=_STATUS_PASS, + detail=f"HuggingFace Hub reachable at {endpoint} (HTTP {status_code}; GET fallback after HEAD 405).", + extras={ + "reachable": True, + "endpoint": endpoint, + "status_code": status_code, + "fallback_method": "GET", + }, + ) + except _requests.RequestException: # pragma: no cover — second-attempt failure + pass + return _CheckResult( + name=_PROBE_HF_HUB_REACHABLE, + status=_STATUS_WARN, + detail=f"HuggingFace Hub at {endpoint} returned HTTP {status_code}.", + extras={"reachable": False, "endpoint": endpoint, "status_code": status_code}, + ) + + +# Cache-walk safety cap. Real HF caches reach 50+ GiB on long-lived +# workstations; an unbounded walk on NFS-mounted caches takes 30+s. +# Phase 34 doesn't need an exact size — "is there *something* cached?" +# is enough. We cap depth + file count and report whether the cap fired. +_HF_CACHE_WALK_DEPTH = 4 +_HF_CACHE_WALK_FILE_LIMIT = 5_000 + + +def _resolve_hf_cache_dir() -> str: + """Resolve the HF Hub cache directory honouring the standard env vars. + + Priority (matches huggingface_hub's own resolution): + + 1. ``HF_HUB_CACHE`` — newer dedicated env var, wins outright. + 2. ``HF_HOME/hub`` — the *hub* sub-directory (huggingface_hub + partitions cache by purpose; ``HF_HOME`` sets the *parent*, not + the hub cache directly). Wave 2a Round-1 review (gemini bot) + fix: the original implementation pointed at ``HF_HOME`` + directly which is wrong — that directory contains the hub + + datasets + spaces sub-trees, not just the hub blobs. + 3. ``~/.cache/huggingface/hub`` — the documented default. + """ + hf_hub_cache = os.environ.get("HF_HUB_CACHE") + if hf_hub_cache: + return hf_hub_cache + hf_home = os.environ.get("HF_HOME") + if hf_home: + return os.path.join(hf_home, "hub") + return os.path.expanduser("~/.cache/huggingface/hub") + + +def _accumulate_files_in_dir( + root: str, + files: List[str], + *, + file_count: int, + total_bytes: int, + unreadable_count: int, +) -> Tuple[int, int, int, bool]: + """Sum regular-file sizes in a single directory under the file cap. + + Returns ``(new_file_count, new_total_bytes, new_unreadable_count, + cap_hit)``. ``cap_hit=True`` means the file-count cap fired during + this directory and the caller should stop descending further. Pulled + out of :func:`_walk_hf_cache_bounded` so each function stays under + SonarCloud's S3776 cognitive-complexity ceiling. + """ + cap_hit = False + for filename in files: + if file_count >= _HF_CACHE_WALK_FILE_LIMIT: + cap_hit = True + break + full = os.path.join(root, filename) + if os.path.islink(full): + continue + try: + total_bytes += os.path.getsize(full) + file_count += 1 + except OSError: + unreadable_count += 1 + return file_count, total_bytes, unreadable_count, cap_hit + + +def _walk_hf_cache_bounded(cache_dir_abs: str) -> Tuple[int, int, int, int, bool]: + """Bounded ``os.walk`` over the HF cache. Returns ``(file_count, + total_bytes, unreadable_count, walk_errors, walk_truncated)``. + + Wave 2a Round-2 nit: extracted from :func:`_check_hf_cache_offline` + so the cache-status check function can read top-down (resolve dir → + walk → render result) without the per-file accounting being inlined. + Sums regular-file sizes only; symlinks in the HF cache point at the + blob store and would be double-counted. Depth + file caps keep the + walk bounded on NFS-mounted 50 GiB+ caches so the doctor probe stays + snappy. ``unreadable_count`` is surfaced so the renderer can warn + when permissions broke part of the scan (F-34-OSE). Per-directory + accounting lives in :func:`_accumulate_files_in_dir` so each helper + stays under SonarCloud's S3776 cognitive-complexity ceiling. + + Wave 2a Round-5 (F-R5-04): ``os.walk`` defaults to silently + *swallowing* ``os.scandir`` errors — a chmod-broken cache root or + sub-directory simply vanishes from the walk, so a fully-unreadable + cache reports ``file_count=0, unreadable_count=0`` and falls into + the empty-cache branch. The ``onerror`` callback now bumps + ``walk_errors`` so the caller can distinguish "cache is empty" from + "cache exists but the operator cannot read it" — the latter is a + *fail*, not a warn. + """ + file_count = 0 + total_bytes = 0 + unreadable_count = 0 + walk_errors = 0 + walk_truncated = False + + def _on_walk_error(exc: OSError) -> None: + nonlocal walk_errors + walk_errors += 1 + logger.warning("Doctor HF cache walk hit %s on %s", exc.__class__.__name__, getattr(exc, "filename", "?")) + + base_depth = cache_dir_abs.rstrip(os.sep).count(os.sep) + for root, dirs, files in os.walk(cache_dir_abs, onerror=_on_walk_error): + if (root.count(os.sep) - base_depth) > _HF_CACHE_WALK_DEPTH: + dirs[:] = [] + walk_truncated = True + continue + file_count, total_bytes, unreadable_count, cap_hit = _accumulate_files_in_dir( + root, + files, + file_count=file_count, + total_bytes=total_bytes, + unreadable_count=unreadable_count, + ) + if cap_hit: + walk_truncated = True + break + return file_count, total_bytes, unreadable_count, walk_errors, walk_truncated + + +def _check_hf_cache_offline() -> _CheckResult: + """Inspect the local HF cache (``--offline`` mode replacement for the Hub probe). + + A populated cache means an air-gapped run can satisfy + ``local_files_only=True`` without any network access. An empty cache + is a warning — the operator may need to run ``forgelm cache-models`` + (Phase 35) before training. + + Wave 2a Round-1 review fixes: + + - bot (gemini): cache directory resolution now honours + ``HF_HUB_CACHE`` and the ``hub`` sub-directory of ``HF_HOME`` + (was: pointed at ``HF_HOME`` itself, which is the parent of + multiple cache trees). + - F-27-03: walk is depth-capped + file-capped so an NFS-mounted + 50+ GiB cache doesn't take 30+s; cap-hit is recorded in extras + so downstream tooling can detect partial-scan results. + """ + cache_dir = _resolve_hf_cache_dir() + if not os.path.isdir(cache_dir): + return _CheckResult( + name=_PROBE_HF_HUB_OFFLINE_CACHE, + status=_STATUS_WARN, + detail=( + f"HF cache not found at {cache_dir}. Air-gapped runs need pre-cached models — " + "see `forgelm cache-models` (Phase 35) once available, or set HF_HUB_CACHE / " + "HF_HOME (with /hub subdirectory) and pre-populate." + ), + extras={"cache_dir": cache_dir, "exists": False}, + ) + cache_dir_abs = os.path.abspath(cache_dir) + file_count, total_bytes, unreadable_count, walk_errors, walk_truncated = _walk_hf_cache_bounded(cache_dir_abs) + cache_gib = round(total_bytes / (1024**3), 2) + offline_env = os.environ.get("HF_HUB_OFFLINE") + truncation_note = " (scan truncated at depth/file cap)" if walk_truncated else "" + unreadable_note = f" [{unreadable_count} file(s) unreadable, totals are partial]" if unreadable_count else "" + walk_error_note = f" [{walk_errors} directory walk error(s) — check chmod / mount]" if walk_errors else "" + detail = ( + f"HF cache at {cache_dir}: {cache_gib} GiB across {file_count} file(s)" + f"{truncation_note}{unreadable_note}{walk_error_note}. " + f"HF_HUB_OFFLINE={offline_env or 'unset'}." + ) + # Status policy: + # fail — the operator cannot READ the cache (walk_errors > 0 AND + # no files visible). Air-gapped run will fail later with a + # misleading missing-model error if we let this masquerade as + # "empty cache". (F-R5-04) + # warn — files visible but partial (walk_errors > 0 but some + # files were sized OK), or fully readable but empty cache, or + # individual files unreadable. All operator-actionable but + # do not block the doctor verdict. + # pass — every file visible + sized. + if walk_errors and file_count == 0: + status = _STATUS_FAIL + elif file_count == 0: + status = _STATUS_WARN + elif unreadable_count or walk_errors: + status = _STATUS_WARN + else: + status = _STATUS_PASS + return _CheckResult( + name=_PROBE_HF_HUB_OFFLINE_CACHE, + status=status, + detail=detail, + extras={ + "cache_dir": cache_dir, + "exists": True, + "size_gib": cache_gib, + "file_count": file_count, + "unreadable_count": unreadable_count, + "walk_errors": walk_errors, + "walk_truncated": walk_truncated, + "hf_hub_offline_env": offline_env, + }, + ) + + +def _check_disk_space(path: str = ".") -> _CheckResult: + """Workspace free-space probe. + + Disk space is a frequent source of late-failing training runs (a + fine-tune that crashes mid-checkpoint because the workspace is + full). ``shutil.disk_usage`` works cross-platform and is in the + stdlib so this probe runs without any optional dep. + """ + try: + usage = shutil.disk_usage(path) + except OSError as exc: # pragma: no cover — defensive (permission denied) + return _CheckResult( + name=_PROBE_DISK_WORKSPACE, + status=_STATUS_FAIL, + detail=f"Could not query disk usage at {path!r}: {exc}.", + extras={"path": os.path.abspath(path), "error": str(exc)}, + ) + free_gib = round(usage.free / (1024**3), 1) + total_gib = round(usage.total / (1024**3), 1) + if free_gib < _DISK_FAIL_GB: + status = _STATUS_FAIL + elif free_gib < _DISK_WARN_GB: + status = _STATUS_WARN + else: + status = _STATUS_PASS + return _CheckResult( + name=_PROBE_DISK_WORKSPACE, + status=status, + detail=f"Workspace {os.path.abspath(path)} — {free_gib} GiB free of {total_gib} GiB.", + extras={ + "path": os.path.abspath(path), + "free_gib": free_gib, + "total_gib": total_gib, + "fail_threshold_gib": _DISK_FAIL_GB, + "warn_threshold_gib": _DISK_WARN_GB, + }, + ) + + +# Env-var names that may carry secret material — never echo their *values* +# in the doctor output even when the operator runs --output-format json +# and pipes to a CI log. Pattern matches AuditLogger's own discipline. +_DOCTOR_SECRET_ENV_NAMES: frozenset[str] = frozenset( + { + "FORGELM_AUDIT_SECRET", # HMAC key for audit-log signing + "HF_TOKEN", # HuggingFace Hub auth token + "HUGGING_FACE_HUB_TOKEN", # legacy alias of HF_TOKEN + "HUGGINGFACE_TOKEN", # alternative spelling read by forgelm/utils.py + "FORGELM_RESUME_TOKEN", # API resume token (Phase 13 future) + # Defence-in-depth: third-party API keys ForgeLM accepts via YAML + # interpolation (auth.openai_api_key etc.). No probe surfaces them + # today, but pre-listing them means a future probe that adds env + # visibility (e.g. an "auth provenance" probe) cannot accidentally + # leak them in --output-format json. + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "WANDB_API_KEY", + "COHERE_API_KEY", + } +) + + +def _mask_env_value_for_audit(name: str, value: str) -> str: + """Mask values for env vars whose name implies secret material. + + Wave 2a Round-1 review (F-27-05): probes echo env-var values + verbatim into ``detail`` + ``extras`` today, which would leak any + future probe surfacing ``HF_TOKEN`` or ``FORGELM_AUDIT_SECRET``. + Centralised mask so the policy lives in one place. ``FORGELM_OPERATOR`` + is *not* in the mask list — it is operator identity, not a secret — + but the discipline for adding new probes is "if the env-var name + contains TOKEN / SECRET / KEY / PASSWORD, add it to + _DOCTOR_SECRET_ENV_NAMES first, then surface it". + """ + if name in _DOCTOR_SECRET_ENV_NAMES: + # Show length only so the operator can confirm "yes, set", + # without exposing the value. + return f"" + return value + + +def _check_operator_identity() -> _CheckResult: + """Article 12 record-keeping reminder. + + ``FORGELM_OPERATOR`` set explicitly is the recommended path for + CI / pipelines (so audit events carry a meaningful identity). + Unset is not a failure — the AuditLogger falls back to + ``getuser()@host`` which is fine on a developer workstation — + but warn so a CI deployer is reminded to pin it. + + Wave 2a Round-1 review (qodo bot): the unresolved-username branch + now respects the ``FORGELM_ALLOW_ANONYMOUS_OPERATOR=1`` opt-in the + same way ``AuditLogger.__init__`` does. When that env var is set, + the doctor returns ``warn`` (anonymous identity OK by operator + choice) instead of ``fail``; when it is not set, ``fail`` stands + because ``AuditLogger`` itself would refuse to start. + """ + explicit = os.environ.get("FORGELM_OPERATOR") + if explicit: + masked = _mask_env_value_for_audit("FORGELM_OPERATOR", explicit) + return _CheckResult( + name=_PROBE_OPERATOR_IDENTITY, + status=_STATUS_PASS, + detail=f"FORGELM_OPERATOR set to {masked!r}; audit events will carry this identity.", + extras={"FORGELM_OPERATOR": masked, "source": "env"}, + ) + # Try to resolve the fallback identity AuditLogger would use so + # the operator sees what their audit events would look like. + try: + import getpass + + username = getpass.getuser() + except (KeyError, OSError, ImportError): + username = None + hostname = socket.gethostname() or "unknown-host" + if username: + fallback = f"{username}@{hostname}" + return _CheckResult( + name=_PROBE_OPERATOR_IDENTITY, + status=_STATUS_WARN, + detail=( + f"FORGELM_OPERATOR not set; audit events will fall back to {fallback!r}. " + "Pin FORGELM_OPERATOR= for CI / pipeline runs so the audit log identifies a stable identity." + ), + extras={"FORGELM_OPERATOR": None, "fallback": fallback, "source": "getpass"}, + ) + # No username AND no FORGELM_OPERATOR: AuditLogger refuses to start + # unless the operator explicitly opted in to anonymous audit + # entries. Mirror that opt-in here so doctor's verdict matches the + # actual training-time behaviour (warn-OK-but-suboptimal vs hard-fail). + allow_anonymous = os.environ.get("FORGELM_ALLOW_ANONYMOUS_OPERATOR") == "1" + if allow_anonymous: + anon = f"anonymous@{hostname}" + return _CheckResult( + name=_PROBE_OPERATOR_IDENTITY, + status=_STATUS_WARN, + detail=( + f"FORGELM_OPERATOR not set, getpass.getuser() unavailable, but " + f"FORGELM_ALLOW_ANONYMOUS_OPERATOR=1 is set; audit events will record {anon!r}. " + "Not recommended for EU AI Act Article 12 record-keeping; pin FORGELM_OPERATOR= when possible." + ), + extras={ + "FORGELM_OPERATOR": None, + "fallback": anon, + "source": "anonymous_opt_in", + }, + ) + return _CheckResult( + name=_PROBE_OPERATOR_IDENTITY, + status=_STATUS_FAIL, + detail=( + "FORGELM_OPERATOR not set AND getpass.getuser() could not resolve a username. " + "Set FORGELM_OPERATOR= (or FORGELM_ALLOW_ANONYMOUS_OPERATOR=1 to opt in to " + "anonymous audit entries — not recommended for EU AI Act Article 12)." + ), + extras={"FORGELM_OPERATOR": None, "fallback": None}, + ) + + +# --------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------- + + +# Order matters for the text rendering — keep it grouped (env first, +# hardware next, optional extras, then network, then operator identity). +def _build_check_plan(*, offline: bool) -> List[Tuple[str, Callable[[], _CheckResult]]]: + """Return the ordered list of ``(label, callable)`` probe entries. + + The label is only used in error reporting if a probe itself crashes; + each ``_CheckResult`` carries its own ``name`` for the renderer. + """ + plan: List[Tuple[str, Callable[[], _CheckResult]]] = [ + (_PROBE_PYTHON_VERSION, _check_python_version), + (_PROBE_TORCH_CUDA, _check_torch_cuda), + (_PROBE_GPU_INVENTORY, _check_gpu_inventory), + ] + for extra_name, module, purpose in _OPTIONAL_EXTRAS: + plan.append((f"extras.{extra_name}", _make_extra_probe(extra_name, module, purpose))) + if offline: + plan.append((_PROBE_HF_HUB_OFFLINE_CACHE, _check_hf_cache_offline)) + else: + plan.append((_PROBE_HF_HUB_REACHABLE, _check_hf_hub_reachable)) + plan.append((_PROBE_DISK_WORKSPACE, _check_disk_space)) + plan.append((_PROBE_OPERATOR_IDENTITY, _check_operator_identity)) + return plan + + +def _make_extra_probe(extra: str, module: str, purpose: str) -> Callable[[], _CheckResult]: + """Build a zero-arg closure that probes one optional extra. + + Defined as a top-level helper rather than a lambda so the closure + binding is explicit and the probe is easy to unit-test by name. + """ + + def _probe() -> _CheckResult: + return _check_optional_extra(extra, module, purpose) + + return _probe + + +def _run_all_checks(*, offline: bool) -> List[_CheckResult]: + """Execute every probe in order, catching crashes per-probe. + + A probe that raises an unexpected exception is converted into a + ``fail`` result so the rest of the report still renders — partial + diagnostics are more useful than zero diagnostics when something is + broken. + """ + results: List[_CheckResult] = [] + for label, probe in _build_check_plan(offline=offline): + try: + results.append(probe()) + except Exception as exc: # noqa: BLE001 — best-effort: per-probe catch keeps the rest of the report rendering when one diagnostic crashes; converting to fail lets the operator see the broken probe name + traceback class instead of silently dropping it. + logger.exception("Doctor probe %r raised: %s", label, exc) + results.append( + _CheckResult( + name=label, + status=_STATUS_FAIL, + detail=f"Probe crashed: {exc.__class__.__name__}: {exc}", + extras={"crashed": True, "error_class": exc.__class__.__name__}, + ) + ) + return results + + +# --------------------------------------------------------------------------- +# Renderers +# --------------------------------------------------------------------------- + + +# Wave 2a Round-2 review F-34-ASCII: the renderer docstring promises +# "Plain ASCII" output for redirected logs / non-UTF8 terminals, but the +# original glyphs (✓ U+2713, ✗ U+2717) are Unicode and would raise +# UnicodeEncodeError on a strict ASCII locale (PYTHONIOENCODING=ascii or +# a CI runner with C.US-ASCII). Replaced with single-byte ASCII so the +# output matches the documented contract. ``+`` / ``!`` / ``x`` keep +# the visual asymmetry (`x` for fail reads as "no" at a glance). +_STATUS_GLYPHS: Dict[str, str] = { + _STATUS_PASS: "+", + _STATUS_WARN: "!", + _STATUS_FAIL: "x", +} + + +def _render_text(results: List[_CheckResult]) -> str: + """Render a tabular text report. + + Plain ASCII so the output stays readable in CI logs, redirected + files, and SSH sessions without colour support. One line per check + plus a summary footer. + """ + pass_count = sum(1 for r in results if r.status == _STATUS_PASS) + warn_count = sum(1 for r in results if r.status == _STATUS_WARN) + fail_count = sum(1 for r in results if r.status == _STATUS_FAIL) + + # F-34-ASCII: header was "forgelm doctor — environment check" with an + # em-dash (U+2014). Replaced with a plain ASCII hyphen so the + # docstring's "plain ASCII" promise holds for redirected logs / strict + # ASCII locales. + lines: List[str] = ["forgelm doctor - environment check", ""] + name_width = max(len(r.name) for r in results) if results else 0 + for result in results: + glyph = _STATUS_GLYPHS.get(result.status, "?") + lines.append(f" [{glyph} {result.status}] {result.name:<{name_width}} {result.detail}") + lines.append("") + lines.append(f"Summary: {pass_count} pass, {warn_count} warn, {fail_count} fail.") + return "\n".join(lines) + + +def _render_json(results: List[_CheckResult]) -> str: + """Render the JSON envelope. + + Shape mirrors other ForgeLM subcommand contracts: + ``{"success": bool, "checks": [...], "summary": {"pass": N, "warn": N, "fail": N, "crashed": N}}``. + ``success`` is True iff no check failed AND no probe crashed (warns are + operator-actionable but do not flip the contract; crashes count as + failures *and* surface separately so consumers can distinguish a + misconfigured environment from a doctor bug). Locked schema lives in + ``docs/usermanuals/en/reference/json-output.md``. + """ + payload_checks = [ + { + "name": r.name, + "status": r.status, + "detail": r.detail, + "extras": r.extras, + } + for r in results + ] + crashed_count = sum(1 for r in results if r.extras.get("crashed") is True) + summary = { + "pass": sum(1 for r in results if r.status == _STATUS_PASS), + "warn": sum(1 for r in results if r.status == _STATUS_WARN), + # `fail` includes the crashed count (a crashed probe surfaces as + # status="fail" + extras.crashed=True, see _run_all_checks). + "fail": sum(1 for r in results if r.status == _STATUS_FAIL), + "crashed": crashed_count, + } + envelope = { + "success": summary["fail"] == 0, + "checks": payload_checks, + "summary": summary, + } + # ``default=str`` (Wave 2a Round-5 F-R5-06): ``extras`` is typed + # ``Dict[str, Any]``; today every probe surfaces only JSON-native + # types but a future probe author dropping a ``Path`` / ``datetime`` + # / ``bytes`` value would otherwise crash with ``TypeError`` and + # exit with a Python traceback to stderr instead of the documented + # JSON envelope. ``default=str`` defangs the contract violation by + # falling back to the value's ``__str__`` repr. + return json.dumps(envelope, indent=2, default=str) + + +def _resolve_exit_code(results: List[_CheckResult]) -> int: + """Map the result list to one of the public exit codes. + + Contract: + - 0 (EXIT_SUCCESS) — every check passed (warn included; warns are + operator-actionable but do not block) + - 1 (EXIT_CONFIG_ERROR) — at least one check returned ``fail`` (a + misconfiguration the operator can correct) + - 2 (EXIT_TRAINING_ERROR) — a probe itself crashed (an unexpected + doctor bug, surfaced as runtime-error class so CI/CD retry logic + treats it the same way as other runtime failures) + """ + crashed = any(r.extras.get("crashed") for r in results) + if crashed: + return EXIT_TRAINING_ERROR + has_fail = any(r.status == _STATUS_FAIL for r in results) + return EXIT_CONFIG_ERROR if has_fail else EXIT_SUCCESS + + +def _run_doctor_cmd(args, output_format: str) -> None: + """Top-level dispatcher for ``forgelm doctor``. + + Reads ``args.offline`` (argparse default ``False``) and emits either + the text report or the JSON envelope. Exits with the public + contract code resolved by :func:`_resolve_exit_code`. + + Also honours the standard HuggingFace airgap environment variables — + ``HF_HUB_OFFLINE=1`` or ``TRANSFORMERS_OFFLINE=1`` — so an air-gapped + operator who already has those set in their shell does not need to + remember to also pass ``--offline``. An operator who explicitly + passes ``--offline`` always gets offline behaviour regardless of env. + """ + offline = bool(getattr(args, "offline", False)) + if not offline: + # Standard HF airgap signals — empty string and "0" do NOT + # activate offline mode (the documented HF behaviour). + for env_name in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"): + value = os.environ.get(env_name) + if value and value not in ("0", "false", "False"): + offline = True + break + results = _run_all_checks(offline=offline) + if output_format == "json": + print(_render_json(results)) + else: + print(_render_text(results)) + sys.exit(_resolve_exit_code(results)) + + +# Re-exports for tests / monkeypatch — kept stable. +__all__ = [ + "_CheckResult", + "_check_python_version", + "_check_torch_cuda", + "_check_gpu_inventory", + "_check_optional_extra", + "_check_hf_hub_reachable", + "_check_hf_cache_offline", + "_check_disk_space", + "_check_operator_identity", + "_build_check_plan", + "_run_all_checks", + "_render_text", + "_render_json", + "_resolve_exit_code", + "_run_doctor_cmd", + "_OPTIONAL_EXTRAS", +] diff --git a/forgelm/data_audit/_orchestrator.py b/forgelm/data_audit/_orchestrator.py index dad879a7..ab88d3bb 100644 --- a/forgelm/data_audit/_orchestrator.py +++ b/forgelm/data_audit/_orchestrator.py @@ -11,12 +11,13 @@ import json import logging +import multiprocessing import os import tempfile from dataclasses import asdict from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from ._croissant import _build_croissant_metadata @@ -179,6 +180,36 @@ def _atomic_write_json(target: Path, payload: Dict[str, Any]) -> None: pass +def _process_split_for_pool( + args_tuple: Tuple[str, Path, Dict[str, Any]], +): + """Module-level wrapper: dispatches to :func:`_process_split` from a tuple. + + ``multiprocessing.Pool.map`` requires the worker callable + its arguments + to be picklable. Lambdas and ``functools.partial`` over keyword-only + args don't pickle cleanly across spawn-method workers, so we accept a + single positional tuple and unpack the kwargs inside the worker. + + **Test injection point** (Wave 2a Round-5 F-R5-03): spawn-method + workers do not see monkeypatches applied in the parent test process, + so the only way to exercise the orchestrator's ``pool.map`` re-raise + branch is via an env var that the worker reads on entry. Setting + ``FORGELM_AUDIT_TEST_WORKER_RAISES=`` causes this + wrapper to raise ``RuntimeError`` for the matching split — used by + ``tests/test_data_audit_workers.py::TestWorkersErrorPropagation`` + to verify that an uncaught worker exception surfaces through + ``Pool.map`` instead of being silently swallowed. Production code + should never set this variable; the check is fail-closed (env var + must equal the exact split name being processed) to minimise blast + radius if an operator sets it by mistake. + """ + split_name, path, kwargs = args_tuple + test_raise_split = os.environ.get("FORGELM_AUDIT_TEST_WORKER_RAISES") + if test_raise_split and test_raise_split == split_name: + raise RuntimeError(f"synthetic test-injected worker failure for split {split_name!r}") + return _process_split(split_name, path, **kwargs) + + def audit_dataset( # NOSONAR — cognitive complexity is inherent to the audit orchestration logic; extraction would fragment cohesive pipeline steps source: str, *, @@ -191,6 +222,7 @@ def audit_dataset( # NOSONAR — cognitive complexity is inherent to the audit enable_pii_ml: bool = False, pii_ml_language: str = "en", emit_croissant: bool = False, + workers: int = 1, ) -> AuditReport: """Run the audit pipeline over a JSONL file or split-keyed directory. @@ -233,6 +265,15 @@ def audit_dataset( # NOSONAR — cognitive complexity is inherent to the audit enable_quality_filter: Phase 12 opt-in flag — when ``True``, run the heuristic quality checks (Gopher / C4 / RefinedWeb-style) and surface findings under ``quality_summary``. + workers: Phase 17 — number of worker processes used for the + split-level pipeline. ``1`` (default) keeps the pre-Phase-17 + single-process behaviour exactly. ``> 1`` distributes the + per-split processing over ``min(workers, len(splits))`` + ``multiprocessing.Pool`` workers; the merge step that builds + the final report stays single-threaded so the determinism + contract (byte-identical JSON across worker counts) is + preserved. Speed-up scales with the number of *splits*, not + row count — single-split corpora ignore values >1. Returns: :class:`AuditReport`. JSON-serialize via ``asdict(report)``. @@ -260,6 +301,9 @@ def audit_dataset( # NOSONAR — cognitive complexity is inherent to the audit # for per-row resilience on pathological strings). _require_presidio(language=pii_ml_language) + if not isinstance(workers, int) or isinstance(workers, bool) or workers < 1: + raise ValueError(f"workers must be a positive integer; got {workers!r}.") + splits_paths, resolution_notes = _resolve_input(source) splits_info: Dict[str, Dict[str, Any]] = {} @@ -276,18 +320,71 @@ def audit_dataset( # NOSONAR — cognitive complexity is inherent to the audit parse_errors_total = 0 decode_errors_total = 0 - for split_name, path in splits_paths.items(): - outcome = _process_split( - split_name, - path, - near_dup_threshold=near_dup_threshold, - dedup_method=dedup_method, - minhash_jaccard=minhash_jaccard, - minhash_num_perm=minhash_num_perm, - enable_quality_filter=enable_quality_filter, - enable_pii_ml=enable_pii_ml, - pii_ml_language=pii_ml_language, + # Determinism contract: regardless of ``workers``, every aggregate is + # built up by walking the splits in the exact order ``splits_paths`` + # yields them. Multiprocessing only changes how the per-split outcomes + # are *computed*; the merge step below is single-threaded so byte- + # equivalent output is guaranteed across worker counts. See the + # ``test_data_audit_workers.py`` regression suite for the test that + # pins this invariant. + split_kwargs: Dict[str, Any] = { + "near_dup_threshold": near_dup_threshold, + "dedup_method": dedup_method, + "minhash_jaccard": minhash_jaccard, + "minhash_num_perm": minhash_num_perm, + "enable_quality_filter": enable_quality_filter, + "enable_pii_ml": enable_pii_ml, + "pii_ml_language": pii_ml_language, + } + effective_workers = min(workers, len(splits_paths)) if splits_paths else 1 + if workers > effective_workers: + # Operator asked for more parallelism than we can use; surface + # the clamp so the wall-clock-vs-expected gap doesn't surprise + # them (Wave 2a Round-2 review F-17-01). + logger.info( + "audit: requested workers=%d but only %d split(s) found; running %d worker(s)", + workers, + len(splits_paths) if splits_paths else 0, + effective_workers, ) + if effective_workers > 1: + # Pool.map preserves input order so ``outcomes`` lines up with + # ``splits_paths.items()`` 1:1. We bound the pool to + # ``len(splits_paths)`` — extra workers would idle since the unit + # of work is one split. + # + # Wave 2a Round-1 review (F-26-03): pin the start method to + # ``"spawn"`` so the determinism contract is platform-independent. + # ``Pool()`` defaults to fork on Linux + spawn on macOS / Windows; + # Linux CI never exercises the spawn path otherwise. Spawn also + # gives every worker a fresh interpreter (no shared + # langdetect.DetectorFactory state, no copied open file handles) + # which is exactly what the byte-identical contract requires. + logger.info( + "audit: spawning %d worker process(es) over %d split(s)", + effective_workers, + len(splits_paths), + ) + ctx = multiprocessing.get_context("spawn") + pool_args = [(name, path, split_kwargs) for name, path in splits_paths.items()] + with ctx.Pool(effective_workers) as pool: + try: + outcomes_list = pool.map(_process_split_for_pool, pool_args) + except Exception: # noqa: BLE001 — best-effort: re-raise unchanged so the operator sees the worker traceback through Pool.map's wrapping; we log at ERROR first so the contextual "which split failed" hint lands even when the traceback is opaque. + logger.error( + "audit: parallel worker raised inside the pool; the original " + "traceback is wrapped — re-run with --workers 1 to surface it " + "with full context." + ) + raise + outcomes_iter = zip(splits_paths.keys(), outcomes_list) + else: + logger.info("audit: sequential mode (workers=%d, splits=%d)", workers, len(splits_paths)) + outcomes_iter = ( + (split_name, _process_split(split_name, path, **split_kwargs)) for split_name, path in splits_paths.items() + ) + + for split_name, outcome in outcomes_iter: splits_info[split_name] = outcome.info signatures_by_split[split_name] = outcome.signatures total_samples += outcome.row_count diff --git a/tests/test_approvals_listing.py b/tests/test_approvals_listing.py new file mode 100644 index 00000000..4ee05380 --- /dev/null +++ b/tests/test_approvals_listing.py @@ -0,0 +1,981 @@ +"""Phase 37: ``forgelm approvals`` listing + show subcommand. + +Mirrors the ``test_human_approval_gate.py`` style — synthetic JSONL audit +log fixtures so the tests stay torch-free and run on every CI matrix combo. +""" + +from __future__ import annotations + +import json + +# --------------------------------------------------------------------------- +# Helpers (mirror tests/test_human_approval_gate.py) +# --------------------------------------------------------------------------- +# Wave 2a Round-2 F-TEST-37-03: monotonically-increasing default timestamp +# so any sort-order assertion in the suite has a real signal to compare +# against (the prior single-hardcoded-value made every sort test vacuous). +# Tests that need a specific timestamp pass it via the ``timestamp`` kwarg. +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +_DEFAULT_TS_BASE = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc) +_NEXT_DEFAULT_TS = [_DEFAULT_TS_BASE] + + +def _next_default_ts() -> str: + """Return the next monotonic ISO-8601 timestamp for fixture events.""" + _NEXT_DEFAULT_TS[0] += timedelta(seconds=1) + return _NEXT_DEFAULT_TS[0].isoformat() + + +def _write_event(audit_path: Path, event: str, run_id: str, **fields: object) -> None: + """Append a synthetic event to ``audit_path``. + + Produces the same JSONL shape AuditLogger writes: one event per line, + a ``timestamp`` / ``operator`` / ``event`` / ``run_id`` core, and + whatever extra fields the caller passes (``staging_path``, ``metrics``, + ``approver``, ``comment`` etc.). + + ``timestamp`` defaults to a monotonically-increasing value per call + (Wave 2a Round-2 F-TEST-37-03 fix). Pass ``timestamp="..."`` in + ``**fields`` to override. + """ + if "timestamp" not in fields: + fields = {**fields, "timestamp": _next_default_ts()} + entry = { + "operator": "tester", + "event": event, + "run_id": run_id, + "prev_hash": "genesis", + } + entry.update(fields) + audit_path.parent.mkdir(parents=True, exist_ok=True) + with open(audit_path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(entry) + "\n") + + +def _seed_run( + output_dir: Path, + run_id: str, + *, + decision: str | None = None, + create_staging: bool = True, +) -> Path: + """Create a run layout: staging dir + ``human_approval.required`` event. + + When ``decision`` is given (``"granted"`` / ``"rejected"``), append the + matching terminal event so the run is no longer pending. + """ + output_dir.mkdir(parents=True, exist_ok=True) + staging_dir = output_dir / "final_model.staging" + if create_staging: + staging_dir.mkdir(exist_ok=True) + (staging_dir / "adapter_config.json").write_text('{"r": 8}', encoding="utf-8") + audit_path = output_dir / "audit_log.jsonl" + _write_event( + audit_path, + "human_approval.required", + run_id, + staging_path=str(staging_dir), + gate="final_model", + reason="require_human_approval=true", + metrics={"eval_loss": 0.42}, + ) + if decision == "granted": + _write_event( + audit_path, + "human_approval.granted", + run_id, + approver="alice", + comment="LGTM", + promote_strategy="rename", + ) + elif decision == "rejected": + _write_event( + audit_path, + "human_approval.rejected", + run_id, + approver="bob", + comment="regression", + staging_path=str(staging_dir), + ) + return staging_dir + + +def _build_args(output_dir: Path, *, pending: bool = False, show: str | None = None) -> SimpleNamespace: + """Build a strict argparse-shaped namespace. + + Wave 2a Round-2 nit: was a ``MagicMock`` which silently allowed + misspelled or missing CLI attributes (``args.pendng`` would have + returned a Mock instead of failing the test). ``SimpleNamespace`` + raises ``AttributeError`` on any access the harness did not declare, + so a future refactor that reads a new attribute lights up here + instead of silently passing. + """ + return SimpleNamespace(pending=pending, show=show, output_dir=str(output_dir)) + + +# --------------------------------------------------------------------------- +# --pending mode +# --------------------------------------------------------------------------- + + +class TestApprovalsPending: + def test_empty_audit_log_returns_empty_list(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + output_dir.mkdir() + # No audit_log.jsonl on disk → friendly empty path, exit 0. + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="text") + assert ei.value.code == 0 + captured = capsys.readouterr().out + assert "nothing to list" in captured.lower() or "no pending" in captured.lower() + + def test_three_runs_one_decided_two_pending(self, tmp_path: Path, capsys) -> None: + # Three runs, one already granted → listing returns the other two. + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-aaaa00000000") + _seed_run(output_dir, "fg-bbbb00000000", decision="granted") + _seed_run(output_dir, "fg-cccc00000000") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + assert ei.value.code == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is True + assert payload["count"] == 2 + run_ids = {summary["run_id"] for summary in payload["pending"]} + assert run_ids == {"fg-aaaa00000000", "fg-cccc00000000"} + + def test_rejected_run_also_excluded_from_pending(self, tmp_path: Path, capsys) -> None: + # Reject (like granted) is a terminal decision; rejected runs must + # not appear in --pending even though their staging dir lingers. + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-rejected0000", decision="rejected") + _seed_run(output_dir, "fg-pending00000") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + assert ei.value.code == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload["count"] == 1 + assert payload["pending"][0]["run_id"] == "fg-pending00000" + + def test_pending_summary_carries_metrics_and_staging_path(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + staging_dir = _seed_run(output_dir, "fg-zzzz00000000") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + payload = json.loads(capsys.readouterr().out) + summary = payload["pending"][0] + assert summary["staging_path"] == str(staging_dir) + assert summary["staging_exists"] is True + assert summary["metrics"] == {"eval_loss": 0.42} + assert summary["reason"] == "require_human_approval=true" + + def test_text_output_renders_table(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-xyz000000000") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="text") + out = capsys.readouterr().out + # Table headers + run id present. + assert "RUN_ID" in out + assert "fg-xyz000000000" in out + # The "present" / "MISSING" staging column is rendered. + assert "present" in out + + +# --------------------------------------------------------------------------- +# --show mode +# --------------------------------------------------------------------------- + + +class TestApprovalsShow: + def test_show_pending_run_emits_full_chain(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-pp0000000000") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-pp0000000000"), output_format="json") + assert ei.value.code == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is True + assert payload["status"] == "pending" + assert len(payload["chain"]) == 1 + assert payload["chain"][0]["event"] == "human_approval.required" + # Staging dir was seeded, so contents include adapter_config.json. + assert "adapter_config.json" in payload["staging_contents"] + + def test_show_granted_run_status_granted(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-gr0000000000", decision="granted") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-gr0000000000"), output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "granted" + # Two events: required + granted. + assert len(payload["chain"]) == 2 + assert payload["chain"][-1]["event"] == "human_approval.granted" + + def test_show_unknown_run_id_exits_1(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-real00000000") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-fake00000000"), output_format="text") + assert ei.value.code == 1 + # Error message names the missing run id. + # (Captured in stderr via the logger; capsys captures stdout for the + # JSON path but the text path goes through ``logger.error``; assert + # via caplog instead in a dedicated test below.) + + def test_show_no_audit_log_exits_1(self, tmp_path: Path) -> None: + output_dir = tmp_path / "run" + output_dir.mkdir() + # No audit_log.jsonl on disk; --show cannot operate. + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-anything0000"), output_format="text") + assert ei.value.code == 1 + + def test_show_text_renders_timeline(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-tt0000000000", decision="granted") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, show="fg-tt0000000000"), output_format="text") + out = capsys.readouterr().out + assert "fg-tt0000000000" in out + assert "human_approval.required" in out + assert "human_approval.granted" in out + assert "by alice" in out # approver rendered + + +# --------------------------------------------------------------------------- +# Subcommand registration smoke +# --------------------------------------------------------------------------- + + +class TestApprovalsRegistration: + def test_approvals_subcommand_registered(self) -> None: + """``forgelm approvals --help`` must succeed (subparser exists).""" + import subprocess + import sys + + result = subprocess.run( + [sys.executable, "-m", "forgelm.cli", "approvals", "--help"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + # Both modes are advertised in the help text. + assert "--pending" in result.stdout + assert "--show" in result.stdout + assert "--output-dir" in result.stdout + + +# --------------------------------------------------------------------------- +# Robustness against malformed audit log lines +# --------------------------------------------------------------------------- + + +class TestApprovalsMalformedLines: + def test_skipped_lines_do_not_hide_pending_runs(self, tmp_path: Path, capsys) -> None: + # Simulate a corrupt audit log: a real `required` event followed by + # a malformed line and a JSON list (non-dict root). The pending + # listing must still surface the legitimate run. + output_dir = tmp_path / "run" + output_dir.mkdir() + audit_path = output_dir / "audit_log.jsonl" + _write_event( + audit_path, + "human_approval.required", + "fg-real00000000", + staging_path=str(output_dir / "final_model.staging"), + ) + with open(audit_path, "a", encoding="utf-8") as fh: + fh.write("not even json\n") + fh.write('["array", "root"]\n') + # Create the staging dir so staging_exists is True (defensive — not + # required for the test, just realistic). + (output_dir / "final_model.staging").mkdir() + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + payload = json.loads(capsys.readouterr().out) + assert payload["count"] == 1 + assert payload["pending"][0]["run_id"] == "fg-real00000000" + + +# --------------------------------------------------------------------------- +# Defensive: dispatcher rejects neither-mode args +# --------------------------------------------------------------------------- + + +class TestApprovalsDispatcherDefensive: + def test_neither_pending_nor_show_exits_1(self, tmp_path: Path) -> None: + """argparse normally enforces the mutex; this exercises the + defensive double-check inside the dispatcher in case a future + refactor drops the parser-level guard.""" + output_dir = tmp_path / "run" + output_dir.mkdir() + args = _build_args(output_dir, pending=False, show=None) + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(args, output_format="text") + assert ei.value.code == 1 + + +# --------------------------------------------------------------------------- +# Patch-based test for monkeypatch resolution discipline +# --------------------------------------------------------------------------- + + +class TestApprovalsMonkeyPatchSurface: + def test_helper_reachable_via_facade(self) -> None: + """``forgelm.cli._collect_pending_runs`` must resolve so tests can + ``patch("forgelm.cli._collect_pending_runs", ...)`` if they want.""" + from forgelm import cli as _cli_facade + + assert callable(_cli_facade._collect_pending_runs) + assert callable(_cli_facade._collect_run_audit_chain) + assert callable(_cli_facade._run_approvals_cmd) + + +@pytest.mark.usefixtures("caplog") +class TestApprovalsErrorMessages: + def test_unknown_run_logs_actionable_error(self, tmp_path: Path, caplog) -> None: + import logging + + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-real00000000") + + from forgelm.cli import _run_approvals_cmd + + with caplog.at_level(logging.ERROR, logger="forgelm.cli"): + with patch("forgelm.cli._collect_pending_runs"): # ensure not called by --show + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-fake00000000"), output_format="text") + assert ei.value.code == 1 + assert "fg-fake00000000" in caplog.text + + +# --------------------------------------------------------------------------- +# Wave 2a Round-1 review fixes — security + bot-suggested coverage +# --------------------------------------------------------------------------- + + +class TestApprovalsPathTraversalGuard: + """F-25-01: tampered audit log with staging_path outside output_dir + must not produce a directory-listing oracle via _staging_contents.""" + + def test_show_refuses_external_staging_path(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + output_dir.mkdir() + # Malicious audit event: staging_path points outside output_dir. + _write_event( + output_dir / "audit_log.jsonl", + "human_approval.required", + "fg-tampered00000", + staging_path="/etc", # would expose /etc listing if guard absent + ) + # Provide a benign fallback so the test can verify the listing + # is empty (not /etc/...). + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, show="fg-tampered00000"), output_format="json") + payload = json.loads(capsys.readouterr().out) + # The dispatcher must NOT have leaked /etc entries into staging_contents. + assert all("etc" not in entry for entry in payload["staging_contents"]), ( + f"path-traversal guard failed: leaked /etc entries: {payload['staging_contents']}" + ) + + def test_pending_refuses_external_staging_path(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + output_dir.mkdir() + _write_event( + output_dir / "audit_log.jsonl", + "human_approval.required", + "fg-tampered00000", + staging_path="/etc", + ) + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + payload = json.loads(capsys.readouterr().out) + # staging_path field falls back to None / canonical when the + # declared one is rejected; staging_exists must be False. + summary = payload["pending"][0] + assert summary["staging_exists"] is False + # staging_path is either None (no fallback found) or the + # canonical final_model.staging — never the rejected /etc. + assert summary["staging_path"] != "/etc" + + +class TestApprovalsLatestWinsRunIdReuse: + """F-25-03: latest-wins semantics for re-staged runs. + + A run that was rejected and then re-staged with the same run_id + (a second human_approval.required event) must show as PENDING + again; the older terminal decision is no longer operative.""" + + def test_re_staged_run_shows_as_pending(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-restaged00000", decision="rejected") + # Now re-stage: append another human_approval.required AFTER the + # rejection. Latest-wins → pending again. + _write_event( + output_dir / "audit_log.jsonl", + "human_approval.required", + "fg-restaged00000", + staging_path=str(output_dir / "final_model.staging"), + metrics={"eval_loss": 0.30}, + ) + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + payload = json.loads(capsys.readouterr().out) + run_ids = {summary["run_id"] for summary in payload["pending"]} + assert "fg-restaged00000" in run_ids + + +class TestApprovalsShowRejectedAndMissingStaging: + """Bot-suggested coverage: rejected --show + MISSING staging table.""" + + def test_show_rejected_run_status_rejected(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-rj0000000000", decision="rejected") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-rj0000000000"), output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "rejected" + # Two events: required + rejected. + assert len(payload["chain"]) == 2 + assert payload["chain"][-1]["event"] == "human_approval.rejected" + + def test_pending_table_renders_missing_when_staging_absent(self, tmp_path: Path, capsys) -> None: + # Seed without creating the staging directory. + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-nostaging000", create_staging=False) + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="text") + out = capsys.readouterr().out + assert "MISSING" in out + + def test_pending_empty_audit_log_json_envelope(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + output_dir.mkdir() + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload == {"success": True, "pending": [], "count": 0} + + def test_show_unknown_run_id_json_envelope(self, tmp_path: Path, capsys) -> None: + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-real00000000") + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-fake00000000"), output_format="json") + assert ei.value.code == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is False + assert "fg-fake00000000" in payload["error"] + + def test_show_robust_to_malformed_lines(self, tmp_path: Path, capsys) -> None: + # --show must skip malformed lines the same way --pending does. + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-real00000000") + with open(output_dir / "audit_log.jsonl", "a", encoding="utf-8") as fh: + fh.write("not even json\n") + fh.write('["array", "root"]\n') + + from forgelm.cli import _run_approvals_cmd + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-real00000000"), output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + # Real human_approval.required event is still surfaced. + assert any(e.get("event") == "human_approval.required" for e in payload["chain"]) + + +# --------------------------------------------------------------------------- +# Wave 2a Round-2 hardening: sort order, latest-wins, path-traversal, +# format helpers, classify-chain re-stage semantics. +# --------------------------------------------------------------------------- + + +class TestApprovalsPendingSortOrder: + """F-TEST-37-02: pending list is sorted newest-first.""" + + def test_pending_sorted_newest_first(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._approvals import _run_approvals_cmd + + output_dir = tmp_path / "run" + output_dir.mkdir() + # Three runs with monotonically-increasing default timestamps. + # Default _next_default_ts is monotonically incrementing across + # _write_event calls — so writing in order [old, mid, new] produces + # ascending timestamps and the pending list should reverse to + # [new, mid, old]. + _seed_run(output_dir, "fg-old0000000000") + _seed_run(output_dir, "fg-mid0000000000") + _seed_run(output_dir, "fg-new0000000000") + with pytest.raises(SystemExit): + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + payload = json.loads(capsys.readouterr().out) + run_ids = [s["run_id"] for s in payload["pending"]] + assert run_ids == ["fg-new0000000000", "fg-mid0000000000", "fg-old0000000000"], ( + f"pending should be newest-first, got {run_ids!r}" + ) + + +class TestFormatAge: + """F-TEST-37-06: _format_age bucket rendering.""" + + @pytest.mark.parametrize( + "seconds,expected", + [ + (None, "unknown"), + (5, "5s"), + (59, "59s"), + (60, "1m"), + (90, "1m"), + (3599, "59m"), + (3600, "1h"), + (7200, "2h"), + (7260, "2h 1m"), + (86399, "23h 59m"), + (86400, "1d"), + (90000, "1d"), + ], + ) + def test_buckets(self, seconds, expected) -> None: + from forgelm.cli.subcommands._approvals import _format_age + + assert _format_age(seconds) == expected + + +class TestClassifyChainLatestWins: + """F-37-01 + user inline: _classify_chain agrees with _collect_pending_runs.""" + + def test_re_stage_after_grant_returns_pending(self, tmp_path: Path) -> None: + """A run that was granted, then re-staged for a second review, + must classify as ``pending`` — not ``granted``. The previous + implementation only looked at decisions and would silently + report a stale ``granted`` even though the operator's pending + list correctly surfaced the run.""" + from forgelm.cli.subcommands._approvals import ( + _classify_chain, + _collect_run_audit_chain, + ) + + output_dir = tmp_path / "run" + output_dir.mkdir() + audit = output_dir / "audit_log.jsonl" + _write_event(audit, "human_approval.required", "fg-x", staging_path=str(output_dir / "final_model.staging")) + _write_event(audit, "human_approval.granted", "fg-x", approver="alice") + _write_event(audit, "human_approval.required", "fg-x", staging_path=str(output_dir / "final_model.staging.2")) + chain = _collect_run_audit_chain(str(audit), "fg-x") + assert _classify_chain(chain) == "pending" + + def test_re_stage_after_reject_returns_pending(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._approvals import ( + _classify_chain, + _collect_run_audit_chain, + ) + + output_dir = tmp_path / "run" + output_dir.mkdir() + audit = output_dir / "audit_log.jsonl" + _write_event(audit, "human_approval.required", "fg-x") + _write_event(audit, "human_approval.rejected", "fg-x", approver="alice") + _write_event(audit, "human_approval.required", "fg-x") + chain = _collect_run_audit_chain(str(audit), "fg-x") + assert _classify_chain(chain) == "pending" + + def test_required_then_granted_returns_granted(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._approvals import ( + _classify_chain, + _collect_run_audit_chain, + ) + + output_dir = tmp_path / "run" + output_dir.mkdir() + audit = output_dir / "audit_log.jsonl" + _write_event(audit, "human_approval.required", "fg-x") + _write_event(audit, "human_approval.granted", "fg-x", approver="alice") + chain = _collect_run_audit_chain(str(audit), "fg-x") + assert _classify_chain(chain) == "granted" + + def test_only_required_returns_pending(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._approvals import ( + _classify_chain, + _collect_run_audit_chain, + ) + + output_dir = tmp_path / "run" + output_dir.mkdir() + audit = output_dir / "audit_log.jsonl" + _write_event(audit, "human_approval.required", "fg-x") + chain = _collect_run_audit_chain(str(audit), "fg-x") + assert _classify_chain(chain) == "pending" + + def test_empty_chain_returns_unknown(self) -> None: + from forgelm.cli.subcommands._approvals import _classify_chain + + assert _classify_chain([]) == "unknown" + + +class TestStagingPathTraversalSandwich: + """F-TEST-37-01: path-traversal guard is the load-bearing protection. + + The previous test could NOT distinguish a working guard from a no-op + because the canonical fallback dir didn't exist — so an empty + staging_contents could mean either "guard fired and fell back to + nothing" OR "guard was a no-op and the listdir returned nothing". + The fix: create the fallback dir with a sentinel file so the test + SEES the difference (guard fires → sentinel surfaces; guard no-ops + → /etc contents leak). Also assert the warning was logged. + """ + + def test_show_refuses_external_staging_path_with_caplog(self, tmp_path: Path, capsys, caplog) -> None: + import logging + + from forgelm.cli.subcommands._approvals import _run_approvals_cmd + + output_dir = tmp_path / "run" + output_dir.mkdir() + # Create the canonical fallback with a sentinel so a working + # guard surfaces ["fallback_marker.txt"] (NOT /etc contents). + fallback = output_dir / "final_model.staging" + fallback.mkdir() + (fallback / "fallback_marker.txt").write_text("ok") + audit = output_dir / "audit_log.jsonl" + # Plant an attacker-controlled staging_path pointing outside output_dir. + _write_event( + audit, + "human_approval.required", + "fg-attack0000000", + staging_path="/etc", + ) + + with caplog.at_level(logging.WARNING): + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd( + _build_args(output_dir, show="fg-attack0000000"), + output_format="json", + ) + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + # Sentinel proves the fallback path was used (not /etc contents). + assert payload["staging_contents"] == ["fallback_marker.txt"] + # Warning proves the guard actually fired. + assert any("Refusing staging_path" in r.message for r in caplog.records), ( + f"path-traversal guard must log a warning; got: {[r.message for r in caplog.records]}" + ) + assert any("/etc" in r.message for r in caplog.records) + + +class TestShowUsesLatestRequired: + """User inline: _run_approvals_show should pick latest required, not first. + + Previously the code did ``next((e for e in chain if ... required), {})`` + which returned the FIRST required event. After a re-stage with a fresh + staging directory, --show would surface the STALE original directory. + The fix walks reversed(chain). + """ + + def test_show_re_staged_run_surfaces_latest_staging_dir(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._approvals import _run_approvals_cmd + + output_dir = tmp_path / "run" + output_dir.mkdir() + old_staging = output_dir / "final_model.staging.old" + old_staging.mkdir() + (old_staging / "old_marker.txt").write_text("v1") + new_staging = output_dir / "final_model.staging.new" + new_staging.mkdir() + (new_staging / "new_marker.txt").write_text("v2") + audit = output_dir / "audit_log.jsonl" + _write_event( + audit, + "human_approval.required", + "fg-restage000000", + staging_path=str(old_staging), + ) + _write_event(audit, "human_approval.rejected", "fg-restage000000", approver="alice") + _write_event( + audit, + "human_approval.required", + "fg-restage000000", + staging_path=str(new_staging), + ) + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd( + _build_args(output_dir, show="fg-restage000000"), + output_format="json", + ) + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + # Latest staging dir wins (the re-stage), not the original. + assert payload["staging_contents"] == ["new_marker.txt"], ( + f"--show must surface the LATEST staging dir, got {payload['staging_contents']}" + ) + # And classify_chain agrees (latest required > latest decision = pending). + assert payload["status"] == "pending" + + +class TestAuditLogReaderStrictMode: + """User inline + Phase D2: strict mode raises AuditLogParseError on malformed lines.""" + + def test_strict_mode_raises_on_malformed_json(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._audit_log_reader import ( + AuditLogParseError, + iter_audit_events, + ) + + audit_path = tmp_path / "audit_log.jsonl" + audit_path.write_text("not json at all\n", encoding="utf-8") + + with pytest.raises(AuditLogParseError) as exc_info: + list(iter_audit_events(str(audit_path), strict=True)) + assert exc_info.value.line_number == 1 + assert str(audit_path) in str(exc_info.value) + + def test_strict_mode_raises_on_non_dict_root(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._audit_log_reader import ( + AuditLogParseError, + iter_audit_events, + ) + + audit_path = tmp_path / "audit_log.jsonl" + audit_path.write_text('["list", "not", "dict"]\n', encoding="utf-8") + + with pytest.raises(AuditLogParseError) as exc_info: + list(iter_audit_events(str(audit_path), strict=True)) + assert exc_info.value.line_number == 1 + assert "not dict" in str(exc_info.value) or "list" in str(exc_info.value) + + def test_lenient_mode_skips_silently(self, tmp_path: Path, caplog) -> None: + import logging + + from forgelm.cli.subcommands._audit_log_reader import iter_audit_events + + audit_path = tmp_path / "audit_log.jsonl" + audit_path.write_text( + 'not json\n{"event": "ok", "run_id": "fg-x"}\n["list"]\n', + encoding="utf-8", + ) + with caplog.at_level(logging.WARNING): + events = list(iter_audit_events(str(audit_path), strict=False)) + assert len(events) == 1 + # Summary warning fires for the 2 skipped lines. + assert any("Skipped 2 malformed line" in r.message for r in caplog.records) + + +class TestApproveStrictModeOnCorruptLog: + """User inline + Phase D2: approve / reject use strict mode and surface + a clear error rather than silently skipping a corrupted decision record.""" + + def test_approve_aborts_on_corrupted_audit_log(self, tmp_path: Path, capsys) -> None: + """A malformed line in the audit log must produce + EXIT_CONFIG_ERROR with an actionable message — NOT silently skip + the line and let the operator double-grant.""" + + from forgelm.cli.subcommands._approve import _run_approve_cmd + + output_dir = tmp_path / "run" + output_dir.mkdir() + audit = output_dir / "audit_log.jsonl" + # Real required event followed by a corrupt line — strict mode + # should fail-fast on the corruption rather than skip past it. + _write_event(audit, "human_approval.required", "fg-x", staging_path=str(output_dir / "final_model.staging")) + with open(audit, "a", encoding="utf-8") as fh: + fh.write("CORRUPTED LINE NOT JSON\n") + + args = MagicMock() + args.run_id = "fg-x" + args.output_dir = str(output_dir) + args.comment = None + with pytest.raises(SystemExit) as ei: + _run_approve_cmd(args, output_format="json") + # EXIT_CONFIG_ERROR (1) — operator must repair log, not retry. + assert ei.value.code == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is False + assert "corrupted" in payload["error"].lower() + assert "line" in payload["error"].lower() + + +class TestApprovalsTimestampTypeSafety: + """F-37-TS-TYPE: tampered / hand-rolled audit log carrying a non-string + timestamp (e.g. epoch int) must not crash --pending sort. + + The previous sort key was ``e.get("timestamp") or ""`` which only + replaces *falsy* values; an int timestamp would crash sorted() with + TypeError when compared against a real ISO-8601 string from another + event. + """ + + def test_pending_with_int_timestamp_does_not_crash(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._approvals import _run_approvals_cmd + + output_dir = tmp_path / "run" + output_dir.mkdir() + audit = output_dir / "audit_log.jsonl" + # Run A: legitimate ISO-8601 string timestamp. + _write_event( + audit, + "human_approval.required", + "fg-string0000000", + staging_path=str(output_dir / "final_model.staging"), + ) + # Run B: hand-rolled / tampered audit event with int timestamp. + _write_event( + audit, + "human_approval.required", + "fg-numeric000000", + staging_path=str(output_dir / "final_model.staging.b"), + timestamp=1730500000, # epoch int — not a string + ) + # Run C: non-string non-int (list) — also must not crash. + _write_event( + audit, + "human_approval.required", + "fg-listts0000000", + staging_path=str(output_dir / "final_model.staging.c"), + timestamp=["bogus", "shape"], + ) + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + run_ids = {s["run_id"] for s in payload["pending"]} + # All three runs must surface — the type-safe sort key must not + # have hidden any of them. + assert run_ids == {"fg-string0000000", "fg-numeric000000", "fg-listts0000000"}, ( + f"non-string timestamp must not hide pending runs; got {run_ids!r}" + ) + + def test_safe_timestamp_key_returns_empty_for_non_strings(self) -> None: + from forgelm.cli.subcommands._approvals import _safe_timestamp_key + + assert _safe_timestamp_key({"timestamp": "2026-01-01T00:00:00+00:00"}) == "2026-01-01T00:00:00+00:00" + assert _safe_timestamp_key({"timestamp": 1730500000}) == "" + assert _safe_timestamp_key({"timestamp": None}) == "" + assert _safe_timestamp_key({"timestamp": ["bogus"]}) == "" + assert _safe_timestamp_key({}) == "" + + +class TestApprovalsAuditLogUnreadable: + """Wave 2a Round-2 review (CodeRabbit): a permission-denied audit log + must surface a clear EXIT_CONFIG_ERROR rather than masquerade as + "no pending approvals" / "no events for this run". + + ``iter_audit_events`` swallows OSError-on-open + logs at ERROR + yields + nothing. Without the explicit os.access readability check in + ``_run_approvals_list_pending`` and ``_run_approvals_show``, an + operator with a chmod-broken audit log would see a healthy-looking + empty result and miss a real pending decision. + """ + + def test_pending_exits_config_error_when_audit_log_unreadable(self, tmp_path: Path, capsys, monkeypatch) -> None: + import os + + from forgelm.cli.subcommands._approvals import _run_approvals_cmd + + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-real00000000") + audit_path = output_dir / "audit_log.jsonl" + # Simulate an unreadable file via os.access monkeypatch (chmod 000 + # behaves differently in CI containers; the access predicate is + # what the code under test actually consults). + original_access = os.access + monkeypatch.setattr( + os, + "access", + lambda path, mode: False if str(path) == str(audit_path) else original_access(path, mode), + ) + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, pending=True), output_format="json") + assert ei.value.code == 1, "unreadable audit log must EXIT_CONFIG_ERROR (1), not silently exit 0" + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is False + assert "not readable" in payload["error"].lower() + assert "audit_log.jsonl" in payload["error"] + + def test_show_exits_config_error_when_audit_log_unreadable(self, tmp_path: Path, capsys, monkeypatch) -> None: + import os + + from forgelm.cli.subcommands._approvals import _run_approvals_cmd + + output_dir = tmp_path / "run" + _seed_run(output_dir, "fg-real00000000") + audit_path = output_dir / "audit_log.jsonl" + original_access = os.access + monkeypatch.setattr( + os, + "access", + lambda path, mode: False if str(path) == str(audit_path) else original_access(path, mode), + ) + + with pytest.raises(SystemExit) as ei: + _run_approvals_cmd(_build_args(output_dir, show="fg-real00000000"), output_format="json") + assert ei.value.code == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is False + assert "not readable" in payload["error"].lower() diff --git a/tests/test_data_audit_workers.py b/tests/test_data_audit_workers.py new file mode 100644 index 00000000..fcef5569 --- /dev/null +++ b/tests/test_data_audit_workers.py @@ -0,0 +1,720 @@ +"""Phase 17: ``audit_dataset(..., workers=N)`` determinism contract. + +Pins the per-Phase-17 invariants: + +1. The audit JSON is byte-identical regardless of ``workers``. Operators + relying on ``data_audit_report.json`` for the EU AI Act Article 10 + governance bundle MUST be able to swap ``--workers 1`` for + ``--workers 4`` without the artefact's hash changing. +2. ``lang_sample`` (random snippets per split) is byte-identical across + worker counts — the only random-ish field in the report and the most + likely place a parallel run would diverge. +3. PII / secrets / quality / near-duplicate counts are identical across + worker counts. +4. The CLI ``--workers`` flag rejects 0 / negative values at parse time. +5. ``audit_dataset(workers=0)`` raises a typed ``ValueError`` so library + callers that bypass argparse still see the validation. + +The fixture corpus is intentionally small (3 splits × 12 rows) — the +suite runs on every CI matrix combo without budgeting for multi-second +multiprocessing spin-up costs. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import asdict +from pathlib import Path +from unittest.mock import patch + +import pytest + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def _seed_three_split_corpus(tmp_path: Path) -> Path: + """Build a deterministic 3-split corpus suitable for parallel audit.""" + corpus = tmp_path / "corpus" + corpus.mkdir() + # Same row text across splits would trip cross-split-leakage detection; + # we want a clean run so per-split assertions stay focused. Different + # subjects per split, deterministic content per row index. + train_rows = [{"text": f"Training sample number {i} about machine learning topics."} for i in range(12)] + val_rows = [{"text": f"Validation sample {i} on natural language processing benchmarks."} for i in range(8)] + test_rows = [{"text": f"Test sample {i} for evaluation pipeline regression coverage."} for i in range(6)] + _write_jsonl(corpus / "train.jsonl", train_rows) + _write_jsonl(corpus / "validation.jsonl", val_rows) + _write_jsonl(corpus / "test.jsonl", test_rows) + return corpus + + +def _audit_to_canonical_json(corpus: Path, workers: int, output_dir: Path) -> str: + """Run ``audit_dataset`` and return the on-disk JSON as a canonical string. + + Re-reading the on-disk file (rather than ``asdict(report)``) closes + the loop on what an operator-facing CI gate actually compares. + """ + from forgelm.data_audit import audit_dataset + + report = audit_dataset(str(corpus), output_dir=str(output_dir), workers=workers) + # Sanity: the report dataclass is also self-consistent across runs. + asdict_payload = asdict(report) + assert isinstance(asdict_payload, dict) + written = (output_dir / "data_audit_report.json").read_text(encoding="utf-8") + return written + + +# --------------------------------------------------------------------------- +# Determinism contract — workers=1 vs workers=2 vs workers=4 +# --------------------------------------------------------------------------- + + +def _strip_generated_at_for_hash(text: str) -> str: + """Remove the top-level ``generated_at`` line so file hashes can compare. + + The audit report's only intentionally non-deterministic field is the + ISO-8601 timestamp captured at write time. We strip it textually + before SHA-256-ing so the rest of the file can be compared + byte-for-byte across worker counts. + + Wave 2a Round-2 F-TEST-17-07: anchored to the **top-level** key + position only. The audit JSON is written via ``json.dumps(..., indent=2)`` + so top-level keys live at indent level 2 (`` "key": value``). A + future schema addition that puts a per-split ``generated_at`` deeper + in the tree (e.g. ``"croissant": {"dateCreated": ...}``) is NOT + stripped — only the report-level timestamp is. This protects against + the false-negative where a real determinism regression would be + silently masked by an over-broad regex. + """ + import re + + # ^ "generated_at": "..." anchored to start-of-line + 2-space indent. + # MULTILINE so ^ matches every line, not just file start. Sonar S6326: + # the explicit `{2}` quantifier reads better than two literal spaces. + return re.sub( + r'^ {2}"generated_at"\s*:\s*"[^"]+"', + ' "generated_at": ""', + text, + flags=re.MULTILINE, + ) + + +def _file_sha256(path: Path) -> str: + """SHA-256 of an on-disk file's bytes (with generated_at stripped first). + + Mirrors what an EU AI Act Article 10 governance-bundle CI gate + actually checks against: the operator pins the hash of + ``data_audit_report.json`` so a regression that changes formatting, + key ordering, float repr, or Unicode normalisation flips the gate + even when ``json.loads`` round-trips to the same dict. + """ + import hashlib + + text = path.read_text(encoding="utf-8") + stripped = _strip_generated_at_for_hash(text) + return hashlib.sha256(stripped.encode("utf-8")).hexdigest() + + +class TestWorkersDeterminism: + """The audit JSON must be byte-identical across worker counts. + + F-26-02 fix: assert SHA-256 of the on-disk file equality (with the + wall-clock ``generated_at`` field stripped textually first), not + ``dict == dict``. A parsed-dict comparison tolerates key ordering / + whitespace / Unicode-normalisation / float-repr drift that a real + file-hash CI gate would flip on. Both checks now run side-by-side + so a regression on either layer surfaces. + """ + + @pytest.mark.parametrize("worker_count", [2, 4]) + def test_audit_json_byte_identical_to_sequential(self, tmp_path: Path, worker_count: int) -> None: + corpus = _seed_three_split_corpus(tmp_path) + + baseline_dir = tmp_path / "out-w1" + baseline_dir.mkdir() + _audit_to_canonical_json(corpus, workers=1, output_dir=baseline_dir) + + parallel_dir = tmp_path / f"out-w{worker_count}" + parallel_dir.mkdir() + _audit_to_canonical_json(corpus, workers=worker_count, output_dir=parallel_dir) + + baseline_path = baseline_dir / "data_audit_report.json" + parallel_path = parallel_dir / "data_audit_report.json" + + # **Primary contract**: byte-for-byte file hash equality (with + # generated_at stripped). This is what an Article 10 governance- + # bundle CI gate actually compares. + baseline_hash = _file_sha256(baseline_path) + parallel_hash = _file_sha256(parallel_path) + assert baseline_hash == parallel_hash, ( + f"data_audit_report.json SHA-256 differs between workers=1 and " + f"workers={worker_count} (baseline={baseline_hash[:12]}..., " + f"parallel={parallel_hash[:12]}...) — operators pin this hash " + f"in CI; the determinism contract is broken." + ) + + # **Secondary, looser contract**: parsed dict equality (catches + # any rare case where the file hashes accidentally agree on + # different content — e.g. two symmetric drift sources cancelling). + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + parallel = json.loads(parallel_path.read_text(encoding="utf-8")) + baseline.pop("generated_at", None) + parallel.pop("generated_at", None) + assert baseline == parallel + + def test_languages_top3_byte_identical(self, tmp_path: Path) -> None: + """``languages_top3`` is the *persisted* derivative of the + per-split language-detection sample (the in-memory + ``lang_sample`` field on ``_StreamingAggregator`` is never + serialised, so the previous ``lang_sample == lang_sample`` + assertion was vacuous None == None). + + F-26-01 fix: compare the actual on-disk field that operators + and CI gates can see. This is the most-likely-to-diverge field + under non-deterministic ordering since langdetect picks a sample + deterministically per-process but the worker spawn order can + affect which sample lands first. + + Wave 2a Round-2 F-TEST-17-01: gated on ``langdetect`` being + installed. Without the optional ``[ingestion]`` extra, + ``_detect_language`` returns the literal ``"unknown"`` for every + row, so seq vs par equality is structurally trivial and a real + ordering regression would not surface. CI runs ``[dev]`` only, + so this test runs locally for developers with ``[ingestion]`` + installed; CI exercises the determinism contract via + ``test_audit_json_byte_identical_to_sequential`` (which is also + SHA-256 byte-equal at the file level). + """ + pytest.importorskip( + "langdetect", + reason="languages_top3 is a flat 'unknown' constant without langdetect; " + "install '[ingestion]' extra to exercise the actual ordering contract.", + ) + corpus = _seed_three_split_corpus(tmp_path) + + baseline_dir = tmp_path / "out-w1" + baseline_dir.mkdir() + _audit_to_canonical_json(corpus, workers=1, output_dir=baseline_dir) + + parallel_dir = tmp_path / "out-w4" + parallel_dir.mkdir() + _audit_to_canonical_json(corpus, workers=4, output_dir=parallel_dir) + + seq = json.loads((baseline_dir / "data_audit_report.json").read_text(encoding="utf-8")) + par = json.loads((parallel_dir / "data_audit_report.json").read_text(encoding="utf-8")) + for split_name in ("train", "validation", "test"): + seq_split = seq["splits"][split_name] + par_split = par["splits"][split_name] + assert seq_split.get("languages_top3") == par_split.get("languages_top3"), ( + f"languages_top3 for split {split_name!r} differs between " + f"workers=1 and workers=4 (seq={seq_split.get('languages_top3')}, " + f"par={par_split.get('languages_top3')})" + ) + + def test_split_iteration_order_pinned(self, tmp_path: Path) -> None: + """F-26-06: the merge step relies on ``splits_paths`` yielding + keys in the canonical train → validation → test order. A future + refactor that swaps the dict for a set or sorts alphabetically + breaks the byte-identical contract; pin the order explicitly so + the regression surfaces here, not in the field.""" + corpus = _seed_three_split_corpus(tmp_path) + + from forgelm.data_audit import audit_dataset + + report = audit_dataset(str(corpus), workers=1) + assert list(report.splits.keys()) == ["train", "validation", "test"], ( + f"split iteration order regression: got {list(report.splits.keys())}, " + f"expected ['train', 'validation', 'test'] (canonical Wave 1 _SPLIT_ALIASES order)" + ) + + +# --------------------------------------------------------------------------- +# Per-component invariants +# --------------------------------------------------------------------------- + + +class TestWorkersComponentInvariants: + def test_pii_summary_identical(self, tmp_path: Path) -> None: + # Inject a PII-bearing row so the summary is non-empty. + corpus = _seed_three_split_corpus(tmp_path) + train = corpus / "train.jsonl" + existing = train.read_text(encoding="utf-8") + train.write_text( + existing + json.dumps({"text": "Contact alice@example.com or call 555-123-4567."}) + "\n", + encoding="utf-8", + ) + + from forgelm.data_audit import audit_dataset + + seq = audit_dataset(str(corpus), workers=1) + par = audit_dataset(str(corpus), workers=4) + assert seq.pii_summary == par.pii_summary + assert seq.pii_severity == par.pii_severity + + def test_near_duplicate_summary_identical(self, tmp_path: Path) -> None: + # Inject duplicate rows in train to exercise the simhash detector. + corpus = _seed_three_split_corpus(tmp_path) + train = corpus / "train.jsonl" + existing = train.read_text(encoding="utf-8") + # Two near-identical rows — the simhash detector should flag the pair. + dup_text = "Customer support handles refund requests promptly." + train.write_text( + existing + json.dumps({"text": dup_text}) + "\n" + json.dumps({"text": dup_text + " "}) + "\n", + encoding="utf-8", + ) + + from forgelm.data_audit import audit_dataset + + seq = audit_dataset(str(corpus), workers=1) + par = audit_dataset(str(corpus), workers=4) + assert seq.near_duplicate_summary == par.near_duplicate_summary + + def test_total_samples_identical(self, tmp_path: Path) -> None: + corpus = _seed_three_split_corpus(tmp_path) + from forgelm.data_audit import audit_dataset + + seq = audit_dataset(str(corpus), workers=1) + par = audit_dataset(str(corpus), workers=4) + assert seq.total_samples == par.total_samples == 26 # 12 + 8 + 6 + + def test_secrets_summary_identical(self, tmp_path: Path) -> None: + corpus = _seed_three_split_corpus(tmp_path) + # Inject a fake AWS key pattern so the secrets scanner has something + # to find. Pattern is intentionally fake (does not start with the + # real AWS prefix) to avoid the gitleaks pre-commit hook flagging + # the test fixture itself. + train = corpus / "train.jsonl" + existing = train.read_text(encoding="utf-8") + # AKIA + 16 hex-style characters trips the AWS access-key regex. + fake_secret = "AKIA" + "1234567890ABCDEF" + train.write_text( + existing + json.dumps({"text": f"Sample with AWS-like token {fake_secret}."}) + "\n", + encoding="utf-8", + ) + + from forgelm.data_audit import audit_dataset + + seq = audit_dataset(str(corpus), workers=1) + par = audit_dataset(str(corpus), workers=4) + assert seq.secrets_summary == par.secrets_summary + + +# --------------------------------------------------------------------------- +# Edge cases — single split, sequential default, validation +# --------------------------------------------------------------------------- + + +class TestWorkersEdgeCases: + def test_single_split_corpus_ignores_workers_above_one(self, tmp_path: Path) -> None: + """A single-split corpus has nothing to parallelise; ``workers > 1`` + must still produce a valid report.""" + corpus_file = tmp_path / "single.jsonl" + _write_jsonl(corpus_file, [{"text": f"Row {i}"} for i in range(20)]) + + from forgelm.data_audit import audit_dataset + + report_seq = audit_dataset(str(corpus_file), workers=1) + report_par = audit_dataset(str(corpus_file), workers=4) + + assert report_seq.total_samples == report_par.total_samples == 20 + assert set(report_seq.splits) == set(report_par.splits) == {"train"} + + def test_default_workers_is_one(self, tmp_path: Path) -> None: + """Backwards compatibility: omitting ``workers`` must produce the + sequential path so the default behaviour for every existing caller + is unchanged. + + Wave 2a Round-2 F-TEST-17-06: previously this only asserted + ``total_samples == 26`` which is a sanity check, not a contract + test. The CHANGELOG-promised contract is "the default produces + byte-identical output to ``--workers 1``". Now compares the + canonical JSON file content. + """ + from dataclasses import asdict + + from forgelm.data_audit import audit_dataset + + corpus = _seed_three_split_corpus(tmp_path) + + # Compare the canonical report dicts (modulo generated_at) so a + # silent default-flip from 1 → e.g. 2 would surface here. + default_report = audit_dataset(str(corpus)) + explicit_report = audit_dataset(str(corpus), workers=1) + default_dict = asdict(default_report) + explicit_dict = asdict(explicit_report) + # generated_at is wall-clock; strip per the determinism contract. + default_dict.pop("generated_at", None) + explicit_dict.pop("generated_at", None) + assert default_dict == explicit_dict, "audit_dataset() default behaviour drifted from explicit workers=1" + + @pytest.mark.parametrize("invalid", [0, -1, -10]) + def test_workers_below_one_raises_valueerror(self, tmp_path: Path, invalid: int) -> None: + """``workers < 1`` is caller error, not a runtime fault.""" + corpus = _seed_three_split_corpus(tmp_path) + + from forgelm.data_audit import audit_dataset + + with pytest.raises(ValueError, match="workers"): + audit_dataset(str(corpus), workers=invalid) + + def test_workers_non_int_raises_valueerror(self, tmp_path: Path) -> None: + corpus = _seed_three_split_corpus(tmp_path) + + from forgelm.data_audit import audit_dataset + + with pytest.raises(ValueError, match="workers"): + audit_dataset(str(corpus), workers="four") # type: ignore[arg-type] + + def test_workers_bool_raises_valueerror(self, tmp_path: Path) -> None: + """``True`` would int-coerce to 1 but is the wrong type — reject + explicitly so a caller can't accidentally pass a boolean.""" + corpus = _seed_three_split_corpus(tmp_path) + + from forgelm.data_audit import audit_dataset + + with pytest.raises(ValueError, match="workers"): + audit_dataset(str(corpus), workers=True) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# CLI smoke — --workers flag end-to-end +# --------------------------------------------------------------------------- + + +class TestWorkersCLI: + def test_cli_workers_flag_help_text(self) -> None: + """``forgelm audit --help`` exposes the new flag with the expected + metavar so operators can discover it.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "forgelm.cli", "audit", "--help"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert "--workers" in result.stdout + assert "default: 1" in result.stdout.lower() or "(default: 1" in result.stdout + + def test_cli_workers_zero_rejected_at_parse_time(self) -> None: + """``--workers 0`` exits with argparse usage-error rather than + propagating into the audit pipeline.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "forgelm.cli", "audit", "/nonexistent", "--workers", "0"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + # argparse error path uses "invalid X value" or our custom + # "value must be >= 1" message. + combined = result.stderr.lower() + assert ">= 1" in combined or "value must" in combined or "invalid" in combined + + def test_cli_workers_negative_rejected_at_parse_time(self) -> None: + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "forgelm.cli", "audit", "/nonexistent", "--workers", "-2"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + + def test_cli_workers_default_when_omitted(self, tmp_path: Path) -> None: + """Running ``forgelm audit`` without ``--workers`` produces the same + report as ``--workers 1``.""" + import subprocess + + corpus = _seed_three_split_corpus(tmp_path) + + out_default = tmp_path / "default" + out_default.mkdir() + result = subprocess.run( + [ + sys.executable, + "-m", + "forgelm.cli", + "audit", + str(corpus), + "--output", + str(out_default), + ], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + + out_explicit = tmp_path / "explicit" + out_explicit.mkdir() + result = subprocess.run( + [ + sys.executable, + "-m", + "forgelm.cli", + "audit", + str(corpus), + "--output", + str(out_explicit), + "--workers", + "1", + ], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + + default_payload = json.loads((out_default / "data_audit_report.json").read_text(encoding="utf-8")) + explicit_payload = json.loads((out_explicit / "data_audit_report.json").read_text(encoding="utf-8")) + default_payload.pop("generated_at", None) + explicit_payload.pop("generated_at", None) + assert default_payload == explicit_payload + + @pytest.mark.parametrize( + "invalid", + ["four", "1.5", "True", "False", "1e2", "0x1", ""], + ids=["word", "float-string", "bool-True", "bool-False", "scientific", "hex", "empty"], + ) + def test_cli_workers_non_integer_rejected_at_parse_time(self, invalid: str) -> None: + """``--workers `` must trip ``_positive_int`` at parse time. + + Wave 2a Round-2 F-TEST-17-02: parametrise to cover the canonical + Python "looks like an int but isn't" cases — float-strings, + bool-strings, scientific notation, hex. A future refactor that + swapped ``int(value)`` for ``float(value)`` truncation (a + well-meaning but wrong simplification) would silently start + accepting ``--workers 1.5 → 1``; this parametrisation pins the + strict-int contract.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "forgelm.cli", "audit", "/nonexistent", "--workers", invalid], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0, f"--workers {invalid!r} should be rejected, got returncode 0" + assert ( + "invalid integer" in result.stderr.lower() + or "invalid" in result.stderr.lower() + or "must be" in result.stderr.lower() + ) + + +class TestWorkersSpawnPinning: + """Wave 2a Round-2 F-TEST-17-03: regression-pin spawn context.""" + + def test_orchestrator_uses_spawn_context(self, tmp_path: Path) -> None: + """The determinism contract requires the spawn start method + unconditionally so Linux fork-by-default does not silently slip + through (langdetect.DetectorFactory state, file-descriptor + inheritance, etc.). Spy on ``multiprocessing.get_context`` to + confirm 'spawn' is what the orchestrator asks for.""" + from forgelm.data_audit import _orchestrator, audit_dataset + + called_with: list[str] = [] + original = _orchestrator.multiprocessing.get_context + + def _spy(method: str): + called_with.append(method) + return original(method) + + corpus = _seed_three_split_corpus(tmp_path) + with patch.object(_orchestrator.multiprocessing, "get_context", _spy): + audit_dataset(str(corpus), workers=2, output_dir=str(tmp_path / "out")) + + assert "spawn" in called_with, f"orchestrator must request 'spawn' start method, got {called_with!r}" + + +class TestWorkersClamp: + """Wave 2a Round-2 F-TEST-17-08: workers > num_splits clamps.""" + + def test_workers_above_split_count_logged(self, tmp_path: Path, caplog) -> None: + """When operator passes ``--workers 10`` on a 3-split corpus, the + orchestrator clamps to 3 and logs the reduction so the + wall-clock-vs-expected gap doesn't surprise the operator.""" + import logging + + from forgelm.data_audit import audit_dataset + + corpus = _seed_three_split_corpus(tmp_path) + with caplog.at_level(logging.INFO): + audit_dataset(str(corpus), workers=10, output_dir=str(tmp_path / "out")) + assert any("requested workers=10" in r.message for r in caplog.records), ( + f"expected workers-reduction log, got: {[r.message for r in caplog.records]}" + ) + + +# --------------------------------------------------------------------------- +# F-26-04: dedup_method="minhash" × workers > 1 (the path operators +# explicitly recommend for >50K-row corpora) +# --------------------------------------------------------------------------- + + +class TestWorkersWithMinHash: + """The byte-identical contract must hold for the minhash dedup path + too — that's the path the documentation recommends operators use + when --workers matters most (large corpora).""" + + def test_minhash_byte_identical_across_workers(self, tmp_path: Path) -> None: + pytest.importorskip("datasketch") + + corpus = _seed_three_split_corpus(tmp_path) + + from forgelm.data_audit import audit_dataset + + baseline_dir = tmp_path / "out-w1" + baseline_dir.mkdir() + audit_dataset( + str(corpus), + output_dir=str(baseline_dir), + dedup_method="minhash", + minhash_jaccard=0.85, + workers=1, + ) + + parallel_dir = tmp_path / "out-w4" + parallel_dir.mkdir() + audit_dataset( + str(corpus), + output_dir=str(parallel_dir), + dedup_method="minhash", + minhash_jaccard=0.85, + workers=4, + ) + + baseline_path = baseline_dir / "data_audit_report.json" + parallel_path = parallel_dir / "data_audit_report.json" + assert _file_sha256(baseline_path) == _file_sha256(parallel_path), ( + "minhash + workers determinism contract failed: data_audit_report.json SHA-256 differs" + ) + + +# --------------------------------------------------------------------------- +# F-26-05: error-propagation contract — a worker exception must not be +# silently swallowed; the operator must learn which split crashed. +# --------------------------------------------------------------------------- + + +class TestWorkersErrorPropagation: + """A failing worker must surface the exception instead of producing a + silently-incomplete report. + + Spawn-method workers cannot see test-process monkeypatches (they + re-import the target module fresh), so the parallel path is + exercised end-to-end via an actually-failing fixture (corrupt JSONL + that ``_audit_split`` raises on) rather than a patched stub. The + sequential path is tested with a normal monkeypatch. + """ + + def test_sequential_split_failure_propagates(self, tmp_path: Path) -> None: + """Sequential path: a per-split failure must bubble up unchanged.""" + from unittest.mock import patch + + corpus = _seed_three_split_corpus(tmp_path) + + # Patch the orchestrator's bound name so the internal `for` + # loop sees the failing version. + from forgelm.data_audit import _orchestrator, audit_dataset + + original = _orchestrator._process_split + + def _flaky(*args, **kwargs): + if args and args[0] == "validation": + raise RuntimeError("synthetic per-split failure for test") + return original(*args, **kwargs) + + with patch.object(_orchestrator, "_process_split", _flaky): + with pytest.raises(RuntimeError, match="synthetic per-split failure"): + audit_dataset(str(corpus), workers=1) + + def test_parallel_path_raises_when_worker_function_raises_uncaught( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Parallel path: an uncaught worker exception (one that escapes + ``_process_split``'s own catches — e.g. a worker pickling error, + ImportError on a worker process, segfault, OOM) must surface + through ``Pool.map`` rather than being silently swallowed. + + Wave 2a Round-5 F-R5-03 fix: the previous parallel-failure test + used corrupt UTF-8 bytes that ``_process_split`` catches + internally and returns as a degraded outcome — the orchestrator's + ``try: pool.map(...) except Exception: logger.error(...); raise`` + branch (``forgelm/data_audit/_orchestrator.py:355-363``) was + therefore dead in tests. This test exercises it via the + worker-side env-var test injection point in + ``_process_split_for_pool``: spawn-method workers see env vars + inherited at spawn time, so ``FORGELM_AUDIT_TEST_WORKER_RAISES=validation`` + causes the validation worker to raise unconditionally. + """ + corpus = _seed_three_split_corpus(tmp_path) + monkeypatch.setenv("FORGELM_AUDIT_TEST_WORKER_RAISES", "validation") + + from forgelm.data_audit import audit_dataset + + # Pool.map wraps the worker exception; we expect the orchestrator's + # broad re-raise to surface it (any subclass of Exception is + # acceptable — the contract is "fails loud, not silent"). + with pytest.raises(Exception) as exc_info: + audit_dataset(str(corpus), workers=2) + # The original exception class + message survive the Pool wrap; + # assert the synthetic marker so we know we hit the injected + # raise, not some unrelated failure. + assert "synthetic test-injected worker failure" in str(exc_info.value), ( + f"orchestrator did not surface the worker-side RuntimeError; got: {exc_info.value!r}" + ) + + def test_parallel_path_does_not_silently_complete_on_split_failure(self, tmp_path: Path) -> None: + """Parallel path: a per-split read failure must surface as a + structured per-split error, not a silent completion that drops + the failed split, AND the other splits must finish cleanly. + + Wave 2a Round-2 F-TEST-17-04 fix: previously this test ``unlink``'d + the validation.jsonl which let split discovery drop the file + BEFORE scheduling — bypassing the worker error path entirely. + The fix preserves the file (so split discovery schedules it) + but corrupts the contents with bytes the streaming JSON reader + cannot decode, exercising the actual ``_process_split`` OSError + catch on the worker side.""" + corpus = _seed_three_split_corpus(tmp_path) + # Corrupt the validation split so split discovery still sees it + # and schedules a worker, but the streaming reader inside the + # worker hits a decode failure on first read. Invalid UTF-8 + # bytes plus a NUL byte ensure both ``json.loads`` and any + # raw-bytes decode path fail. + (corpus / "validation.jsonl").write_bytes(b"\xff\xfe not json at all \x00\x01\x02\n") + + from forgelm.data_audit import audit_dataset + + report = audit_dataset(str(corpus), workers=2) + # Train + test must still land cleanly — the parallel path's + # error isolation is the contract being defended. + assert "train" in report.splits + assert "test" in report.splits + # Validation must appear in the report (the file existed at + # discovery time) with either an explicit error marker or a + # zero sample count — both are documented `read_failed` shapes. + assert "validation" in report.splits, ( + "validation split must appear in report.splits — " + "the parallel worker path must not silently drop a corrupted split" + ) + validation = report.splits["validation"] + assert "error" in validation or validation.get("sample_count", 0) == 0 diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 00000000..0a4f0b64 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,928 @@ +"""Phase 34: ``forgelm doctor`` env-check subcommand. + +Heavy on lightweight unit tests of individual probes (so a CI runner +without torch / GPU / network access can still exercise the doctor +surface), with one CLI subprocess smoke at the bottom. +""" + +from __future__ import annotations + +import json +import sys +from collections import namedtuple +from unittest.mock import MagicMock, patch + +import pytest + +# ``sys.version_info`` is a named tuple (`.major`/`.minor`/`.micro`/...); +# patching with a plain tuple breaks the attribute access in the probe. +# Build a structurally-equivalent fake we can pass to ``patch``. +_FakeVersionInfo = namedtuple("_FakeVersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) + + +def _make_version(major: int, minor: int, micro: int = 0) -> _FakeVersionInfo: + return _FakeVersionInfo(major, minor, micro, "final", 0) + + +# ``shutil.disk_usage`` returns a ``_ntuple_diskusage`` named tuple +# (`.total` / `.used` / `.free`). Mirror it for the disk-space tests. +_FakeDiskUsage = namedtuple("_FakeDiskUsage", ["total", "used", "free"]) + +# --------------------------------------------------------------------------- +# Individual probes +# --------------------------------------------------------------------------- + + +class TestPythonVersionCheck: + def test_python_310_warns(self) -> None: + from forgelm.cli.subcommands._doctor import _check_python_version + + with patch("sys.version_info", _make_version(3, 10, 0)): + result = _check_python_version() + assert result.status == "warn" + assert "3.11" in result.detail # recommendation appears + + def test_python_311_passes(self) -> None: + from forgelm.cli.subcommands._doctor import _check_python_version + + with patch("sys.version_info", _make_version(3, 11, 5)): + result = _check_python_version() + assert result.status == "pass" + assert "3.11.5" in result.detail + + def test_python_312_passes(self) -> None: + from forgelm.cli.subcommands._doctor import _check_python_version + + with patch("sys.version_info", _make_version(3, 12, 1)): + result = _check_python_version() + assert result.status == "pass" + + def test_python_39_fails(self) -> None: + from forgelm.cli.subcommands._doctor import _check_python_version + + with patch("sys.version_info", _make_version(3, 9, 7)): + result = _check_python_version() + assert result.status == "fail" + assert "3.10" in result.detail + assert "below" in result.detail.lower() + + +class TestTorchCudaCheck: + def test_no_torch_fails(self) -> None: + """When torch is not importable doctor must surface a clear fail + (not crash).""" + import builtins + + from forgelm.cli.subcommands import _doctor + + original_import = builtins.__import__ + + def _block_torch(name, *args, **kwargs): + if name == "torch": + raise ImportError("No module named 'torch'") + return original_import(name, *args, **kwargs) + + with patch.object(builtins, "__import__", _block_torch): + result = _doctor._check_torch_cuda() + assert result.status == "fail" + assert "torch" in result.detail.lower() + + def test_torch_cpu_only_warns(self) -> None: + """CPU-only torch is supported but warned (training will be slow).""" + from forgelm.cli.subcommands._doctor import _check_torch_cuda + + fake_torch = MagicMock() + fake_torch.__version__ = "2.5.0" + fake_torch.cuda.is_available.return_value = False + fake_torch.version.cuda = None + with patch.dict("sys.modules", {"torch": fake_torch}): + result = _check_torch_cuda() + assert result.status == "warn" + assert "CPU-only" in result.detail or "cpu-only" in result.detail.lower() + + def test_torch_cuda_passes(self) -> None: + from forgelm.cli.subcommands._doctor import _check_torch_cuda + + fake_torch = MagicMock() + fake_torch.__version__ = "2.5.0" + fake_torch.cuda.is_available.return_value = True + fake_torch.version.cuda = "12.4" + with patch.dict("sys.modules", {"torch": fake_torch}): + result = _check_torch_cuda() + assert result.status == "pass" + assert "12.4" in result.detail + + +class TestGpuInventoryCheck: + def test_no_torch_fails(self) -> None: + import builtins + + from forgelm.cli.subcommands import _doctor + + original_import = builtins.__import__ + + def _block_torch(name, *args, **kwargs): + if name == "torch": + raise ImportError("nope") + return original_import(name, *args, **kwargs) + + with patch.object(builtins, "__import__", _block_torch): + result = _doctor._check_gpu_inventory() + assert result.status == "fail" + + def test_no_cuda_warns(self) -> None: + from forgelm.cli.subcommands._doctor import _check_gpu_inventory + + fake_torch = MagicMock() + fake_torch.cuda.is_available.return_value = False + with patch.dict("sys.modules", {"torch": fake_torch}): + result = _check_gpu_inventory() + assert result.status == "warn" + assert result.extras["device_count"] == 0 + + def test_two_gpus_pass_with_inventory(self) -> None: + from forgelm.cli.subcommands._doctor import _check_gpu_inventory + + fake_torch = MagicMock() + fake_torch.cuda.is_available.return_value = True + fake_torch.cuda.device_count.return_value = 2 + # 24 GiB and 80 GiB devices. + props_24 = MagicMock(name="A10", total_memory=24 * (1024**3)) + props_80 = MagicMock(name="A100", total_memory=80 * (1024**3)) + # MagicMock attribute trick: ``name`` is special, set explicitly. + props_24.name = "NVIDIA A10" + props_80.name = "NVIDIA A100" + fake_torch.cuda.get_device_properties.side_effect = [props_24, props_80] + with patch.dict("sys.modules", {"torch": fake_torch}): + result = _check_gpu_inventory() + assert result.status == "pass" + assert result.extras["device_count"] == 2 + assert len(result.extras["devices"]) == 2 + assert result.extras["devices"][0]["vram_gib"] == pytest.approx(24.0) + assert result.extras["devices"][1]["vram_gib"] == pytest.approx(80.0) + + +class TestOptionalExtraCheck: + def test_present_module_passes(self) -> None: + from forgelm.cli.subcommands._doctor import _check_optional_extra + + # ``json`` is always installed in stdlib — use it as the + # "definitely present" probe target. + result = _check_optional_extra("fakextra", "json", "stdlib JSON") + assert result.status == "pass" + assert "json" in result.detail + assert result.extras["installed"] is True + + def test_missing_module_warns_with_install_hint(self) -> None: + from forgelm.cli.subcommands._doctor import _check_optional_extra + + result = _check_optional_extra("ghost", "definitely_not_installed_xyz123", "fake purpose") + assert result.status == "warn" + assert "pip install 'forgelm[ghost]'" in result.detail + assert result.extras["installed"] is False + + +class TestHfHubReachableCheck: + """Probe verifies the HF Hub is reachable. + + Wave 2a Round-2 (F-XPR-02-01): the probe was migrated from raw + ``urllib.request.urlopen`` to :func:`forgelm._http.safe_get` so it + inherits the project HTTP discipline (SSRF guard, scheme policy, + timeout floor, secret-mask). Tests now monkeypatch ``safe_get`` at + its module location. + """ + + def test_unreachable_warns_not_fails(self) -> None: + """A network outage must NOT flip the gate to fail; doctor exists + precisely to surface that fact.""" + import requests as _requests + + from forgelm.cli.subcommands._doctor import _check_hf_hub_reachable + + with patch( + "forgelm._http.safe_get", + side_effect=_requests.ConnectionError("DNS lookup failed"), + ): + result = _check_hf_hub_reachable(timeout_seconds=5.0) + assert result.status == "warn" + assert result.extras["reachable"] is False + + def test_200_response_passes(self) -> None: + from forgelm.cli.subcommands._doctor import _check_hf_hub_reachable + + fake_response = MagicMock() + fake_response.status_code = 200 + with patch("forgelm._http.safe_get", return_value=fake_response): + result = _check_hf_hub_reachable(timeout_seconds=5.0) + assert result.status == "pass" + assert result.extras["status_code"] == 200 + + def test_http_discipline_rejection_fails(self) -> None: + """Wave 2a Round-2 F-XPR-02-01: when the HTTP discipline rejects + the URL (e.g. http:// without opt-in, private IP without opt-in), + the probe should emit ``fail`` with an actionable detail — + operator misconfigured something the policy blocks.""" + from forgelm._http import HttpSafetyError + from forgelm.cli.subcommands._doctor import _check_hf_hub_reachable + + with patch( + "forgelm._http.safe_get", + side_effect=HttpSafetyError("Private/loopback/IMDS destination blocked: host=10.0.0.1"), + ): + result = _check_hf_hub_reachable(timeout_seconds=5.0) + assert result.status == "fail" + assert result.extras["reachable"] is False + assert "Private" in result.extras["error"] or "blocked" in result.extras["error"] + + def test_hf_hub_probe_uses_safe_get_layer(self) -> None: + """Wave 2a Round-2 F-XPR-02-01: regression-pin that the doctor + probe routes through forgelm._http.safe_get rather than calling + urllib / requests directly. Catches a future refactor that + reverts to undisciplined HTTP.""" + from forgelm.cli.subcommands._doctor import _check_hf_hub_reachable + + fake_response = MagicMock() + fake_response.status_code = 200 + with patch("forgelm._http.safe_get", return_value=fake_response) as spy: + _check_hf_hub_reachable(timeout_seconds=5.0) + assert spy.call_count == 1 + call = spy.call_args + # Method must be HEAD by default (no body download); UA header set. + assert call.kwargs["method"] == "HEAD" + assert "User-Agent" in call.kwargs["headers"] + + +class TestHfCacheOfflineCheck: + """Wave 2a Round-1 review (gemini bot): HF cache resolution honours + HF_HUB_CACHE > HF_HOME/hub > ~/.cache/huggingface/hub. Tests must + set up the *correct* cache dir layout (HF_HOME/hub subdirectory) + or use HF_HUB_CACHE directly.""" + + def test_missing_cache_warns(self, tmp_path, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _check_hf_cache_offline + + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "nonexistent_cache")) + monkeypatch.delenv("HF_HOME", raising=False) + result = _check_hf_cache_offline() + assert result.status == "warn" + assert result.extras["exists"] is False + + def test_populated_cache_passes_via_hf_hub_cache(self, tmp_path, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _check_hf_cache_offline + + cache_dir = tmp_path / "hub_cache" + cache_dir.mkdir() + (cache_dir / "model_blob").write_bytes(b"x" * 1024) + monkeypatch.setenv("HF_HUB_CACHE", str(cache_dir)) + monkeypatch.delenv("HF_HOME", raising=False) + result = _check_hf_cache_offline() + assert result.status == "pass" + assert result.extras["file_count"] == 1 + assert result.extras["size_gib"] >= 0 + + def test_populated_cache_passes_via_hf_home_hub_subdir(self, tmp_path, monkeypatch) -> None: + """gemini bot fix: HF_HOME → ``HF_HOME/hub`` (sub-directory).""" + from forgelm.cli.subcommands._doctor import _check_hf_cache_offline + + hf_home = tmp_path / "hf_home" + hub_dir = hf_home / "hub" + hub_dir.mkdir(parents=True) + (hub_dir / "model_blob").write_bytes(b"x" * 1024) + monkeypatch.delenv("HF_HUB_CACHE", raising=False) + monkeypatch.setenv("HF_HOME", str(hf_home)) + result = _check_hf_cache_offline() + assert result.status == "pass" + assert "hub" in result.extras["cache_dir"] + + def test_empty_cache_dir_warns(self, tmp_path, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _check_hf_cache_offline + + cache_dir = tmp_path / "hf_cache_empty" + cache_dir.mkdir() + monkeypatch.setenv("HF_HUB_CACHE", str(cache_dir)) + monkeypatch.delenv("HF_HOME", raising=False) + result = _check_hf_cache_offline() + assert result.status == "warn" + assert result.extras["file_count"] == 0 + + def test_unreadable_files_surface_as_warn_not_pass(self, tmp_path, monkeypatch) -> None: + """F-34-OSE: previously OSError on getsize was swallowed silently + and the doctor reported a clean ``pass`` with a misleading total. + After the fix, any unreadable file flips the verdict to ``warn`` + and surfaces ``unreadable_count`` so the operator sees the issue. + """ + import os + + from forgelm.cli.subcommands import _doctor + + cache_dir = tmp_path / "hub_cache" + cache_dir.mkdir() + # One readable, one unreadable. + (cache_dir / "readable_blob").write_bytes(b"x" * 256) + (cache_dir / "unreadable_blob").write_bytes(b"y" * 256) + monkeypatch.setenv("HF_HUB_CACHE", str(cache_dir)) + monkeypatch.delenv("HF_HOME", raising=False) + + original_getsize = os.path.getsize + + def _fake_getsize(path: str) -> int: + if path.endswith("unreadable_blob"): + raise OSError("simulated permission denied") + return original_getsize(path) + + monkeypatch.setattr(os.path, "getsize", _fake_getsize) + result = _doctor._check_hf_cache_offline() + assert result.status == "warn", f"unreadable file in cache must downgrade verdict to warn, got {result.status}" + assert result.extras["unreadable_count"] == 1 + assert result.extras["file_count"] == 1 # only the readable one counted + assert "unreadable" in result.detail.lower(), ( + f"detail must surface unreadable count to the operator, got: {result.detail!r}" + ) + + +class TestHfEndpointResolution: + """Wave 2a Round-1 (gemini bot): HF_ENDPOINT must be respected for + self-hosted mirrors / enterprise installs.""" + + def test_default_endpoint_when_unset(self, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _resolve_hf_endpoint + + monkeypatch.delenv("HF_ENDPOINT", raising=False) + assert _resolve_hf_endpoint() == "https://huggingface.co" + + def test_env_var_override(self, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _resolve_hf_endpoint + + monkeypatch.setenv("HF_ENDPOINT", "https://internal-mirror.example/") + assert _resolve_hf_endpoint() == "https://internal-mirror.example" + + +class TestOperatorIdentityAnonymousOptIn: + """Wave 2a Round-1 (qodo bot): respect FORGELM_ALLOW_ANONYMOUS_OPERATOR + like AuditLogger.__init__ does — no-username + opt-in => warn (not fail).""" + + def test_no_username_with_opt_in_warns(self, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _check_operator_identity + + monkeypatch.delenv("FORGELM_OPERATOR", raising=False) + monkeypatch.setenv("FORGELM_ALLOW_ANONYMOUS_OPERATOR", "1") + with patch("getpass.getuser", side_effect=OSError("no user")): + result = _check_operator_identity() + assert result.status == "warn" + assert "anonymous" in result.detail.lower() + assert result.extras["source"] == "anonymous_opt_in" + + def test_no_username_without_opt_in_fails(self, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _check_operator_identity + + monkeypatch.delenv("FORGELM_OPERATOR", raising=False) + monkeypatch.delenv("FORGELM_ALLOW_ANONYMOUS_OPERATOR", raising=False) + with patch("getpass.getuser", side_effect=OSError("no user")): + result = _check_operator_identity() + assert result.status == "fail" + + +class TestSecretEnvMasking: + """Wave 2a Round-1 (F-27-05): secret env-var values must not echo + into doctor output.""" + + def test_mask_helper_redacts_secret_names(self) -> None: + from forgelm.cli.subcommands._doctor import _mask_env_value_for_audit + + masked = _mask_env_value_for_audit("FORGELM_AUDIT_SECRET", "super-secret-key-32-chars-long-x") + assert "super-secret" not in masked + assert " None: + from forgelm.cli.subcommands._doctor import _mask_env_value_for_audit + + passthrough = _mask_env_value_for_audit("FORGELM_OPERATOR", "alice") + assert passthrough == "alice" + + +class TestDiskSpaceCheck: + def test_plenty_passes(self, tmp_path) -> None: + from forgelm.cli.subcommands._doctor import _check_disk_space + + result = _check_disk_space(str(tmp_path)) + # On a CI runner free space typically > 50 GiB; ensure at least + # one of the three valid statuses comes back. + assert result.status in ("pass", "warn", "fail") + assert result.extras["free_gib"] >= 0 + + def test_low_disk_fails(self, tmp_path) -> None: + from forgelm.cli.subcommands import _doctor + + # Build a fake disk_usage that reports 5 GiB free. + fake_usage = _FakeDiskUsage( + total=1000 * (1024**3), + used=950 * (1024**3), + free=5 * (1024**3), + ) + with patch("shutil.disk_usage", return_value=fake_usage): + result = _doctor._check_disk_space(str(tmp_path)) + assert result.status == "fail" + + def test_warn_threshold(self, tmp_path) -> None: + from forgelm.cli.subcommands import _doctor + + # 30 GiB free → warn (between 10 and 50). + fake_usage = _FakeDiskUsage( + total=1000 * (1024**3), + used=970 * (1024**3), + free=30 * (1024**3), + ) + with patch("shutil.disk_usage", return_value=fake_usage): + result = _doctor._check_disk_space(str(tmp_path)) + assert result.status == "warn" + + +class TestOperatorIdentityCheck: + def test_explicit_env_passes(self, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _check_operator_identity + + monkeypatch.setenv("FORGELM_OPERATOR", "ci-pipeline-prod") + result = _check_operator_identity() + assert result.status == "pass" + assert "ci-pipeline-prod" in result.detail + + def test_missing_env_warns_with_fallback(self, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import _check_operator_identity + + monkeypatch.delenv("FORGELM_OPERATOR", raising=False) + result = _check_operator_identity() + # On a normal dev workstation getpass.getuser() resolves so we + # get warn (not fail). + assert result.status in ("warn", "fail") + assert "FORGELM_OPERATOR" in result.detail + + +# --------------------------------------------------------------------------- +# Renderers + exit-code mapping +# --------------------------------------------------------------------------- + + +def _make_results(*statuses: str): + from forgelm.cli.subcommands._doctor import _CheckResult + + return [_CheckResult(name=f"test.{i}", status=s, detail=f"d{i}") for i, s in enumerate(statuses)] + + +class TestExitCodeMapping: + def test_all_pass_returns_zero(self) -> None: + from forgelm.cli.subcommands._doctor import _resolve_exit_code + + assert _resolve_exit_code(_make_results("pass", "pass", "pass")) == 0 + + def test_warn_only_returns_zero(self) -> None: + from forgelm.cli.subcommands._doctor import _resolve_exit_code + + # Warns are operator-actionable but do not flip the gate. + assert _resolve_exit_code(_make_results("pass", "warn", "warn")) == 0 + + def test_fail_returns_one(self) -> None: + from forgelm.cli.subcommands._doctor import _resolve_exit_code + + assert _resolve_exit_code(_make_results("pass", "fail", "warn")) == 1 + + def test_crashed_probe_returns_two(self) -> None: + from forgelm.cli.subcommands._doctor import _CheckResult, _resolve_exit_code + + crashed = _CheckResult( + name="crash.probe", + status="fail", + detail="boom", + extras={"crashed": True, "error_class": "RuntimeError"}, + ) + assert _resolve_exit_code([crashed]) == 2 + + +class TestRenderers: + def test_text_renders_summary_line(self) -> None: + from forgelm.cli.subcommands._doctor import _render_text + + results = _make_results("pass", "warn", "fail") + out = _render_text(results) + assert "1 pass" in out + assert "1 warn" in out + assert "1 fail" in out + # Each check is rendered. + assert "test.0" in out and "test.1" in out and "test.2" in out + + def test_json_envelope_shape(self) -> None: + from forgelm.cli.subcommands._doctor import _render_json + + results = _make_results("pass", "fail") + payload = json.loads(_render_json(results)) + assert payload["success"] is False # has a fail + assert payload["summary"] == {"pass": 1, "warn": 0, "fail": 1, "crashed": 0} + assert len(payload["checks"]) == 2 + + def test_json_success_true_when_only_passes_and_warns(self) -> None: + from forgelm.cli.subcommands._doctor import _render_json + + results = _make_results("pass", "warn", "pass") + payload = json.loads(_render_json(results)) + assert payload["success"] is True + + def test_text_output_is_pure_ascii(self) -> None: + """F-34-ASCII: the docstring promises plain ASCII for redirected + logs and non-UTF8 terminals. Previously used ✓ / ✗ (Unicode) + which would crash with UnicodeEncodeError on PYTHONIOENCODING=ascii. + Pinning the contract: every byte of the rendered text must encode + cleanly as ASCII. + """ + from forgelm.cli.subcommands._doctor import _render_text + + results = _make_results("pass", "warn", "fail") + out = _render_text(results) + # If a Unicode glyph leaks back in, this raises UnicodeEncodeError + # exactly the way an ASCII-locale terminal would. + out.encode("ascii") # must not raise + # And the glyphs are the documented ASCII tokens. + assert "[+ pass]" in out + assert "[! warn]" in out + assert "[x fail]" in out + + +# --------------------------------------------------------------------------- +# Crash isolation +# --------------------------------------------------------------------------- + + +class TestProbeCrashIsolation: + def test_one_crashing_probe_does_not_abort_the_run(self, monkeypatch) -> None: + """If one probe raises an unexpected exception, the rest must + still execute and the failed probe must be converted to a fail + result with a ``crashed`` marker.""" + from forgelm.cli.subcommands import _doctor + + def _boom() -> _doctor._CheckResult: + raise RuntimeError("synthetic crash") + + # Sandwich pattern (Wave 2a Round-2 F-TEST-34-01): a [ok, crash] + # pair would silently pass even if the dispatcher aborted on the + # crash, because nothing comes after it. Putting an `ok_after` + # probe at the end is what actually proves "the crash did not + # truncate the rest of the plan". + def _fake_plan(*, offline: bool): + return [ + ("ok_before", lambda: _doctor._CheckResult(name="ok_before", status="pass", detail="a")), + ("middle.crash", _boom), + ("ok_after", lambda: _doctor._CheckResult(name="ok_after", status="pass", detail="b")), + ] + + monkeypatch.setattr(_doctor, "_build_check_plan", _fake_plan) + results = _doctor._run_all_checks(offline=False) + assert [r.name for r in results] == ["ok_before", "middle.crash", "ok_after"] + ok_before = next(r for r in results if r.name == "ok_before") + ok_after = next(r for r in results if r.name == "ok_after") + crash_result = next(r for r in results if r.name == "middle.crash") + assert ok_before.status == "pass" + assert ok_after.status == "pass" + assert crash_result.status == "fail" + assert crash_result.extras.get("crashed") is True + assert "RuntimeError" in crash_result.detail + + +# --------------------------------------------------------------------------- +# Plan composition +# --------------------------------------------------------------------------- + + +class TestCheckPlan: + def test_offline_uses_cache_probe_not_hub_probe(self) -> None: + from forgelm.cli.subcommands._doctor import _build_check_plan + + plan_offline = _build_check_plan(offline=True) + plan_online = _build_check_plan(offline=False) + names_offline = [name for name, _ in plan_offline] + names_online = [name for name, _ in plan_online] + assert "hf_hub.offline_cache" in names_offline + assert "hf_hub.reachable" not in names_offline + assert "hf_hub.reachable" in names_online + assert "hf_hub.offline_cache" not in names_online + + def test_extras_in_plan(self) -> None: + """All optional extras advertised in pyproject.toml are probed.""" + from forgelm.cli.subcommands._doctor import _OPTIONAL_EXTRAS, _build_check_plan + + plan = _build_check_plan(offline=False) + names = {name for name, _ in plan} + for extra, _module, _purpose in _OPTIONAL_EXTRAS: + assert f"extras.{extra}" in names + + +# --------------------------------------------------------------------------- +# Top-level dispatcher +# --------------------------------------------------------------------------- + + +class TestDispatcher: + def test_text_output_prints_summary(self, capsys) -> None: + from forgelm.cli.subcommands._doctor import _run_doctor_cmd + + args = MagicMock() + args.offline = True # avoids the network probe + with pytest.raises(SystemExit): + _run_doctor_cmd(args, output_format="text") + out = capsys.readouterr().out + assert "Summary:" in out + assert "forgelm doctor" in out + + def test_json_output_emits_envelope(self, capsys) -> None: + from forgelm.cli.subcommands._doctor import _run_doctor_cmd + + args = MagicMock() + args.offline = True + with pytest.raises(SystemExit): + _run_doctor_cmd(args, output_format="json") + payload = json.loads(capsys.readouterr().out) + assert "success" in payload + assert "checks" in payload + assert "summary" in payload + assert all(set(c) >= {"name", "status", "detail", "extras"} for c in payload["checks"]) + + def test_dispatcher_exits_with_resolved_code(self, capsys) -> None: + from forgelm.cli.subcommands import _doctor + + # Force every probe to pass so exit code is 0. + def _all_pass_plan(*, offline: bool): + return [("only.probe", lambda: _doctor._CheckResult(name="only.probe", status="pass", detail="ok"))] + + with patch.object(_doctor, "_build_check_plan", _all_pass_plan): + args = MagicMock() + args.offline = False + with pytest.raises(SystemExit) as ei: + _doctor._run_doctor_cmd(args, output_format="text") + assert ei.value.code == 0 + + +# --------------------------------------------------------------------------- +# CLI subprocess smoke +# --------------------------------------------------------------------------- + + +class TestDoctorCLISmoke: + def test_doctor_subcommand_registered(self) -> None: + """`forgelm doctor --help` exits 0 and advertises --offline.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "forgelm.cli", "doctor", "--help"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert "--offline" in result.stdout + # Common-flag inheritance: --output-format / --quiet / --log-level. + assert "--output-format" in result.stdout + + def test_doctor_offline_runs_end_to_end(self) -> None: + """`forgelm doctor --offline --output-format json` produces a valid + JSON envelope and exits with one of the public exit codes. + + Smoke-level scope (Wave 2a Round-2 F-TEST-34-02): the runtime + environment is unconstrained (CI runners may lack a populated HF + cache, may set FORGELM_OPERATOR or not, etc.), so this test + only pins (a) the JSON envelope shape and (b) that the exit + code is one of the documented contract values. Strict per- + scenario assertions live in :class:`TestDispatcherStrictExit` + below, where the plan is monkeypatched.""" + import subprocess + + result = subprocess.run( + [ + sys.executable, + "-m", + "forgelm.cli", + "doctor", + "--offline", + "--output-format", + "json", + ], + capture_output=True, + text=True, + timeout=120, + ) + # Public exit codes: 0 (all pass+warn) / 1 (some fail) / 2 (probe crashed). + assert result.returncode in (0, 1, 2), result.stderr + # The JSON envelope is on stdout regardless. + payload = json.loads(result.stdout) + # Contract: top-level keys are stable; consumers read these names. + assert set(payload.keys()) == {"success", "checks", "summary"} + assert isinstance(payload["success"], bool) + assert isinstance(payload["checks"], list) + assert set(payload["summary"].keys()) == {"pass", "warn", "fail", "crashed"} + # success: bool aligns with exit code per docs/standards/error-handling.md + if payload["success"]: + assert result.returncode == 0 + else: + assert result.returncode in (1, 2) + + +class TestDispatcherStrictExit: + """Strict per-scenario exit code assertions (monkeypatched plan).""" + + def test_all_pass_exits_zero(self, capsys, monkeypatch) -> None: + from forgelm.cli.subcommands import _doctor + + def _all_pass_plan(*, offline: bool): + return [("a.pass", lambda: _doctor._CheckResult(name="a.pass", status="pass", detail="ok"))] + + monkeypatch.setattr(_doctor, "_build_check_plan", _all_pass_plan) + args = MagicMock() + args.offline = True + with pytest.raises(SystemExit) as exc_info: + _doctor._run_doctor_cmd(args, output_format="json") + assert exc_info.value.code == 0 + + def test_any_fail_exits_one(self, capsys, monkeypatch) -> None: + from forgelm.cli.subcommands import _doctor + + def _has_fail_plan(*, offline: bool): + return [ + ("a.pass", lambda: _doctor._CheckResult(name="a.pass", status="pass", detail="ok")), + ("b.fail", lambda: _doctor._CheckResult(name="b.fail", status="fail", detail="bad")), + ] + + monkeypatch.setattr(_doctor, "_build_check_plan", _has_fail_plan) + args = MagicMock() + args.offline = True + with pytest.raises(SystemExit) as exc_info: + _doctor._run_doctor_cmd(args, output_format="json") + assert exc_info.value.code == 1 + + def test_any_crashed_exits_two(self, capsys, monkeypatch) -> None: + from forgelm.cli.subcommands import _doctor + + def _boom() -> _doctor._CheckResult: + raise RuntimeError("boom") + + def _has_crash_plan(*, offline: bool): + return [ + ("a.pass", lambda: _doctor._CheckResult(name="a.pass", status="pass", detail="ok")), + ("b.crash", _boom), + ] + + monkeypatch.setattr(_doctor, "_build_check_plan", _has_crash_plan) + args = MagicMock() + args.offline = True + with pytest.raises(SystemExit) as exc_info: + _doctor._run_doctor_cmd(args, output_format="json") + assert exc_info.value.code == 2 + + def test_secrets_never_appear_in_json_envelope(self, monkeypatch, capsys) -> None: + """Wave 2a Round-2 F-TEST-34-03: end-to-end secret-masking proof. + + Sets a sentinel value for HF_TOKEN and confirms it never surfaces + anywhere in the JSON envelope, even though FORGELM_OPERATOR (a + non-secret env) is allowed to surface its value. Pins the + masking discipline against the *full dispatcher path*, not just + the helper function in isolation.""" + from forgelm.cli.subcommands import _doctor + + sentinel = "ghs_test_token_DO_NOT_LEAK_42" + monkeypatch.setenv("HF_TOKEN", sentinel) + + # Use a minimal plan that includes the operator-identity probe + # (which is the one most likely to surface env values). + def _identity_only_plan(*, offline: bool): + return [("operator.identity", _doctor._check_operator_identity)] + + monkeypatch.setattr(_doctor, "_build_check_plan", _identity_only_plan) + args = MagicMock() + args.offline = True + with pytest.raises(SystemExit): + _doctor._run_doctor_cmd(args, output_format="json") + captured = capsys.readouterr().out + assert sentinel not in captured, "HF_TOKEN value must be masked in JSON envelope" + + def test_offline_inferred_from_hf_hub_offline_env(self, capsys, monkeypatch) -> None: + """Wave 2a Round-2 F-XPR-07-01: HF_HUB_OFFLINE=1 should imply --offline. + + Without an explicit --offline flag, the dispatcher resolves the + offline mode from HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE. This + spares air-gapped operators from having to pass --offline on + every doctor invocation when their shell already has the standard + HF airgap envs set.""" + from forgelm.cli.subcommands import _doctor + + captured_offline = [] + + def _spy_plan(*, offline: bool): + captured_offline.append(offline) + return [("a.pass", lambda: _doctor._CheckResult(name="a.pass", status="pass", detail="ok"))] + + monkeypatch.setattr(_doctor, "_build_check_plan", _spy_plan) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + # Argparse default is offline=False but env should flip it. + args = MagicMock() + args.offline = False + with pytest.raises(SystemExit): + _doctor._run_doctor_cmd(args, output_format="json") + assert captured_offline == [True], "HF_HUB_OFFLINE=1 must promote dispatcher to offline mode" + + +class TestHfCacheWalkBoundaries: + """Wave 2a Round-2 F-TEST-34-04: depth + file-count cap boundaries.""" + + def test_walk_truncated_at_file_cap(self, tmp_path, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import ( + _HF_CACHE_WALK_FILE_LIMIT, + _check_hf_cache_offline, + ) + + cache = tmp_path / "cache" + cache.mkdir() + for i in range(_HF_CACHE_WALK_FILE_LIMIT + 50): + (cache / f"f{i:06d}").write_bytes(b"x") + monkeypatch.setenv("HF_HUB_CACHE", str(cache)) + monkeypatch.delenv("HF_HOME", raising=False) + result = _check_hf_cache_offline() + # The walk should have hit the file cap and flagged truncation. + assert result.extras.get("walk_truncated") is True + assert result.extras.get("file_count") == _HF_CACHE_WALK_FILE_LIMIT + + def test_walk_not_truncated_at_exactly_file_cap(self, tmp_path, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import ( + _HF_CACHE_WALK_FILE_LIMIT, + _check_hf_cache_offline, + ) + + cache = tmp_path / "cache" + cache.mkdir() + for i in range(_HF_CACHE_WALK_FILE_LIMIT): + (cache / f"f{i:06d}").write_bytes(b"x") + monkeypatch.setenv("HF_HUB_CACHE", str(cache)) + monkeypatch.delenv("HF_HOME", raising=False) + result = _check_hf_cache_offline() + # Exactly at cap: walked clean, no truncation. + assert result.extras.get("walk_truncated") is False + assert result.extras.get("file_count") == _HF_CACHE_WALK_FILE_LIMIT + + def test_walk_truncated_at_depth_cap(self, tmp_path, monkeypatch) -> None: + from forgelm.cli.subcommands._doctor import ( + _HF_CACHE_WALK_DEPTH, + _check_hf_cache_offline, + ) + + cache = tmp_path / "cache" + # Build a tree deeper than the cap with a file at the bottom; the + # bottom file is below the depth cap so it should NOT be counted. + deep = cache.joinpath(*[f"d{i}" for i in range(_HF_CACHE_WALK_DEPTH + 2)]) + deep.mkdir(parents=True) + (deep / "blob").write_bytes(b"x") + monkeypatch.setenv("HF_HUB_CACHE", str(cache)) + monkeypatch.delenv("HF_HOME", raising=False) + result = _check_hf_cache_offline() + # The walk should report truncation since a non-empty subtree was + # below the cap. + assert result.extras.get("walk_truncated") is True + + +class TestDoctorSecretEnvNames: + """Wave 2a Round-2 F-34-02: secret-env mask covers third-party tokens.""" + + @pytest.mark.parametrize( + "name", + [ + "FORGELM_AUDIT_SECRET", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "HUGGINGFACE_TOKEN", + "FORGELM_RESUME_TOKEN", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "WANDB_API_KEY", + "COHERE_API_KEY", + ], + ) + def test_known_secret_env_names_are_masked(self, name: str) -> None: + from forgelm.cli.subcommands._doctor import _DOCTOR_SECRET_ENV_NAMES + + assert name in _DOCTOR_SECRET_ENV_NAMES + + +# --------------------------------------------------------------------------- +# Facade re-exports +# --------------------------------------------------------------------------- + + +class TestFacadeReExports: + def test_doctor_helpers_reachable_via_facade(self) -> None: + """Tests / monkeypatches reach doctor helpers via ``forgelm.cli``.""" + from forgelm import cli as _cli_facade + + for name in ( + "_run_doctor_cmd", + "_run_all_checks", + "_render_json", + "_render_text", + "_resolve_exit_code", + "_check_python_version", + "_check_torch_cuda", + "_check_operator_identity", + ): + assert hasattr(_cli_facade, name), f"forgelm.cli must re-export {name!r}"