diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c3ee26d..ded461b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,14 @@ on: pull_request: branches: [main] +# Coverage gate (--cov-fail-under=40) is enforced via pyproject.toml's +# [tool.pytest.ini_options].addopts; the test job intentionally does not +# duplicate the flag here. FORGELM_OPERATOR is set so future operator-identity +# checks (Phase 3 hardening) do not raise on distroless runners where USER is +# unset. +env: + FORGELM_OPERATOR: ci-smoke + jobs: # --- Job 1: Lint (fast, no heavy deps) --- lint: @@ -30,6 +38,7 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: @@ -46,7 +55,10 @@ jobs: python -m pip install -e ".[dev]" - name: Run tests with coverage - run: pytest -q --tb=short --cov=forgelm --cov-report=term-missing --cov-report=xml:coverage.xml + # --cov, --cov-report=term-missing, and --cov-fail-under=40 come from + # pyproject.toml addopts (single source of truth). Only the XML report + # is added here because it is CI-specific (artifact upload). + run: pytest -q --tb=short --cov-report=xml:coverage.xml - name: Upload coverage (3.11 only) if: matrix.python-version == '3.11' && always() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 385949d5..ab9d1195 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -7,6 +7,11 @@ on: # No workflow-level permissions — each job declares the minimal set it needs. +# Set FORGELM_OPERATOR so future operator-identity checks (Phase 3 hardening) +# do not raise on distroless runners where USER is unset. +env: + FORGELM_OPERATOR: ci-smoke + jobs: test-latest-deps: name: Test with latest dependencies @@ -71,6 +76,18 @@ jobs: forgelm --config "/tmp/qs-$tpl.yaml" --dry-run done + - name: Forbid git+https in notebooks + run: | + # Notebooks must install ForgeLM from PyPI with a pinned version, + # never from a git+https URL. Closure plan Faz 5 — git+https makes + # Colab installs depend on a specific branch HEAD, breaking + # reproducibility and confusing users when main moves ahead of + # the latest released wheel. + if grep -rn 'git+https://github.com/cemililik/ForgeLM' notebooks/; then + echo "::error::Notebooks must install from PyPI, not git+https" + exit 1 + fi + - name: Ingestion + audit smoke test (Phase 11) run: | # Minimal end-to-end: TXT in → JSONL out → audit report out. diff --git a/.github/workflows/usermanuals-validate.yml b/.github/workflows/usermanuals-validate.yml index 985beb0a..5b89e29f 100644 --- a/.github/workflows/usermanuals-validate.yml +++ b/.github/workflows/usermanuals-validate.yml @@ -6,12 +6,22 @@ name: Validate user-manuals # source. Catches structural problems before they hit main. on: + push: + branches: [main, development] + paths: + - "docs/usermanuals/**" + - "tools/build_usermanuals.py" pull_request: branches: [main] paths: - "docs/usermanuals/**" - "tools/build_usermanuals.py" +# Mirror CI's operator identity so any future pytest invocation does not +# raise on distroless runners where USER is unset. +env: + FORGELM_OPERATOR: ci-smoke + jobs: validate: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 4cdef8af..27786533 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,26 @@ # Internal strategy, marketing, and analysis docs (not public) docs/marketing/ docs/analysis/ +# Explicit exceptions: a small set of v0.5.5-closure planning artifacts +# is exposed publicly (referenced from closure PRs and the roadmap). The +# rest of docs/analysis/** stays internal — including all subdirectories +# (QKV-Core, Trion, autoresearch, proposals, ...) and every other file +# under code_reviews/. +# +# Pattern explanation: +# 1. docs/analysis/ — ignore everything under this dir +# 2. !docs/analysis/ — re-allow the directory itself (so we can descend) +# 3. docs/analysis/* — ignore every direct entry (subdirs + files) +# 4. !docs/analysis/code_reviews/ — re-allow code_reviews/ subdir only +# 5. docs/analysis/code_reviews/* — ignore every entry in code_reviews/ +# 6. ! — re-include the 3 closure planning artifacts +!docs/analysis/ +docs/analysis/* +!docs/analysis/code_reviews/ +docs/analysis/code_reviews/* +!docs/analysis/code_reviews/closure-plan-202604300906.md +!docs/analysis/code_reviews/master-review-opus-202604300906.md +!docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md # Generated by tools/build_usermanuals.py — produced from # docs/usermanuals//**/*.md during the CI deploy. Not committed; @@ -70,3 +90,4 @@ logs/ # Claude Code runtime artifacts .claude/scheduled_tasks.lock .claude/state/ +.claude/worktrees/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0107b0ff..b82451cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,89 @@ All notable changes to ForgeLM are documented here. ## [Unreleased] -### Added +> **Active cycle:** v0.5.5 closure — a single-release consolidation of +> the master review's 175 findings + 4 new feature tracks (Library API, +> ISO 27001 / SOC 2 alignment, GDPR right-to-erasure, Article 14 real +> staging directory). Detailed plan: +> [closure-plan-202604300906.md](docs/analysis/code_reviews/closure-plan-202604300906.md). +> No interim releases; v0.5.5 ships once Faz 1-33 are complete. +> Per-PR CHANGELOG entries below collapse into the v0.5.5 release +> notes at tag time. + +### Added — Foundation bundle (PR #19, Faz 1-8) + +- `forgelm verify-audit` subcommand + library function + `forgelm.compliance.verify_audit_log` (Faz 6 — closes F-compliance-103 + Critical). +- `forgelm._http.safe_post` — single boundary for outbound HTTP with + SSRF guard, redirect refusal, scheme policy, timeout floor, TLS + pinning, secret-mask error reasons (Faz 7 — closes M-201 Major). + Migrated webhook + judge + synthetic call sites. +- `WebhookNotifier.notify_reverted` and `notify_awaiting_approval` — + paired with `training.reverted` and `approval.required` events + (Faz 8 — closes F-compliance-104 Major). +- `SafetyEvalThresholds` dataclass — bundles five Phase 9 knobs so + `run_safety_evaluation` stays under the 13-param ceiling (Faz 4 v2 + + complexity refactor). +- `audit.classifier_load_failed` audit event with `audit_logger=` + parameter on `run_safety_evaluation` (Faz 3 — closes F-compliance-120 + Minor). +- `tests/test_lazy_imports.py` — regression test pinning that + `import forgelm.trainer` / `import forgelm.model` do not eagerly + load torch (Faz 4 — closes F-performance-101 Major). +- `docs/reference/audit_event_catalog.md` + TR mirror — comprehensive + event vocabulary with payload schemas (Faz 3 + Faz 8 union). +- `docs/standards/release.md` "Deprecation cadence" section (Faz 2 + — closes F-business-011 Major). + +### Changed + +- `AuditLogger` — operator identity raises `ConfigError` instead of + falling back to literal `"unknown"` (Faz 3 — closes F-compliance-102 + Critical). `getpass.getuser()@socket.gethostname()` chain with + `FORGELM_ALLOW_ANONYMOUS_OPERATOR=1` escape hatch. +- `AuditLogger.log_event` — `os.fsync(f.fileno())` after flush; chain + durability across power-cut (Faz 3 — closes F-compliance-114 Major). +- `compute_dataset_fingerprint` — split into three helpers (local file + / HF metadata / HF revision); HF Hub revision SHA pinned (Faz 3 + — closes F-compliance-117 Minor + complexity refactor). +- `_generate_safety_responses` and `_generate_responses_batched` — + `batch_size=8` default with token-pad-longest + per-batch CUDA-OOM + fallback to single-prompt (Faz 4 — closes F-performance-102 Major). + Per-batch error handling extracted to `_generate_*_batch_with_oom_retry` + helpers. +- `_chunk_paragraph_tokens` — single-encode + offset slicing (Faz 4 + — closes F-performance-103 Major). +- `_post_payload` — delegates to `safe_post` with `min_timeout=1.0` + for back-compat (Faz 7). +- 7 notebooks — install from PyPI (`forgelm[qlora]==0.5.0`) instead + of `git+https://...` (Faz 5 — closes F-business-005 Major). +- CI now enforces `pytest --cov-fail-under=40` via `pyproject.toml` + `addopts` (Faz 2 — closes F-test-001 Critical). +- CI matrix `fail-fast: false`; `usermanuals-validate.yml` runs on + push + PR (Faz 2 — closes F-test-006 + F-test-017 Major). +- Site honesty: `compliance.html` artefact tree, `quickstart.html` + template names, GPU stat (16 vs claimed 18) — all refreshed against + real code (Faz 1 — closes F-business-001/002/004 Critical+Major). +- QMS `sop_data_management.md` — single v0.5.0 story; v0.5.1+/v0.5.2 + splits removed (Faz 1 — closes M-DOC-001 Critical). +- Roadmap (`roadmap.md`, `roadmap-tr.md`, `releases.md`) — v0.5.0 + marked released; tristate status legend added (Faz 1 — closes + F-business-003 Critical). +- `forgelm.webhook` exports `_is_private_destination` via `__all__` + for back-compat (Faz 7). + +### Documentation -- (next entry goes here — Phase 14 / Multi-Stage Pipeline Chains is the active phase; see [docs/roadmap.md](docs/roadmap.md)) +- Full Faz 1-8 closure plan: + [closure-plan-202604300906.md](docs/analysis/code_reviews/closure-plan-202604300906.md) + (33 phases, ~47 PRs). +- Master review: + [master-review-opus-202604300906.md](docs/analysis/code_reviews/master-review-opus-202604300906.md) + (175 findings). +- `data_audit/` + `cli/` package split design: + [split-design-data_audit-cli-202604300906.md](docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md) + (Faz 14-15 forward-looking). --- @@ -153,9 +233,7 @@ Touch points (so the next reviewer can audit blast radius quickly): Inline-comment batches landing on top of PR #13 (now merged to `main`). Same review surface as rounds 4-7; further hardening on top of the -`v0.5.2` content. The `v0.5.2` git tag + PyPI publish are the -remaining release-engineering steps before this section can be -frozen into a release. +`v0.5.2` content. - **Audit log hardening** (`forgelm/compliance.py`) — HMAC `_hmac` field is now emitted only when `FORGELM_AUDIT_SECRET` is set; without a secret, a key diff --git a/CLAUDE.md b/CLAUDE.md index d9c8d64f..80232345 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ ForgeLM/ │ ├── webhook.py # Slack/Teams notifications │ ├── grpo_rewards.py # Built-in GRPO format/length shaping reward fallback │ └── ... # benchmark, judge, merging, synthetic, wizard, ... -├── tests/ # pytest, 26 test modules +├── tests/ # pytest test modules ├── docs/ │ ├── roadmap.md # Public roadmap (short index) │ ├── roadmap/ # Detailed phase files + archive diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0cad91d..ca99966a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,30 +59,10 @@ Push your branch and open a Pull Request against `main`. ### Project Structure -``` -forgelm/ -├── cli.py # CLI entry point -├── config.py # Pydantic config models -├── data.py # Dataset loading -├── model.py # Model + LoRA setup -├── trainer.py # Training orchestration -├── results.py # TrainResult dataclass -├── benchmark.py # lm-eval-harness -├── safety.py # Safety evaluation -├── judge.py # LLM-as-Judge -├── compliance.py # EU AI Act artifacts -├── model_card.py # Model card generation -├── merging.py # Model merging (TIES/DARE/SLERP) -├── wizard.py # Interactive config wizard -├── synthetic.py # Synthetic data pipeline -├── webhook.py # Notifications -└── utils.py # Auth & checkpoints - -tests/ # 47 test files -notebooks/ # 10 Colab notebooks -configs/deepspeed/ # ZeRO presets -docs/guides/ # user guides -``` +ForgeLM is a single-package layout: ~26 single-file modules under `forgelm/`, +~47 test files under `tests/`, plus `configs/`, `docs/`, and `notebooks/`. For +the authoritative module-by-module map (purpose, public surface, dependency +arrows), see [`docs/reference/architecture.md`](docs/reference/architecture.md). ### Running Tests diff --git a/README.md b/README.md index 7323eeb3..e2cb5f0b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - **Synthetic Data Pipeline**: Teacher-to-student distillation with `--generate-data` CLI flag - **DeepSpeed & FSDP**: Multi-GPU distributed training with ZeRO-2/3 presets - **MoE Support**: Fine-tune Mixture of Experts models (Qwen3, Mixtral, DeepSeek) -- **GPU Cost Estimation**: Auto-detection for 18 GPU models with per-run cost tracking +- **GPU Cost Estimation**: Auto-detection for 16 GPU models with per-run cost tracking ### Evaluation & Safety - **Automated Benchmarking**: Post-training evaluation via `lm-evaluation-harness` @@ -39,7 +39,7 @@ - **Wizard "audit first"**: when the wizard resolves a JSONL (typed or produced by `forgelm ingest`) it offers to run `forgelm audit` inline and prints the verdict before continuing — closes the BYOD audit loop end-to-end. ### Quickstart Layer (v0.4.5) -- **One-Command Templates**: `forgelm quickstart customer-support` — 4 bundled templates (SFT customer-support, code-assistant, medical-qa-tr, GRPO math-reasoning) plus a bring-your-own-data domain-expert scaffold. Auto-downsizes models on small GPUs. +- **One-Command Templates**: `forgelm quickstart customer-support` — 5 bundled templates (SFT customer-support, code-assistant, medical-qa-tr, domain-expert, GRPO grpo-math). Auto-downsizes models on small GPUs. - **Conservative Defaults**: Every template ships QLoRA 4-bit, rank=8, batch=1, gradient checkpointing on — designed to run on a single 12 GB GPU. - **Wizard Integration**: `forgelm --wizard` opens with "Start from a template?" — same code paths, same YAML schema as a hand-written config. @@ -231,7 +231,7 @@ forgelm --version # Show version ``` forgelm/ -├── cli.py # CLI with 10+ modes (train, dry-run, merge, benchmark, wizard, ingest, audit, ...) +├── cli.py # CLI entry point (train, dry-run, merge, benchmark, wizard, ingest, audit, ...) ├── config.py # Pydantic config (19 models: training, evaluation, distributed, ...) ├── data.py # Dataset loading (SFT, DPO, KTO, GRPO formats + multi-dataset) ├── data_audit.py # Audit pipeline (length / language / dedup / leakage / PII / secrets / quality) — `forgelm audit` diff --git a/docs/analysis/code_reviews/closure-plan-202604300906.md b/docs/analysis/code_reviews/closure-plan-202604300906.md new file mode 100644 index 00000000..58f4d0ee --- /dev/null +++ b/docs/analysis/code_reviews/closure-plan-202604300906.md @@ -0,0 +1,1644 @@ +# ForgeLM Comprehensive Closure Plan — Path to v0.5.5 ★★★★★ + +**Plan ID:** `closure-plan-202604300906` +**Created:** 2026-04-30 +**Last revised:** 2026-04-30 (v2 — kullanıcı kararları sonrası tek-sürüm modeline geçiş) +**Branch:** `main` +**Commit (baseline):** `6b515ed912f8f22304194c1b3f55ed07a26f519c` +**Source review:** [`master-review-opus-202604300906.md`](./master-review-opus-202604300906.md) +**Companion design:** [`split-design-data_audit-cli-202604300906.md`](./split-design-data_audit-cli-202604300906.md) +**Owner:** Cemil ILIK (proje sahibi) +**Release model:** **TEK SÜRÜM v0.5.5** — bütün fazlar tamamlanmadan ara sürüm yayınlanmaz +**Hedef tarih:** Yok (kullanıcı kararı). Sıralama dependency-driven. + +--- + +## 0. Önsöz — Bu plan neyi kapsar, neyi kapsamaz + +Bu doküman master review'un 175 bulgusunu (8 Critical + 67 Major + 60 Minor + 40 Nit) **artı** kullanıcı tarafından eklenen 4 yeni kapsamı (Library API, ISO 27001/SOC 2, GDPR full implementation, Article 14 real staging) ★★★★★/5'e ulaşacak şekilde **fazlara dağıtılmış, her fazın sonu PR ile kapanan, risk-yönetilmiş** bir kapanış planına dönüştürür. + +### Kullanıcı kararları (2026-04-30) + +1. **Tek sürüm:** Bütün bu çalışma **v0.5.5** olarak çıkacak. Ara sürüm yok. Tüm fazlar tamamlanmadan tag atılmaz. +2. **Hedef tarih yok:** Sıralama dependency-driven. Her faz bir önceki faz tamamlanınca başlar. +3. **Faz = PR:** Her faz tek bir PR ile (veya çok-PR'lık coherent bir seri ile) kapanır. Faz içinde task'lar olur. +4. **Library API support:** `forgelm.ForgeTrainer` library API olarak desteklenecek. Önce **detaylı analiz fazı**, sonra **implementation fazı**. +5. **ISO 27001 / SOC 2 Type II:** Pro CLI hedefliyor (Cloud SaaS ayrı ürün, ForgeLM içinde değil). Bu plan'a alignment fazı ekleniyor — analiz + implementation. Açık iş bırakmıyoruz. +6. **GDPR right-to-erasure:** Stub-and-document yerine **gerçek implementation**. Açık iş yok. +7. **Article 14:** Path (a) honesty fix + Path (b) real staging directory + `forgelm approve` subcommand — ikisi birden, bir fazda. +8. **Cross-OS testing:** ubuntu/macos/windows matrix sadece **release-tag** trigger'ında çalışır. Cross-OS yeşil olmadan release publish çalışmaz. Her commit'te değil, sadece release'de. +9. **Pre-commit hooks:** **Opsiyonel** (Soru 8 için kullanıcının önerisini istediği opsiyon; gerekçe §11.7'de). +10. **Final documentation + site sweep:** Tüm fonksiyonel iş bittikten sonra `docs/` (EN+TR) + `docs/usermanuals/{en,tr}/` + `site/*` ayrı bir fazda (Faz 30) v0.5.5 nihai durumuna senkronlanır. **DE/FR/ES/ZH user-manual çevirisi bu fazda yapılmaz** (kullanıcı kararı: "diğer diller için daha sonra çeviri yapacağız"). + +### Plan disiplini + +1. **Tek seferde doğru.** "Şunu da unutmuşuz" yok. +2. **Bisectable PR'lar.** Her faz PR'ı bağımsızca yeşil; revert güvenli. +3. **Sıralı bağımlılıklar açıkça işaretli.** §10 dependency map. +4. **5/5 ne demek?** Her boyut için exit kriteri yazılı (§9). +5. **Açık iş bırakmıyoruz.** GDPR + ISO + Library API gerçek implementation; Article 14 path-b dahil. + +### Kapsam dışı (tek satır gerekçeli) + +- **DE/FR/ES/ZH user-manual çevirisi** (kullanıcı kararı: deferred — EN+TR yeterli; picker honesty + standards update yapılır; çeviri yok). +- **Cloud SaaS ürün** — ForgeLM içinde değil; ayrı ürün (kullanıcı açıklaması). +- **Yeni roadmap feature'ları** (Phase 13 Pro CLI work, Phase 14 pipeline chains) — bu plan sadece kapanış. +- **Marketing / GTM expansion.** + +--- + +## 1. Mevcut durum (HEAD `6b515ed`, post-v0.5.0) + +| Boyut | Şu an | 5/5 farkı | Ana eksikler (özet) | +|---|---|---|---| +| Business | ★★★½ | -1.5 | site/compliance.html sahte artefakt, site/quickstart.html mock, PyPI publish drift, GPU count drift | +| Code | ★★★½ | -1.5 | data_audit.py 3098, cli.py 1756, Literal 6/12, description= 1/170, ~25 silent except, wizard print/coverage | +| Compliance | ★★★★ | -1.0 | Article 14 staging, verify-audit, operator "unknown", audit fsync, judge/synthetic provenance, retention block, QMS bilingual, ISO/SOC 2, GDPR full | +| Documentation | ★★★½ | -1.5 | QMS contradiction, configuration-tr H4 drift, count drift, compliance_summary, design status blockquotes | +| Localization | ★★★ | -2.0 (kullanıcı kararıyla -1.0) | EN+TR mirror H2 drift, site picker honesty, locale-aware lower-case, orphan-md CI guard | +| Performance | ★★★★ | -1.0 | Lazy torch, safety+judge batching, paragraph chunker, --workers, build_usermanuals serial | +| Security | ★★★★ | -1.0 | Judge/synthetic SSRF, GRPO classifier trust_remote_code, FORGELM_GGUF_CONVERTER ack, _sanitize_md list bypass, Ollama injection | +| Testing & CI/CD | ★★★½ | -1.5 | --cov-fail-under devre dışı, runtime_smoke disorder, wizard omit, _minimal_config 4 yerde, fail-fast asimetri, cross-OS yok, pre-commit yok | + +**Overall:** ★★★¾ → hedef ★★★★★. v0.5.5 release tek seferde bu mesafeyi kapatır. + +--- + +## 2. Faz haritası (özet) + +Her faz bir veya birden çok PR'la kapanır. Sıralama dependency-driven; paralel-safe pair'lar §10'da. + +| # | Faz | Tip | PR sayısı | Bağımlılık | +|---|---|---|---|---| +| **1** | Site & doc honesty + count drift sweep | Tek PR | 1 | — | +| **2** | CI gates & standards drift cleanup | Tek PR | 1 | Yok (her şeyden önce) | +| **3** | Operator identity + audit forensic completeness | Tek PR | 1 | — | +| **4** | Performance: lazy torch + safety/judge batching + paragraph chunker | Tek PR | 1 | — | +| **5** | Notebook PyPI pin + nightly grep guard | Tek PR | 1 | — | +| **6** | `forgelm verify-audit` subcommand + library function | Tek PR | 1 | — | +| **7** | `safe_post` HTTP discipline extension (Theme ε) | Tek PR | 1 | — | +| **8** | Webhook lifecycle vocabulary | Tek PR | 1 | — | +| **9** | Article 14: honesty fix + real staging + `forgelm approve` | Tek PR | 1 | Faz 6 (verify-audit benzeri pattern) | +| **10** | Pydantic Literal sweep (6 kalan field) | Tek PR | 1 | — | +| **11** | Wizard `_print` indirection + drop coverage omit | Tek PR | 1 | — | +| **12** | Fixture consolidation (`tests/_helpers/`) | Tek PR | 1 | — | +| **13** | `--data-audit` deprecation discipline | Tek PR | 1 | — | +| **14** | `data_audit/` package split (multi-PR series) | 5 PR (D-1..D-5) | 5 | — | +| **15** | `cli/` package split (multi-PR series) | 6 PR (C-1..C-6) | 6 | — | +| **16** | Pydantic `description=` migration with CI guard (multi-PR) | 4 PR | 4 | Faz 10 (Literal'lar tamamen önce) | +| **17** | `--workers N` audit determinism | Tek PR | 1 | Faz 14 (data_audit/) + Faz 15 (cli/) | +| **18** | **Library API support — Analysis & design** | Tek PR (design dosyası) | 1 | Faz 14 + 15 | +| **19** | **Library API support — Implementation** | Tek PR | 1 | Faz 18 | +| **20** | **GDPR right-to-erasure — Analysis & design** | Tek PR | 1 | Faz 6 + 9 (audit + staging) | +| **21** | **GDPR right-to-erasure — Implementation** | Tek PR | 1 | Faz 20 | +| **22** | **ISO 27001 / SOC 2 Type II — Analysis & gap assessment** | Tek PR | 1 | Faz 3, 6, 7, 8, 9, 21 (security + compliance baseline) | +| **23** | **ISO 27001 / SOC 2 Type II — Implementation** | Tek PR (büyük) | 1 | Faz 22 | +| **24** | Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN/TR user-manual H2 drift) | Tek PR | 1 | Faz 16 (description= migration ile config docs auto-gen) | +| **25** | Site picker honesty + site-as-tested-surface CI guard | Tek PR | 1 | Faz 1 (site honesty fixes önce) | +| **26** | QMS bilingual EN+TR mirror + compliance docs cleanup | Tek PR | 1 | Faz 23 (ISO/SOC 2 dokümanları QMS ile lockstep) | +| **27** | Silent `except Exception:` sweep (Theme D residual) | Tek PR | 1 | — | +| **28** | Remaining Major + Minor cleanup | Tek PR | 1 | Hepsi öncesindekilere bağlı | +| **29** | Nit sweep (40 madde mass-edit) | Tek PR | 1 | — | +| **30** | **Final documentation + site finalization (EN+TR only)** — tüm `docs/` + `docs/usermanuals/{en,tr}/` + `site/*` v0.5.5 nihai durumuna senkron | Tek PR | 1 | **Faz 1-29** (tüm fonksiyonel iş bitmiş olmalı) | +| **31** | **Cross-OS release-tag matrix workflow** | Tek PR | 1 | — (release öncesi her şeyden) | +| **32** | `.pre-commit-config.yaml` (opsiyonel — kullanıcı tercihi) | Tek PR | 1 | — | +| **33** | **v0.5.5 RELEASE** (CHANGELOG, version bump, tag, PyPI) | Tek PR | 1 | **Tüm önceki fazlar** | + +**Toplam:** 33 faz, ~47 PR (multi-PR seri'ler dahil). + +--- + +## 3. Foundation fazları (Faz 1-5) + +Diğer her şeyden bağımsız. Paralel açılabilir. + +### Faz 1 — Site & doc honesty + count drift sweep (Tek PR) + +**Closes:** F-business-001 (Critical), F-business-002 (Critical), F-business-003 (Critical), F-business-004 (Major), F-business-007 (Major), F-business-010 (Major), F-business-014 (Minor), F-business-020 (Nit), F-code-130 (Minor), F-code-137 (Minor), M-DOC-009 (Major), M-DOC-011 (Major), M-DOC-020 (Minor), M-DOC-021 (Minor), m-DOC-018 (Minor) + +**Tasks:** +1. `site/compliance.html:102, 160-176` — artefakt ağacını gerçek `compliance.py::_export_compliance_artifacts` çıktısıyla rewrite et: + - `audit_log.jsonl` + `.manifest.json` + - `data_audit_report.json` + - `data_governance_report.json` + - `annex_iv_metadata.json` (markdown değil flat JSON — açıkça) + - `model_card.md`, `model_integrity.json`, `deployer_instructions.md` +2. `site/quickstart.html:111-116` — gerçek 5 template (`code-assistant`, `customer-support`, `domain-expert`, `grpo-math`, `medical-qa-tr`). +3. PyPI publish status sweep — `docs/roadmap.md:12,16`, `docs/roadmap-tr.md:12,16`, `docs/roadmap/releases.md:85`, `CHANGELOG.md:156-158`'den "PyPI publish pending" cümlesini sil; ✅ Done işaretle (ama bu plan v0.5.5'i hedeflediği için v0.5.0 PyPI status zaten true → ✅). +4. `README.md:22` "18 GPU models" → "16" (gerçek `_GPU_PRICING` count). +5. `docs/qms/sop_data_management.md:74-99` — v0.5.1+/v0.5.2 ifadelerini sil; v0.5.0 consolidation hikayesi tek; `forgelm audit` ile lead-with. +6. `CONTRIBUTING.md:62-79` project structure tree → architecture.md pointer. +7. `CLAUDE.md:60` "26 test modules" → "47 test modules". +8. `docs/standards/testing.md:8` aynı düzeltme; L127 `--cov-fail-under=25` → `--cov-fail-under=40`. +9. Quick wins bundle: organisations→organizations (`data_audit.py:104`), `✓`→`OK` (`build_usermanuals.py:404`), Tür→Type (roadmap), domain-expert "(requires Phase 11)" parantezini sil, "10+ modes" literal'ı sil. +10. `docs/roadmap.md:7-14` üstüne tristate symbol legend. +11. `docs/roadmap/risks-and-decisions.md` — 2 yeni Decision Log satırı (v0.5.0 consolidation + PyPI publish 2026-04-30). + +**Faz 1 Acceptance:** +- `grep -rn "publish pending\|byod-domain-expert\|math-reasoning\|annex_iv\.json\|26 test modules\|18 GPU\|organisations\b\|Tür\b" site/ docs/ README.md CLAUDE.md CONTRIBUTING.md` boş (legitimate locale strings hariç). +- `pytest tests/` yeşil; coverage düşmemiş. + +--- + +### Faz 2 — CI gates & standards drift cleanup (Tek PR) + +**Closes:** F-test-001 (Critical), F-test-006 (Major), F-test-010 (Major), F-test-017 (Major), F-business-011 (Major), F-business-012 (Minor) + +**Tasks:** +1. `pyproject.toml [tool.pytest.ini_options]` — `addopts = "--cov=forgelm --cov-report=term-missing --cov-fail-under=40"` ekle. Veya `.github/workflows/ci.yml:49`'a flag. +2. `docs/standards/testing.md:127` `--cov-fail-under=25` → `--cov-fail-under=40` (Faz 1'de yapılmadıysa bu fazda). +3. `.github/workflows/ci.yml:30-34` matrix block'una `fail-fast: false` ekle. +4. `.github/workflows/usermanuals-validate.yml` — `on: [push, pull_request]` (sadece pull_request değil → main-direct push'lar da validate edilsin). +5. `docs/standards/release.md`'e "Deprecation cadence" alt-bölümü ekle (10 satır; cli.py:1424-1428 worked example): + - Minimum 1 minor release overlap before removal + - DeprecationWarning mandatory + - Removal version in help text + - Tracking issue link +6. (Opsiyonel optimization) — şimdilik ertelendi: pytest-xdist enable Faz 28'de (çünkü description= migration testleri pytest-xdist'le interaction etkileyebilir). + +**Faz 2 Acceptance:** +- Coverage 40% altına düşürmeye çalışan bir test sil → CI fail. +- `usermanuals-validate.yml` push trigger'ında çalışır. +- `docs/standards/release.md` deprecation cadence section mevcut. + +--- + +### Faz 3 — Operator identity + audit forensic completeness (Tek PR) + +**Closes:** F-compliance-102 (Critical), F-compliance-114 (Major), F-compliance-117 (Minor), F-compliance-119 (Minor), F-compliance-120 (Minor), F-compliance-121 (Minor) + +**Tasks:** +1. `forgelm/compliance.py:56` — operator identity logic'i değiştir: + ``` + FORGELM_OPERATOR > getpass.getuser() + socket.gethostname() > raise ConfigError unless FORGELM_ALLOW_ANONYMOUS_OPERATOR=1 + ``` +2. `forgelm/compliance.py:262-265` — `f.flush()` sonrasına `os.fsync(f.fileno())` ekle. +3. HF Hub revision pinning (F-compliance-117) — `data_audit.py` fingerprint, dataset HF'den geliyorsa `revision` kaydet. +4. `docs/reference/audit_event_catalog.md` (yeni) — tüm audit event vocabulary'sini belgele: + - `training.started`, `pipeline.completed`, `human_approval.required`, `human_approval.granted`, `human_approval.rejected`, `model.reverted`, `compliance.governance_exported`, `compliance.governance_failed`, `compliance.artifacts_exported`, `cli.legacy_flag_invoked`, `audit.classifier_load_failed`, `audit.cross_run_continuity` +5. `forgelm/safety.py:387/499` — classifier-load-failure path'inde `audit.classifier_load_failed` event emit (F-compliance-120). +6. `forgelm/compliance.py` — cross-run continuity claim'ini `output_dir` scope'una daralt (F-compliance-121). +7. `tests/test_compliance.py` — yeni 8 test: + - operator FORGELM_OPERATOR set + - operator getpass.getuser() resolves + - operator both fail without flag → raise + - operator anonymous flag set → "anonymous@host" + - audit.fsync called after flush (mock'lu) + - HF revision pinned + - classifier-load-failed event emitted + - cross-run continuity scope test +8. CI'a `FORGELM_OPERATOR=ci-smoke` env var set (.github/workflows/*.yml — yoksa CI distroless'te raise eder). + +**Faz 3 Acceptance:** +- `grep -n "\"unknown\"" forgelm/compliance.py:56-60` artık yok. +- `os.fsync` `compliance.py`'da çağrılıyor. +- `docs/reference/audit_event_catalog.md` mevcut + bilingual mirror (varsa F-compliance-112 lockstep'i). + +--- + +### Faz 4 — Performance: lazy torch + safety/judge batching + paragraph chunker (Tek PR) + +**Closes:** F-performance-101 (Major), F-performance-102 (Major), F-performance-103 (Major), F-performance-105 (Major), F-performance-106 (Major) + +**Tasks:** +1. `forgelm/trainer.py:8-10` — module-top imports method body'lerine taşı. +2. `forgelm/model.py:4-6` — aynı. +3. Regression test: `tests/test_lazy_imports.py` (yeni) — `import forgelm.trainer` torch yüklememeli. +4. `forgelm/safety.py:86-102` — `evaluate_*` fonksiyonlarına `batch_size: int = 8` parametresi; token-pad-longest; per-batch OOM fallback. +5. `forgelm/judge.py:139-151, 301` — aynı pattern. +6. `forgelm/config.py` — `EvaluationConfig.safety.batch_size: int = 8` config field (Pydantic). +7. `forgelm/ingestion.py:902` — `_chunk_paragraph_tokens` single-encode + offset slicing. +8. `forgelm/data_audit.py:1706` — `agg.lang_sample.append(payload[:512])` slice. +9. `tools/build_usermanuals.py` — `markdown.Markdown` instance reuse with `.reset()`; mtime/hash cache; `ProcessPoolExecutor` cross-language. +10. Tests: safety batching deterministic (same input → same output regardless of batch_size); paragraph chunker token count unchanged. + +**Faz 4 Acceptance:** +- `python -c "import forgelm.trainer; import sys; assert 'torch' not in sys.modules"` yeşil. +- 100-prompt safety eval ölçülebilir hızlanma (ölçüm: `pytest tests/test_safety_advanced.py` runtime). +- `lang_sample` per-row max 512 char. + +--- + +### Faz 5 — Notebook PyPI pin + nightly grep guard (Tek PR) + +**Closes:** F-business-005 (Major), M-DOC-005 (Major) + +**Tasks:** +1. 7 notebook (`dpo_alignment`, `safety_evaluation`, `grpo_reasoning`, `kto_binary_feedback`, `galore_memory_optimization`, `multi_dataset`, `synthetic_data_training`) — `git+https://github.com/cemililik/ForgeLM.git` → `'forgelm[qlora]==0.5.0'` (veya bağımlı extra'ya göre `'forgelm==0.5.0'`). **NOT:** v0.5.5 PyPI'da olmadığı için `==0.5.0` pin (mevcut PyPI latest) kullanılır; Faz 33 release PR'ı `==0.5.5`'e bump eder. Plan §15.5 C-4 deferred. +2. `quickstart_sft.ipynb` ve `data_curation.ipynb` zaten PyPI'da; mevcut pin `==0.5.0` korunur (Faz 33'te `==0.5.5`'e bump). +3. `.github/workflows/nightly.yml` — yeni step: + ``` + - name: Forbid git+https in notebooks + run: | + if grep -rn 'git+https://github.com/cemililik/ForgeLM' notebooks/; then + echo "::error::Notebooks must install from PyPI" + exit 1 + fi + ``` + +**Faz 5 Acceptance:** +- `grep -rn "git+https://" notebooks/` boş. +- Nightly CI step çalışır + yeşil. + +--- + +## 4. Compliance closure fazları (Faz 6-9) + +### Faz 6 — `forgelm verify-audit` subcommand + library function (Tek PR) + +**Closes:** F-compliance-103 (Critical) + +**Tasks:** +1. `forgelm/compliance.py` — `verify_audit_log(path: Path | str, *, hmac_secret: str | None = None) -> VerifyResult`. SHA-256 chain check + opsiyonel HMAC check. `VerifyResult` dataclass. +2. `forgelm/cli.py` (Faz 14/15 split öncesi) — `_add_verify_audit_subcommand` registrar + `_run_verify_audit_cmd` dispatcher. +3. Argparse: `forgelm verify-audit [--hmac-secret-env FORGELM_AUDIT_SECRET] [--require-hmac]`. +4. Exit codes: 0 valid, 1 invalid, 2 file/permission error. +5. Tests `tests/test_compliance.py`'a 8 case: + - valid chain pass + - tampered line fail + - missing manifest fail + - HMAC valid pass + - HMAC invalid fail + - HMAC required but key absent → `--require-hmac` ile fail + - empty log file edge case + - corrupt JSONL line edge case +6. `docs/guides/safety_compliance.md` + `-tr.md` mirror — verify-audit kullanım örneği. + +**Faz 6 Acceptance:** +- `forgelm verify-audit run123/audit_log.jsonl` valid için exit 0; tampered için exit 1 + reason. +- 8 test yeşil. + +--- + +### Faz 7 — `safe_post` HTTP discipline extension (Theme ε) (Tek PR) + +**Closes:** M-201 (Major), M-202 (Major) + +**Tasks:** +1. `forgelm/_http.py` (yeni) — `safe_post(url, *, json=None, headers=None, timeout=10.0, verify=True, ca_bundle=None) -> requests.Response`: + - SSRF guard (`_is_private_destination`) — webhook.py'den extract. + - Redirect refusal (`allow_redirects=False`). + - http:// refusal (allowed only with `--allow-insecure-http` config). + - Timeout floor (≥10s). + - Secret-mask header values in errors via `mask_secrets`. +2. `forgelm/webhook.py:_post_payload` — `safe_post`'u kullansın. +3. `forgelm/judge.py:85` — outbound POST `safe_post` üzerinden. +4. `forgelm/synthetic.py:207-212` — aynı. +5. `forgelm/trainer.py:639-640` — `_AutoTok.from_pretrained` ve `AutoModelForSequenceClassification.from_pretrained` çağrılarına explicit `trust_remote_code=False`. +6. Tests: + - SSRF block (RFC1918, loopback, link-local, IMDS, multicast) + - Redirect block + - http:// block (config opt-in olmadan) + - Timeout floor enforcement + - Header masking on error + - Response truncation + - GRPO classifier reward `trust_remote_code=False` doğrulandı + +**Faz 7 Acceptance:** +- `grep -rn "requests\.post\|requests\.get" forgelm/ | grep -v _http\.py | grep -v test_` boş. +- `grep -rn "from_pretrained" forgelm/ | grep -v "trust_remote_code=False"` ya tüm `from_pretrained` çağrıları explicit `trust_remote_code=False`'la ya `# noqa: trust-remote-code` yorumlu. + +--- + +### Faz 8 — Webhook lifecycle vocabulary (Tek PR) + +**Closes:** F-compliance-104 (Major) + +**Tasks:** +1. `forgelm/webhook.py:Notifier` — yeni method'lar: + - `notify_reverted(run_name: str, reason: str) -> None` + - `notify_awaiting_approval(run_name: str, model_path: str) -> None` +2. `forgelm/trainer.py` — `_revert_model` (varsa) sonrası `notify_reverted` çağrı; `_handle_human_approval_gate` içinde `notify_awaiting_approval` çağrı. +3. `docs/standards/logging-observability.md` (varsa lifecycle vocabulary listesi) — 5 event'i sıralı belgele. +4. `docs/reference/audit_event_catalog.md` (Faz 3'te oluşturulan) — webhook event'lerini de belgele. +5. Tests `tests/test_webhook.py` (`requests_mock`'la): + - `notify_reverted` payload doğru (model_path, reason, masked) + - `notify_awaiting_approval` payload doğru + - Existing `notify_success`/`notify_failure` regresyon yok + +**Faz 8 Acceptance:** +- 5 lifecycle event hepsi webhook'tan emit ediliyor. +- audit_event_catalog.md ve logging-observability.md ile uyumlu. + +--- + +### Faz 9 — Article 14: honesty fix + real staging directory + `forgelm approve` (Tek PR) + +**Closes:** F-compliance-101 (Critical) — **PATH (a) + PATH (b)** birlikte + +**Tasks:** + +#### Part 1 — Honesty fix (path a) +1. `forgelm/trainer.py:865-878` — `_handle_human_approval_gate` çağrısının pozisyonunu **save_final_model'dan ÖNCEYE** taşı. Yapısı şöyle olur: + ``` + if self.config.evaluation.require_human_approval: + staging_path = final_path + ".staging" + self.save_final_model(staging_path) + audit.log_event("human_approval.required", staging_path=staging_path, run_id=...) + notifier.notify_awaiting_approval(run_name, staging_path) + train_result.success = True + train_result.staging_path = staging_path + return train_result # exit code 4 + else: + self.save_final_model(final_path) + # ... continue pipeline + ``` + +#### Part 2 — `forgelm approve` subcommand (path b) +2. Yeni `forgelm/cli.py` (Faz 14/15 öncesi) `_add_approve_subcommand`: + ``` + forgelm approve --output-dir + ``` +3. Approve dispatcher logic: + - `/final_model.staging/` var mı kontrol et + - Audit log'tan `human_approval.required` event'ini doğrula (run_id eşleşiyor) + - `final_model.staging/` → `final_model/` rename (atomic) + - audit `human_approval.granted` event emit + - `notify_success` webhook çağır +4. `forgelm reject ` subcommand (decline path): + - `final_model.staging/` olduğu yerde kalır (forensics için) + - audit `human_approval.rejected` event emit + - `notify_failure` webhook çağır +5. `forgelm/config.py` — `EvaluationConfig.staging_ttl_days: int = 7` (otomatik silme için, sadece doc; Faz 21 GDPR'da implement edilir). + +#### Part 3 — Marketing claim alignment +6. `docs/product_strategy.md:68` + `-tr.md` mirror — "Human approval gate (exit code 4) defers production deployment; model lands in `final_model.staging/`; operators run `forgelm approve ` to promote." +7. `CHANGELOG.md [Unreleased]` — staging directory landing notu. + +#### Part 4 — Tests +8. `tests/test_compliance.py` veya yeni `tests/test_human_approval_gate.py`: + - Gate fires → save lands in `.staging/` (NOT `final_model/`) + - Approve path: `.staging/` → `final_model/` atomic rename, audit event emitted + - Reject path: `.staging/` preserved, audit event emitted + - Stale staging detection (run_id eşleşmiyor → error) + - Concurrent approve/reject → atomic, no race + - Notifier `notify_awaiting_approval` çağrıldı (`requests_mock`) + +**Faz 9 Acceptance:** +- Test: `evaluation.require_human_approval=true` ile training; `final_model/` boş, `final_model.staging/` dolu. +- `forgelm approve ` sonrası `final_model/` dolu, `.staging/` yok. +- `forgelm reject ` sonrası `.staging/` korunur. +- Marketing claim staging hikayesini doğru anlatır. + +--- + +## 5. Code hygiene fazları (Faz 10-13) + +### Faz 10 — Pydantic Literal sweep (Tek PR) + +**Closes:** F-code-101 (Major), F-compliance-105 (Major) + +**Tasks:** +1. `forgelm/config.py`'da 6 enum-shaped field `Literal[...]`'e: + - `LoraConfig.bias: str = "none"` → `Literal["none", "all", "lora_only"]` + - `DistributedConfig.fsdp_backward_prefetch` → `Literal["backward_pre", "backward_post"]` + - `DistributedConfig.fsdp_state_dict_type` → `Literal["FULL_STATE_DICT", "SHARDED_STATE_DICT"]` + - `SafetyConfig.scoring` → `Literal["binary", "confidence_weighted"]` + - `ComplianceMetadataConfig.risk_classification` → `Literal["unknown", "minimal", "limited", "high-risk", "unacceptable"]` + - `GaLoreConfig.galore_optim` ve `galore_proj_type` → gerçek seçenekleri grep'le bul, Literal'e çevir. +2. Bespoke runtime validators sil (Pydantic Literal otomatik validate eder). +3. `config_template.yaml` `--dry-run` yeşil olduğunu doğrula. +4. Tests: invalid Literal → ValidationError raise. + +**Faz 10 Acceptance:** +- `grep -nE "^\s+\w+: str = \"[a-z_]+\"" forgelm/config.py` enum-shape görünen 0 (legitimate non-enum hariç). +- `forgelm --config config_template.yaml --dry-run` exit 0. + +--- + +### Faz 11 — Wizard `_print` indirection + drop coverage omit (Tek PR) + +**Closes:** F-code-105 (Major), F-test-003 (Major), F-code-019 (Major carry-over) + +**Tasks:** +1. `forgelm/chat.py`'deki `_print()` indirection pattern'ı `forgelm/wizard.py`'a port et. +2. 85 `print(...)` çağrısını `_print(...)` ile değiştir. +3. `_print` testable (capsys uyumlu). +4. `pyproject.toml [tool.coverage.run] omit` listesinden `forgelm/wizard.py` çıkar. +5. Wizard interactive `input()` çağrılarına `# pragma: no cover` (line-level). +6. `tests/test_wizard_byod.py`, `test_wizard_phase11_5.py` — `_print` ile capsys assertion'ları ekle (en azından 5-10 case). +7. Coverage'ın hâlâ ≥40% olduğunu doğrula. + +**Faz 11 Acceptance:** +- `pyproject.toml`'da omit boş. +- `pytest --cov=forgelm.wizard` >0% coverage gösterir. +- `grep -n "print(" forgelm/wizard.py` 0 (sadece `_print` çağrıları). + +--- + +### Faz 12 — Fixture consolidation (`tests/_helpers/`) (Tek PR) + +**Closes:** F-test-004 (Major), F-test-005 (Major), F-code-015 (Major carry-over) + +**Tasks:** +1. `tests/_helpers/__init__.py` (yeni; empty). +2. `tests/_helpers/factories.py` (yeni) — `minimal_config(**overrides) -> Dict` factory. +3. `tests/conftest.py`'a fixture-as-factory: + ``` + @pytest.fixture + def minimal_config(): + from tests._helpers.factories import minimal_config as _factory + return _factory + ``` +4. 4 lokal `_minimal_config` tanımı (`test_cli.py`, `test_cli_phase10.py`, `test_fit_check.py`, `test_integration_smoke.py`) sil; testleri factory kullanacak şekilde değiştir. +5. 7 `from conftest import minimal_config` site → fixture-injection. + +**Faz 12 Acceptance:** +- `grep -rn "_minimal_config\s*=\s*lambda\|def _minimal_config\|from conftest import" tests/` boş. +- 47 test modülü hâlâ yeşil. + +--- + +### Faz 13 — `--data-audit` deprecation discipline (Tek PR) + +**Closes:** F-code-107 (Major), F-business-024 (Minor) + +**Tasks:** +1. `forgelm/cli.py:604-615, 1424-1428` — `--data-audit` flag yakalandığında: + ``` + warnings.warn( + "--data-audit is deprecated and will be removed in v0.7.0. " + "Use 'forgelm audit' subcommand instead. See " + "https://github.com/cemililik/ForgeLM/issues/", + DeprecationWarning, stacklevel=2, + ) + audit.log_event("cli.legacy_flag_invoked", flag="--data-audit") + ``` +2. Tracking issue aç (#); link'i kod yorumunda + CHANGELOG'da. +3. `docs/standards/release.md` Deprecation cadence section (Faz 2'de eklenmiş) cite. + +**Faz 13 Acceptance:** +- `forgelm --data-audit /path` invocation `DeprecationWarning` üretir + audit event yazar. +- Tracking issue açık. + +--- + +## 6. Architectural splits (Faz 14-16) + +### Faz 14 — `data_audit/` package split (5 PR series) + +**Closes:** F-code-103 (Major) + +**Tam tasarım:** [`split-design-data_audit-cli-202604300906.md`](./split-design-data_audit-cli-202604300906.md) §1. + +**Sub-PRs:** +- **PR D-1** — Extract `_streaming.py` + `_quality.py` + `_secrets.py` + `_pii_regex.py` + - **C-53 inline cleanup:** `[ingestion-secrets]` extra `pyproject.toml`'dan kaldır; `_secrets.py`'da "fallback regex set" → "regex set" reframe; `README.md` install satırını sil; CHANGELOG Unreleased entry. (Detayları §15.5 C-53) +- **PR D-2** — Extract `_optional.py` + `_types.py` + `_simhash.py` + `_minhash.py` +- **PR D-3** — Extract `_pii_ml.py` (Presidio) + test patch updates +- **PR D-4** — Extract `_aggregator.py` + `_splits.py` + `_croissant.py` + `_summary.py` +- **PR D-5** — Convert `data_audit.py` → `data_audit/_orchestrator.py`, finalize `__init__.py` + +**Faz 14 Acceptance:** split-design §4'teki 13 acceptance criterion + import-stability test + `[ingestion-secrets]` extra repo'da hiçbir yerde referans edilmiyor (grep guard). + +--- + +### Faz 15 — `cli/` package split (6 PR series) + +**Closes:** F-code-104 (Major) + +**Tam tasarım:** [`split-design-data_audit-cli-202604300906.md`](./split-design-data_audit-cli-202604300906.md) §2. + +**Sub-PRs:** +- **PR C-1** — leaf helpers (`_exit_codes.py`, `_logging.py`, `_argparse_types.py`, `_resume.py`, `_result.py`) +- **PR C-2** — `_parser.py` (move all registrars + `parse_args`) +- **PR C-3** — `_dry_run.py`, `_fit_check.py`, `_no_train_modes.py`, `_wizard.py`, `_config_load.py`, `_training.py` +- **PR C-4** — `subcommands/_chat.py`, `_export.py`, `_deploy.py`, `_ingest.py` +- **PR C-5** — `subcommands/_quickstart.py` + `subcommands/_audit.py` +- **PR C-6** — `_dispatch.py` + `__init__.py` final + **`__main__.py` (load-bearing)** + import-stability test + +**Faz 15 Acceptance:** split-design §4 + `python -m forgelm.cli --help` byte-identical + quickstart subprocess test yeşil. + +**Note:** Faz 14 ve Faz 15 PR'ları paralel-safe pair'lar halinde (D-1||C-1, D-2||C-2, D-4||C-3, D-5||C-4) açılabilir. + +--- + +### Faz 16 — Pydantic `description=` migration with CI guard (4 PR series) + +**Closes:** F-code-102 (Major), M-DOC-006 (Major) + +**Tasks:** +- **PR DESC-1** — CI guard + ilk batch (~30 field): + - `tools/check_field_descriptions.py` (yeni) — `--strict` mode'da description= eksik field'ları listele + 1 ile çık. + - `.github/workflows/ci.yml`'a step ekle. + - 3 config class güncelle (`ModelConfig`, `LoraConfig`, `MultimodalConfig`). + - `tools/regenerate_config_doc.py` (yeni) — Pydantic schema'dan `docs/reference/configuration.md` table generate. + - CI diff guard: schema ↔ markdown. +- **PR DESC-2** — `TrainingConfig`, `DistributedConfig`, `GaLoreConfig` (~50 field). +- **PR DESC-3** — `EvaluationConfig`, `SafetyConfig`, `JudgeConfig`, `BenchmarkConfig` (~40 field). +- **PR DESC-4** — `ComplianceConfig`, `ComplianceMetadataConfig`, `RiskAssessmentConfig`, `WebhookConfig`, top-level (~50 field) + bilingual `configuration.md` + `configuration-tr.md` regenerated. + +**Faz 16 Acceptance:** `python tools/check_field_descriptions.py --strict forgelm/config.py` exit 0; configuration.md ↔ schema CI guard yeşil. + +**Bağımlılık:** Faz 10 (Literal sweep) bittikten sonra başlar — Literal type'lardan sonra description= eklemek daha temiz. + +--- + +## 7. Architecture-dependent fazlar (Faz 17-19) + +### Faz 17 — `--workers N` audit determinism (Tek PR) + +**Closes:** F-performance-104 (Major) + +**Bağımlılık:** Faz 14 (data_audit/) + Faz 15 (cli/). + +**Tasks:** +1. `forgelm/data_audit/_orchestrator.py` — `multiprocessing.Pool.imap` over chunked rows. +2. `forgelm/cli/_parser.py` — `_add_audit_subcommand`'a `--workers N` flag. +3. **Determinism contract** (master review §5.1): + - `lang_sample` byte-identical across worker counts. + - Chunk-index ordering preserved. + - Near-dup pairs sorted at end. + - PII counts identical. +4. Tests: `tests/test_data_audit_workers.py` (yeni): + - N=1 vs N=2 vs N=4: `lang_sample` byte-identical + - Same fixture: `pii_summary`, `near_duplicate_summary`, `quality_summary` identical + - Performance assertion: N=4 ≥ 2× faster than N=1 on synthetic 10K-row corpus + +**Faz 17 Acceptance:** +- `forgelm audit --workers 4 corpus/` 4× speedup; output JSON byte-identical to `--workers 1`. +- Determinism testleri yeşil. + +--- + +### Faz 18 — Library API support — Analysis & design (Tek PR; design dosyası) + +**Closes:** Master review Open Q5 (Performance Q1: `forgelm.ForgeTrainer` library API) + +**Bağımlılık:** Faz 14 + Faz 15 (cleaner public surface). + +**Tasks:** +1. Yeni dosya: `docs/analysis/code_reviews/library-api-design-202604YYMMHHMM.md`. İçerik: + - **Mevcut public surface** — `forgelm/__init__.py`'da `__getattr__` lazy ile exposed olanları enumerate et. + - **Hangi class/function library API olarak destekleyecek?** + - `ForgeTrainer` (training entry) + - `audit_dataset` (data audit one-shot) + - `Notifier` (webhook'lar) + - `AuditLogger` (compliance log writer) + - `verify_audit_log` (chain verification) + - `compute_simhash`, `detect_pii`, `mask_pii`, `detect_secrets`, `mask_secrets` (utility belt) + - **API stability tier'ları:** + - Stable (semver-protected): `ForgeTrainer.__init__`, `ForgeTrainer.train`, `audit_dataset`, `verify_audit_log`, mask_* + - Experimental: `Notifier`, advanced trainer hooks + - Internal: `_*` her şey + - **Type stub strategy** — `forgelm/py.typed` marker; type hints PEP 561 compliant; `mypy --strict` pass. + - **Lazy import discipline** — `forgelm.ForgeTrainer` access torch'u tetikler ama `import forgelm` etmemeli (mevcut `__getattr__` paterni doğru). + - **Versioned API contract** — semver explicit document; breaking change detection (CI guard?). + - **Integration test strategy** — `tests/test_library_api.py` (yeni); CLI'siz pure Python integration paths. + - **Documentation surface** — `docs/reference/library_api.md` + `-tr.md` mirror; example notebook (`notebooks/library_api_example.ipynb`). + - **Cross-cutting concerns** — logging configuration, config loading from dict (not just YAML), error handling at library boundary. +2. PR description'ı bu design dosyasını review için sunar; merge sonrası implementation Faz 19 başlar. + +**Faz 18 Acceptance:** +- Design dosyası ≥400 satır, scope + decision points + alternatives complete. +- Code review'da onay var. + +--- + +### Faz 19 — Library API support — Implementation (Tek PR) + +**Closes:** Library API support fully realized. + +**Bağımlılık:** Faz 18. + +**Tasks (Faz 18 design output'una göre; aşağıdaki muhtemel iskeleti):** +1. `forgelm/__init__.py` — `__all__` + `__getattr__` lazy public symbols ile finalize. +2. `forgelm/py.typed` (boş dosya — PEP 561 marker). +3. `forgelm/api.py` (yeni; opsiyonel) — top-level `train()`, `audit()`, `verify_audit()` convenience functions. +4. `forgelm/_version.py` (yeni) — `API_VERSION` ve `CLI_VERSION` ayrı semver string'leri. +5. Type hints completeness audit — public surface'te `Any` kalmasın; `mypy --strict` pass. +6. Integration tests `tests/test_library_api.py`: + - `from forgelm import ForgeTrainer, ForgeConfig; ForgeTrainer(config=ForgeConfig.load_dict({...})).train()` + - `from forgelm import audit_dataset; report = audit_dataset("corpus.jsonl"); summarize_report(report)` + - `from forgelm import verify_audit_log; result = verify_audit_log("audit.jsonl"); assert result.valid` + - Lazy import: `import forgelm` torch yüklemiyor. +7. `docs/reference/library_api.md` + `-tr.md` mirror. +8. `notebooks/library_api_example.ipynb` (yeni) — library kullanımı example. +9. CHANGELOG'a "Library API support" notu. + +**Faz 19 Acceptance:** +- `import forgelm; forgelm.ForgeTrainer` çalışır; type hints IDE autocomplete sağlar. +- `mypy --strict forgelm/` clean (en azından public surface). +- Integration test'ler yeşil. + +--- + +## 8. Compliance vertical fazlar (Faz 20-23) + +### Faz 20 — GDPR right-to-erasure — Analysis & design (Tek PR; design dosyası) + +**Closes:** F-compliance-107 (Major) — Analysis part. + +**Bağımlılık:** Faz 6 (verify-audit) + Faz 9 (staging directory) — audit chain integrity baseline. + +**Tasks:** +Yeni dosya: `docs/analysis/code_reviews/gdpr-erasure-design-202604YYMMHHMM.md`. İçerik: + +1. **GDPR Article 17 detaylı haritalama:** + - "Right to erasure" tetikleyiciler: data subject request, lawful basis withdrawal, processing no longer necessary + - Scope: training data, model weights (memorization risk), audit logs, derived artefacts +2. **ForgeLM'da silinmesi gerekenler:** + - **Training corpus rows** — JSONL'den satır silinmeli; downstream re-train gerekebilir + - **Audit log entries** — append-only ile çelişki; resolution: "redaction event" yazılır (orjinal silinmez ama yeni event'le geçersiz işaretlenir) + - **Model weights memorization** — full retraining only proper resolution; documented escalation path + - **Cached artefacts** — `final_model/`, `final_model.staging/`, audit reports +3. **Retention policy mekanizması:** + - `forgelm/config.py` `RetentionConfig` Pydantic block (yeni): + - `audit_log_retention_days: int = 1825` (5 yıl) + - `staging_ttl_days: int = 7` + - `ephemeral_artefact_retention_days: int = 90` + - `enforce: Literal["log_only", "warn_on_excess", "block_on_excess"] = "log_only"` +4. **Erasure subcommand tasarımı:** + - `forgelm purge --row-id --corpus `: JSONL'den satır siler, audit event yazar + - `forgelm purge --run-id --kind {staging,artefacts,logs}`: staging directory veya derived artefact silme + - `forgelm purge --check-policy`: retention policy ihlallerini list eder +5. **Audit chain ile uyumluluk:** + - Append-only ile çelişki yok: silme bir "deletion event" olarak yazılır; orjinal entry'ler chain integrity için kalır + - Hash chain re-verify post-deletion +6. **Marketing claim revizyonu:** + - `safety_compliance.md`'a GDPR section + - Operator-facing how-to-comply guide +7. **Test strategy:** + - Erasure subcommand atomic + - Audit event correctly emitted + - Chain still verifies post-erasure + - Retention policy detection +8. **Out of scope:** + - Model retraining automation (operator decision) + - Distributed corpus deletion (single-node only) + +**Faz 20 Acceptance:** Design ≥400 satır; tüm GDPR Article 17 senaryoları için karar verilmiş. + +--- + +### Faz 21 — GDPR right-to-erasure — Implementation (Tek PR) + +**Closes:** F-compliance-107 (Major) — Implementation part. + +**Bağımlılık:** Faz 20. + +**Tasks (Faz 20 design output'una göre):** +1. `forgelm/config.py` — `RetentionConfig` Pydantic block (yukarıda tarif). +2. `forgelm/cli.py` (Faz 15 sonrası `forgelm/cli/subcommands/_purge.py`) — `forgelm purge` subcommand: + - `--row-id`, `--corpus`, `--run-id`, `--kind`, `--check-policy` flag'leri + - Atomic operations +3. `forgelm/compliance.py` — `record_erasure_event(target_kind, target_id, justification)` helper. +4. Audit event vocabulary'sine ekle: `data.erasure_requested`, `data.erasure_completed`, `data.erasure_failed`. +5. `docs/guides/gdpr_erasure.md` + `-tr.md` mirror — operator-facing how-to. +6. `docs/qms/sop_data_management.md` — retention + erasure procedure güncelleme (Faz 1'deki rewrite üstüne). +7. Tests `tests/test_gdpr_erasure.py` (yeni): + - Row erasure: JSONL satır silindi, audit event yazıldı + - Run-id-based staging deletion + - Run-id-based artefact deletion + - Audit chain post-erasure still verifies + - Retention policy `--check-policy` doğru rapor üretir + - Concurrent erasure → atomic + - Unknown row-id / run-id → clear error message +8. `safety_compliance.md` GDPR section. + +**Faz 21 Acceptance:** +- `forgelm purge --row-id 42 --corpus train.jsonl` çalışır; satır silinir; audit event yazar; chain valid kalır. +- 7 test yeşil. +- GDPR docs bilingual. + +--- + +### Faz 22 — ISO 27001 / SOC 2 Type II alignment — Analysis & gap assessment (Tek PR; design dosyası) + +**Closes:** Master review §8.8 strategic open question + kullanıcı kararı (Pro CLI hedeflemeyi destekleyecek baseline). + +**Bağımlılık:** Faz 3 (operator identity), Faz 6 (verify-audit), Faz 7 (safe_post), Faz 8 (webhook lifecycle), Faz 9 (Article 14 staging), Faz 21 (GDPR full). + +**Önemli not:** ForgeLM "ISO 27001 sertifikalı" olamaz — yazılım kütüphanesi sertifika sahibi değil organizasyonlar olur. Yapacağımız şey: **deployer'ın ISO 27001 / SOC 2 audit'inde ForgeLM tarafından sağlanan kontrolleri kanıtlamasını kolaylaştıran teknik + dokümantasyon altyapısı kurmak.** + +**Tasks:** +Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.md`. İçerik: + +1. **ISO 27001 Annex A control mapping (~93 control)** — her control için: + - `A.5.x — A.18.x` her madde için + - "ForgeLM bu control'ü nasıl destekler?" (e.g., `A.12.4 Logging and monitoring` → audit chain + verify-audit) + - "Deployer'ın bu control için yapması gerekenler" + - "ForgeLM'ın doldurmadığı kısımlar (out-of-scope)" +2. **SOC 2 Trust Services Criteria mapping:** + - **Security:** access control (operator identity), encryption-at-rest (deployer-side), incident response (`docs/qms/sop_incident_response.md` expand) + - **Availability:** out-of-scope (ForgeLM CLI; deployer infra) + - **Processing Integrity:** audit chain + verify-audit + dataset fingerprint TOCTOU defence (Theme B closures) + - **Confidentiality:** PII detection + masking + secrets detection + - **Privacy:** GDPR Article 17 (Faz 21) + retention policy +3. **SBOM (Software Bill of Materials) generation:** + - `tools/generate_sbom.py` — pip dependencies + package metadata → SPDX/CycloneDX format + - CI step: SBOM generated on each release; published as artifact +4. **Vulnerability scanning automation:** + - `pip-audit` integration in nightly.yml + - `bandit` integration in ci.yml (low-severity warnings only; high-severity fail) +5. **Encryption-at-rest documentation:** + - `docs/qms/encryption_at_rest.md` — operator-facing guidance for model weights, audit logs, training data +6. **Access control documentation:** + - `docs/qms/access_control.md` — multi-user pipelines; OS-level user isolation; operator identity +7. **Incident response runbook expansion:** + - `docs/qms/sop_incident_response.md` — ISO A.16 mapping; security incident playbook +8. **Change management formal process:** + - `docs/qms/sop_change_management.md` — ISO A.12.1 mapping; CI gates as control +9. **Risk treatment plan (RTP):** + - `docs/qms/risk_treatment_plan.md` — identified risks (data poisoning, supply chain, credential leak), mitigations, residual risk +10. **Statement of Applicability (SoA):** + - `docs/qms/statement_of_applicability.md` — ISO Annex A controls × applicable/excluded × justification +11. **Deployer-side audit cookbook:** + - `docs/guides/iso_soc2_deployer_guide.md` + `-tr.md` mirror — "ForgeLM deployment'ınızı ISO 27001 audit'ine nasıl hazırlarsınız" step-by-step +12. **CI/CD integration:** + - SBOM generation on release tag + - pip-audit + bandit in nightly + - SCA report archived as release artifact + +**Faz 22 Acceptance:** Design ≥800 satır (ISO controls × SOC 2 criteria mapping detaylı). Pro CLI gelecek work için clear blueprint. + +--- + +### Faz 23 — ISO 27001 / SOC 2 Type II alignment — Implementation (Tek PR; büyük) + +**Closes:** ISO/SOC 2 alignment baseline implementation. + +**Bağımlılık:** Faz 22. + +**Tasks (Faz 22 design output'una göre):** +1. `tools/generate_sbom.py` — SPDX format SBOM generator. +2. `.github/workflows/publish.yml` — SBOM generation step + archive as artifact. +3. `pyproject.toml [project.optional-dependencies] security` — `pip-audit`, `bandit` dev extras. +4. `.github/workflows/nightly.yml` — `pip-audit` + `bandit` step (high-severity fail; low-severity warn). +5. `docs/qms/` — yeni dosyalar: + - `encryption_at_rest.md` (+`-tr.md`) + - `access_control.md` (+`-tr.md`) + - `risk_treatment_plan.md` (+`-tr.md`) + - `statement_of_applicability.md` (+`-tr.md`) +6. `docs/qms/sop_incident_response.md`, `sop_change_management.md` — expand per Faz 22 design. +7. `docs/guides/iso_soc2_deployer_guide.md` (+`-tr.md`) — deployer-facing audit cookbook. +8. `docs/reference/iso27001_control_mapping.md` (+`-tr.md`) — control table. +9. `docs/reference/soc2_trust_criteria_mapping.md` (+`-tr.md`) — criteria table. +10. README'a "ISO 27001 / SOC 2 alignment" section (1 paragraph + link). +11. Tests: + - SBOM generation produces valid SPDX + - `tools/generate_sbom.py` deterministic (same input → same output) + - pip-audit step doesn't fail on known clean tree + +**Faz 23 Acceptance:** +- SBOM artifact üretiliyor. +- pip-audit + bandit nightly'de çalışıyor. +- 7 yeni QMS dosya + 2 yeni reference + 1 yeni guide bilingual mevcut. +- README ISO/SOC 2 alignment claim'i kanıtla destekli. + +--- + +## 9. Polish + remaining cleanup fazları (Faz 24-29) + +### Faz 24 — Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN+TR user-manual H2 drift) (Tek PR) + +**Closes:** M-DOC-002, M-DOC-007, M-DOC-012, M-DOC-013, M-DOC-014, M-DOC-015, F-loc-005, F-loc-006, F-loc-007, F-loc-008 + +**Bağımlılık:** Faz 16 (description= migration → configuration.md auto-gen → TR mirror reflect). + +**Tasks:** +1. `configuration-tr.md` H4 14 vs 6 drift kapatılır (Faz 16 description= migration sonrası auto-gen + manual review). +2. `distributed_training-tr.md` 3 H3 ekle (Custom DeepSpeed Config, FSDP-vs-DeepSpeed, LoRA+Distributed). +3. `ingestion-tr.md` 2 H3 ekle (Markdown-aware splitter, DOCX→markdown table preservation). +4. `architecture-tr.md`'ye `forgelm/templates/` ekle; EN'den `configs/safety_prompts/` listing'i mirror et veya TR'den çıkar. +5. `data_preparation*.md` — rewrite ~120 satır (mevcut 35-line micro-doc) ya da `alignment.md`'a fold (decide in PR; default rewrite). +6. `alignment.md:230` "v0.5.1" → phase number reference. +7. EN+TR user-manual H2 drift düzeltmeleri: + - `tr/training/sft.md` "Diskte ne elde edersiniz" ekle + - `tr/training/simpo.md` H2 spine 1:1 mirror + - `tr/compliance/overview.md` "Annex IV neyi içerir" ekle + - `tr/concepts/data-formats.md` "Verinizi doğrulama" ekle +8. **Bilingual H3/H4 parity CI guard ekle** — `tools/check_bilingual_parity.py` H2 + H3 + H4 paritesini diff eder. + +**Faz 24 Acceptance:** +- `tools/check_bilingual_parity.py` strict mode exit 0. +- 7 doc çift parite. + +--- + +### Faz 25 — Site picker honesty + site-as-tested-surface CI guard (Tek PR) + +**Closes:** F-loc-001 (Major), F-loc-003 (Major), Theme α (site claim/code drift) + +**Bağımlılık:** Faz 1 (site honesty fixes önce land). + +**Tasks:** +1. `docs/standards/localization.md` güncelle: + ``` + ## Supported languages + - **EN + TR (mandatory).** Tüm user-facing docs ve user-manual sayfaları EN+TR mirror. + - **DE / FR / ES / ZH (deferred).** Site picker'da görünmüyor; user-manual content yok. Translator capacity geldiğinde reverse path. + ``` +2. Site picker'dan DE/FR/ES/ZH'yi geçici çıkar — `site/index.html`, `features.html`, `quickstart.html`, `compliance.html`, `guide.html`, `contact.html`, `privacy.html`, `terms.html` × 8 dosya, picker dropdown 6 → 2. +3. `site/js/i18n.js`, `site/js/translations.js` — 6 dilden 2 dile redükte (deferred dilleri kaldır; ileride geri eklenir). +4. `tools/build_usermanuals.py` — sparse-emit: deferred languages için bag üretme. +5. 8 site sayfasına `og:locale="en_US"` + `og:locale:alternate="tr_TR"` + `` + ``. +6. `YOUR_DOMAIN` placeholder'ı resolve et (gerçek domain ya da ortam değişkeni). +7. **`tools/check_site_claims.py` (yeni)** — site-as-tested-surface CI guard: + - `site/compliance.html` artefakt isimleri ↔ `forgelm/compliance.py::_export_compliance_artifacts` diff + - `site/quickstart.html` template names ↔ `forgelm/quickstart.py::TEMPLATES` diff + - `site/index.html` GPU count ↔ `_GPU_PRICING` count diff + - `site/*` PyPI version mention ↔ `pyproject.toml version` diff +8. `.github/workflows/ci.yml`'a `check_site_claims.py --strict` step. +9. `.github/workflows/usermanuals-validate.yml` — `--strict` mode (orphan markdown detection). +10. `site/js/guide.js:436, 448, 455, 458, 466, 654` — `.toLowerCase()` → `.toLocaleLowerCase(state.lang)` (Türkçe İ/I bozulmasın). + +**Faz 25 Acceptance:** +- Site picker sadece EN+TR. +- `tools/check_site_claims.py --strict` exit 0; bir testte `compliance.py`'da artefact filename değiştir → CI fail. +- Tüm site sayfaları doğru `og:locale` + `hreflang`. +- Standards explicit deferred policy. + +--- + +### Faz 26 — QMS bilingual EN+TR mirror + compliance docs cleanup (Tek PR) + +**Closes:** F-compliance-112 (Major), M-DOC-003 (Major), F-compliance-113 (Major) + +**Bağımlılık:** Faz 23 (ISO/SOC 2 dokümanları QMS ile lockstep — TR mirror'lar onları da kapsar). + +**Tasks:** +1. `docs/qms/` 6 EN dosya için TR mirror: + - `README-tr.md` + - `roles_responsibilities-tr.md` + - `sop_change_management-tr.md` + - `sop_data_management-tr.md` + - `sop_incident_response-tr.md` + - `sop_model_training-tr.md` +2. Faz 23'te eklenen QMS dosyaları için TR mirror'lar (encryption_at_rest, access_control, risk_treatment_plan, statement_of_applicability) — Faz 23'te zaten yapılmış olabilir (ya zaten orada ya bu fazda). +3. `docs/reference/compliance_summary.md` — H1 + scope blockquote ekle, 11 broken relative path düzelt, "50 prompts × 3 categories" → "140 × 6", README'den ve QMS index'inden link ekle. +4. `compliance_summary.md` — stale `#L33`, `#L167` line anchors → symbol-style references (function names) veya regenerate'le CI guard. +5. CI: `tools/check_anchor_resolution.py` (yeni) — markdown anchor'ları resolve edilebilir mi diff. + +**Faz 26 Acceptance:** +- `docs/qms/` 6+ EN/TR mirror tam. +- `compliance_summary.md` reachable + linked + anchors resolve. +- CI anchor checker yeşil. + +--- + +### Faz 27 — Silent `except Exception:` sweep (Theme D residual) (Tek PR) + +**Closes:** F-code-106 (Major) + +**Tasks:** +1. `forgelm/data_audit/` (split sonrası) — `_streaming.py:_detect_language` (eski 1132): narrow class to `langdetect.LangDetectException`. +2. `forgelm/safety.py:99/225/387/499` — narrow each. +3. `forgelm/trainer.py`, `forgelm/judge.py`, `forgelm/compliance.py:292` — review every `except Exception:`. +4. **Üç-pass sweep:** + - **Pass 1:** Narrow class. + - **Pass 2:** Justify with `# noqa: BLE001 — best-effort: ` if narrow class infeasible. + - **Pass 3:** Add tests for the narrow exception path. +5. `docs/standards/error-handling.md` — "best-effort artefact" carve-out documentation. `# noqa: BLE001` tek geçerli sanctioned pattern. +6. **C-55: MoE expert naming compatibility refactor** — `forgelm/model.py:285-290` `_expert_index_in_name` hardcoded string match → `_EXPERT_NAME_PATTERNS` per-architecture dict + regex resolver. Mixtral/DeepSeek/Qwen3 fixture'larıyla unit test. Detayları §15.5 C-55 + +**Faz 27 Acceptance:** +- `grep -rn "except Exception:" forgelm/ | grep -v "# noqa: BLE001"` boş. +- error-handling.md best-effort carve-out clear. +- `_expert_index_in_name` regex-based, 3 architecture fixture testi yeşil. + +--- + +### Faz 28 — Remaining Major + Minor + Nit cleanup (Tek PR) + +**Closes:** Master review tablosundan Faz 1-27'ye girmeyen tüm Major + Minor + Nit (~70+ madde). + +**Tasks (örneklerle):** +- F-code-108: `tools/build_usermanuals.py` `_meta.yaml` Pydantic schema validation +- F-business-006: `--compliance-export` populated bundle + help text +- F-business-008: QMS README artefakt mapping table rewrite +- M-203: FORGELM_GGUF_CONVERTER `--allow-custom-converter` ack flag + SHA-256 manifest +- M-204: `_sanitize_md_list` helper + adapter audit +- M-205: `deploy._ollama_modelfile` newline guard +- F-compliance-106: webhook timeout default 5s → 10s +- F-compliance-108: judge_results.json provenance block +- F-compliance-109: synthetic-data provenance in manifest +- F-compliance-110: high-risk + safety-disabled raise +- F-compliance-111: `_maybe_inline_audit_report` failure → WARNING +- M-DOC-008 / M13: design doc status blockquotes +- M-DOC-004: 6 yerde `v0.3.1rc1` → `v0.4.0` +- F-test-002: `runtime_smoke.py` decision (delete; nightly wheel-install-smoke covers it) +- F-test-009: `pytest-xdist` enable +- F-test-011: `test_integration_smoke.py` → `test_integration.py` rename +- **C-52: `_chunk_semantic` implementation** — yeni opsiyonel `[chunking-semantic]` extra (`sentence-transformers>=2.7.0,<4.0.0`); embedding cosine-similarity threshold-based segmentation; `_require_semantic_chunker` install hint helper (datasketch/Presidio pattern'iyle aynı); `ingestion.md` + `ingestion-tr.md` chunking-strategies tablosuna semantic satırı; unit test (sentence-transformers mock'lu kısa paragraf + multi-topic doc fixture'ları + extra missing → ImportError). Detayları §15.5 C-52 +- **C-54: `_is_private_destination` `__all__` cleanup** — `forgelm/webhook.py:16` re-export'u kaldır; downstream importer kalmadıysa direct removal, varsa `DeprecationWarning` + v0.6.0 removal note. Detayları §15.5 C-54 +- **C-57: GRPO reward token domain honesty** — `forgelm/trainer.py:57,78` `_REWARD_STRIP_TOKENS` + `_BOUNDARY_REQUIRED_TOKENS` docstring'ine GSM8K/MATH-specific olduğunu açıkça yaz; v0.6.0 GRPO config-driven reward plugin migration roadmap'e ekle. Detayları §15.5 C-57 +- 60 Minor + 40 Nit kalan kısımlar. +- Final ruff format + ruff check pass. + +**Faz 28 Acceptance:** +- Master review tablosundan açık olan ≤2 madde (varsa explicit deferred + `risks-and-decisions.md`'ye kaydedilmiş). + +--- + +### Faz 29 — Nit sweep (Tek PR) + +**Closes:** Kalan 40 Nit (Faz 28'da bitmemiş olanlar). + +Çoğunlukla 1-line fix'ler. Mass edit. ruff format pass. + +--- + +## 10. Final documentation + site finalization (Faz 30) + +### Faz 30 — Final documentation + site finalization (EN+TR only) (Tek PR) + +**Closes:** Tüm v0.5.5 user-facing documentation deliverable; Theme C (doc drift) + Theme α (site claim/code) final closure. + +**Bağımlılık:** **Faz 1-29 tamamı** — fonksiyonel iş bittikten sonra son dokümantasyon pass'i. + +**Kapsam (kullanıcı kararı):** +- `docs/` dizinindeki tüm dokümanlar +- `docs/usermanuals/{en,tr}/` — **sadece EN ve TR**; DE/FR/ES/ZH **bu fazda dokunulmaz** (kullanıcı kararı: "diğer diller için daha sonra çeviri yapacağız") +- `site/*` — tüm HTML, CSS, JS + +**Yaklaşım:** Faz 1-29 sırasında her faz kendi user-facing dokümanını ekledi/güncelledi. Bu faz **release-readiness final pass**: kaynaklar arası tutarlılığı doğrular, eksikleri kapatır, build çıktılarını yeniler. + +#### Tasks + +##### A. Yeni feature'lar için user-facing doküman tamamlama (EN + TR) +Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa ekle: + +| Feature | `docs/guides/` | `docs/reference/` | `docs/usermanuals/{en,tr}/` | +|---|---|---|---| +| `forgelm verify-audit` | safety_compliance.md (Faz 6'da) | yeni `verify_audit.md` (+ -tr.md) | `compliance/verify-audit.md` (+ tr) | +| Article 14 staging + approve/reject | `human_approval_gate.md` (yeni; +tr) | yeni `approve_subcommand.md` (+ -tr) | `compliance/human-approval-gate.md` (+ tr) | +| GDPR right-to-erasure | gdpr_erasure.md (Faz 21'de; +tr) | yeni `purge_subcommand.md` (+ -tr) | `compliance/gdpr-erasure.md` (+ tr) | +| `forgelm audit --workers N` | data_audit.md (+tr) güncellenir | configuration.md `audit` section güncellenir | `data/audit.md` workers section (+ tr) | +| Library API support | library_api.md (Faz 19'da; +tr) | yeni `library_api_reference.md` (+ -tr) | `reference/library-api.md` (+ tr) | +| ISO 27001 / SOC 2 alignment | iso_soc2_deployer_guide.md (Faz 23'te; +tr) | iso27001_control_mapping.md, soc2_trust_criteria_mapping.md (Faz 23'te; +tr) | `operations/iso-soc2-deployer.md` (+ tr) | +| `data_audit/` + `cli/` package splits | architecture.md (+tr) güncellenir | architecture_decisions.md changelog | `reference/architecture.md` (+ tr) | +| Pydantic `description=` config docs auto-gen | configuration.md (Faz 16'da auto-gen; +tr) | (auto-gen) | `reference/configuration.md` (+ tr) | +| Lazy torch + safety/judge batching | performance.md (yeni; +tr) | configuration.md `evaluation.safety.batch_size` | `reference/performance.md` (+ tr) | +| SBOM + pip-audit + bandit | iso_soc2_deployer_guide.md (cite) | yeni `supply_chain_security.md` (+ -tr) | `operations/supply-chain.md` (+ tr) | +| **Semantic chunking** (C-52, Faz 28 ships code) | `ingestion.md` chunking-strategies tablosuna `semantic` satırı (+tr) | `configuration.md` `[chunking-semantic]` extra docs | `data/ingestion.md` chunking section (+ tr) | +| **`detect-secrets` extra removal** (C-53, Faz 14 PR D-1 ships code) | `data_audit.md:269-274` + `data_audit-tr.md:269-273` paragrafı sil; `ingestion.md:138` + `ingestion-tr.md:137-138` cümlesi sil | (yok) | `data/secrets.md` regex set tablosunu mevcut `_SECRET_PATTERNS`'a hizala (+tr) — Anthropic/Stripe/SendGrid/Twilio drift kararı (kısalt veya code'a ekle) | +| **GRPO reward token honesty** (C-57, Faz 28 ships docstring) | `alignment.md:177-188` GRPO section'a "built-in reward GSM8K/MATH-tuned" disclaimer + custom reward callable örneği (TR mirror Faz 24 scope'u) | (yok) | `training/grpo.md` aynı disclaimer (+ tr) | + +**Yapılacak:** Yukarıdaki tablodaki her satır için, eksik olan dokümanı yaz veya mevcut olanı v0.5.5'e güncelle. Bilingual (EN+TR) zorunlu. + +##### B. `docs/usermanuals/_meta.yaml` navigation tree update +1. Yeni eklenen sayfalar için `_meta.yaml` `sections`/`pages` ağacına entry ekle. +2. Sadece `en` ve `tr` title'ları zorunlu doldurulur. Diğer 4 dil için (de/fr/es/zh) title field'ı boş bırakılır → `tools/build_usermanuals.py` fallback işaret eder. +3. ~10 yeni sayfa beklenir (yukarıdaki tablodaki "+tr" sütunu sayısı). + +##### C. User manual rebuild +1. `python tools/build_usermanuals.py` çalıştır → `site/js/usermanuals/{en,tr}.js` v0.5.5 içerikle yeniden üretilir. +2. `de.js`, `fr.js`, `es.js`, `zh.js` build çıktılarında üretilmez (Faz 25 sparse-emit kararı + kullanıcı kararı: deferred). +3. Veya alternatif: site picker zaten EN+TR olduğu için diğer 4 dil bag'i de gereksiz; sparse-emit pattern'i confirm. + +##### D. `site/*` HTML pages — v0.5.5 final state +1. **`site/index.html`:** + - Stat block güncelle: 47 test modules / 800+ tests / Library API supported / ISO 27001 + SOC 2 alignment / EU AI Act Articles 9-17 + Annex IV + - Feature highlights: Article 14 staging gate, GDPR erasure, --workers parallelism, library API +2. **`site/features.html`:** + - Yeni feature blokları: Article 14 staging directory, GDPR Article 17 right-to-erasure, ISO 27001 / SOC 2 alignment, Library API support, --workers determinism + - Mevcut blokları v0.5.5 wording'iyle güncelle +3. **`site/compliance.html`:** + - GDPR Article 17 erasure section (yeni) + - ISO 27001 / SOC 2 alignment section (yeni) + - Article 14 staging directory + `forgelm approve` workflow (yeni) + - Mevcut Annex IV / EU AI Act mapping güncel; artefact tree Faz 1'de düzeltilmişti +4. **`site/quickstart.html`:** + - 5 gerçek template (Faz 1'de düzeltilmiş); v0.5.5 install command (`pip install forgelm==0.5.5`) + - Audit + verify-audit + approve flow'unun quickstart'a entegre olduğunu göster +5. **`site/guide.html`:** + - User-manual page list otomatik build'den geliyor (`site/js/usermanuals/{en,tr}.js`); manuel update gerekmez ama search index güncel olduğunu doğrula +6. **`site/contact.html`, `privacy.html`, `terms.html`:** v0.5.5 versiyon mention'ları + GDPR + ISO/SOC 2 referansları varsa güncelle +7. Tüm site sayfaları `og:locale` + `hreflang` (Faz 25'te eklenmişti); doğrulama + +##### E. README + CONTRIBUTING + CLAUDE.md final pass +1. **`README.md`:** + - Feature list v0.5.5 nihai + - Stat block güncel (test count, module count, Library API supported, ISO/SOC 2 alignment claim) + - Quickstart `pip install forgelm==0.5.5` + - Bilingual link (varsa README-tr.md, yoksa eklenmeli mi karar) +2. **`CONTRIBUTING.md`:** + - Faz 1'de project structure tree → architecture.md pointer (already done); doğrulama + - Pre-commit hooks section (Faz 32'de optional; bu fazda da hatırlat) +3. **`CLAUDE.md`:** + - Faz 1'de "26 test modules" → "47" düzeltmiş; doğrulama + - Yeni eklenen modüller (`forgelm.cli/`, `forgelm.data_audit/`, `forgelm/_http.py`, vb.) repository structure section'da yansır + - Yeni eklenen skill listesi (`add-config-field`, `add-trainer-feature`, `cut-release`, vb.) güncel +4. **`CHANGELOG.md`:** + - `[Unreleased]` boş kalır (release Faz 33'te `[0.5.5]` olarak finalize) + - DRAFT format hazır + +##### F. Roadmap final state +1. `docs/roadmap.md` + `docs/roadmap-tr.md`: + - Phase 12.6 closure cycle ✅ Done işareti + - v0.5.5 satırı next-up + - Phase 13 (Pro CLI) hâlâ planned (bu plan kapsam dışı; ama ISO/SOC 2 Phase 13'ün baseline'ı olarak ödüllü olduğu vurgulanır) +2. `docs/roadmap/phase-12-6-closure-cycle.md` (yeni) — bu kapanış planının summary'si; tüm 33 fazı liste halinde +3. `docs/roadmap/releases.md` — v0.5.5 satırı eklenir (içerik Faz 33'te tamamlanır; bu fazda placeholder) +4. `docs/roadmap/risks-and-decisions.md` — v0.5.5 cycle'ı bitiren tüm major decisions (kullanıcı kararları + plan'daki conflict resolutions) Decision Log'a eklenir + +##### G. Standards final pass +1. `docs/standards/README.md` — yeni eklenen standardlar (Faz 2 deprecation cadence, Faz 27 best-effort carve-out, Faz 18 library API stability tier'ları) index'te görünür +2. `docs/standards/architecture.md` — `data_audit/` + `cli/` package'lar architectural overview'da yansır; "module cohesion ~1000 line ceiling" still active rule +3. `docs/standards/documentation.md` — bilingual H3/H4 parity rule formalize (Faz 24'te eklenmişti) +4. `docs/standards/localization.md` — Faz 25'te EN+TR mandatory + DE/FR/ES/ZH deferred policy formalized; cross-check +5. `docs/standards/release.md` — Faz 30 release-prep workflow (cross-OS matrix); Faz 32 pre-commit; v0.5.5 release sequence +6. `docs/standards/code-review.md` — pre-commit optional + CI catches mandatory disipline +7. `docs/standards/testing.md` — cov-fail-under=40 (Faz 2'de düzeltilmiş); cross-OS release-tag testing (Faz 31); pytest-xdist (Faz 28) + +##### H. Bilingual parity strict-mode CI guard run +1. `tools/check_bilingual_parity.py` (Faz 24'te eklenmişti) `--strict` mode tüm `*.md ↔ *-tr.md` çiftleri için H1+H2+H3+H4 parity diff +2. Hatalar varsa düzelt; clean state'i CI guard'la pin + +##### I. Site claim CI guard run +1. `tools/check_site_claims.py` (Faz 25'te eklenmişti) `--strict` mode site/* iddiaları kod ile diff +2. Hatalar varsa düzelt + +##### J. CLI help ↔ guide alignment check +1. `tools/check_cli_help_consistency.py` (Faz 30'da yeni eklenir): + - `forgelm --help` çıktısındaki tüm subcommand'ler `docs/guides/` + `docs/reference/usage.md`'de mention + - Her subcommand için `forgelm --help` flag listesi `docs/reference/configuration.md` ve `docs/usermanuals/{en,tr}/reference/cli.md` ile uyumlu + - Mismatch'ler test failure +2. `.github/workflows/ci.yml`'a step ekle + +##### K. Config docs auto-gen final regenerate +1. `tools/regenerate_config_doc.py` (Faz 16'da eklenmişti) — `docs/reference/configuration.md` + `configuration-tr.md` regenerate +2. CI diff guard zaten vardı — sadece final state'i lock et + +##### L. DE/FR/ES/ZH user-manual policy formalization +1. `docs/usermanuals/_meta.yaml` üst notuna açıkça yaz: "DE/FR/ES/ZH user-manual content deferred to future cycle. EN + TR mandatory; others fallback to EN at build time." +2. `docs/standards/localization.md` (Faz 25'te güncellenmişti) cross-reference + +##### M. Diagram / asset sweep +1. `docs/` içindeki tüm referans verilen diagram + image dosyaları hâlâ var mı kontrol +2. v0.5.0'dan v0.5.5'e architectural diagram'lar update gerektiriyor mu (data_audit/, cli/ split'leri) +3. site/assets/ — varsa logo + screenshot v0.5.5 + +##### N. Spell-check + link integrity +1. `tools/check_anchor_resolution.py` (Faz 26'da eklenmişti) tüm markdown anchor'ları resolve +2. Opsiyonel: `markdownlint-cli2` rule'ları temiz + +#### Faz 30 Acceptance + +- [ ] Tablo A'daki her feature için EN+TR `docs/guides/`, `docs/reference/`, `docs/usermanuals/{en,tr}/` üçlüsü tam. +- [ ] `_meta.yaml` yeni sayfaları içerir; `en` + `tr` title'ları doğru. +- [ ] `python tools/build_usermanuals.py` exit 0; `site/js/usermanuals/{en,tr}.js` v0.5.5 içerik. +- [ ] `tools/check_bilingual_parity.py --strict` exit 0. +- [ ] `tools/check_site_claims.py --strict` exit 0. +- [ ] `tools/check_cli_help_consistency.py` exit 0. +- [ ] `tools/regenerate_config_doc.py` diff yeşil. +- [ ] `tools/check_anchor_resolution.py` exit 0. +- [ ] README + CONTRIBUTING + CLAUDE.md v0.5.5 stat block + feature list ile senkron. +- [ ] `docs/roadmap/` final state (Phase 12.6 ✅ + v0.5.5 satırı + risks-and-decisions Decision Log). +- [ ] Site/* HTML pages v0.5.5 features highlighted (özellikle features.html, compliance.html, index.html). +- [ ] DE/FR/ES/ZH user-manual'lara dokunulmamış (kullanıcı kararı); deferred policy formalized. +- [ ] CHANGELOG `[Unreleased]` `[0.5.5]` draft hazır (Faz 33'te tamamlanacak). + +--- + +## 11. Release prep + release fazları (Faz 31-33) + +### Faz 31 — Cross-OS release-tag matrix workflow (Tek PR) + +**Closes:** F-test-007 (Major) — kullanıcı kararı: release-tag-only matrix. + +**Tasks:** + +> **Design note (post-review revisi):** İlk taslak `publish.yml`'a `needs: cross-os-tests` ekleyerek cross-workflow dependency kurmaya çalışıyordu — bu GitHub Actions'da **geçersiz** (`needs:` sadece aynı workflow içindeki job'lara işaret edebilir). Ayrıca `pip install -e .` editable install yapıyordu — kullanıcının PyPI'dan çekeceği packaged artifact'ı (wheel + sdist) test etmiyordu. Aşağıdaki revize tasarım her iki sorunu çözer: +> - **Tek workflow:** Cross-OS test job'u + publish job'u **aynı `publish.yml`** dosyasında, ikincisi `needs: cross-os-tests` ile birinciye bağlı. +> - **Packaged artifact testi:** `python -m build` ile wheel + sdist üret, sonra wheel'i fresh venv'e `pip install` et, ardından pytest. Editable install yok. + +1. `.github/workflows/publish.yml` — yeniden yapılandır (tek workflow, iki job): + ```yaml + name: Release — Build, cross-OS validate, publish to PyPI + on: + push: + tags: ['v*'] + + jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: '3.11' } + - name: Build wheel + sdist + run: | + python -m pip install --upgrade build + python -m build # outputs dist/*.whl and dist/*.tar.gz + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + cross-os-tests: + needs: build + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ['3.10', '3.11', '3.12', '3.13'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: ${{ matrix.python }} } + - uses: actions/download-artifact@v4 + with: { name: dist, path: dist } + - name: Install packaged wheel (NOT editable) + shell: bash # POSIX shell on Windows runners too + run: | + # Pick the .whl built by the upstream build job; install it + # into a fresh interpreter so the test exercises exactly what + # PyPI users will pip install. + python -m pip install --upgrade pip + python -m pip install dist/*.whl + - name: Install Linux-only extras (qlora/ingestion/eval) + if: runner.os == 'Linux' + # qlora pulls bitsandbytes which only ships Linux wheels. + # ingestion + eval are cross-platform but bundled here so a + # single conditional install covers the breadth-test surface. + # We resolve transitive dependencies from PyPI (no + # ``--no-index --find-links``) — the local ``dist/`` only + # contains the forgelm wheel, not its dependency closure. + run: python -m pip install 'forgelm[qlora,ingestion,eval]' + - name: Test (CPU-only, no-network) + run: pytest tests/ -q --ignore=tests/test_cost_estimation.py + - name: SBOM generation + run: python tools/generate_sbom.py > sbom-${{ matrix.os }}-py${{ matrix.python }}.json + - uses: actions/upload-artifact@v4 + with: + name: sbom-${{ matrix.os }}-py${{ matrix.python }} + path: sbom-${{ matrix.os }}-py${{ matrix.python }}.json + + publish: + needs: cross-os-tests # blocked until cross-OS green + runs-on: ubuntu-latest + environment: pypi # OIDC trusted publishing + permissions: { id-token: write, contents: read } + steps: + - uses: actions/download-artifact@v4 + with: { name: dist, path: dist } + - uses: pypa/gh-action-pypi-publish@release/v1 + ``` +2. `docs/standards/release.md` — yeni section "Release prep workflow": + - Tag push trigger + - `build` job: wheel + sdist + - `cross-os-tests` job (gated on `build`): 3 OS × 4 Python = 12 kombinasyon, **packaged wheel install**, pytest, SBOM artifact upload + - `publish` job (gated on `cross-os-tests`): OIDC trusted publishing + - Hiçbir kombinasyon fail ederse publish çalışmaz +3. README'da "OS Independent" claim'i artık kanıtla destekli (3 OS × 4 Python green CI matrix gerekçesi). + +**Faz 31 Acceptance:** +- `git tag -a v0.5.5-test -m "test"` push edildiğinde `publish.yml` tetiklenir; `build` → `cross-os-tests` (matrix runs + SBOM artifact upload) → `publish` job zinciri akar. +- `publish` job'u `cross-os-tests` herhangi bir kombinasyonda fail ederse `needs:` zinciri kırılır ve PyPI publish çalışmaz. + +--- + +### Faz 32 — `.pre-commit-config.yaml` (opsiyonel) (Tek PR) + +**Closes:** F-test-008 (Major) — kullanıcı kararı için: **A (opsiyonel) öneriliyor.** + +**Önerinin gerekçesi (Soru 8):** +- **A (opsiyonel)** — Geliştirici `pre-commit install` yaparsa aktif; CI hook'ları zaten enforce ediyor. Friction yok ama gitleaks gibi pre-commit-only value korunur. +- **B (zorunlu)** — Daha sıkı, ama dev cycle yavaşlar. CI zaten aynı kontrolleri yapıyor; çift kontrol marjinal değer. +- **C (eklemeyiz)** — gitleaks pre-commit value kaybolur; secret commit-time'da değil push-time'da yakalanır. + +**A en pragmatik:** mandatory disiplin CI'da, opsiyonel ergonomic optimization local'de. + +**Tasks:** +1. `.pre-commit-config.yaml` (yeni): + ```yaml + repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.0 + hooks: + - id: ruff + - id: ruff-format + - repo: https://github.com/gitleaks/gitleaks + rev: v8.18.0 + hooks: + - id: gitleaks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-merge-conflict + ``` +2. `CONTRIBUTING.md`'a "Pre-commit hooks (optional)" section: `pre-commit install`. +3. `docs/standards/code-review.md`'a opsiyonel olduğunu belirten not. +4. CI'a pre-commit run step **eklenmez** (CI zaten ruff + gitleaks ayrıca çalıştırıyor; çift kontrol gereksiz). + +**Faz 32 Acceptance:** +- `.pre-commit-config.yaml` mevcut. +- CONTRIBUTING.md'da install talimatı. +- `pre-commit run --all-files` clean (ya zaten bu PR'da fix edilmiş ya zaten temiz). + +--- + +### Faz 33 — v0.5.5 RELEASE (Tek PR) + +**Closes:** Plan kapanış — v0.5.5 PyPI'da. + +**Bağımlılık:** **Tüm önceki fazlar (1-32)**. + +**Tasks:** +1. `pyproject.toml` version `0.5.1rc1` → `0.5.5`. +2. `forgelm/__init__.py` `importlib.metadata` resolve edilince `0.5.5` döner — manuel değişiklik yok. +3. `CHANGELOG.md`'a `[0.5.5] — YYYY-MM-DD` consolidated section. Tüm Faz 1-32 closure'ları gruplu listele: + - Critical closures (8) + - Architectural splits (data_audit/, cli/) + - Library API support + - GDPR right-to-erasure full implementation + - ISO 27001 / SOC 2 Type II alignment baseline + - Article 14 staging directory + approve subcommand + - Cross-OS release-tag matrix + - Performance closures + - Security closures + - Documentation sweep + - Localization standard policy +4. `docs/roadmap/releases.md`'a v0.5.5 satırı. +5. `docs/roadmap.md` + `-tr.md` — Phase 12.6 (closure cycle) işareti ✅ Done. +6. `cut-release` skill'i ile tag'le. +7. Tag push → Faz 31'deki `publish.yml` tetiklenir → `build` → `cross-os-tests` matrix passes → `publish` job çalışır → PyPI'da `forgelm-0.5.5`. +8. Post-release: `pyproject.toml` `0.5.6rc1` next-cycle bump (aynı PR ya da ayrı follow-up). + +**Faz 33 Acceptance:** +- PyPI'da `forgelm 0.5.5` mevcut. +- `pip install forgelm==0.5.5` her 3 OS × 4 Python kombinasyonunda çalışır. +- README + CHANGELOG + roadmap senkron. +- 175 master review bulgusunun ≥%95'i Closed; ≤2 deferred (varsa) explicit `risks-and-decisions.md`'de kayıtlı. + +--- + +## 11. Per-dimension 5/5 exit kriterleri + +### Business 5/5 +- [ ] Tüm Critical/Major Business bulgular Closed. +- [ ] `tools/check_site_claims.py` CI guard aktif. +- [ ] README, site, CHANGELOG, roadmap, product_strategy tek hikaye. +- [ ] CONTRIBUTING.md project structure auto-generated/pointer. +- [ ] Quickstart 5 template gerçek + test edilebilir. +- [ ] Notebook'lar PyPI'dan install. +- [ ] "What ForgeLM is NOT" disiplini scope creep'siz. +- [ ] `risks-and-decisions.md` güncel. + +### Code 5/5 +- [ ] `data_audit/` package; en büyük dosya <400 satır. +- [ ] `cli/` package; en büyük dosya <600 satır. +- [ ] Hiçbir module-top torch/transformers/trl import. +- [ ] Pydantic Literal sweep 12/12. +- [ ] Pydantic `description=` 100% (170/170) + CI guard. +- [ ] `except Exception:` site sayısı 0 (best-effort'ler `# noqa: BLE001` ile justified). +- [ ] Wizard `_print` indirection; coverage omit yok. +- [ ] Cyclomatic complexity per function <15. +- [ ] Hiçbir karşılıksız TODO/FIXME/XXX. + +### Compliance 5/5 +- [ ] Article 9-17 hepsi kanıtlı (mapping `compliance_summary.md`'de). +- [ ] `forgelm verify-audit` çalışır. +- [ ] Operator identity raise eder; "unknown" yok. +- [ ] Audit fsync; HF revision pin. +- [ ] Article 14 honesty + staging + `forgelm approve` complete. +- [ ] GDPR right-to-erasure implemented (Faz 21). +- [ ] ISO 27001 / SOC 2 alignment baseline (Faz 23). +- [ ] QMS bilingual EN+TR. +- [ ] `audit_event_catalog.md` kapsamlı. + +### Documentation 5/5 +- [ ] EN ↔ TR H1/H2/H3/H4 parity (CI guard'lı). +- [ ] Hiçbir count drift. +- [ ] Hiçbir v0.4.x reference. +- [ ] Pydantic descriptions otomatik configuration.md'ye yansır. +- [ ] CLI help ↔ guide ↔ reference uyumlu. +- [ ] design/*.md status blockquotes. +- [ ] `compliance_summary.md` reachable + linked. +- [ ] QMS bilingual. + +### Localization 5/5 (kullanıcı kararıyla) +- [ ] EN + TR perfect parity (kod + docs + user-manuals + site). +- [ ] Site picker EN + TR only (DE/FR/ES/ZH gizli). +- [ ] `docs/standards/localization.md` explicit deferred policy. +- [ ] H2 + H3 + H4 parity CI guard. +- [ ] Orphan markdown CI guard. +- [ ] Locale-aware `.toLocaleLowerCase`. +- [ ] og:locale + hreflang doğru. +- [ ] Build sparse-emit (deferred dilleri içermiyor). + +### Performance 5/5 +- [ ] Lazy torch/transformers/trl import. +- [ ] Safety + judge batching. +- [ ] `forgelm audit --workers N` deterministic. +- [ ] Paragraph chunker single-encode. +- [ ] `tools/build_usermanuals.py` mtime cache + parallel. +- [ ] `lang_sample[:512]`. +- [ ] No quadratic-on-corpus path. + +### Security 5/5 +- [ ] `_http.py:safe_post` tek noktada SSRF + redirect + http:// + timeout + secret-mask. +- [ ] Tüm `from_pretrained` `trust_remote_code=False` explicit. +- [ ] FORGELM_GGUF_CONVERTER `--allow-custom-converter` + SHA-256 manifest. +- [ ] `_sanitize_md_list` for list interpolations. +- [ ] Ollama modelfile newline guard. +- [ ] `.pre-commit-config.yaml` opsiyonel + gitleaks. +- [ ] CI'da `pip-audit` + `bandit` (Faz 23 ile geldi). +- [ ] SBOM generation. + +### Testing & CI/CD 5/5 +- [ ] `--cov-fail-under=40` enforce. +- [ ] Hiçbir runtime_smoke disorder. +- [ ] Wizard coverage omit kalkmış. +- [ ] Fixture consolidation. +- [ ] **Cross-OS release-tag matrix** (3 OS × 4 Python). +- [ ] `.pre-commit-config.yaml` opsiyonel disiplin. +- [ ] `pytest-xdist` enabled. +- [ ] Site-as-tested-surface CI guard. +- [ ] Bilingual H3/H4 parity CI. +- [ ] `usermanuals-validate.yml` strict mode. +- [ ] Pydantic field description CI guard. +- [ ] No `continue-on-error: true` without justification. +- [ ] No `|| true` in CI. + +### Genel exit gate +- [ ] **Master review §2.1 Critical (8): hepsi Closed.** +- [ ] **Master review §2.2 Major (67): ≥65 Closed; ≤2 explicit deferred + `risks-and-decisions.md`.** +- [ ] **§2.3 Minor (60): ≥58 Closed.** +- [ ] **§2.4 Nit (40): ≥38 Closed.** +- [ ] **8 boyut yukarıdaki per-dimension exit kriterlerini geçti.** +- [ ] v0.5.5 PyPI'da, cross-OS matrix yeşil. + +--- + +## 12. Conflict & trade-off resolution (kullanıcı kararları sonrası) + +| Conflict | Karar | Rationale | +|---|---|---| +| §5.1 — `--workers` vs determinism | Determinism wins | Faz 17 byte-identical `lang_sample` + chunk-index ordering | +| §5.2 — Lazy torch vs library API | Both: lazy + library API supported | Faz 4 lazy import; Faz 18-19 library API support | +| §5.3 — Article 14 staging vs marketing | **Path (a) + Path (b) ikisi birlikte (kullanıcı kararı)** | Faz 9 honesty + real staging + approve | +| §5.4 — DE/FR/ES/ZH picker | **Drop from picker + standards explicit deferred (kullanıcı kararı)** | Faz 25 | +| §5.5 — data_audit/ split vs API | Re-export everything | split-design §1.e | +| §5.6 — GGUF env-var ack flag | One-release deprecation window (Faz 28) | F-business-011 cadence | +| §5.7 — QMS rewrite + TR mirror order | Rewrite EN body first (Faz 1), TR mirror sonra (Faz 26) | M-DOC-001 higher severity | +| **YENİ** — ISO/SOC 2 vs Cloud SaaS scope | **ISO/SOC 2 Pro CLI'a; Cloud SaaS ayrı ürün (kullanıcı kararı)** | Faz 22-23 | +| **YENİ** — GDPR stub vs full | **Full implementation (kullanıcı kararı)** | Faz 20-21 | +| **YENİ** — Cross-OS scope | **Release-tag-only matrix (kullanıcı kararı)** | Faz 31 | +| **YENİ** — Pre-commit policy | **Optional (Soru 8 önerisi A)** | Faz 32 | + +--- + +## 13. Risk register (faz-spesifik) + +### Yüksek-impact riskler + +| # | Risk | Faz | Likelihood | Impact | Mitigation | +|---|---|---|---|---|---| +| RM-1 | data_audit split test patch path migration eksik kalır | 14 | Medium | High | split-design §1.h R-2: aynı PR'da prod + test güncellenir | +| RM-2 | `python -m forgelm.cli` kırılır (`__main__.py` unutulur) | 15 | Medium | Critical | split-design §2.i R-11: PR-C6 explicit; `subprocess.run` smoke test | +| RM-3 | Cov-fail-under enforce edildikten sonra mevcut PR'lar coverage drop | 2 | Low | Medium | Faz 2 öncesi `pytest --cov` baseline ölç; gerekirse threshold ayarla | +| RM-4 | Pydantic Literal sweep config_template.yaml'leri kırar | 10 | Low | High | Her field için `--dry-run` doğrulama | +| RM-5 | description= migration configuration.md auto-gen breaking | 16 | Medium | Medium | PR-DESC-1'de `regenerate_config_doc.py` + CI diff guard | +| RM-6 | Article 14 path-b real staging directory eski testleri kırar | 9 | Medium | High | Faz 9 mevcut test'leri staging-aware yapacak; backward-compat: `evaluation.require_human_approval=false` default'u korur | +| RM-7 | GDPR erasure audit chain integrity'yi etkiler | 21 | Medium | High | Faz 20 design'da explicit: orjinal entry'ler kalır, "deletion event" eklenir; chain re-verify post-erasure | +| RM-8 | ISO/SOC 2 alignment dokümantasyonu marketing claim'le çelişir | 23 | Low | Medium | Faz 22 design "ForgeLM cannot be ISO certified; deployer-side audit support" net belirler | +| RM-9 | Library API support breaking change semver risk | 19 | Medium | High | Faz 18 design'da API stability tier'ları + semver explicit; integration test pin | +| RM-10 | Cross-OS release-prep test'leri Linux-only deps yüzünden Windows'ta fail | 30 | High | Medium | qlora extra Linux-only — `pyproject.toml`'da explicit. Cross-OS matrix qlora'sız base install + pytest | +| RM-11 | Operator identity raise CI distroless smoke kırılır | 3 | High if CI distroless | Medium | Faz 3'te `FORGELM_OPERATOR=ci-smoke` env var set CI'a | +| RM-12 | Site-as-tested-surface CI guard false positive | 25 | Low | Low | structured assertions (filename comparison, not substring); whitelist mechanism | +| RM-13 | Faz 14 + 15 paralel PR'ları main'de yarım merged state | 14, 15 | Low | High | Stack PR strategy; her sub-PR bağımsız yeşil; ara revert güvenli | +| RM-14 | Faz 31 release-prep cross-OS macOS arm64 ortamında numpy/pandas precision drift | 31 | Low | Medium | Floating point assertion'larda `pytest.approx`; arm64 vs x86_64 her ikisinde test | +| RM-18 | Faz 30 final doc sweep'i kaçırılmış doc-gap'leri release sonrası ortaya çıkarır | 30 | Medium | Medium | Tablo A'daki feature × {guide, reference, usermanual} matrisi explicit checklist olarak PR description'ında doğrulanır; `tools/check_cli_help_consistency.py` + bilingual parity strict mode mekanik garantör | +| RM-19 | Faz 30 user-manual rebuild build_usermanuals.py performance regression (336 dosya × yeni page'ler) | 30 | Low | Low | Faz 4 performance closure (incremental cache + ProcessPoolExecutor) zaten landed; sparse-emit DE/FR/ES/ZH'yi atlar | +| RM-15 | Library API integration test'leri trainer'ın torch ihtiyacını tetikler → CI cost | 19 | Medium | Low | Integration test'ler small fixture model + tiny dataset; dakika altı runtime | +| RM-16 | ISO/SOC 2 dokümantasyon hacmi review bandwidth'i aşar | 23 | Medium | Low | Faz 23 büyük PR olabilir; chunked review (per-control review, not big-bang) | +| RM-17 | GDPR erasure subcommand atomic değilse partial state | 21 | Medium | High | tempfile + atomic rename pattern; post-erasure chain verify | + +### Düşük-impact riskler (kayıt için) + +- ruff config Python 3.13 ile uyumsuzluk olabilir (RM-X1) +- gitleaks pre-commit hook test fixture'larını fail edebilir (RM-X2; Faz 27 silent except sweep ile fixture-fragment-build pattern landed) +- pytest-xdist parallel ortam `tmp_path` collision (RM-X3; conftest.py per-worker tmp_path discipline) +- SBOM generator transitive dep version drift (RM-X4; per-release SBOM diff tracking) + +--- + +## 14. Bağımlılık ve sıralama + +### Sıralı zincirler +- **Faz 2 → herşey** (cov gate önce; sonraki PR'lar coverage drop'una karşı korunmuş olur) +- **Faz 14 → Faz 17 → Faz 18-19** (data_audit split → workers → library API) +- **Faz 15 → Faz 17 → Faz 18-19** (cli split → workers → library API) +- **Faz 10 → Faz 16** (Literal'lar önce, description= sonra) +- **Faz 6 + Faz 9 → Faz 20 → Faz 21** (audit + staging → GDPR analysis → impl) +- **Faz 3 + 6 + 7 + 8 + 9 + 21 → Faz 22 → Faz 23** (security + compliance baseline → ISO analysis → impl) +- **Faz 23 → Faz 26** (QMS yeni dosyalar TR mirror'la birlikte) +- **Faz 1 → Faz 25** (site honesty fixes önce, sonra site CI guard) +- **Faz 16 → Faz 24** (description= migration → configuration-tr.md auto-gen) +- **Faz 1-29 → Faz 30** (final doc sweep tüm fonksiyonel iş bittikten sonra) +- **Faz 30 → Faz 31, 32 → Faz 33** (final docs → release prep workflows → release) + +### Paralel-safe pair'lar +- Faz 1, 2, 3, 4, 5 → tamamen bağımsız +- Faz 6, 7, 8 → bağımsız +- Faz 10, 11, 12, 13 → bağımsız +- Faz 14 ↔ Faz 15 (split-design §3'teki D-X || C-X pair'lar) +- Faz 24, 27, 28, 29 → büyük ölçüde bağımsız +- Faz 31, 32 → bağımsız (her ikisi de Faz 30 sonrası release öncesi) +- Faz 30 → bağımsız değil; tüm fonksiyonel iş bitmiş olmalı (final sweep) + +--- + +## 15. Per-PR template + +Her faz PR'ı şu format'ı kullanır: + +```markdown +# Phase N — + +## What +<1 paragraph scope> + +## Why +- <Master review finding ID>: <severity> <one-liner> +- ... + +## How +<bullet list of tasks> + +## Testing +- `pytest tests/<modules>` yeşil +- Coverage etkisi: <0% / +X% / -Y% — gerekçe> +- Manuel doğrulama: + - `forgelm <command>` çalıştır → beklenen çıktı + - <other manual checks> + +## Risk +- RM-N: <mitigation> + +## Acceptance (faz exit kriteri) +- [ ] <kriter 1> +- [ ] <kriter 2> +- [ ] ... + +## Closes +- F-business-NNN +- F-code-NNN +- ... + +## Companion +- [closure-plan-202604300906.md §N](docs/analysis/code_reviews/closure-plan-202604300906.md) +``` + +### Self-review komutu (her PR öncesi) +```bash +ruff format . && ruff check . && pytest tests/ && \ + forgelm --config config_template.yaml --dry-run && \ + python -m forgelm.cli --help > /tmp/help.txt && \ + diff /tmp/help.txt tests/fixtures/help-baseline.txt +``` + +--- + +## 15.5 Foundation PR (Faz 1-8) carry-over registry + +Foundation PR (`closure/foundation-faz1-8`, GitHub PR #19) merge sonrası tespit edilen carry-over maddeleri. Plan'ın "geride iş yok" disiplini için tek nokta kaydı; her satır gerçekten yapıldığından/erteleneceğinden emin olunmuş bir madde. + +| # | Faz | Madde | Hedef faz | Sebep / Durum | +|---|---|---|---|---| +| C-1 | 1 | `docs/reference/usage.md:300` + `usage-tr.md:276` "18 GPU models" → "16" (count drift) | **Faz 30** | Faz 1 ajanı allow-list dışı bıraktı; Faz 30 Tablo A "Performance" feature × `docs/reference/` satırına dahil edilmeli | +| C-2 | 4 v2 | `tools/build_usermanuals.py` incremental mtime cache + ProcessPoolExecutor cross-language (F-performance-105 Major) | **Faz 30** | İlk Faz 4 ajanı stalled olunca tighter scope ile re-spawn edildi; Faz 30 zaten user-manual rebuild için bu dosyaya dokunuyor | +| C-3 | 4 v2 | Safety + judge **deterministic batching** test'i (same input + farklı `batch_size` = byte-identical output) | **Faz 28** (Remaining cleanup) | Faz 4 v2 spec'inde madde #10 olarak listeli ama explicit determinism test eklenmedi; mevcut batching tests yeşil ama determinism contract pin'lemesi yok | +| C-4 | 5 | Notebook PyPI pin'leri `==0.5.0` → `==0.5.5` bump (7 closure-touched + `post_training_workflow.ipynb` `>=0.5.2`) | **Faz 33** | v0.5.5 PyPI'da olmadan pinlemek Colab install'ını kırar; release PR'ında bump edilir | +| C-5 | 6 | `docs/guides/safety_compliance-tr.md` full structural mirror (verify-audit'e ek olarak tüm guide'ı TR'ye taşıma) | **Faz 30** | Faz 6 ajanı feature-scoped excerpt yarattı (header'da explicit not düştü); full mirror Faz 30 Tablo A "verify-audit" satırının `docs/guides/` × TR sütunu | +| C-6 | 8 | `Notifier` → `AuditLogger` dependency injection (`notify_reverted` event'inden bir `webhook.reverted_sent` audit event'i de emit etsin) | **Faz 22** (ISO/SOC 2 analysis) | Faz 8 ajanı kasıtlı skip etti — Notifier'a audit dependency eklemek scope creep; `_revert_model` zaten `model.reverted` audit event'i emit ediyor; Faz 22'nin "audit event coverage matrix"'inde tartışılacak | +| C-7 | 3 | `safety.py:499` plan'da "classifier-load-failure event" diye listeli; ama line aslında trend-entry-write | **n/a — kapatıldı** | Faz 3 ajanı doğru karar verdi: line 387 (gerçek classifier-load path) emit ediyor; line 499 farklı bir context. Plan task referansı yanlıştı; kapanış doğru | +| C-8 | 3 | Plan task #3 "data_audit.py'da fingerprint" yanlış konum; `compute_dataset_fingerprint` aslında `compliance.py:491`'de | **n/a — kapatıldı** | Faz 3 ajanı doğru fonksiyona HF revision pin uyguladı (compliance.py'de). Plan referans hatası; kapanış doğru | +| C-9 | 7+8 merge | Webhook test patch path migration `forgelm.webhook.requests.post` → `forgelm._http.requests.post` | **n/a — kapatıldı (foundation PR'a inline)** | Faz 7 ajanı 11 testi migrate etti; Faz 8 ajanı eski path kullandı; merge sırasında orchestrator (Claude) 4 Faz 8 testini fragment-build slack token + path migrate ile tamamladı | +| C-10 | pre-existing | `tests/test_cost_estimation.py:7` `from tests.conftest import minimal_config` collection error (Phase 10'dan beri) | **Faz 12** (Fixture consolidation) | F-test-005 fixture fragmentation cluster; lokal `pytest` invocation'ı için kırık ama CI'da `pip install -e .` cwd ile resolve oluyor. Faz 12 `tests/_helpers/` migration'ında kapanır | +| C-11 | foundation PR review (`foundation-pr-review-20250504-opus.md` F-001..F-005, F-007, F-008, F-011) | `except Exception` cluster: `safety.py:96, 458`, `judge.py:101, 117, 158, 203`, `webhook.py:130`, `compliance.py:82, 349`, `cli.py:1075, 1806` + ~15 trainer.py site (toplam ~25 site) | **Faz 27** (Silent except sweep — Theme D residual) | Review agent bu cluster'ı Critical/Major olarak flag etti; plan zaten Faz 27'yi bu sweep için tahsis etmişti. PR #19 scope'u dışı bilinçli. Faz 27'de three-pass sweep (narrow class / `# noqa: BLE001` justify / per-path test) uygulanır | +| C-12 | foundation PR review (F-009 nit) | `data_audit.py:_TOKEN_PATTERN = re.compile(r"\w+", re.UNICODE)` — `regex.md` §1'in önerdiği `\b[\w']+\b` pattern'ı yerine | **n/a — kapatıldı (docstring justification)** | Pattern intent simhash/MinHash dedup için tasarımsal doğru (underscore-bearing identifier'ların token paylaşması istenen davranış). Foundation PR'a inline justification yorumu eklendi | +| C-13 | foundation PR review (F-010) | CHANGELOG `[Unreleased]` Phase 14 placeholder; v0.5.5 closure cycle anchor olmalı | **n/a — kapatıldı (foundation PR'a inline)** | `[Unreleased]` v0.5.5 closure cycle scope notu + foundation PR (Faz 1-8) entry'si ile güncellendi | +| C-14 | foundation PR review (F-006) | `tests/test_judge_functions.py` 7 yerde `@patch("requests.post")` → `@patch("forgelm._http.requests.post")` | **n/a — kapatıldı (foundation PR'a inline)** | Faz 7 ajanı webhook patch'lerini migrate etti; judge testleri kaçırıldı. Foundation PR review fix commit'inde 7 patch path düzeltildi | +| C-15 | foundation PR review (F-compliance-111) | Review iddiası: `_COMMONMARK_SPECIALS` `<`/`>` içermiyor → markdown injection riski | **n/a — REJECTED (verify failed)** | Verify edildi: `frozenset` 33 karakterli CommonMark special set'i (`!`, `"`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `-`, `.`, `/`, `:`, `;`, `<`, `=`, `>`, `?`, `@`, `[`, `\`, `]`, `^`, `_`, backtick, `{`, pipe, `}`, `~`) içeriyor — `<` ve `>` literal mevcut. Review agent yanılmış; aksiyon yok | +| C-16 | foundation-pr-review-202604301557 (R-Major-001) | `compliance.py:82` `except Exception:` → `except OSError:` daraltma | **n/a — kapatıldı (foundation PR'a inline)** | Comment zaten `OSError` diyordu; catch genişliği yorum ile uyumsuzdu. 3 karakterlik diff, runtime davranışı değişmez | +| C-17 | foundation-pr-review-202604301557 (R-Major-002) | `webhook.py:130` catch-all `except Exception:` `requests.RequestException`'dan sonra redundant | **n/a — kapatıldı (foundation PR'a inline)** | Block kaldırıldı; `requests.RequestException` library'nin tüm transport hata hiyerarşisinin tabanı, ek catch gereksiz. Programlama bug'ları (TypeError, vb.) artık propagate eder | +| C-18 | foundation-pr-review-202604301557 (R-Major-003) | Audit-vs-webhook event vocabulary asimetrisi (`training.started` ↔ `training.start` vb.) | **Faz 22** (ISO/SOC 2 analysis design) | Plan §8 Faz 22 design dosyasına "audit ↔ webhook event vocabulary alignment" başlığı eklenecek. Üç opsiyon: (a) audit'i webhook'a hizala (b) webhook'u audit'e hizala (c) "Rosetta stone" lookup tablosu. C-6 ile aynı concern, ek detail | +| C-19 | external bot review (CodeRabbit/gemini round 2) | `closure-plan-202604300906.md` markdown table pipe escape (L1464), Faz 5 pin spec `==0.5.0`, phase numbers (Faz 31/32), Cross-OS workflow design (single workflow + packaged wheel test) | **n/a — kapatıldı (plan revisi inline)** | 4 plan editorial fix; Cross-OS workflow design originally cross-workflow `needs:` invalid + `pip install -e .` editable instead of packaged wheel — revize edildi | +| C-20 | external bot review (CodeRabbit/gemini round 2) | `audit_event_catalog.md` + TR mirror — `training.failed`/`training.completed` events emit edilmiyor; gerçek event `pipeline.failed`/`pipeline.completed` | **n/a — kapatıldı (foundation PR'a inline)** | Catalog code'a hizalandı: `training.completed` satırı silindi (kod emit etmiyor), `training.failed` → `pipeline.failed` rename. EN+TR mirror tutarlı | +| C-21 | external bot review (round 2) | `_fingerprint_hf_revision` ImportError vs HF API errors ayrımı | **n/a — kapatıldı (foundation PR'a inline)** | Two-layer error handling: ImportError ayrı catch + dataset_info çağrısı `# noqa: BLE001` justified broad catch (HF Hub error tail uzun) | +| C-22 | external bot review (round 2) | `_revert_model` audit event eksik (NaN/Inf + threshold paths) | **n/a — kapatıldı (foundation PR'a inline)** | `_EVT_REVERT_TRIGGERED` audit emit'i `_revert_model` içine taşındı; tüm 5 caller (NaN/Inf, threshold, benchmark, safety, judge) artık audit event üretiyor. `source` keyword arg ile gate identity korunur. DRY iyileştirmesi + I2 closure | +| C-23 | external bot review (round 2) | `tests/test_compliance.py:_build_log` env var leak (try/finally yok) | **n/a — kapatıldı (foundation PR'a inline)** | Try/finally eklendi; FORGELM_AUDIT_SECRET prior state restore (pop or restore) edilir. AuditLogger raise etse bile env var leak etmez | +| C-24 | external bot review (round 2) | `tests/test_webhook.py:308` ve `test_compliance.py:598` `# noqa: S105 NOSONAR` kombine syntax Sonar parse etmiyor | **n/a — kapatıldı (foundation PR'a inline)** | Annotation'lar ayrı satırlara split edildi: `# NOSONAR ...` üst satırda, `# noqa: S105` line-level. Her iki linter parse edebiliyor | +| C-25 | external bot review (round 2) — DEFERRED | `_http.py:_mask_secrets_in_text` URL-substring redaction (Authorization header URL'de embed olabilir) | **Faz 27** (Silent except sweep) Theme genişletme: error-message hardening | Header value masking var; URL-substring masking yok. Risk: bearer token URL path/query'de sızabilir. Faz 27 + ISO/SOC 2 alignment kapsamında değerlendirilecek | +| C-26 | external bot review (round 2) — DEFERRED | `compliance.py` `FORGELM_ALLOW_ANONYMOUS_OPERATOR` env var → validated config field | **Faz 22** (ISO/SOC 2 design) | Ambient env Article 12 davranışını değiştiriyor; ISO 27001 A.12.4 perspektifinden config-driven olmalı. Plan §8 Faz 22 task list'ine "operator identity opt-in: env var vs config" tartışması eklenecek | +| C-27 | external bot review (round 2) — DEFERRED | `webhook.py:101` `safe_post(allow_insecure_http=True)` hard-coded | **Faz 28** (Remaining cleanup) — config-driven flag | Şu an webhook back-compat için zorunlu; explicit config field (`webhook.allow_insecure_http: bool = False`) Faz 28'de eklenecek; YAML opt-in ile gate'lenir | +| C-28 | external bot review (round 2) — DEFERRED | `webhook.py:90` default timeout 5s → 10s + min_timeout simplification | **Faz 28** (Remaining cleanup) — F-compliance-106 closure | Master review F-compliance-106 zaten Faz 28 task list'inde; bu review explicit confirm ediyor | +| C-29 | external bot review (round 2) — DEFERRED | `audit_event_catalog.md` ↔ codebase parity CI guard | **Faz 30** (Final doc sweep) Tablo A güncellemesi | `tools/check_audit_event_catalog.py` (yeni) — catalog'da listeli her event için `forgelm/` içinde literal grep; eksik olan CI fail. Faz 30'un code-claim alignment kapsamında | +| C-30 | external bot review (round 2) — DEFERRED | `# NOSONAR` kullanım koşulları `docs/standards/coding.md`'de belgelenmemiş | **Faz 28** veya 29 — coding.md "Static-analysis suppressions" alt-bölümü | `# noqa`, `# type: ignore`, `# NOSONAR`, `# pragma: no cover` — her birinin ne zaman kabul edilebilir + zorunlu açıklama formatı | +| C-31 | external bot review (round 2) — DEFERRED | F-compliance-101 honesty fix Path (a) "Faz 9'a defer" → master'da yanlış davranış sürecek | **Faz 9** (her iki path birlikte) | Master `_handle_human_approval_gate` "Model saved to staging" log atıp gerçekte `final_model/`'a yazıyor — operatör için belirsizlik. Plan §3 Faz 9 PR'ı bu PR'dan hemen sonra açılmalı | +| C-32 | external bot review (round 2) — REJECTED | `judge.py:213` `_generate_responses_batched` runtime `batch_size > 0` validation | **n/a — REJECTED (Pydantic ge=1 yeterli)** | Config parse-time `Field(default=8, ge=1)` zaten enforce ediyor. Runtime defansif check redundant. CodeRabbit önerisi haksız; reject | +| C-33 | external bot review (round 2) — REJECTED | `safety.py:464` audit_logger.log_event failure'ı re-raise edilmeli (downgrade etmemeli) | **n/a — REJECTED (intentional best-effort)** | Audit emission failure orijinal classifier-load hatası kayıp etmemeli; mevcut pattern (`logger.warning + raise original RuntimeError from e`) doğru. Audit emission "best-effort" en üst-üst layer için yutulabilir; original error chain `from e` ile preserve. `# pragma: no cover — defensive` zaten mevcut | +| C-34 | external bot review (round 2) — REJECTED | `trainer.py:888` `notify_awaiting_approval(model_path=final_path)` staging path olmalı | **n/a — defer to Faz 9** | F-compliance-101 Path (a) honesty fix'in alt-bulgusu; Faz 9 PR'ında staging directory + approve subcommand ile birlikte ele alınacak. Foundation PR scope dışı (master review'da explicit Faz 9 deferred) | +| C-35 | external bot review (round 3) | closure-plan §11 Faz 31 acceptance "release-prep.yml tetiklenir" diyordu — workflow konsolide edildi, `publish.yml` olmalı | **n/a — kapatıldı (plan revisi inline)** | Faz 31 acceptance + design note `publish.yml` build → cross-os-tests → publish job zinciri ile aligned | +| C-36 | external bot review (round 3) | Cross-OS Install step Windows'ta POSIX shell + `--no-index --find-links` extras kırıyor | **n/a — kapatıldı (plan revisi inline)** | İki değişiklik: `shell: bash` ekle (Windows runner'da POSIX shell garantisi), extras install ayrı `if: runner.os == 'Linux'` step'e taşındı (qlora bitsandbytes Linux-only), `--no-index --find-links` kaldırıldı (transitive deps PyPI'dan resolve) | +| C-37 | external bot review (round 3) | foundation-pr-review-20250504 F-006 stale (closed olarak işaretlenmeli) | **n/a — kapatıldı (foundation PR'a inline)** | F-006 entry'si "CLOSED (commit 2dabd83)" başlığı ile güncellendi; resolution metni 7 patch site'ın `forgelm._http.requests.post`'a migrate edildiğini belgeliyor (C-14 kayıt referansı) | +| C-38 | external bot review (round 3) | foundation-pr-review-20250504 front matter timestamp anti-chronological (2025-05-04 < 2026-04-30 commit) | **n/a — REMOVED** | Dosya kullanıcı tarafından "yanlışlıkla commit'lenmiş" olarak işaretlendi ve `git rm`'le tamamen kaldırıldı. Timestamp düzeltmesi de bu kararla obsolete oldu | +| C-39 | external bot review (round 3) | split-design fenced code blocks dilsiz (markdownlint MD040) | **n/a — kapatıldı (plan revisi inline)** | 3 dilsiz fence (data_audit/cli package layout + week-by-week sequence) `text` dil tag'iyle aligned | +| C-40 | external bot review (round 3) | `compliance.py:_verify_genesis_manifest` manifest read failure → warn+success (security risk) | **n/a — kapatıldı (foundation PR'a inline)** | Behavior değişti: manifest present-but-unreadable → `VerifyResult(valid=False, ...)`; manifest missing required fields (first_entry_sha256 / run_id) → aynı failure. Truncate-and-resume detector korumaya yönelik bir attacker manifest corruption'ını gizleyemez | +| C-41 | external bot review (round 3) | `safety.py:run_safety_evaluation` library API `batch_size` validation eksik (Pydantic ge=1 bypass edilebilir) | **n/a — kapatıldı (foundation PR'a inline)** | C-32'deki reject internal helper'lar içindi (`_generate_responses_batched` Pydantic-fed); bu satır public library API. `isinstance + < 1` check eklendi: invalid `batch_size`'da `ValueError` raise. Direct caller batch_size=0 artık silent no-op değil | +| C-42 | external bot review (round 3) | `trainer.py:_revert_model` "Adapters discarded." → artifact-neutral | **n/a — kapatıldı (foundation PR'a inline)** | "Artifacts discarded." — adapter / merged model / başka output ayırmıyor | +| C-43 | external bot review (round 3, outside-diff) | `compliance.py:_read_chain_head` 4 KiB tail entry > 4 KiB durumunda truncated record hash'liyor | **n/a — kapatıldı (foundation PR'a inline)** | Progressive-widen pattern: tail window 4 KiB'tan başlar, her empty result'ta 2× büyür, full-file'a kadar. Trailing-newline guard zaten present-but-truncated'i yakalar; bu fix oversized records için doğru chain head sağlar | +| C-44 | external bot review (round 4) | `audit_event_catalog{,-tr}.md` operator alanı sadece 2 kaynağı listeliyordu (`$FORGELM_OPERATOR` + `getpass.getuser()@hostname`) — `anonymous@<hostname>` üçüncü conditional yol mevcut ama belgesizdi | **n/a — kapatıldı (foundation PR'a inline)** | Her iki kataloğa 3-yollu çözüm sıralaması eklendi: (1) `$FORGELM_OPERATOR`, (2) `<getpass.getuser()>@<hostname>`, (3) sadece `getpass.getuser()` fail VE `FORGELM_ALLOW_ANONYMOUS_OPERATOR=1` setse `anonymous@<hostname>`; aksi halde loud abort. Compliance.py L93-103 davranışını birebir aynalar | +| C-45 | external bot review (round 4) | `trainer.py` 3 audit emit'i kataloglu şemadan sapıyordu: `human_approval.required(model_path)`, `compliance.governance_exported(path)`, `compliance.artifacts_exported(directory)` vs doc `gate/reason/metrics`, `output_path/dataset_count`, `output_dir/files` | **n/a — kapatıldı (foundation PR'a inline)** | Doc canonical contract; emitterlar dokumanla aligned: `human_approval.required` artık `gate="final_model"` + `reason` + `metrics` + `model_path` (downstream tooling için backward-compat keep); `governance_exported` `output_path` + `dataset_count`; `artifacts_exported` `output_dir` + `files` (sorted listdir). Article 12 contract artık tek source of truth | +| C-46 | external bot review (round 4) | `compliance.py:_verify_chain_walk` JSON scalar/array satırda `entry.get("prev_hash")` AttributeError raise ediyordu | **n/a — kapatıldı (foundation PR'a inline)** | `isinstance(entry, dict)` early-return ile `VerifyResult(valid=False, reason="line N is not a JSON object")` döndürüyor — verifier artık malformed satırlarda crash etmek yerine hatlı failure raporluyor | +| C-47 | external bot review (round 4) | `judge.py:_call_api_judge` `HttpSafetyError`'ı yutup `{"score": None}` dönüyordu — SSRF/config bug'ı per-prompt soft-fail olarak gizliyordu | **n/a — kapatıldı (foundation PR'a inline)** | `_call_api_judge` artık `HttpSafetyError`'ı re-raise ediyor; `run_judge_evaluation` outer try/except ile hard `JudgeResult(passed=False, failure_reason="judge endpoint rejected by HTTP safety policy: ...")`'a map ediyor. Trainer auto-revert / approval gate misconfigured judge endpoint'e tepki verebiliyor. Test `test_judge_ssrf_block_for_private_url` pytest.raises(HttpSafetyError) kontratına geçti | +| C-48 | external bot review (round 4) | `safety.py:SafetyEvalThresholds` docstring İngilizce dışı "Kondenses" başlatma | **n/a — kapatıldı (foundation PR'a inline)** | "Kondenses" → "Condenses" (typo); CLAUDE.md "code/comments English only" kuralına uygun | +| C-49 | external bot review (round 4) — **C-32 reversal** | C-32'de `judge.py:run_judge_evaluation` library API `batch_size` boundary check'i Pydantic redundancy gerekçesiyle reject edilmişti — fakat `safety.py` muadili (C-41) accept edilmişti ve direct API caller path için aynı argüman geçerli | **n/a — kapatıldı (foundation PR'a inline)** | `run_judge_evaluation` başına `isinstance(batch_size, int) and batch_size >= 1` check eklendi (C-41 ile birebir aynı pattern). C-32 reject pozisyonu obsolete; tutarlılık için library API boundary genel kural — Pydantic schema sadece YAML-fed path için yeterli | +| C-50 | external bot review (round 4) | `trainer.py:_run_judge_evaluation_if_needed` call site `batch_size` field'ını `JudgeConfig`'tan geçirmiyordu — `JudgeConfig.batch_size` field'ı zaten yoktu, YAML batch tuning silent no-op'tu | **n/a — kapatıldı (foundation PR'a inline)** | İki değişiklik: (a) `JudgeConfig`'a `batch_size: int = Field(default=8, ge=1)` field'ı eklendi (SafetyConfig ile aynı pattern), (b) trainer call site `batch_size=judge_cfg.batch_size` geçiriyor. YAML `evaluation.llm_judge.batch_size: N` artık etkili | +| C-51 | external bot review (round 4) — DEFER | `tests/test_data_audit_import_stability.py` paket-level Presidio sentinel coverage (`_HAS_PRESIDIO`, `_PresidioAnalyzer` aliases) eksik | **defer to Faz 30** | Test dosyası henüz yok; oluşturulması yeni test fixture sayılır. Faz 30 doc/test gap closure scope'unda — paket-level alias coverage + `_optional` modül symbol coverage hep birlikte yeni dosyada toplanır | +| C-52 | external evaluation report (2026-04-30) — DEFER | `forgelm/ingestion.py:487` `_chunk_semantic` `NotImplementedError` raise ediyor ama `CHUNK_STRATEGIES` listede `"semantic"` var; kullanıcı flag'i seçerse runtime crash | **defer to Faz 28** + **Faz 30 doc sweep** | Implementation kararı verildi (kullanıcı: "implement edelim"). Yeni opsiyonel `[chunking-semantic]` extra (`sentence-transformers>=2.7.0,<4.0.0`); embedding similarity threshold-based segmentation. **Acceptance (Faz 28 code):** `_chunk_semantic` real implementation + unit test (sentence-transformers mock'lu + olmadan ImportError install hint pattern). **Doc/usermanual sweep (Faz 30 Task A):** `docs/guides/ingestion.md` + `ingestion-tr.md` chunking-strategies tablosuna `semantic` satırı; `docs/usermanuals/en/data/ingestion.md` + `tr/data/ingestion.md` chunking section güncellenir; `README.md` extras listesine `[chunking-semantic]` install satırı; CHANGELOG entry. Test: short paragraf + multi-topic doc fixture'larıyla segment count beklentisi | +| C-53 | external evaluation report (2026-04-30) — DEFER | `pyproject.toml:108-113` `[ingestion-secrets]` extra `detect-secrets>=1.5.0` indiriyor ama `forgelm/data_audit.py:1305` "**not** invoked here" diyor — ölü bağımlılık (CHANGELOG.md:397-401 zaten doğruluyor) | **defer to Faz 14 PR D-1** + **Faz 30 doc sweep** | Karar: extra'dan tamamen çıkar (kullanıcı: "çıkartalım"). PR D-1 `_secrets.py` extract'i sırasında natural fit. **Code touch points (Faz 14 PR D-1):** (a) `pyproject.toml:107-113` `[ingestion-secrets]` block sil; (b) `data_audit.py:1229-1308` (split sonrası `_secrets.py`) "fallback regex set" → "regex set" reframe; (c) `README.md:177` install satırını sil; (d) `CHANGELOG.md` Unreleased'a "remove unused `detect-secrets` extra (was reserved for follow-up release that never materialized; regex set is the sole detection backend)" entry. **Doc sweep (Faz 30 Task A):** (e) `docs/guides/data_audit.md:269-274` "Optional / forward-compatibility" paragrafını sil; (f) `docs/guides/data_audit-tr.md:269-273` aynı paragrafı sil; (g) `docs/guides/ingestion.md:138` ve `ingestion-tr.md:137-138` `[ingestion-secrets]` reservation cümlesini sil. **Drift bonus:** `docs/usermanuals/en/data/secrets.md` (+ TR) tablosu mevcut regex set'in **kapsamadığı** kategorileri (Anthropic, Stripe, SendGrid, Twilio) listeliyor — `_SECRET_PATTERNS`'a hizalamak için ya tablo'yu code'a göre kısalt, ya code'a yeni pattern'ler ekle (ayrı karar; doc-vs-code drift findings'e ekle). Wire-etme yolu reddedildi: detect-secrets API'sı file-path-based, ForgeLM'in row-level JSONL audit'iyle uyumsuz; regex set zaten 9-aile coverage sağlıyor | +| C-54 | external evaluation report (2026-04-30) — DEFER | `forgelm/webhook.py:16` `__all__` listesinde `_is_private_destination` underscore-prefix'li internal helper public API olarak re-export ediliyor (round-2 Codacy F401 fix'inin yan etkisi); intentional smell | **defer to Faz 28** | Faz 28 webhook hardening sweep'inde ele alınır. Karar matrisi: (a) Faz 7 split'inden sonra downstream importer kalmadı varsayımıyla `__all__`'den çıkar — direct importer'lar `forgelm._http`'den almalı; (b) downstream importer var ise `DeprecationWarning` ekle ve removal v0.6.0'a planlanır. Acceptance: `grep -rn "from forgelm.webhook import _is_private_destination"` external referans yok; varsa deprecate path'i kullan | +| C-55 | external evaluation report (2026-04-30) — DEFER | `forgelm/model.py:285-290` `_expert_index_in_name` MoE expert tespit'i hardcoded string match (`f"experts.{i}." in name or f"expert_{i}." in name`); DeepSeek/Qwen3/Mixtral farklı naming convention kullanırsa brittle | **defer to Faz 27** | Faz 27 silent-except sweep'ine ekle (mini-task scope). Refactor: (a) `_EXPERT_NAME_PATTERNS` dict per-architecture key'lerle (mixtral/deepseek/qwen3); (b) regex-based resolver `re.compile(r"(?:experts|expert)[_\.](\d+)[._]")`; (c) unit test fixture'ları her 3 architecture için gerçek state_dict key sample'larıyla; (d) bilinmeyen architecture'da loud warning + fallback regex. Risk: Mixtral test fixture'ı yoksa düşük; üç architecture'ın da OSS state_dict listing'i README'de var | +| C-56 | external evaluation report (2026-04-30) — BACKLOG | `forgelm/wizard.py:28` `POPULAR_MODELS` hardcoded liste; periyodik refresh mekanizması yok (Llama 3.2/3.3, Qwen 3, DeepSeek-V3 vs. eklenirken manuel düzenleme) | **defer to v0.6.0 backlog** | v0.5.5 closure scope'u dışı. v0.6.0+ için: (a) HF trending API-driven generator (cron-driven `tools/refresh_popular_models.py` quarterly); (b) `MODEL_LIST.md` source-of-truth + dev script; (c) `/schedule` agent çeyreklik PR açar. Karar v0.6.0 planlama döngüsünde | +| C-57 | external evaluation report (2026-04-30) — DEFER | `forgelm/trainer.py:57,78,83,93,123` GRPO math reward `_REWARD_STRIP_TOKENS` (tuple) + `_BOUNDARY_REQUIRED_TOKENS` (frozenset) domain-specific unit-suffix strip mantığı; sadece "m" boundary required, GSM8K/MATH benchmark'larına dar — başka math domain'lerine genişletmek karmaşık. _NOT_: agent atıfı `grpo_rewards.py` idi, gerçek konum `trainer.py` | **defer to Faz 28** + **Faz 30 doc sweep** | Faz 28 cleanup sweep'ine refactor candidate olarak ekle. Karar matrisi: (a) genelleştirilebilir mi? (math vs. code vs. logic GRPO için ortak suffix-strip pattern'i çıkar) — non-trivial; (b) status quo + comment improvements (mevcut tuple/frozenset'in GSM8K/MATH-specific olduğunu docstring'de açıkça belirt); (c) v0.6.0'da GRPO config-driven reward function plugin pattern'iyle çöz. Tavsiye (b) — minimal cost, risk yok, doc honesty kazanılır. **Doc sweep (Faz 30 Task A):** `docs/guides/alignment.md:177-188` GRPO reward section'a "built-in math reward GSM8K/MATH-tuned" disclaimer + custom reward callable yazma örneği. _Not_: `alignment-tr.md` mirror'ı henüz yok — TR mirror creation Faz 24 bilingual sweep scope'u, bu C-57 ile coupled değil | + +### Faz 30 task A güncellemesi (C-1, C-2, C-5 absorption) + +Plan §10 Faz 30 Task A "User-facing doc tamamlama" tablosuna aşağıdaki satırlar **eklenmeli** (foundation PR kapanırken kalan kuyruk): + +| Feature | `docs/guides/` | `docs/reference/` | `docs/usermanuals/{en,tr}/` | +|---|---|---|---| +| **GPU count refresh** (C-1) | — | `usage.md:300` + `usage-tr.md:276` "18" → "16" | usermanual rebuild zaten EN/TR sayfaları yeniden üretir | +| **`tools/build_usermanuals.py` cache + parallel** (C-2) | — | architecture.md performans notu | — (build pipeline performance) | +| **`safety_compliance-tr.md` full mirror** (C-5) | `safety_compliance.md` ile bit-bit yapı paritesi | (Faz 6 ile zaten) | (Faz 6 ile zaten) | + +### Faz 28 task güncellemesi (C-3 absorption) + +Plan §9 Faz 28 "Remaining Major + Minor + Nit cleanup" task listesine eklenmeli: + +- **Faz 4 v2 carry-over (C-3):** `tests/test_safety_advanced.py` ve `tests/test_judge_functions.py`'a deterministic batching test'i ekle: + - Aynı 16 prompt fixture, `batch_size=1`, `2`, `4`, `8`, `16` ile koşulduğunda — output JSON byte-identical olmalı (model.generate temperature=0 + deterministic seed varsa). + - F-performance-102 closure'ının **determinism contract**'ını CI'da pin'ler. + +### Faz 33 task güncellemesi (C-4 absorption) + +Plan §10 Faz 33 "v0.5.5 RELEASE" task listesine eklenmeli (zaten implicit ama explicit yapalım): + +- **Foundation PR carry-over (C-4):** Notebook pin'lerini bump et: + - 7 closure-touched notebook (`dpo_alignment`, `safety_evaluation`, `grpo_reasoning`, `kto_binary_feedback`, `galore_memory_optimization`, `multi_dataset`, `synthetic_data_training`): `==0.5.0` → `==0.5.5` + - `post_training_workflow.ipynb`: `forgelm>=0.5.2` → `forgelm==0.5.5` (range pin → exact pin) + - `quickstart_sft.ipynb`, `data_curation.ipynb`: `==0.5.0` → `==0.5.5` + - Yorum: `# Pinned to v0.5.5; bump on each release` + +### Faz 22 task güncellemesi (C-6 absorption) + +Plan §8 Faz 22 "ISO 27001 / SOC 2 Type II — Analysis & gap assessment" design dosyasına eklenmeli: + +- **Foundation PR carry-over (C-6):** "Audit event coverage matrix" başlığı altında — `Notifier` → `AuditLogger` dependency tartışması. Mevcut durum: `_revert_model` `model.reverted` audit event'i emit ediyor; webhook ayrıca `training.reverted` event'i emit ediyor. Çift kayıt mu, single source of truth mu? ISO 27001 A.12.4 Logging perspektifinden karar. + +### Foundation PR closure-coverage özeti + +Foundation PR'da tamamlanan toplam: **8 Critical + 30+ Major** (master review ID'leri foundation PR commit message'da listeli). 2 madde explicit-skip (C-7, C-8 — plan referans hataları) ve 7 carry-over (C-1, C-2, C-3, C-4, C-5, C-6, C-9, C-10) plan'da yukarıda explicit yazıldı. **Atlanan veya gözden kaçan iş yok.** + +--- + +## 16. Versiyonlama ve kayıt + +Bu plan v2 (revised 2026-04-30 kullanıcı kararları sonrası). +**v2.1 (2026-04-30):** §15.5 Foundation PR carry-over registry eklendi. + +Plan dosyası: `docs/analysis/code_reviews/closure-plan-202604300906.md`. Her PR description'ında bu plana referans verir (`§N` faz numarası ile). + +Faz tamamlanmaları takibi için (opsiyonel): `docs/analysis/code_reviews/closure-progress.md` — checkboxlu liste, her PR merge sonrası güncelleme. + +v0.5.5 release sonrası: yeni master review turu (`master-review-opus-202604YYMMHHMM.md`) plan'ın kapanışını valide eder. + +--- + +## 17. Özet — Çalışma sırası + +1. **Foundation (Faz 1-5):** Site honesty + CI gates + operator identity + lazy torch + notebook pin. Birbirinden bağımsız, paralel açılabilir. + +2. **Compliance closures (Faz 6-9):** verify-audit + safe_post + webhook lifecycle + Article 14 (honesty + staging + approve). + +3. **Code hygiene (Faz 10-13):** Pydantic Literal + wizard _print + fixture consolidation + deprecation discipline. + +4. **Architectural splits (Faz 14-16):** data_audit/ + cli/ + description= migration. Multi-PR series; D-X || C-X paralel. + +5. **Architecture-dependent (Faz 17-19):** --workers + Library API analysis + Library API implementation. + +6. **Compliance verticals (Faz 20-23):** GDPR analysis + GDPR impl + ISO/SOC 2 analysis + ISO/SOC 2 impl. + +7. **Polish + cleanup (Faz 24-29):** Bilingual sweep + site picker + QMS bilingual + silent except + remaining + nit. + +8. **Final documentation + site (Faz 30):** Tüm `docs/` (EN+TR) + `docs/usermanuals/{en,tr}/` + `site/*` v0.5.5 nihai durumuna senkronlanır. Tablo A feature×{guide,reference,usermanual} matrisinin tamamı doldurulur. DE/FR/ES/ZH dokunulmaz (kullanıcı kararı: deferred). + +9. **Release prep (Faz 31-32):** Cross-OS matrix + pre-commit. + +10. **Release (Faz 33):** v0.5.5 PyPI publish. + +**Toplam:** 33 faz, ~47 PR. **Hedef tarih yok**, dependency-driven sıralama. Her faz PR ile kapanır. **Geride iş yok. Tekrarlamadık. Stub kalmadı. Tüm doküman + site EN+TR'de v0.5.5 ile senkron.** + +--- + +*End of closure plan v2 — 202604300906.* diff --git a/docs/analysis/code_reviews/foundation-pr-review-prompt.md b/docs/analysis/code_reviews/foundation-pr-review-prompt.md new file mode 100644 index 00000000..50de59ce --- /dev/null +++ b/docs/analysis/code_reviews/foundation-pr-review-prompt.md @@ -0,0 +1,211 @@ +# Foundation PR Review Agent Prompt + +**Hedef:** GitHub PR [#19](https://github.com/cemililik/ForgeLM/pull/19) (`closure/foundation-faz1-8` → `development`) için **bağımsız ve detaylı** bir kod review. + +**Bu prompt'un disiplini:** Yönlendirici değildir. Hangi bulguları arayacağını, hangi severity'yi vereceğini, neyi onaylayıp neyi reddedeceğini söylemez. Sen ajan olarak kendi değerlendirmeni oluşturursun — PR sahibi'nin anlatısı (commit message, PR description, plan dosyaları) **doğrulanması gereken iddialardır**, kabul edilecek hakikat değildir. + +--- + +## 0. Sen kimsin + +ForgeLM kod tabanını dışarıdan inceleyen kıdemli bir reviewer'sın. Bu bir foundation PR'ı — 8 fazlık bir kapanış cycle'ının ilk büyük yığını. Senden istenen: + +1. PR'da yapılan iddiaları **bağımsızca doğrula**. +2. PR'da **gözden kaçırılmış / yanlış yapılmış / yetersiz yapılmış** noktaları bul. +3. Bulduklarını severity-sorted, evidence-backed bir markdown raporuna döşe. +4. **Kod yazma.** Yalnızca raporunu yaz. + +PR sahibi'nin kendi self-assessment'i (commit message + PR body + closure plan §15.5 carry-over registry) sana referans olarak verilmiştir. Bunu kabul etmek zorunda **değilsin** — kendi iddialarınla çakışan her noktayı tartış. + +--- + +## 1. Proje bağlamı (zorunlu okuma) + +ForgeLM = config-driven enterprise-grade LLM fine-tuning toolkit. YAML in → fine-tuned model + EU AI Act compliance artifacts out. CI/CD pipeline'lar için tasarlandı, notebook için değil. v0.5.0 yayınlandı; bu PR v0.5.5 closure cycle'ının ilk fazlarını içeriyor. + +**PR meta:** +- Working directory: `/Users/dev/Documents/Projects/ModelTrainer/ForgeLM` +- Branch (checkout edilmiş): `closure/foundation-faz1-8` +- Base: `development` +- Commit count: 2 (`885395d` foundation bundle + `269637b` planning artifacts publicly tracked) +- Toplam diff: 58 dosya, +5418 / -296 satır + +**Çalıştırılabilen kontrol komutları (önerilen, zorunlu değil):** +```bash +gh pr view 19 +gh pr diff 19 | less +git log --stat development..HEAD +pytest tests/ --ignore=tests/test_cost_estimation.py --no-cov +ruff format --check forgelm/ tests/ tools/ +ruff check forgelm/ tests/ tools/ +forgelm --config config_template.yaml --dry-run +forgelm --help +forgelm verify-audit --help +python -c "import forgelm.trainer; import sys; assert 'torch' not in sys.modules" +``` + +Bu komutları sen seçersin; hangilerini koşturup hangilerini atlayacağına karar ver. + +--- + +## 2. Reviewable artifact'ler (öncelik sırasıyla) + +### 2.1 Birinci derece (PR'ın kendisi) +- `gh pr diff 19` (veya `git diff development..HEAD`) — kod ve doküman değişikliklerinin tamamı +- `git log development..HEAD --format="%H %s%n%n%b%n---"` — commit message'larındaki iddialar + +### 2.2 İkinci derece (PR'ın iddia ettiği kontrat) +- `docs/analysis/code_reviews/closure-plan-202604300906.md` — bu PR'ın "execute ediyorum" dediği plan + - Özellikle §3 Faz 1-5 ve §4 Faz 6-8 task listeleri + - §13 Risk register + - §15 Per-PR template (her PR'ın taşıması gereken acceptance signals) + - **§15.5 Foundation PR carry-over registry — PR sahibi'nin "atlanan veya gözden kaçan iş yok" iddiasının somut listesi.** Bunu literal kabul etme; her satır için bağımsızca doğrula. +- `docs/analysis/code_reviews/master-review-opus-202604300906.md` — closure plan'ın hedef aldığı 175 bulgu (8 Critical + 67 Major + 60 Minor + 40 Nit) +- `docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md` — bu PR'ın scope'u dışındaki Faz 14-15 design (referans) + +### 2.3 Üçüncü derece (proje standartları — ihlal iddiaların buradan kanıtlanmalı) +- `docs/standards/README.md` (index) +- `docs/standards/coding.md` — Python kalite kuralları +- `docs/standards/architecture.md` — modül cohesion, lazy import disiplini, ~1000 satır ceiling +- `docs/standards/error-handling.md` — bare except, no-silent-fail, exit codes +- `docs/standards/logging-observability.md` — operator identity, log seviyeleri, webhook lifecycle vocabulary +- `docs/standards/regex.md` — 8 hard rules (Rule 4 ReDoS, Rule 7 fragment-built fixtures özellikle önemli) +- `docs/standards/testing.md` — fixture izolasyonu, no GPU + no network, coverage gate +- `docs/standards/documentation.md` — bilingual mirror disiplini, code-claim alignment +- `docs/standards/localization.md` — EN+TR mandatory; DE/FR/ES/ZH deferred policy +- `docs/standards/release.md` — semver, deprecation cadence, CHANGELOG hijyeni +- `docs/standards/code-review.md` — review disiplini (sen bu standardı uygularken aynı zamanda denetliyorsun) +- `CLAUDE.md` ve `CONTRIBUTING.md` — proje rehberi + +### 2.4 Çevresel referans (gerek olursa) +- `pyproject.toml` — extras, build, ruff, pytest, coverage config +- `.github/workflows/*.yml` — CI/CD kontratı +- `config_template.yaml` — canonical YAML +- `README.md` — public marketing yüzeyi + +--- + +## 3. Çıktı + +**Dosya:** `docs/analysis/code_reviews/foundation-pr-review-{TIMESTAMP}-{MODELNAME}.md` +- `{TIMESTAMP}`: review başlatma zamanı, `YYYYMMDDhhmm` UTC. (Örn. `202604301930`.) +- `{MODELNAME}`: hangi model olarak koşturuluyorsun (örn. `opus`, `sonnet`, `haiku`, `gpt-5`). + +Bu dosyaya **yalnızca kendi raporunu** yaz. Başka hiçbir dosyayı (kod, test, doküman) değiştirme. + +### 3.1 Şablon (kullanmak zorunda değilsin ama bu seviyede ayrıntı bekleniyor) + +```markdown +# ForgeLM Foundation PR (Faz 1-8) Independent Review — {TIMESTAMP} + +**PR:** https://github.com/cemililik/ForgeLM/pull/19 +**Branch:** closure/foundation-faz1-8 +**Base:** development +**HEAD:** 269637b +**Reviewer:** {MODELNAME} +**Review tarihi:** YYYY-MM-DD +**Verified against current code at commit 269637b** + +## 0. Executive summary +1-2 paragraf. PR'ın genel sağlığı, ★/5 puan, en kritik 3 bulgunun özeti. Senin görüşün — PR sahibi'nin "881 test yeşil + 8 Critical kapandı" anlatısını kabul edip etmediğin. + +## 1. Findings (severity-sorted) + +### 1.1 Critical +Her bulgu zorunlu alanlarla: +- **F-NNN** (kendi ID şeman; foundation-001, -002, ... veya başka bir scheme) +- **Severity:** Critical +- **Phase / scope:** Hangi Faz'ın task'ı / hangi cross-cut tema / standards section +- **File:** `path/to/file:line` (commit 269637b'deki konum) +- **Issue:** ne yanlış (1-2 cümle) +- **Rationale:** neden Critical (standards ref + impact) +- **Recommendation:** somut düzeltme (kod yazma — sadece tarif) +- **Verifies plan claim?** Plan'daki bir task'ın incomplete/incorrect olduğunu mu kanıtlıyor (eğer öyleyse plan satır numarası); yoksa plan-dışı yeni bulgu mu + +### 1.2 Major +### 1.3 Minor +### 1.4 Nit + +## 2. Positive observations +PR'ın iyi yaptığı şeyler — file:line ile kanıt. Bu bölüm zorunlu; denge için. + +## 3. Verification of plan claims +Plan'ın (closure-plan-202604300906.md) §15.5 carry-over registry'sindeki 10 maddenin (C-1 .. C-10) her biri için durum tespiti: +- Closed-as-claimed / Closed-but-with-caveats / Open / Reopened / Not Verifiable + +Plan'ın §3 Faz 1-5 ve §4 Faz 6-8 task listelerinde belirtilen her task için (yaklaşık 60+ task) per-task verification matrisi: +- Done & evidence-backed / Done but spec drift / Skipped silently / Skipped with rationale / Not done + +## 4. Cross-cutting concerns +PR'ın boyut + yapı + bisectability + reviewability açısından değerlendirilmesi. 16 commit squashed-into-1 disiplini doğru tercihti mi? PR scope'u (8 faz tek PR'da) review'i için makul mü? + +## 5. Conflicts with project standards +Hangi `docs/standards/X.md` kuralı sistemic olarak kırılıyor (varsa). Plan §6 "Standards posture" tablosuyla uyumlu mu yoksa yeni ihlaller mi var? + +## 6. Open questions +Cevap veremediğin / project owner kararı bekleyen maddeler. + +## 7. Recommendations +Severity'sini özetleyen değerlendirme + sıralı action list. PR'ı: +- (a) merge ediyor musun olduğu gibi +- (b) küçük revisyon (≤1 saat) sonrası merge +- (c) büyük revisyon (≥1 gün) sonrası merge +- (d) reddet, parçalara böl + +## 8. Coverage notes +Hangi dosya/dizinleri okudun, hangilerini scope dışı bıraktın. Hangi `pytest` / `ruff` / smoke testlerini çalıştırdın, sonuçlar. + +## 9. Methodology +Bu review'i nasıl yaptığını açıkla — review sırası, hangi araçları kullandın, kaç saat sürdü. +``` + +### 3.2 Format kuralları + +- Her bulguda **dosya:satır** referansı zorunlu. `forgelm/compliance.py:56` formatında. Satır 269637b commit'ine göre. +- Her ihlal iddiasında **standards reference** zorunlu. `docs/standards/X.md` veya `CLAUDE.md` bölümü ile. +- Spekülasyon yok. "Belki", "muhtemelen", "olabilir" yerine ya kanıt göster ya yazma. +- Pozitif gözlem bölümü zorunlu. PR'ın ne yaptığı doğru, ne işlendi düzgün — bunlar da raporun parçası. +- Min uzunluk yaklaşık 500 satır markdown. Daha fazlası uygun, daha azı yetersiz. +- **Önceki master review'a yeni bir bulgu ekleme.** Eğer Faz 1-8 dışında bir alan'da bir şey gözüne çarpıyorsa (örn. Faz 14 split scope'u, Faz 22 ISO, Faz 30 docs) kısaca işaret et ama bu PR review'ün scope'u değildir — `## 6. Open questions` altında işaretle. + +--- + +## 4. Anti-bias guardrails + +PR sahibi'nin özet mesajı seni etkilememeli. Spesifik anti-bias direktifler: + +1. **Self-test sayıları (881 passed, 13 skipped) iddia, kanıt değil.** Kendin koştur. Sayıları kendi gözlemlerinle karşılaştır. +2. **"8 Critical kapandı" iddiası master review'in Critical listesiyle eşlemli mi?** Plan §3-4'teki "Closes" alanlarında belirtilen ID'leri master review §2.1'in 8 Critical'ı ile cross-check et. Hangi ID'ler PR'da gerçekten file:line evidence ile kapatılmış, hangileri sadece "Closes:" başlığında listelenmiş ama implementation kanıtı zayıf? +3. **Plan §15.5 carry-over registry "atlanan iş yok" diyor.** 10 carry-over'ı sırayla doğrula. Plan-iddia ↔ kod-gerçek arasında drift var mı? +4. **Conflict resolution kalitesi:** Plan §3-4'te belirtilen file ownership disiplini merge sırasında korundu mu? `safety.py` (Faz 3+4 conflict), `test_compliance.py` (Faz 3+6), `audit_event_catalog.md` (Faz 3+8), `test_webhook.py` (Faz 7+8) merge'lerinde **veri kaybı** veya **logic drift** oldu mu? +5. **Squash + force-push** disiplini: original 16 commit'in semantik bilgisi squash sonrası tek commit message'a tam taşındı mı? Bisect kabiliyeti kayıp önemli mi? +6. **PR içinde stage'lenmiş ama plan'da olmayan iş var mı?** (Scope creep) Veya plan'da olup PR'da olmayan bir task var mı? (Scope shrink) + +Bu listeyi kapsam değil, **örnek kontrol açısı** olarak gör. Kendi açılarını da bul. + +--- + +## 5. Yapma: + +- Kod / test / standards / config / doküman dosyalarını **değiştirme**. Yalnızca raporunu yaz. +- "Düzelteyim" deme. Düzeltme önerisi yaz, düzeltmeyi PR sahibi yapacak. +- Plan'a ya da master review'a yeni bulgu **ekleme**. Onlar tarihsel kayıt; senin raporun yan-yana yaşar. +- PR sahibi'nin self-assessment'ini onaylamak için yazma. **Bağımsız bir görüş istiyoruz.** + +--- + +## 6. Çalışma serbest + +- İstediğin sırayla oku. +- İstediğin kadar tool çağrısı yap. +- İstediğin kadar uzun süre çalış. +- İstediğin sub-agent (Explore vb.) çağır. +- İstediğin pytest invocation'ı koştur. + +Tek kısıtlama: çıktı tek bir markdown raporu olarak yukarıdaki dosya yoluna yazılacak; başka hiçbir dosya değişmeyecek. + +--- + +## 7. Başla + +Hazırsan başla. Önce PR'ı oku, sonra plan + master review + standartlar, sonra kodu sample'la, sonra raporunu yaz. Sıra senin. diff --git a/docs/analysis/code_reviews/master-review-opus-202604300906.md b/docs/analysis/code_reviews/master-review-opus-202604300906.md new file mode 100644 index 00000000..78c283e2 --- /dev/null +++ b/docs/analysis/code_reviews/master-review-opus-202604300906.md @@ -0,0 +1,725 @@ +# ForgeLM Master Review (post-v0.5.0) — opus — 202604300906 + +**Verified against synthesized sub-reports at commit 6b515ed912f8f22304194c1b3f55ed07a26f519c** + +**Commit:** 6b515ed912f8f22304194c1b3f55ed07a26f519c +**Branch:** main +**Consolidator model:** opus (Opus 4.7, 1M context) +**Sub-reports synthesized:** Business, Code, Compliance, Documentation, Localization, Performance, Security, Testing-CICD, Regression-Check +**Prior master:** [`master-review-opus-202604281313.md`](./master-review-opus-202604281313.md) (delta: 52 commits, v0.4.5 → v0.5.0 → 0.5.1rc1) + +--- + +## 0. Executive summary + +v0.5.0 ("Document Ingestion + Data Curation Pipeline") shipped on 2026-04-30 as a consolidated release covering Phases 11 / 11.5 / 12 / 12.5. The release engineering itself was clean: PyPI confirms `forgelm 0.5.0`; the runtime `__version__` resolves through `importlib.metadata`; CHANGELOG carries a properly-dated `[0.5.0] — 2026-04-30` section; HEAD is now `0.5.1rc1` with `[Unreleased]` correctly empty and pointing at Phase 14. The two highest-leverage cross-cutting themes from the prior turn — Theme A (public version drift across runtime / manifest / changelog) and Theme B (audit/compliance trust path) — are **both closed at 100 % closure**. Theme D (silent-failure cluster) is also at 100 % closure for the load-bearing audit-trust paths, with sweep work continuing on best-effort artefact paths. Theme H (webhook security hardening) closed at 100 % including SSRF guard, redirect refusal, secret-masked failure reasons, and TLS-bundle pinning. The Critical-only closure across the prior master's 10-Critical surface is **100 %**. + +The new findings cluster materially around the **marketing / site / QMS surface**: three new Critical/Major problems emerge directly *from* the v0.5.0 ship (`site/compliance.html` listing artefact filenames the code does not produce; `site/quickstart.html` mock output naming non-existent templates; `docs/qms/sop_data_management.md` body still asserting `v0.5.1+` / `v0.5.2` feature pins after the consolidation note in CHANGELOG explicitly retconned that timeline; `tests/runtime_smoke.py` violating four standards at once; CI `--cov-fail-under` gate documented but not actually enforced by `pytest-cov`; trainer.py still imports `torch` / `transformers` / `trl` at module top against the lazy-import discipline its sister modules already follow). Localization is now a separately-tracked dimension and exposes the single largest delivered-vs-advertised gap on the site: DE/FR/ES/ZH chrome translation is native-quality across 286 keys × 6 languages, but the user-manual content for those four languages is **0 of 56 pages** — site-chrome promises six languages of content and delivers two. + +**Project health (this round): ★★★¾ / 5 overall.** Net trend post-v0.5.0: **improving on integrity / security / regression closure; stable on code / documentation / performance / business; new attack surface on the public marketing site that was lower-traffic in the prior round.** The ten remaining Open items from the prior round map cleanly to Phase 13/14 sprints; none are individually release-blocking. Closing the top three new Critical findings (site compliance artefacts, QMS version contradiction, CI cov gate) plus the two highest-leverage carry-overs (lazy torch import, Pydantic `description=` migration) lifts the overall to **★★★★ / 5** within a single sprint. + +| Dimension | Rating | Direction vs prior | One-line | +|---|---|---|---| +| Business | ★★★½ / 5 | flat | Theme A & B closures balance new site-drift Critical findings; close site compliance artefacts to lift to ★★★★½ | +| Code | ★★★½ / 5 | flat | Stylistic standards posture (Literal sweep, descriptions, except-Exception) deferred; load-bearing trio closed | +| Compliance | ★★★★ / 5 | up (★★★½ → ★★★★) | Theme B closure earned the fourth star; Article 14 ordering + verifier subcommand are next-tier debt | +| Documentation | ★★★½ / 5 | flat | EN ~★★★★½; TR-mirror ~★★★ drags it; QMS sop_data_management critical and three count-drift reopens | +| Localization | ★★★ / 5 | new axis | EN/TR full content parity (4 H2 drift items); DE/FR/ES/ZH 0 % page coverage with 1.79 MB EN-stuffed bags | +| Performance | ★★★★ / 5 | flat | 5 of 9 prior cliffs closed; lazy-torch + workers + safety batching remain | +| Security | ★★★★ / 5 | up (★★★½ → ★★★★) | Webhook hardening + audit chain + dataset fingerprint TOCTOU all closed; SSRF guard now needs to extend to judge.py + synthetic.py | +| Testing & CI/CD | ★★★½ / 5 | new axis | Strong test corpus (47 modules, 800+ tests, 12.3 K LOC); cov-fail-under gate not enforced + runtime_smoke disorder + fixture fragmentation reopened | + +--- + +## 1. Regression posture vs prior master review + +The prior master review listed **10 Critical** + **47 Major** findings (57 total in scope). Status across the 57 at HEAD `6b515ed`: + +| Status | Count | % | +|---|---|---| +| Closed | **38** | 66.7 % | +| Partially Closed | **9** | 15.8 % | +| Open | **10** | 17.5 % | +| Reopened | **0** | 0 % | +| Not Verifiable | **0** | 0 % | + +**Critical-only closure: 10 / 10 = 100 %.** Major closure: 28 / 47 = 59.6 % outright; +19.1 % partially-closed. **Combined Closed-or-Partially: 47 / 57 = 82.5 %.** + +### 1.1 Themes fully closed in this cycle + +- **Theme A — Public version drift (4 / 4 sub-findings closed).** `forgelm/__init__.py` resolves via `importlib.metadata`; `compliance._get_version()` follows the same path; CHANGELOG `[0.5.0] — 2026-04-30` section present; `pyproject.toml` and runtime stamp share one source. Single-source contract realigned. Commits: `b97b971`, `e5ba1d9`, `c78040b`. +- **Theme B — Audit/compliance trust path (7 / 7 sub-findings closed).** `_load_last_hash` now distinguishes file-missing from file-unreadable and raises on the latter. `log_event` writes under `flock`, re-reads chain head from disk under the same lock to defeat multi-writer fork race. `_prev_hash` advances only after the line lands on disk. Genesis manifest sidecar (`*.manifest.json`) detects truncate-and-resume attacks. `compute_dataset_fingerprint` lru_cache dropped; `os.path.realpath` resolves symlinks; `os.fstat(f.fileno())` captures stat from the same fd as the SHA-256 stream. `generate_data_governance_report` wired into `_export_compliance_if_needed` with `compliance.governance_exported` / `compliance.governance_failed` audit events gating the rollup `compliance.artifacts_exported` event. Annex IV markdown promise resolved by *renaming* references to `annex_iv_metadata.json` (the rename path, not the build path — see §3 caveats). Commits: `6143321`, `7db47bc`. +- **Theme D — Silent-failure cluster (4 / 4 closed).** `data.py:_process_messages_format` narrowed catch + raise on malformed rows; `safety.py:_release_model_from_gpu` narrowed to RuntimeError with partial-cleanup logging; `cli.py:_load_config_or_exit` distinct branches for FileNotFoundError / ConfigError / yaml.YAMLError / ValidationError / OSError; `config.py:load_config` lets ValidationError propagate. Six load-bearing audit-trust silent-fail sites now raise. Commit: `fb93ebd`. +- **Theme F — Pydantic schema discipline (1 / 1 in scope closed).** Six target fields converted to `Literal[...]`; bespoke `_validate_trainer_type` runtime validator deleted. Commit: `fb93ebd`. +- **Theme H — Webhook security hardening (3 / 3 closed).** SSRF guard via `_is_private_destination` (rejects RFC1918 / loopback / link-local / metadata-service / multicast); explicit `verify=True` + `tls_ca_bundle` opt-in; timeout floor enforced; redirect-following disabled; `notify_failure(reason)` runs through `mask_secrets()` with 2 KB truncation. Commit: `3da2810`. + +### 1.2 Themes partially closed + +- **Theme C — Documentation drift (9 closed / 3 partial / 3 open of 15).** Mirror-parity sweep shipped (configuration-tr / usage-tr / architecture-tr / distributed_training-tr corrected). Residual: `data_preparation*.md` 35-line micro-doc still under-covers messages schema + mix_ratio (Open M5); `compliance_summary.md` fake CLI removed but H1 / scope blockquote / 11 broken relative paths still open (Partial M7); design-doc status blockquotes still missing on `wizard_mode.md` + `blackwell_optimized.md` (Open M13); Pydantic `description=` 1 of 172 fields (Open M9); README "18 GPU models" vs `_GPU_PRICING` 16 / 17 (Open F-business-020); `architecture.md` EN missing `configs/safety_prompts/` (Partial M10); roadmap.md Phase 12.5 row consolidated rather than dedicated (Partial M12). +- **Theme E — Performance scaling cliffs (5 closed / 4 open of 9).** All three Critical perf items closed (`agg.minhashes` double-copy, bidirectional MinHash, `text_lengths` reservoir). Numpy simhash vectorisation + markdown chunker batch-encode landed. Open: F-performance-003 (lazy torch import — quick win not done), F-performance-004 (combined PII regex — deferred per master verdict), F-performance-007 (`--workers` audit flag — deferred to v0.5.3), F-performance-012 (safety eval batching — deferred for determinism contract). +- **Theme G — Standards drift (2 closed / 1 open of 3).** Coverage `fail_under` toml/standard mismatch closed (both at 40 now). `ci.yml` `continue-on-error: true` removed. `wizard.py` whole-module coverage `omit` still open (coupled with print-ban deferred decision). + +### 1.3 What did NOT land between rounds (still Open after 52 commits) + +These 10 master-tracked Major items remain Open at HEAD: + +1. **F-code-013** — `print()` in `wizard.py` (85 calls) — chat.py landed `_print` indirection but wizard didn't follow → consistency problem on top of standards problem. +2. **F-code-019** — `pyproject.toml` `omit = ["forgelm/wizard.py"]` — coupled with #1. +3. **M5** — `data_preparation*.md` rewrite or fold — content decision not made. +4. **M9** — Pydantic `description=` migration (1 of 172 fields). +5. **M13** — design-doc status blockquotes (`wizard_mode.md`, `blackwell_optimized.md`). +6. **F-business-020** — README "18 GPU models" vs code (16 entries per code review) → trivial off-by-one count drift. +7. **F-performance-003** — lazy `torch` / `transformers` / `trl` import in `trainer.py:8-10` (~30 min fix). +8. **F-performance-004** — combined PII+secrets+tokenize regex (deferred per master §4.1). +9. **F-performance-007** — `--workers N` audit flag (deferred to v0.5.3 per master §7). +10. **F-performance-012** — safety eval batching. + +**Trend:** 7 of 10 Open are *intentional* or *strategic-deferred*; 3 (F-performance-003, F-business-020, M13) are quick wins that fell through the cracks. + +--- + +## 2. Severity-aggregated findings table + +This round's findings only — prior round's open carry-overs are documented in §1.3 and the regression-check report. Numbering preserves source-report IDs for traceability. + +### 2.1 Critical + +| # | Severity | Dimension | File:Line | Title | Recommendation | Source report | +|---|---|---|---|---|---|---| +| 1 | Critical | Business | `site/compliance.html:102, 160-176` | Compliance footprint tree advertises 5 artefacts code does not produce (`annex_iv.json`, `safety_report.json`, `benchmark_results.json`, `conformity_declaration.md`, `manifest.json`) | Rewrite tree against `compliance.py::export_compliance_artifacts` real filenames; add CI diff check | F-business-001 | +| 2 | Critical | Business | `site/quickstart.html:111-116` | `--list` mock output uses non-existent template names (`byod-domain-expert`, `math-reasoning`); descriptions claim SFT+DPO chains templates do not ship | Patch `<pre>` to match `forgelm/quickstart.py::TEMPLATES` actual handles + descriptions | F-business-002 | +| 3 | Critical | Business+Doc | `docs/roadmap.md:12,16`; `docs/roadmap-tr.md:12,16`; `docs/roadmap/releases.md:85`; `CHANGELOG.md:156-158` | Three high-traffic surfaces still claim "PyPI publish pending" for v0.5.0 after the tag shipped | Flip to `✅ Done`; update callout to "PyPI 2026-04-30"; delete vestigial CHANGELOG sentence | F-business-003 | +| 4 | Critical | Documentation | `docs/qms/sop_data_management.md:74-99` | QMS template asserts audit pipeline is "v0.5.1+", LSH dedup arrived in "v0.5.2", PII tiers in "v0.5.1", secrets-mask in "v0.5.2" — none shipped on PyPI; CHANGELOG explicitly consolidates Phases 11/11.5/12/12.5 into v0.5.0 | Rewrite to `forgelm audit` (v0.5.0+) covering full Phase 11+11.5+12+12.5 surface; delete v0.5.1/v0.5.2 per-feature splits | M-DOC-001 | +| 5 | Critical | Compliance | `forgelm/trainer.py:894-919, 865-878` | Article 14 human-approval gate still saves model BEFORE gate triggers (`save_final_model` → `_finalize_artifacts` → `_handle_human_approval_gate`); log says "Model saved to staging" but path is literal `final_model/` | Either ship literal `final_model.staging/` directory + `forgelm approve` subcommand, OR fix the misleading log message + add CHANGELOG note that operators must gate on exit code 4 | F-compliance-101 | +| 6 | Critical | Compliance | `forgelm/compliance.py:56` | Operator identity falls back to literal `"unknown"` despite the standard explicitly forbidding it; in distroless CI containers `USER` is unset → all audit entries attributed to `"unknown"` | Raise on missing operator identity unless `--allow-anonymous-operator` flag is passed; replace chained `os.getenv` with `getpass.getuser()` + `socket.gethostname()` | F-compliance-102 | +| 7 | Critical | Compliance | `forgelm/cli.py` (no `verify-audit` subcommand) | `forgelm verify-audit` not shipped despite docs continuing to promise integrity verification; HMAC-when-keyed write path landed but read path is documentation only | Add `forgelm verify-audit <path>` subcommand + `forgelm.compliance.verify_audit_log()` library function | F-compliance-103 | +| 8 | Critical | Testing/CI | `.github/workflows/ci.yml:49`; `pyproject.toml:188-189` | CI runs `pytest --cov=forgelm` but does NOT pass `--cov-fail-under`; `pyproject.toml`'s `fail_under = 40` is not auto-enforced by pytest-cov → coverage can silently drop and CI stays green | Add `[tool.pytest.ini_options].addopts = "--cov-fail-under=40"`, OR add flag to `ci.yml:49` directly | F-test-001 | + +### 2.2 Major + +| # | Severity | Dimension | File:Line | Title | Recommendation | Source report | +|---|---|---|---|---|---|---| +| 9 | Major | Business | `site/index.html:153-154`; `README.md:22` | "18 GPU profiles auto-detected" claim while `_GPU_PRICING` ships 16 entries | Bump to 16 across site stats + 6 locales + README, OR drop the literal | F-business-004 (= F-business-020 carry-over) | +| 10 | Major | Business | 7 of 10 Colab notebooks | `git+https://...` install bypasses PyPI smoke job (dpo_alignment, safety_evaluation, grpo_reasoning, kto_binary_feedback, galore_memory_optimization, multi_dataset, synthetic_data_training) | Pin each to `forgelm[qlora]==0.5.0`; add CI grep-and-fail step | F-business-005, M-DOC-005 | +| 11 | Major | Business | `forgelm/cli.py:594-602, 856-869` | `--compliance-export` writes empty-metrics manifest with help text that implies populated bundle | Either inline post-training artefacts from `output_dir` or reword help to cliff-edge | F-business-006 | +| 12 | Major | Business | `CONTRIBUTING.md:62-79` | Project structure tree lists 17 modules; package ships 26 (missing chat, quickstart, inference, data_audit, ingestion, export, deploy, fit_check, grpo_rewards) | Replace with one-line pointer to `architecture.md`'s Directory Layout, OR regenerated 26-entry block | F-business-007, M-DOC-010 | +| 13 | Major | Business | `docs/qms/README.md:23-33` | QMS "Maps to QMS" table cites `safety_results.json` and `benchmark_results.json` that don't exist as standalone artefacts | Rewrite against `compliance.py::export_compliance_artifacts` actual filenames | F-business-008 | +| 14 | Major | Business | site (8 HTML pages × 6 locales) | No Pro CLI / OSS scaffold on site (README has it); no expectation-setting for Phase 13 | Add 5-line callout or `site/pricing.html` page | F-business-009 | +| 15 | Major | Business | `docs/roadmap.md:31`; `docs/roadmap-tr.md:31` | "17 phases complete" sentence contradicts top-of-page "Merged, PyPI publish pending" symbol | After F-business-003 flips status, the L31 sentence becomes accurate | F-business-010 | +| 16 | Major | Business | none (gap to fill) | `docs/standards/release.md` has no documented deprecation cadence; `--data-audit` removal in v0.7.0 is the live precedent but not codified | Add 10-line "Deprecation cadence" subsection citing `cli.py:1424-1428` as worked example | F-business-011 | +| 17 | Major | Business | `docs/roadmap/risks-and-decisions.md:71-91` | Decision Log stops at 2026-04-25; v0.5.0 consolidation + PyPI publish decisions absent | Append two rows | F-business-012 | +| 18 | Major | Code | `forgelm/config.py:77, 149-153, 179, 186-187, 248, 318` | Six Pydantic enum-shaped fields still bare `str` not `Literal[...]` (`LoraConfig.bias`, `DistributedConfig.strategy`, `SafetyConfig.scoring`, `ComplianceMetadataConfig.risk_classification`, FSDP fields, GaLore fields) | Convert to `Literal[...]`; lift shared `RiskCategory` to module-level alias | F-code-101, F-compliance-105 | +| 19 | Major | Code | `forgelm/config.py` (170+ fields) | Pydantic `description=` migration unstarted: 1 of ~170 fields | Incremental: PR template gate for new fields + dimension-scoped batches + CI guard diffing schema vs configuration.md | F-code-102, M-DOC-006 | +| 20 | Major | Code | `forgelm/data_audit.py` (3098 lines) | Single module owns six concerns past architecture standard's ~1000-line split threshold | Plan Phase 13 `data_audit/` package split (pii / pii_ml / dedup_simhash / dedup_minhash / quality / croissant / streaming) | F-code-103 | +| 21 | Major | Code | `forgelm/cli.py` (1756 lines) | Six subcommands' definers + dispatchers in one file; CLI-as-thin-shim principle eroded | Split into `forgelm/cli/` package with one parser+dispatcher pair per subcommand | F-code-104 | +| 22 | Major | Code | `forgelm/wizard.py` (85 print calls) | print-vs-logger conflict §4.4 unresolved; chat.py landed `_print` indirection, wizard didn't | Either port chat's `_print` pattern OR update standard with named interactive carve-out | F-code-105 | +| 23 | Major | Code | `forgelm/data_audit.py:1132`; `forgelm/safety.py:99/225/387/499`; trainer/judge/etc. (~25 sites across 13 modules) | Silent `except Exception:` sweep partially landed; non-fatal best-effort paths uncommented | Three-pass sweep: narrow class / comment-or-justify / add tests | F-code-106 | +| 24 | Major | Code | `forgelm/cli.py:604-615, 1424-1428` | `--data-audit` deprecation lacks DeprecationWarning + tracking issue + audit-log of legacy invocation | Emit `warnings.warn(..., DeprecationWarning)`, audit-log, link tracking issue | F-code-107, F-business-024 | +| 25 | Major | Code | `tools/build_usermanuals.py:106-119, 441` | `_meta.yaml` schema validation missing — malformed YAML produces opaque `KeyError` | Add Pydantic validation OR defensive `validate_meta(data)` walker | F-code-108 | +| 26 | Major | Code | `forgelm/cli.py:1742` (and 28 print sites) | Operator output mostly bypasses logger; `cli.py:1742` writes to stderr via print | Update standard to "cli.py may print to stdout for operator-visible text/JSON; errors via logger" | F-code-109 | +| 27 | Major | Compliance | `forgelm/webhook.py:206-264`; `forgelm/trainer.py:519-531, 865-878` | Webhook lifecycle vocabulary missing `training.reverted` and `approval.required` events; standard lists 5 events, code implements 3 | Add `notify_reverted` + `notify_awaiting_approval`; wire `_revert_model` and `_handle_human_approval_gate` | F-compliance-104 | +| 28 | Major | Compliance | `forgelm/config.py:367`; `forgelm/webhook.py:134-140` | Webhook timeout default 5s against standard floor of 10s | Bump default to 10; clamp floor to 10 not 1 | F-compliance-106 | +| 29 | Major | Compliance | none (no `RetentionConfig`); `docs/qms/sop_data_management.md:107-110` | QMS claims 5-year retention but no `retention:` config block, no archival hook, no GDPR right-to-erasure path | Stub-and-document is acceptable v1: add `retention:` Pydantic block with `enforce: log_only` default + document right-to-erasure procedure | F-compliance-107 | +| 30 | Major | Compliance | `forgelm/judge.py:172-195` | `judge_results.json` does not record `judge_model` / `api_base` / `rubric_sha256` / `min_score` provenance (Annex IV §2(d) gap) | Extend `_save_judge_results` to persist `judge_provenance` block | F-compliance-108 | +| 31 | Major | Compliance | `forgelm/synthetic.py`; `forgelm/compliance.py:523-616` | Synthetic-data provenance absent from training manifest (teacher_model, api_base, generation_params not recorded) | Inline `data_provenance.synthetic_data` block when `synthetic.enabled` | F-compliance-109 | +| 32 | Major | Compliance | `forgelm/config.py:282-291, 451-452` | Article 9 risk-management still declarative-only; `risk_classification: high-risk` + `evaluation.safety.enabled: false` is a WARNING not a hard block | Raise on high-risk + safety-disabled at runtime; soft-warn during `--dry-run` only | F-compliance-110 | +| 33 | Major | Compliance | `forgelm/compliance.py:331-360` | `_maybe_inline_audit_report` failure mode is INFO; missing Article 10 evidence section silently drops out of governance bundle | Promote to WARNING; emit `governance.data_audit_missing` audit event; raise on high-risk | F-compliance-111 | +| 34 | Major | Compliance | `docs/qms/` (no `-tr.md` mirrors) | All 6 QMS files (README, 5 SOPs) are EN-only despite localization standard requiring user-facing docs to be EN+TR mirrored | Add `*-tr.md` mirrors for all 6 QMS files via sync-bilingual-docs skill | F-compliance-112 | +| 35 | Major | Compliance | `docs/reference/compliance_summary.md:21,25,28,32,36` | Stale line anchors point to non-existent line ranges in current 873-line `compliance.py` (`#L33`, `#L167`, etc.) | Switch to symbol-style references (function names) or regenerate against HEAD with CI guard | F-compliance-113 | +| 36 | Major | Compliance | `forgelm/compliance.py:262-265` | `audit_log.jsonl` lacks `os.fsync()` after flush; chain advances on `flush()` even if power-cut precedes kernel writeback | Add `os.fsync(f.fileno())` immediately after `f.flush()` | F-compliance-114 | +| 37 | Major | Documentation | `docs/reference/configuration-tr.md:1-260` vs `configuration.md:1-318` | TR mirror H4 14 vs 6; missing `model.multimodal`, bnb_4bit fields, lora dropout/bias/task_type, 11 training fields, evaluation.benchmark sub-block, 5 compliance fields, 5 H2 sections out of order | Run `sync-bilingual-docs` skill end-to-end | M-DOC-002 | +| 38 | Major | Documentation | `docs/reference/compliance_summary.md:1-123` | No H1, no scope blockquote, 11 broken relative paths, "50 prompts in 3 categories" while README says "140 × 6"; unreachable from index | Promote (rewrite + link from README/QMS) OR delete + merge into safety_compliance.md | M-DOC-003 | +| 39 | Major | Documentation | 6 guide locations | Stale `v0.3.1rc1` references in enterprise_deployment, cicd_pipeline, safety_compliance (×3), troubleshooting | Mass-replace with `v0.4.0` or "Phase 10" prose | M-DOC-004 | +| 40 | Major | Documentation | `docs/reference/data_preparation*.md` | 35-line micro-doc; only legacy columnar form documented; messages schema, mix_ratio, governance fields absent; TR uses parenthetical-translation MT style | Rewrite to ~120 lines OR fold into alignment.md | M-DOC-007 (= M5 carry-over) | +| 41 | Major | Documentation | `docs/design/wizard_mode.md`, `blackwell_optimized.md` | Both lack scope blockquote + status indicator; future-tense wizard description while shipped, present-imperative for unimplemented blackwell | Add `> **Status:** Shipped` / `> **Status:** Proposed` blockquote | M-DOC-008 (= M13 carry-over) | +| 42 | Major | Documentation | `docs/standards/testing.md:8, 127` | Doc claims "26 test modules" (reality 47); CI command shows `--cov-fail-under=25` while pyproject is 40 | Update count to 47 + drop enumeration; sync command to 40 | M-DOC-009 | +| 43 | Major | Documentation | `CLAUDE.md:60` | Repo-orientation block says "26 test modules"; reality 47 | One-line fix | M-DOC-011 | +| 44 | Major | Documentation | `docs/reference/distributed_training-tr.md` | TR missing 3 EN H3 sections (Custom DeepSpeed Config, When to Choose FSDP over DeepSpeed, LoRA + Distributed); FSDP fields backward_prefetch + state_dict_type missing | Translate the 3 H3 sections; deepen CI bilingual check to H3 | M-DOC-012 | +| 45 | Major | Documentation | `docs/guides/ingestion-tr.md` | TR missing Phase 12 H3 sections: Markdown-aware splitter, DOCX table preservation | Translate the 2 H3 sections | M-DOC-013 | +| 46 | Major | Documentation | `docs/reference/architecture-tr.md:60-61` vs `architecture.md:59-69` | TR has `configs/safety_prompts/`, EN doesn't; both omit `forgelm/templates/` | Add to EN; add templates to both | M-DOC-014 (= M10 carry-over) | +| 47 | Major | Documentation | `docs/guides/alignment.md:230` | "Coming in v0.5.1 (Phase 14)" prose fragile to consolidation pattern that produced v0.5.0 | Pin to phase number not version tag | M-DOC-015 | +| 48 | Major | Localization | `docs/usermanuals/{de,fr,es,zh}/` (all empty); site language picker (8 HTML pages × 6 locales) | DE/FR/ES/ZH user-manual content is **0 of 56 pages**; site picker advertises 6 languages; bags ship 1.79 MB redundant English text under fallback flag | Drop from picker until ≥1 critical-path subset translated, OR render coverage badge ("0/56 — beta"), OR sparse-fallback emit | F-loc-001 | +| 49 | Major | Localization | `.github/workflows/usermanuals-validate.yml:31-48`; `tools/build_usermanuals.py:284-331` | Orphan markdown silently dropped — undeclared `*.md` under `docs/usermanuals/<lang>/` produces neither warning nor CI failure | Add orphan walk + `--strict` mode in CI | F-loc-002 | +| 50 | Major | Localization | All 8 site HTML pages | `og:locale="en_US"` hard-coded; no `og:locale:alternate` siblings; no `<link rel="alternate" hreflang>` block | Add hreflang + `data-i18n-attr="content:meta.og.locale"`; pre-resolve `YOUR_DOMAIN` placeholder | F-loc-003 | +| 51 | Major | Localization | `site/js/guide.js:436, 448, 455, 458, 466, 654` | Six `.toLowerCase()` call-sites are locale-unaware; Turkish `İ`/`I` parity drops in search; spec is unambiguous | Replace with `.toLocaleLowerCase(state.lang)` and invalidate `_searchText` cache on lang switch | F-loc-004 | +| 52 | Major | Localization | `docs/usermanuals/tr/training/sft.md` | TR drops "What you get on disk" section (H2 8 → 7) | Add section with TR heading "Diskte ne elde edersiniz" | F-loc-005 | +| 53 | Major | Localization | `docs/usermanuals/tr/training/simpo.md` | TR collapses 3 H2s; drops "Common pitfalls" / "Compute and memory" / splits "Veri formatı" + "Konfigürasyon parametreleri" | Restructure TR to mirror EN H2 spine 1:1 | F-loc-006 | +| 54 | Major | Localization | `docs/usermanuals/tr/compliance/overview.md` | TR drops "What goes into Annex IV" section (H2 7 → 6) | Translate as "Annex IV neyi içerir" | F-loc-007 | +| 55 | Major | Localization | `docs/usermanuals/tr/concepts/data-formats.md` | TR drops "Validating your data" section (H2 10 → 9) | Translate as "Verinizi doğrulama" | F-loc-008 | +| 56 | Major | Performance | `forgelm/trainer.py:8-10` | Eager `import torch` / `from transformers import EarlyStoppingCallback` / `from trl import SFTConfig, SFTTrainer` at module top — prior round's F-performance-003, ~30 min fix unaddressed | Move imports into method bodies; add regression test that `import forgelm.trainer` doesn't pull torch | F-performance-101 | +| 57 | Major | Performance | `forgelm/safety.py:86-102`; `forgelm/judge.py:139-151, 301` | Per-prompt `model.generate()` loop in safety + judge; GPU 70-90% idle on each step; 100-prompt eval ~30-60s instead of ~2-5s | Add `batch_size: int = 8` parameter; tokenise with `padding="longest"`; per-batch OOM fallback to single-prompt | F-performance-102 | +| 58 | Major | Performance | `forgelm/ingestion.py:902` | `_chunk_paragraph_tokens` re-encodes every paragraph in a loop (markdown twin's other half closed in c1af7a8) | Extract `_count_section_tokens` into `_batch_count_tokens`; reuse from both chunkers | F-performance-103 | +| 59 | Major | Performance | `forgelm/data_audit.py:1703-1733`; `forgelm/cli.py` (no `--workers`) | Per-row PII + secrets + simhash + Presidio walk same payload three or four times serially; no `--workers` → Presidio 1 M-row corpus 1.5-5 hours single-core | Add `--workers N` flag; `multiprocessing.Pool.imap` over chunked rows; preserve determinism via chunk index | F-performance-104 | +| 60 | Major | Performance | `tools/build_usermanuals.py:212-244, 284-331, 397-407` | 6 lang × 56 pages = 336 file builds with fresh `markdown.Markdown` instance per page; no incremental cache; serial across languages | `md.reset()` between calls + mtime/hash cache + `ProcessPoolExecutor` across languages | F-performance-105 | +| 61 | Major | Performance | `forgelm/data_audit.py:1664, 1706, 1838` | `lang_sample` stores full payload string up to N=200; ~512 chars suffices for langdetect; ~10 MB resident set on long-payload corpora | Slice to first 512 chars: `agg.lang_sample.append(payload[:512])` | F-performance-106 | +| 62 | Major | Security | `forgelm/judge.py:85`; `forgelm/synthetic.py:207-212` | Outbound POST without SSRF guard, explicit `verify=True`, HTTP refusal, redirect ban, or timeout floor; `Authorization: Bearer` header can leak to IMDS endpoint | Extract shared `forgelm/_http.py:safe_post` helper from `webhook._post_payload`; route judge + synthetic + webhook through it | M-201 | +| 63 | Major | Security | `forgelm/trainer.py:639-640` | `_AutoTok.from_pretrained` and `AutoModelForSequenceClassification.from_pretrained` for GRPO classifier reward miss explicit `trust_remote_code=False` (M-6 fix applied to safety.py only) | Pass `trust_remote_code=False` explicitly; add unit test mirroring safety pattern | M-202 | +| 64 | Major | Security | `forgelm/export.py:73-104, 261-321, 430-431` | `FORGELM_GGUF_CONVERTER` override partially closed via `.py` allow-list + warning, but no SHA-256 capture, no CLI ack flag, no docs | Add `--allow-custom-converter` flag; record `converter_sha256` in `model_integrity.json`; document as privileged knob | M-203 (= M-5 partial carry-over) | +| 65 | Major | Security | `forgelm/compliance.py:835-851, 664` | `_describe_adapter_method` interpolates `config.lora.target_modules` list via str() — bypasses `_sanitize_md` (regression-shaped) | Add `_sanitize_md_list` helper; audit `generate_deployer_instructions` for any list-valued interpolation | M-204 | +| 66 | Major | Security | `forgelm/deploy.py:99-103` | `_ollama_modelfile` `system_prompt` only escapes `"`; newline injection bypasses wrapper | Switch to triple-quoted form OR refuse newlines with actionable error | M-205 | +| 67 | Major | Testing/CI | `tests/runtime_smoke.py:11-103` | Violates 4 standards: writes to repo cwd (no `tmp_path`), cleanup commented out, real HF Hub fetch (no mock), Turkish comments + emoji | Either delete (nightly wheel-install-smoke covers it) OR move to `tools/runtime_smoke.py` and re-discipline | F-test-002, F-test-019, F-test-020 | +| 68 | Major | Testing/CI | `pyproject.toml:184-186`; `tests/test_wizard_byod.py`, `test_wizard_phase11_5.py`, `test_phase12_5.py:108-167` | `omit = ["forgelm/wizard.py"]` still active despite 3 wizard test modules existing | Drop module-level omit; use line-level `pragma: no cover` for interactive `input()` only | F-test-003 (= F-code-019 carry-over) | +| 69 | Major | Testing/CI | 4 test files (`test_cli.py`, `test_cli_phase10.py`, `test_fit_check.py`, `test_integration_smoke.py`) | `_minimal_config` lokal kopya in 4 places; F-code-015 partially closed | Move factory to `tests/_helpers/` module; replace 4 lokal copies + 7 `from conftest import` calls | F-test-004, F-test-005 | +| 70 | Major | Testing/CI | `tests/` (7 files) | `from conftest import minimal_config` is anti-idiom — pytest fixture system bypassed via plain module import | Move to `tests/_helpers.py` OR convert to factory-as-fixture | F-test-005 | +| 71 | Major | Testing/CI | `.github/workflows/ci.yml:30-34` vs `nightly.yml:16-19` | `fail-fast` policy asymmetric: nightly explicit `false`, ci.yml uses default `true` | Pick one + document in `docs/standards/testing.md` "CI gates" section | F-test-006 | +| 72 | Major | Testing/CI | `.github/workflows/ci.yml:31`; `pyproject.toml:23` | All CI on `ubuntu-latest`; pyproject claims "OS Independent"; cross-OS testing absent | Add `os: [ubuntu-latest, macos-latest, windows-latest]` axis to nightly wheel-install-smoke | F-test-007 | +| 73 | Major | Testing/CI | `.pre-commit-config.yaml` (absent) | No pre-commit hooks; CI catches lint/format only at PR-time | Add minimal `.pre-commit-config.yaml` with ruff + gitleaks + standard hooks | F-test-008 | +| 74 | Major | Testing/CI | `pyproject.toml:131-135`; `.github/workflows/ci.yml:49` | `pytest-xdist` not in dev extra; 47 modules / 800+ tests run serially | Add `pytest-xdist>=3.0`; CI `pytest -n auto`; trial in nightly first | F-test-009 | +| 75 | Major | Testing/CI | `pyproject.toml:189` (40) vs `docs/standards/testing.md:127` (25) | Standard says fail_under=25, pyproject says 40 — F-code-016 partially closed | Bump standard to 40 to match pyproject | F-test-010 | +| 76 | Major | Testing/CI | 3 separate "smoke" files (`test_smoke.py:14`, `test_integration_smoke.py:426`, `runtime_smoke.py:110`) | Naming inconsistency; `test_integration_smoke.py` is integration not smoke; `runtime_smoke.py` is not even a pytest test | Rename `test_integration_smoke.py` → `test_integration.py`; expand `test_smoke.py` to real `--help` smoke | F-test-011 | + +### 2.3 Minor (summary count by dimension) + +| Dimension | Minor count | Source(s) | +|---|---|---| +| Business | 6 | F-business-013 to F-business-018 | +| Code | 11 | F-code-130 through F-code-143 | +| Compliance | 7 | F-compliance-115 through F-compliance-121 | +| Documentation | 7 | m-DOC-016 through m-DOC-022 | +| Localization | 6 | F-loc-009 through F-loc-014 | +| Performance | 7 | F-performance-107 through F-performance-113 | +| Security | 10 | m-201 through m-210 | +| Testing/CI | 6 | F-test-012 through F-test-018 | + +**Total Minor: 60.** + +### 2.4 Nit (summary count by dimension) + +| Dimension | Nit count | Source(s) | +|---|---|---| +| Business | 6 | F-business-019 through F-business-024 | +| Code | 7 | F-code-160 through F-code-166 | +| Compliance | 3 | F-compliance-122, 123, 124 | +| Documentation | 5 | n-DOC-023 through n-DOC-027 | +| Localization | 4 | F-loc-015 through F-loc-018 | +| Performance | 5 | F-performance-114 through F-performance-118 | +| Security | 6 | n-201 through n-206 | +| Testing/CI | 4 | F-test-019 through F-test-022 | + +**Total Nit: 40.** + +### 2.5 Severity matrix totals + +| Dimension | Critical | Major | Minor | Nit | Total | +|---|---|---|---|---|---| +| Business | 3 | 7 | 6 | 6 | 22 | +| Code | 0 | 9 | 11 | 7 | 27 | +| Compliance | 3 | 11 | 7 | 3 | 24 | +| Documentation | 1 | 11 | 7 | 5 | 24 | +| Localization | 0 | 8 | 6 | 4 | 18 | +| Performance | 0 | 6 | 7 | 5 | 18 | +| Security | 0 | 5 | 10 | 6 | 21 | +| Testing/CI | 1 | 10 | 6 | 4 | 21 | +| **Total** | **8** | **67** | **60** | **40** | **175** | + +(Cross-cut findings — e.g., F-business-007 = M-DOC-010 — counted once in primary dimension.) + +--- + +## 3. Cross-cutting themes + +These themes have ≥ 2 sub-reports surfacing the same root cause. + +### Theme α — Site/marketing surface as a divergent code-claim boundary + +**Sources:** Business (F-business-001/002/003/004/008/009/010/017/023), Documentation (M-DOC-005), Localization (F-loc-001/003/011/012/013). + +**Root cause:** The public site (`site/*.html` × 6 locales × 286 i18n keys) is a separately-tested surface that does not yet have the doc-vs-code parity discipline applied to `docs/*.md ↔ docs/*-tr.md`. v0.5.0 widened the gap in five visible places: artefact filenames the code does not produce; quickstart template names that don't resolve; PyPI-publish-status lies in three high-traffic locations; GPU profile counts off by 1-2 across site/README/code; six-language picker promising 56 pages × 6 of content while delivering 56 pages × 2. + +**Verdict:** The strongest single fix in this review cycle is a one-pass `site/*.html` rewrite against the actual code — `compliance.py::export_compliance_artifacts` filenames, `quickstart.py::TEMPLATES` handles, `_GPU_PRICING` count, PyPI status flip — followed by adding a CI guard equivalent to the bilingual H2 parity check that already exists for the `docs/` tree. The `site/` surface needs its own per-page-against-code linter. + +### Theme β — QMS / compliance-summary documentation as a regulatory deliverable that contradicts itself or the code + +**Sources:** Documentation (M-DOC-001, M-DOC-003), Compliance (F-compliance-112, F-compliance-113, F-compliance-119, F-compliance-124), Business (F-business-008). + +**Root cause:** The QMS pack (`docs/qms/`) and the reference `compliance_summary.md` are exactly the artefacts a regulated-industry adopter hands to their audit lead alongside ForgeLM's own compliance bundle. Three drift classes accumulate here: (a) `sop_data_management.md` body still cites `v0.5.1+` / `v0.5.2` per-feature splits that the consolidation note in CHANGELOG explicitly retconned; (b) `compliance_summary.md` opens at H2 (no H1), has 11 broken relative paths to `forgelm/...`, lists `safety_results.json` and `benchmark_results.json` that don't exist as standalone artefacts, and stale line anchors point into wrong code regions; (c) audit-event vocabulary scattered across compliance.py and trainer.py with no `audit_event_catalog.md` for an external auditor; (d) all 6 QMS files EN-only despite localization standard requiring bilingual mirroring. Cumulative effect: the regulated-industry deliverable surface now visibly under-delivers vs the marketing thesis. + +**Verdict:** Promote `compliance_summary.md` to a real reference doc OR delete + merge into `safety_compliance.md`. Rewrite `sop_data_management.md:74-99` against consolidated v0.5.0 reality. Add `docs/reference/audit_event_catalog.md` (+ `-tr.md` mirror). Decision in §5 below resolves: ship QMS as bilingual or formalise the EN-only carve-out. + +### Theme γ — Module cohesion drift past architecture standard's split threshold + +**Sources:** Code (F-code-103, F-code-104), Performance (E.4 carry-over implicitly). + +**Root cause:** Two modules clear the architecture standard's ~1000-line ceiling by significant margins: `forgelm/data_audit.py` at 3098 lines (six concerns: PII regex + Presidio adapter + simhash + MinHash + streaming aggregator + quality filter + Croissant emitter); `forgelm/cli.py` at 1756 lines (six subcommands' definers + dispatchers + training-mode entry + helpers). v0.5.0 added Phase 12.5 features additively, growing both files past the prior round's already-flagged ceilings without splits. + +**Verdict:** Plan Phase 13 `data_audit/` package split (PII / pii_ml / dedup_simhash / dedup_minhash / quality / croissant / streaming) and `cli/` package split (subcommands/{chat,export,deploy,quickstart,ingest,audit}.py + parser glue + helpers). The two splits should land in the same sprint — they are mechanically similar and both unblock targeted Phase 13 perf work. + +### Theme δ — Pydantic schema discipline gaps (Literal sweep + descriptions migration) + +**Sources:** Code (F-code-101, F-code-102), Compliance (F-compliance-105), Documentation (M-DOC-006). + +**Root cause:** The prior round's F-code-014 closed 6 of 12 enum-shaped fields to `Literal[...]`; six remain bare `str` (`LoraConfig.bias`, `DistributedConfig.strategy/fsdp_backward_prefetch/fsdp_state_dict_type`, `SafetyConfig.scoring`, `ComplianceMetadataConfig.risk_classification`, GaLore fields). Same shape: `risk_classification` parallel field on `RiskAssessmentConfig` IS Literal — so the GTM-load-bearing classification field can drift via typo to a no-match string, silently downgrading high-risk to no-match. Separately, only 1 of ~170 fields carries `description=`, blocking config-doc generation and forcing the markdown configuration table to drift. + +**Verdict:** One-day sweep for Literal[] (mechanical). Multi-day sprint for `description=` migration with CI guard as the load-bearing piece (without it the migration regresses the moment a contributor forgets). Both are master §7 strategic items that have aged a round. + +### Theme ε — Outbound HTTP discipline (SSRF + TLS) extension to non-webhook call sites + +**Sources:** Security (M-201), Compliance (implicit via webhook lifecycle). + +**Root cause:** Webhook hardening landed in commit `3da2810` set the project-wide bar for outbound HTTP: SSRF guard via `_is_private_destination`, redirect refusal, explicit `verify=True`, timeout floor, `notify_failure(reason)` masked through `mask_secrets`. Two new outbound HTTP call sites (`judge.py:85`, `synthetic.py:207`) that pre-date that bar were not part of the sweep. Both honour an operator-supplied `api_base`, both ship `Authorization: Bearer <api_key>` headers, neither guards against IMDS endpoints, neither rejects `http://` URLs, neither has a redirect ban or timeout floor. + +**Verdict:** Extract shared `forgelm/_http.py:safe_post` from `webhook._post_payload`; route judge + synthetic + webhook through it. Mirror config fields. ~80 LOC + tests. **Single PR.** + +### Theme ζ — Documentation count drift across CONTRIBUTING / CLAUDE / standards + +**Sources:** Documentation (M-DOC-009, M-DOC-010, M-DOC-011), Business (F-business-007), Code (none — count-drift is doc problem, not code). + +**Root cause:** v0.5.0's prior-round count-refresh sweep landed in `architecture.md` (26 modules / 47 test files / 800+ tests / 10 notebooks) and partially in CONTRIBUTING.md (test count corrected; module enumeration missed). Three orientation surfaces still drift: CLAUDE.md says "26 test modules" (reality 47); CONTRIBUTING.md project-structure lists 17 modules (reality 26); `docs/standards/testing.md:8` says "26 test modules" + L127 cites `--cov-fail-under=25` while pyproject is 40. Each is a 1-line fix; cumulative effect is "the docs don't notice when they go stale," which is the single most documentation-drift-shaped finding in the standards posture. + +**Verdict:** One-PR low-effort cleanup. Add a CI step that fails on stale module / test / notebook counts in any markdown that mentions them — same pattern as the H2 parity check. + +### Theme η — Operator identity + audit-log forensic completeness + +**Sources:** Compliance (F-compliance-102, F-compliance-114, F-compliance-117, F-compliance-119, F-compliance-120, F-compliance-121), Code (F-code-143). + +**Root cause:** Theme B closure makes the audit chain tamper-evident; downstream gaps weaken its forensic value: (a) operator falls back to literal `"unknown"` when env unset (Critical, in distroless containers always fires); (b) audit log lacks `os.fsync` after flush (Major, narrow durability window); (c) HF Hub dataset fingerprint silently falls back without a `revision` pin (Major); (d) no `audit_event_catalog.md` documents the vocabulary; (e) safety classifier-load failure returns `passed=False` without an audit-log event distinguishing "model unavailable" from "classifier returned unsafe output"; (f) cross-run continuity claim broader than reality (only within same `output_dir`). + +**Verdict:** Three of these (operator fallback, fsync, classifier-load-failure event) are <1 hour fixes that materially strengthen Article 12 record-keeping evidence. The audit_event_catalog is a Phase 13 docs deliverable. The HF Hub revision pin is a Phase 12.6 mini-sprint. + +### Theme θ — Performance: lazy import + batching + workers as the three remaining wins + +**Sources:** Performance (F-performance-101 lazy torch, F-performance-102 safety+judge batching, F-performance-103 paragraph chunker, F-performance-104 `--workers`, F-performance-105 build_usermanuals). + +**Root cause:** v0.5.0 closed all three Critical perf items (Theme E.1/E.2/E.3) plus E.6 (numpy simhash) and E.7 (markdown chunker). Four of nine prior cliffs remain Open. Two — F-performance-003 (lazy torch) and F-performance-004 (combined regex) — are deferred per master verdict; F-performance-007 (`--workers`) is deferred to v0.5.3; F-performance-012 (safety batching) is unaddressed. Add the new findings: paragraph token chunker re-encodes (the markdown twin's other half), `lang_sample` stores full payloads, build_usermanuals runs serial with no incremental cache. + +**Verdict:** F-performance-101 is a 30-min quick win that fell through the cracks of an otherwise thorough sweep. F-performance-102 (safety + judge batching) is half-day work with 10-20× wall-clock reduction. F-performance-104 (`--workers`) is the deciding factor for whether `--pii-ml` is usable on >100K-row corpora. + +### Theme ι — CI gate enforcement gap (cov-fail-under, fixture fragmentation, cross-OS, pre-commit) + +**Sources:** Testing/CI (F-test-001, F-test-004, F-test-005, F-test-007, F-test-008, F-test-009, F-test-010, F-test-017). + +**Root cause:** The test corpus is nominally strong (47 modules / 800+ tests / 12.3 K LOC / disciplined skip-if guards / no GPU + no network in unit tests / Llama Guard / TRL / datasketch mocked). The mechanical enforcement layer underneath has gaps: (a) `pytest --cov-fail-under` not actually enforced; (b) `wizard.py` whole-module coverage omit; (c) `_minimal_config` fixture in 4 lokal copies + 7 anti-idiom `from conftest import` callers; (d) cross-OS testing absent despite "OS Independent" claim; (e) no `.pre-commit-config.yaml`; (f) `usermanuals-validate.yml` triggers on PR-only (main-direct push bypasses validation); (g) `pytest-xdist` parallelism not enabled. Each gap individually low-impact; cumulatively they explain why the project ships strong tests without a corresponding strong "are the tests actually enforced" surface. + +**Verdict:** Single-sprint hardening PR that bundles cov-fail-under enforcement + drop wizard omit + fixture consolidation + pre-commit + cross-OS axis is the right shape. + +--- + +## 4. Top-N priority queue (next sprint) + +Ordered by leverage × effort. Items 1-4 close the new Critical findings introduced by v0.5.0. Items 5-10 are quick wins or single-sprint hardening. Items 11-15 are medium-effort decisions. + +1. **Site compliance.html artefact rewrite** (Critical, F-business-001) — 30 minutes per locale + 6 locales × ~5 min = ~1 hour. Rewrite the artefact tree against `compliance.py::export_compliance_artifacts` real filenames. Add CI guard. **Highest-leverage Critical.** + +2. **Site quickstart.html template names** (Critical, F-business-002) — 5 lines × 6 locales = 30 minutes. Patch `<pre>` mock against `forgelm/quickstart.py::TEMPLATES`. + +3. **QMS sop_data_management body rewrite** (Critical, M-DOC-001) — 30 minutes. Drop the v0.5.1+/v0.5.2 per-feature splits; align with the consolidated `[0.5.0]` story. The QMS pack is the project's load-bearing regulatory-deliverable surface; the contradiction is the most embarrassing single doc artefact at HEAD. + +4. **CI cov-fail-under enforcement** (Critical, F-test-001) — 1-line `addopts = "--cov-fail-under=40"` + sync `docs/standards/testing.md:127` to 40. **5 minutes.** Closes the silent CI-coverage gap that would have masked any future regression. + +5. **PyPI publish status sweep** (Critical, F-business-003) — 4 files, 15 minutes. Flip roadmap.md/roadmap-tr.md status icons; rewrite L16 callout; promote releases.md L85; delete CHANGELOG vestigial sentence at L156-158. + +6. **Lazy `torch` / `transformers` / `trl` import in trainer.py** (Major, F-performance-101) — 30 minutes. Sister fix in `model.py`. ~700-1500 ms cold-start drop. **Quick win that fell through prior sweep.** + +7. **Operator identity fallback fix** (Critical, F-compliance-102) — 30 minutes + CI smoke test. Replace chained `os.getenv` with `getpass.getuser()` + `socket.gethostname()`; raise without explicit opt-out flag in fully-anonymous envs. + +8. **`runtime_smoke.py` decision** (Major, F-test-002) — 30 minutes if delete; 1-2 hours if move-to-tools. Single biggest-symbol violation in the test corpus; nightly wheel-install-smoke already provides equivalent garanti. + +9. **Notebook install pin sweep** (Major, F-business-005, M-DOC-005) — 30 minutes for the 7 notebooks + CI grep-and-fail step in nightly.yml. + +10. **Drop wizard.py coverage omit + sweep `print()` to `_print` indirection** (Major, F-test-003 + F-code-105 = F-code-019/F-code-013 carry-over) — 3-4 hours. Coupled decision; chat.py is the canonical pattern. + +11. **`forgelm verify-audit` subcommand** (Critical, F-compliance-103) — ~4 hours. Without it the Theme B HMAC code is dead-write-only and an external auditor cannot exercise the chain. + +12. **Webhook lifecycle vocabulary** (Major, F-compliance-104) — half-day. Add `notify_reverted` + `notify_awaiting_approval`; wire `_revert_model` and `_handle_human_approval_gate`; tests via `requests_mock`. + +13. **Article 14 staging directory OR honesty-fix log** (Critical, F-compliance-101) — 5 min for honesty fix; 2-3 days for real staging. Decision-gated (see §5 conflicts). + +14. **Pydantic Literal sweep (six remaining fields)** (Major, F-code-101) — 1 hour mechanical; closes Theme δ to 100 %. + +15. **Fixture consolidation** (Major, F-test-004 + F-test-005) — 4-6 hours. Move `minimal_config` to `tests/_helpers/`; replace 4 lokal copies + 7 `from conftest import` callers; bundle GRPO trainer helpers (F-test-012) into the same module. + +**Sprint capacity:** items 1-9 are ~1 day total; items 10-15 are ~5 days; with parallel execution this is a single 1-week sprint that lifts each affected dimension by half a star. + +--- + +## 5. Conflicts & trade-offs + +### 5.1 Performance "F-performance-104 add `--workers`" vs Compliance "audit determinism contract" + +**Conflict:** Performance review recommends `--workers N` for the audit row loop (5-7× wall-clock improvement; required for `--pii-ml` to be usable on >100K-row corpora). Compliance review's audit chain promise is "across all runs that share the same `training.output_dir`" (F-compliance-121); the audit's `lang_sample` is "first 200" not "random sample" — order-sensitive across worker counts. A multi-worker run that produces different `lang_sample` than a single-worker run on the same corpus would break the audit's reproducibility evidence. + +**Resolution:** Multi-worker variant must reproduce single-worker's `lang_sample` byte-for-byte by preserving chunk order via deterministic chunk index. Cross-worker output is sorted at end (`data_audit.py:889` already does this for near-dup pairs); same discipline extended to `lang_sample`. The audit determinism contract wins; the performance flag is fine if it adheres. + +### 5.2 Performance "F-performance-101 lazy torch import" vs Code "library API contract" + +**Conflict:** Lazy-import discipline lives in `cli.py` (subcommand handlers do `from .trainer import ForgeTrainer` only inside the train branch) — operator-visible `forgelm --help` already pays no torch cost. But `forgelm/__init__.py:39-77` `__getattr__` exposes `ForgeTrainer` as a public API; library-API users (`import forgelm; forgelm.ForgeTrainer(...)`) trigger the lazy `__getattr__` which pulls trainer.py which eagerly imports torch. + +**Resolution:** The library-API contract is real and matters for low-latency Python tools. Per Performance review's open question 4.1, the priority depends on whether `forgelm.ForgeTrainer` is a supported integration surface. The current package surface argues yes. **Verdict:** ship the lazy-import fix in trainer.py + model.py; the priority is Major not Hygiene. + +### 5.3 Compliance "F-compliance-101 staging directory" vs Documentation "marketing claim" + +**Conflict:** F-compliance-101 says the human-approval gate fires AFTER the model is on disk in `final_model/`, and the log message lies ("Model saved to staging" but path is `final_model/`). Master review's compliance verdict frames this as either a 5-min log honesty fix OR a 2-3 day real-staging implementation. Marketing claim (`docs/product_strategy.md:68`) reads the gate as effective: "Human approval gate (exit code 4) fits into existing governance workflows". + +**Resolution:** Two paths are valid: **(a)** Ship the honesty fix in v0.5.1 (5 min), promote staging in v0.5.2 (2-3 days); **(b)** Ship both together. Path (a) is the lower-risk one; path (b) is more work but lands the regulator-facing fix in one tag. Recommendation: **path (a)** — the honesty fix unblocks the regulated-industry diligence concern immediately; the staging promotion can be planned separately with proper test coverage. Update `docs/product_strategy.md:68` and the deployer-instructions text in the same PR. + +### 5.4 Localization "F-loc-001 drop DE/FR/ES/ZH from picker" vs Localization "F-loc-011 sparse fallback" + +**Conflict:** Localization review offers three paths for the 0/56-page coverage gap: (a) drop DE/FR/ES/ZH from picker until ≥1 critical-path subset translated; (b) render coverage badge in picker ("0/56 — beta"); (c) make build skip empty languages. F-loc-011 separately recommends sparse-fallback emit (saves 1.79 MB of redundant English text but keeps all 6 languages in the picker). + +**Resolution:** Path (a) is the most honest; path (b) is the cheapest; path (c) is the build-side-only fix. The strategic call belongs with the project owner (open question §5.7 in localization review). **Recommended:** Path (a) for marketing surface honesty + Path (c) on the build side to also save the bandwidth — these compose. Reverse path (a) once any ≥8-page critical-path subset lands for a given language. Document the policy decision in `docs/standards/localization.md` (currently the standard says "Spanish translations — not planned for 2026" which already contradicts the picker including ES). + +### 5.5 Code "F-code-103 data_audit/ split" vs Compliance "stable public API contract" + +**Conflict:** Code review recommends Phase 13 `data_audit/` package split. Compliance + Performance reviews both reference `forgelm.data_audit.SECRET_TYPES`, `audit_dataset`, `summarize_report`, `AuditReport`, `detect_pii`, `mask_pii`, `detect_secrets`, `mask_secrets`, `compute_simhash` — these are the public-API symbols tests + external consumers depend on. + +**Resolution:** Re-export every public symbol from `data_audit/__init__.py` so import paths don't break. Same pattern data_audit currently uses internally for the subsections. Document the split as non-breaking in the PR description; pin a regression test asserting `from forgelm.data_audit import AuditReport, audit_dataset, ...` still resolves. + +### 5.6 Security "M-203 CLI ack flag for FORGELM_GGUF_CONVERTER" vs Operator-Convenience + +**Conflict:** Security review recommends gating `FORGELM_GGUF_CONVERTER` env var on `--allow-custom-converter` CLI flag. Operators currently using the env var as a pure convenience knob will see a hard break. + +**Resolution:** Ship the CLI ack flag with a one-release deprecation window: until v0.6.0 the env-var-only path emits a `DeprecationWarning` recommending `--allow-custom-converter`; from v0.6.0 the flag is required. Document under the existing `--data-audit` deprecation cadence (which F-business-011 wants codified anyway). Same standard. + +### 5.7 Documentation "M-DOC-001 rewrite QMS body" vs Compliance "F-compliance-112 add TR mirrors" + +**Conflict:** M-DOC-001 says rewrite `sop_data_management.md` body NOW. F-compliance-112 says ALL 6 QMS files need TR mirrors, including `sop_data_management.md`. Order matters: if you rewrite EN body now, then sync TR mirror, you double the work; if you add TR mirror first, the contradiction in EN body propagates to TR. + +**Resolution:** Rewrite EN body first (M-DOC-001) — it's the higher-severity drift. Then run `sync-bilingual-docs` over the corrected EN body to produce TR mirror. Two PRs landing in sequence; do not parallelise. + +--- + +## 6. Standards posture + +### 6.1 Standards held systemically + +These standards are visibly enforced across the codebase at HEAD and improved this round: + +- **`docs/standards/regex.md` 8 hard rules** — every regex landed in v0.5.0 carries the rule citations. `_TOKEN_PATTERN` explicit Unicode (Rule 1); `_SECRET_PATTERNS` `re.ASCII` on `\w`-bearing tokens (Rule 1); PEM markers split via concatenation (Rule 7); `_MARKDOWN_HEADING_PATTERN` anchored on non-whitespace (Rule 4); state-machine fence parsers in `_strip_code_fences` and `tools/build_usermanuals.py:preprocess_admonitions` (Rule 6). The `tools/build_usermanuals.py` preprocessor docstring even cites `regex.md` rule 6 by number — exemplary cross-reference. +- **`docs/standards/error-handling.md` no-silent-fail (audit-trust paths)** — Theme D closure: 6/6 audit-trust silent-fail sites narrowed and re-raise. Audit log writes raise OSError; data.py messages raises ValueError with row index; safety GPU release narrows to RuntimeError; CLI config loader has 5 distinct catch branches. +- **`docs/standards/release.md` Keep-a-Changelog discipline** — `[0.5.0] — 2026-04-30` properly dated; `[Unreleased]` correctly empty post-release pointing at Phase 14; version single-sourced via `importlib.metadata`. +- **`docs/standards/architecture.md` lazy-import for optional extras** — `data_audit.py:56-111` `_HAS_XXHASH` / `_HAS_DATASKETCH` / `_HAS_NUMPY` / `_HAS_PRESIDIO` sentinel pattern is consistent across the audit module; `ingestion.py:233-237` for pypdf etc. +- **CI security best-practice** — `.github/workflows/publish.yml` OIDC trusted publishing (no PYPI_API_TOKEN); per-job least-privilege permissions; no `pull_request_target`; no `continue-on-error: true`; no `|| true` fake-green patterns; SRI on the single external CDN script (mermaid). +- **Outbound HTTP discipline (webhook only)** — Theme H closure: SSRF guard, redirect refusal, explicit `verify=True`, timeout floor, secret-masked failure reasons, response body suppression on 4xx/5xx. + +### 6.2 Systemic violations (pattern, not slips) + +These are standards the project documents but does not actually enforce mechanically: + +- **`docs/standards/logging-observability.md` Rule 2 ("Never use `print()` in library code")** — 86 violation sites at HEAD (chat.py: 1 with `_print` indirection that resolves the conflict for chat; wizard.py: 85 raw print calls; the print-vs-logger conflict §4.4 from the prior round is unresolved). The standard has not been updated with an interactive-subcommand carve-out; the modules have not been refactored to a `_print` shim. **Worst-of-both** state. +- **`docs/standards/testing.md` coverage (`fail_under` enforcement)** — `pyproject.toml` says `fail_under = 40`; CI does NOT pass `--cov-fail-under=40`; `pytest-cov` does not auto-read the toml setting. The gate is documented but not enforced. F-test-001 closes this. +- **`docs/standards/testing.md` `omit` policy (whole-module exclusions need justification)** — `pyproject.toml:186` `omit = ["forgelm/wizard.py"]` is a 951-LOC whole-module exclusion despite 3 wizard test modules existing. The standard does not list this exception. +- **`docs/standards/coding.md` Pydantic `Literal[]` rule** — 6 of 12 enum-shaped fields drift (closed 6 in prior round, 6 remain). Same shape for `description=` migration: 1 of 170 fields. +- **`docs/standards/architecture.md` module cohesion (~1000-line ceiling)** — `forgelm/data_audit.py` at 3098 lines (3× threshold), `forgelm/cli.py` at 1756 lines (1.7× threshold) for a third consecutive round. No split planned in current roadmap. +- **`docs/standards/localization.md` structural mirror rule** — H2 parity holds for the 7 EN/TR doc pairs covered by the bilingual H2 CI check; H3/H4 drift visible in 4 places (configuration-tr H4 14 vs 6; ingestion-tr 2 missing H3; distributed_training-tr 3 missing H3; 4 user-manual TR pages with H2 drift). Standard says H2/H3 parity required; CI only checks H2. +- **`docs/standards/release.md` deprecation cadence** — `--data-audit` removal in v0.7.0 is the live precedent but the standard does not codify minimum overlap, removal-version-in-help, tracking-issue link. Future deprecations will follow ad-hoc patterns. +- **`docs/standards/logging-observability.md` operator identity rule** ("Operator is required. Either from env, config, or fall back to OS username. Never 'unknown'") — directly violated by `compliance.py:56` chained `os.getenv` that resolves to literal `"unknown"` in distroless containers. + +### 6.3 Standards drift score + +| Standard | Held? | Trend | +|---|---|---| +| `coding.md` Type hints / Literal | Partial (6/12) | improving (was 0/12) | +| `coding.md` Imports / lazy patterns | Held | stable | +| `coding.md` Pydantic models | Partial (descriptions) | stable | +| `regex.md` 8 hard rules | Held | improving (build_usermanuals cites rule 6) | +| `error-handling.md` no-silent-fail (audit-trust) | Held | improving (Theme D closed) | +| `error-handling.md` no-silent-fail (best-effort) | Partial (~25 sites) | stable | +| `logging-observability.md` no print in library | Violated systemically | stable | +| `logging-observability.md` operator identity | Violated systemically | stable | +| `logging-observability.md` log levels | Mostly held | stable | +| `testing.md` coverage gates | Documented not enforced | stable (worse — pyproject + standard now inconsistent) | +| `testing.md` no GPU + no network | Held | stable | +| `testing.md` fixture factory pattern | Partial (4 dup + 7 anti-idiom) | improving | +| `testing.md` 47-modules count | Doc says 26 | doc-drift | +| `architecture.md` module topology | Violated (data_audit.py 3098, cli.py 1756) | regressing (grew this round) | +| `architecture.md` lazy-import optional extras | Held | stable | +| `architecture.md` outbound HTTP discipline | Held for webhook | needs extension (judge.py, synthetic.py) | +| `documentation.md` claim/code alignment | Partial (site drift) | regressing on site, improving on docs/ | +| `documentation.md` bilingual mirror H2 parity | Held | stable | +| `documentation.md` bilingual mirror H3/H4 parity | Violated | stable | +| `localization.md` structural mirror | Partial | stable | +| `localization.md` quality bar (TR) | Held (TR native quality) | stable | +| `localization.md` user-facing 6-language coverage | Violated systemically (DE/FR/ES/ZH at 0%) | stable | +| `release.md` Keep-a-Changelog | Held | improving | +| `release.md` deprecation cadence | Standard gap | stable | +| `code-review.md` lint discipline | Held in CI | stable | + +**Net standards posture trend:** 6 of 8 master-tracked items improved; 2 unchanged (wizard print + wizard coverage omit, coupled). Three items regressed (module cohesion +200 lines on data_audit; site claim/code drift; testing coverage doc-vs-pyproject inconsistency). + +--- + +## 7. Quick wins (Pareto) + +≤ 30 minutes each, high signal-to-noise. All are mechanical. + +1. **`addopts = "--cov-fail-under=40"`** — F-test-001. 1 line in pyproject.toml. Closes Critical CI gate. +2. **`Tür` → `Type`** in `docs/roadmap.md:7` — F-business-020 minor / m-DOC-016. 1-character fix. +3. **`organisations` → `organizations`** in `data_audit.py:104` — F-code-130. 1-character fix. +4. **`✓` → `OK`** in `tools/build_usermanuals.py:404` — F-code-137. 1-line fix; Windows cp1252 portability. +5. **CLAUDE.md test count** `26` → `47` — M-DOC-011. 1-line fix. +6. **`testing.md` test count + cov flag** `26` → `47`, `--cov-fail-under=25` → `40` — M-DOC-009. 2-line fix. +7. **`fail-fast: false`** in `ci.yml:30-34` — F-test-006. 1-line fix. +8. **`usermanuals-validate.yml` push trigger** — F-test-017. 4-line fix. +9. **`forgelm/templates/domain-expert/README.md:13`** — F-business-014. Drop "(requires Phase 11)" parenthetical. +10. **`docs/roadmap.md:7-14` legend** — m-DOC-021. Add tristate symbol legend above status table. +11. **README "10+ modes" → drop literal** — m-DOC-020. +12. **PyPI publish status flip** — F-business-003. 4 files, ~15 minutes. +13. **`safety.py:381-386` `trust_remote_code=False`** mirror to `trainer.py:639-640` — M-202. 2 lines. +14. **`compliance.py:262-265` `os.fsync` after flush** — F-compliance-114. 1 line. +15. **`risk_classification` `str` → `Literal[...]`** — F-compliance-105. 1 line. +16. **Lazy `torch` / `transformers` / `trl` import in trainer.py** — F-performance-101. ~10 lines moved into method bodies. +17. **`lang_sample[:512]`** truncation in `data_audit.py:1706` — F-performance-106. 1-line slice. +18. **`docs/qms/sop_data_management.md` lead-with `forgelm audit`** — m-DOC-018. ~5 lines (rolls up into M-DOC-001 Critical when M-DOC-001 lands). +19. **`docs/standards/release.md` deprecation cadence subsection** — F-business-011. ~10 lines citing cli.py:1424-1428. +20. **`risks-and-decisions.md` 2 new Decision Log rows** — F-business-012. + +**Total quick-win effort: ~3 hours of mechanical work; closes 1 Critical + 4 Major + 6 Minor + 9 Nit findings.** + +--- + +## 8. Strategic items (planning) + +These are roadmap-level decisions; not single-PR work. + +### 8.1 Site-as-tested-surface CI (mod Theme α) + +The site (`site/*.html` × 6 locales) is now a divergent code-claim boundary. The bilingual H2 parity check covers `docs/` but not `site/`. Three new Critical findings (F-business-001/002/003) all live on the site. **Decision needed:** either build a `site/`-against-code linter (artefact filenames vs `compliance.py`; template names vs `quickstart.py::TEMPLATES`; GPU count vs `_GPU_PRICING`; PyPI status vs PyPI metadata), or accept that the site requires manual review per release. The first costs ~1 day to write; the second costs ~30 min per release in disciplined eyes. **Recommended: build the linter.** Single source of truth for "what does the site claim, what does the code do". + +### 8.2 6-language picker honesty (Theme α + Localization F-loc-001) + +DE/FR/ES/ZH user-manual content is 0/56 pages. Picker advertises 6 languages. Three resolution paths in §5.4 above. **Decision needed:** owner intent. If "pre-announce, fill later" → coverage badges in picker. If "unfinished work, hide until ready" → drop from picker. If "community will translate" → name the owner + cadence. The current state is the worst of all three. The `docs/standards/localization.md` "Future (not today)" section explicitly says Spanish is not planned for 2026 — yet ES is in the picker. Resolve in the standard first; then either promote (with content) or pare back. + +### 8.3 `data_audit/` + `cli/` package splits (Theme γ) + +Both modules clear architecture standard's split threshold by significant margins. Plan a single sprint that splits both — they are mechanically similar. **Decision needed:** v0.5.x or v0.6.0? The public API contract is the load-bearing constraint; re-export from `__init__.py` keeps imports stable. The risk is multi-day, multi-PR work that will produce some merge-conflict pain on parallel PRs. **Recommended: target v0.5.3** (after the `--workers` flag in v0.5.2) so the perf work has the clean surface to land against. + +### 8.4 Pydantic `description=` migration with CI guard (Theme δ) + +3-5 days for the backfill across ~170 fields. The CI guard (diff `model_fields[name].description` vs configuration.md table cells) is the load-bearing piece — without it the migration regresses the moment a contributor forgets. **Decision needed:** Phase 12.6 docs-tooling sprint vs incremental. **Recommended: incremental** — add `description=` to every newly-added or newly-modified field starting now (PR template gate); backfill in dimension-scoped batches (one PR per top-level config section); CI guard lands with the first batch. Sequence note: this should land **before** any TR-mirror-config sweep so the TR markdown can be regenerated, not hand-translated. + +### 8.5 Webhook lifecycle vocabulary + Article 14 staging directory (Compliance Theme β) + +Two mid-effort items that together close the Article 14 effective-oversight gap: F-compliance-104 (`notify_reverted` + `notify_awaiting_approval` events) is half-day work. F-compliance-101 (real staging directory + `forgelm approve <run_id>` subcommand) is 2-3 days. **Decision needed:** ship both in v0.5.1 vs v0.5.1 (vocabulary) + v0.5.2 (staging). Coupled with the marketing claim on `docs/product_strategy.md:68`. **Recommended:** bundle both in v0.5.2 with the `forgelm verify-audit` subcommand (F-compliance-103) — these three together close the regulatory-diligence gap to ★★★★½. + +### 8.6 QMS bilingual policy (Compliance F-compliance-112) + +`docs/qms/` is currently EN-only (6 files: README + 5 SOPs). Localization standard says bilingual user-facing required. **Decision needed:** is QMS deliberately EN-only (standard QMS-template practice; provider customises per-deployer) or oversight? Both interpretations are defensible; the current ambiguity is the issue. **Recommended:** record the decision in `risks-and-decisions.md`. If EN-only is the policy, update `docs/standards/localization.md` "What's translated" table to formalise the QMS exception. If TR mirror is needed, schedule under `add-localization-task` skill. + +### 8.7 `--workers N` audit flag determinism contract (Compliance + Performance overlap) + +Per §5.1 above, the multi-worker variant must reproduce single-worker `lang_sample` byte-for-byte. **Decision needed:** confirm the determinism contract, then ship `--workers N` to v0.5.2 or v0.5.3 per master verdict. The `lang_sample` is "first 200" not "random sample" so byte-for-byte preservation across worker counts is non-trivial but feasible via deterministic chunk-index ordering. + +### 8.8 ISO 27001 / SOC 2 future Cloud SaaS path (Compliance open question 8) + +Open question in both prior + current rounds. The answer affects whether F-compliance-011-style explicit non-claims should appear in `safety_compliance.md`. **Decision needed:** does Phase 13 Pro CLI / Cloud SaaS pursue ISO 27001 conformance? If yes, this changes the security review's posture (currently "credible for trusted-internal pipelines + regulated-industry diligence"). If no, document the explicit non-claim. **Owner-decision** belonging with the `docs/marketing/` strategy (gitignored). + +--- + +## 9. Star ratings recap + +| Dimension | Rating (this round) | Rating (prior round) | Trend | Path to next half-star | +|---|---|---|---|---| +| Business | ★★★½ / 5 | ★★★½ / 5 | flat | Close site compliance.html + quickstart.html + PyPI status (1-3 hours) → ★★★★ | +| Code | ★★★½ / 5 | ★★★¾ / 5 | flat (-0.25, within rounding) | Pydantic Literal sweep + silent-fail narrowing + lazy torch import → ★★★★ | +| Compliance | ★★★★ / 5 | ★★★½ / 5 | up | Article 14 staging + verify-audit + operator identity → ★★★★½ | +| Documentation | ★★★½ / 5 | ★★★½ / 5 | flat | QMS rewrite + count drift + design-doc blockquotes → ★★★★ | +| Localization | ★★★ / 5 | (new axis) | new | F-loc-005/007/008 TR sections + F-loc-001 picker honesty → ★★★½ | +| Performance | ★★★★ / 5 | ★★★★ / 5 | flat | Lazy torch + safety batching + paragraph chunker → ★★★★½ | +| Security | ★★★★ / 5 | ★★★½ / 5 | up | M-201 (judge/synthetic SSRF) + M-202 (trainer trust_remote) + M-203 (GGUF SHA-256) → ★★★★½ | +| Testing & CI/CD | ★★★½ / 5 | (new axis) | new | F-test-001 cov gate + drop wizard omit + fixture consolidation → ★★★★ | + +**Overall this round:** ★★★¾ / 5 (weighted-by-dimension). **Prior round:** ★★★½ / 5. **Net: +0.25 stars** despite two new axes (Localization, Testing/CI) lowering the average against a smaller prior denominator. + +--- + +## 10. v0.5.0 release post-mortem + +### 10.1 Where v0.5.0 brought measurable improvement + +1. **Critical-only closure: 100 % across 10 prior Critical findings.** Theme A (version drift) and Theme B (audit-chain integrity) — the two highest-leverage cross-cuts — closed at 100 %. This is the biggest single trust signal in the cycle. The audit chain is now genuinely tamper-evident: flock + post-write hash advancement + genesis manifest + HMAC-when-keyed. +2. **Webhook security hardening** — Theme H closed at 100 %. SSRF guard + redirect refusal + explicit verify=True + timeout floor + secret-masked failure reasons. The webhook is now best-in-class for outbound HTTP among comparable OSS tools. +3. **Performance scaling cliffs at 100K-1M-row tier** — three of nine prior findings closed structurally: `agg.minhashes` double copy eliminated; bidirectional MinHash builds each LSH once; `text_lengths` reservoir sampling (Algorithm R, 100K cap, ~800 KB regardless of corpus size). Audit pipeline is materially faster for wedge corpora. +4. **Numpy simhash vectorisation + markdown chunker batch-encode** — F-performance-005 + F-performance-006 closed via vector dispatch; pure-Python fallback retained for fixture stability. +5. **Pydantic `Literal[]` discipline** — 6 of 12 enum-shaped fields converted; bespoke `_validate_trainer_type` runtime validator deleted in favour of schema-level enforcement. +6. **Documentation count refresh** — `architecture.md` / `architecture-tr.md` / CONTRIBUTING.md test-count + notebook-count corrected. Three of five count-drift surfaces fixed. +7. **Bilingual H2 parity** — 4 TR mirrors at H2 count parity (configuration-tr, usage-tr, architecture-tr, distributed_training-tr); architectural drift halved. +8. **CHANGELOG hygiene** — `[0.5.0] — 2026-04-30` properly dated section; consolidation of Phases 11/11.5/12/12.5 explicitly explained at L17-25; `[Unreleased]` correctly empty post-release pointing at Phase 14. +9. **Phase 12.5 surface** — `forgelm ingest --all-mask`, wizard "audit first" entry point, `forgelm audit --croissant`, Presidio ML-NER PII (with fail-loud language pre-flight) — all four backlog items shipped with dedicated regression coverage in `tests/test_phase12_5.py` (745 lines). +10. **Audit-chain integrity hardening landed end-to-end** with disciplined commit lineage (`6143321` → `8228faa` → `7db47bc`) — the most disciplined multi-commit hardening sequence in the project's history. + +### 10.2 Where v0.5.0 missed or regressed + +1. **Site / marketing surface** — `site/compliance.html` lists 5 artefact filenames the code does not produce (F-business-001 Critical); `site/quickstart.html` mock uses non-existent template names (F-business-002 Critical); GPU count off-by-2 (F-business-004); 7 of 10 notebooks still install via `git+https://...` (F-business-005). The site grew faster than the parity discipline that protects `docs/`. +2. **QMS body** — `docs/qms/sop_data_management.md:74-99` still asserts `v0.5.1+` / `v0.5.2` per-feature splits, contradicting the consolidation note in CHANGELOG (M-DOC-001 Critical). The QMS pack is the regulatory deliverable; the contradiction is the most embarrassing single doc artefact at HEAD. +3. **CI cov-fail-under** — `pyproject.toml`'s `fail_under = 40` is documented but `pytest-cov` does not auto-enforce it; CI command does not pass `--cov-fail-under=40` as a flag. Coverage can silently drop and CI stays green (F-test-001 Critical). +4. **Operator identity** — falls back to literal `"unknown"` in distroless containers despite the project's own standard explicitly forbidding it (F-compliance-102 Critical). +5. **`forgelm verify-audit`** — the HMAC-when-keyed write path landed (Theme B); the read path remains documentation only. External auditors cannot exercise the chain without the verifier subcommand (F-compliance-103 Critical). +6. **Article 14 staging directory** — gate fires AFTER model is written to `final_model/`; log message lies ("Model saved to staging" but path is final). F-compliance-101 Critical, carried over from prior round. +7. **Lazy `torch` import** — F-performance-003 still open in trainer.py:8-10 (~30 min fix). Quick win that fell through the otherwise-thorough sweep. +8. **`runtime_smoke.py`** — violates 4 standards at once (no tmp_path; cleanup commented; real HF fetch; Turkish comments + emoji). Fresh finding from the new Testing/CI axis. +9. **DE/FR/ES/ZH user-manual content** — 0/56 pages while site picker advertises 6 languages. Fresh finding from the new Localization axis. +10. **`forgelm/data_audit.py` cohesion** — grew from ~2.6K to 3098 lines via Phase 12.5 additive features; `forgelm/cli.py` grew from ~1.5K to 1756 lines. Module cohesion regressing for a third consecutive round. +11. **TR-mirror H3/H4 drift** — `configuration-tr.md` H4 14 vs 6 (12 missing fields in `training` block alone); `distributed_training-tr.md` 3 missing H3 sections; `ingestion-tr.md` 2 missing H3 sections (Phase 12 surface). Bilingual H2 CI check holds; H3/H4 drift hides below the check's resolution. +12. **`description=` migration unstarted** — 1 of 170 fields. Master §7 strategic item #2 aged a round. +13. **`wizard.py` print-vs-logger conflict** — chat.py landed `_print` indirection; wizard didn't follow → standards violation + internal consistency problem. + +### 10.3 Net trend + +**Improving:** Integrity (audit chain, version drift, dataset fingerprint TOCTOU). Security (SSRF, redirect refusal, secret masking). Compliance (Theme B closure earned the fourth star). Performance (3 of 3 Critical perf items closed). + +**Stable:** Code (standards-posture work deferred). Documentation (mirror sweep landed but new drift opened on QMS). Business (Theme A/B closures balance new site-drift findings). + +**Regressing:** Module cohesion (data_audit.py grew 500 lines past threshold). Site claim/code alignment (4 new findings on a surface that had fewer in prior round). Testing/CI mechanical gates (cov-fail-under not enforced — gap visible only with the new axis added). + +**Bottom line:** v0.5.0 is a *substantively better* release than v0.4.5 along all six axes the prior review tracked. Two new axes (Localization, Testing/CI) surfaced gaps that pre-dated v0.5.0 but now have visibility. The release engineering itself was disciplined; the "what landed" delivers; the "what we said about it" surface (site, QMS, CHANGELOG roadmap claims) is where the residual friction lives. + +--- + +## 11. Appendix — Per-dimension report links + +- [Business]( ./business-review-202604300906.md ) +- [Code]( ./code-review-202604300906.md ) +- [Compliance]( ./compliance-review-202604300906.md ) +- [Documentation]( ./documentation-review-202604300906.md ) +- [Localization]( ./localization-review-202604300906.md ) +- [Performance]( ./performance-review-202604300906.md ) +- [Security]( ./security-review-202604300906.md ) +- [Testing & CI/CD]( ./testing-cicd-review-202604300906.md ) +- [Regression Check]( ./regression-check-202604300906.md ) + +### A.1 Methodology notes + +- **Synthesis source:** the 9 sub-reports listed above were read line-by-line; cross-cuts identified where ≥ 2 sub-reports surfaced the same root cause; severity preserved from source-report verdicts unless reconciled in §5 conflicts. +- **Severity matrix:** 8 dimensions × 4 severities = 32 cells; total findings 175. Cross-cut findings counted once in primary dimension (e.g., F-business-007 = M-DOC-010 counted in Business only). +- **Star rating reconciliation:** sub-report ratings accepted unchanged for Business (★★★½), Compliance (★★★★), Documentation (★★★½), Localization (★★★), Performance (★★★★), Security (★★★★), Testing/CI (★★★½). Code adjusted from the sub-report's ★★★½ — sub-report executive summary explicitly says "★★★½ / 5 (unchanged from prior round)"; prior round was ★★★¾ — kept at ★★★½ within rounding tolerance. +- **Closure rate from regression-check:** 38 Closed / 9 Partially Closed / 10 Open / 0 Reopened / 0 Not Verifiable. Critical-only closure: 10 / 10 = 100 %. +- **Prior round comparison surface:** master-review-opus-202604281313.md §1 severity table (rows 1-54 in scope; rows 55-60 are Minor/Nit roll-ups, out of scope per the regression-check brief). +- **No new findings introduced by this master report:** all severity-tagged items trace to source sub-reports. + +### A.2 Sprint-1 PR roadmap (recommended sequencing) + +| PR | Scope | Effort | Closes | +|---|---|---|---| +| PR-1 | Site compliance.html + quickstart.html artefact rewrite + 6 locales | 2 hours | 2 Critical (F-business-001, F-business-002) | +| PR-2 | PyPI publish status sweep + roadmap.md/-tr.md flips | 30 min | 1 Critical (F-business-003) | +| PR-3 | `docs/qms/sop_data_management.md` body rewrite | 30 min | 1 Critical (M-DOC-001) | +| PR-4 | CI cov-fail-under enforcement + standards/testing.md sync | 10 min | 1 Critical (F-test-001) | +| PR-5 | Operator identity fallback fix + CI smoke test | 1 hour | 1 Critical (F-compliance-102) | +| PR-6 | Lazy torch import in trainer.py + model.py + regression test | 1 hour | 1 Major (F-performance-101) | +| PR-7 | Notebook install pin sweep (7 notebooks) + nightly grep step | 1 hour | 1 Major (F-business-005) | +| PR-8 | Pydantic Literal sweep (6 fields) + drop redundant runtime validators | 1 hour | 1 Major + 1 Major (F-code-101 + F-compliance-105) | +| PR-9 | Quick wins bundle (10-15 small fixes) | 2 hours | ~10 Minor + Nit | +| PR-10 | Webhook lifecycle vocabulary (notify_reverted + notify_awaiting_approval) | half-day | 1 Major (F-compliance-104) | +| PR-11 | `forgelm verify-audit` subcommand + library function | half-day | 1 Critical (F-compliance-103) | +| PR-12 | Wizard `_print` indirection + drop coverage omit | half-day | 1 Major (F-test-003 + F-code-105) | +| PR-13 | `--data-audit` deprecation discipline (DeprecationWarning + tracking issue) | 30 min | 1 Major (F-code-107) | +| PR-14 | Fixture consolidation (`tests/_helpers/`) + remove 4 lokal copies | 4 hours | 1 Major (F-test-004 + F-test-005) | +| PR-15 | safe_post helper extraction (judge.py / synthetic.py / webhook.py) | half-day | 1 Major (M-201) | + +**PR-1 through PR-9: ~1 day total of mechanical work.** +**PR-10 through PR-15: ~3 days of medium-effort hardening.** +**Sprint total: ~4-5 days; closes 8 Critical + 9 Major + ~10 Minor/Nit.** + +After this sprint, the rating projection lifts each affected dimension by half a star: Business ★★★★, Compliance ★★★★½, Documentation ★★★★, Code ★★★★, Performance ★★★★½, Security ★★★★½, Testing/CI ★★★★. Localization remains a separate strategic decision per §8.2. + +### A.3 Dimensional findings density vs prior round + +| Dimension | Prior round Critical | This round Critical | Prior Major | This Major | Net surface delta | +|---|---|---|---|---|---| +| Business | 3 | 3 | ~12 | 7 | -5 (closures dominated; 2 carry-over Critical resolved, 3 new Critical) | +| Code | 4 | 0 | ~14 | 9 | -9 (Theme A + B + D + F closures; Major bucket halved) | +| Compliance | 2 | 3 | ~10 | 11 | +2 (Theme B closed; new gaps surfaced — Article 14, verifier, retention) | +| Documentation | 1 | 1 | ~13 | 11 | -2 (mirror sweep landed; QMS body emerged as new Critical) | +| Localization | n/a | 0 | n/a | 8 | new axis — coverage gap was always there; now visible | +| Performance | 3 | 0 | 6 | 6 | -3 Critical (closed) — Major count flat with new findings replacing old | +| Security | 2 | 0 | 7 | 5 | -4 (webhook + audit + fingerprint TOCTOU + sanitisation closures) | +| Testing/CI | n/a | 1 | n/a | 10 | new axis — gap was always there; now visible | + +**Direction read:** four dimensions show net surface reduction (Business, Code, Documentation, Security). Two dimensions show flat surface (Compliance, Performance) because every prior closure was matched by a new finding of comparable severity emerging from the v0.5.0 surface itself. Two new axes added — both surface gaps that pre-dated v0.5.0 but were not in scope for the prior round. + +### A.4 Effort budget for full closure (all severities) + +| Tier | Findings count | Estimated effort | +|---|---|---| +| Critical (8) | 8 | 1.5 days | +| Major (67) | 67 | 12-15 days for full closure; 5 days for Top-15 priority queue | +| Minor (60) | 60 | 4-6 days drive-by cleanup; most fold into adjacent PRs | +| Nit (40) | 40 | 1-2 days mass-edit | + +**All-severity closure target: ~3-4 weeks of focused work.** Realistic next-sprint target (Sprint-1 PR roadmap above): **1 week.** Lifts overall to ★★★★ within that window. + +### A.5 Findings exclusively from new dimensions + +Adding the Localization and Testing/CI axes for this round surfaced 18 + 21 = 39 findings that did not exist as tracked items in the prior round. Of these: + +- **Localization:** 0 Critical, 8 Major, 6 Minor, 4 Nit. Most-load-bearing: F-loc-001 (DE/FR/ES/ZH 0/56 page coverage with active picker promise) and F-loc-005/006/007/008 (TR mirror H2 drift on 4 user-manual pages). The chrome-translation discipline (286 keys × 6 languages, native quality) is exemplary; the user-manual-content discipline did not follow. +- **Testing/CI:** 1 Critical, 10 Major, 6 Minor, 4 Nit. Most-load-bearing: F-test-001 (cov-fail-under not actually enforced in CI), F-test-002 (`runtime_smoke.py` violates 4 standards at once), F-test-003 (wizard.py coverage omit blocks visibility on 951 LOC + 3 test modules), F-test-004 + F-test-005 (fixture fragmentation + anti-idiom `from conftest import`). + +These two axes are net-new visibility, not net-new bug surface. The sub-reports' rating ★★★ (Localization) and ★★★½ (Testing/CI) are honest given the gaps; both can lift quickly with disciplined sprints (per §A.2). + +### A.6 Closure-vs-discovery balance + +- **Closed in this cycle: 38 prior Major/Critical findings** (66.7 % outright). +- **Newly discovered Critical: 8** (3 Business, 3 Compliance, 1 Documentation, 1 Testing/CI). Of the 8: 4 are direct consequences of v0.5.0 ship (site/QMS/CHANGELOG drift); 3 are pre-existing gaps now visible (operator identity, verify-audit subcommand, Article 14 ordering — all flagged in prior round but partially-closed or unaddressed); 1 is the new Testing/CI axis (cov gate). +- **Newly discovered Major: 67** across all 8 dimensions. Of these: ~25 are carry-overs renamed (e.g., F-code-014 → F-code-101, F-business-003 carry-overs, prior round's F-performance-003 surfaces as F-performance-101); ~25 are fresh discoveries from the new axes (Localization 8 + Testing/CI 10) and Phase 12.5 surface; ~17 are net-new from a cleaner-eye second-pass review. + +**Closure rate analysis:** 38 closed / (38 + 8 newly discovered Critical) = 82.6 % effective Critical-tier net improvement. The rating uptick on Compliance (★★★½ → ★★★★) and Security (★★★½ → ★★★★) reflects this net improvement; the flat ratings on Code / Documentation / Business reflect the offsetting new findings. + +### A.7 Trend signals worth watching for the next master review + +1. **Will site/-against-code parity discipline land?** §8.1. Without it, Theme α will recur with each subsequent release. +2. **Will the wizard print/coverage decision land?** Coupled F-code-105 + F-test-003 are the longest-running open items; both wait on the same wizard refactor sprint. +3. **Will Pydantic `description=` migration start?** Without the CI guard the migration will leak back the moment any new field is added without it. +4. **Will Article 14 staging directory ship?** F-compliance-101 is the highest-severity carry-over from the prior round still open. +5. **Will DE/FR/ES/ZH user-manual content land or the picker pare back?** F-loc-001 strategic call belongs with the project owner. +6. **Will `data_audit/` + `cli/` package splits happen before v0.6.0?** Module cohesion regressing for a third consecutive round; v0.7.0 risk profile worsens otherwise. +7. **Will `--workers N` audit flag preserve `lang_sample` determinism?** §5.1 conflict resolution has the right shape; implementation needs the contract pinned in tests. +8. **Will operator identity raise instead of falling back to `"unknown"`?** F-compliance-102 is a 30-min fix that materially strengthens Article 12 forensic value. + +If half of these eight signals close by the next master review, the project trajectory clears the EU AI Act August 2, 2026 enforcement bar with material runway. The audit chain is now tamper-evident; the residual gaps are individually narrow; the cumulative risk profile is materially better than at the prior round. + +### A.8 Reading order recommendations + +Different readers should enter at different points: + +- **5-minute exec read:** §0 Executive summary + §9 Star ratings recap + §4 Top-10 priority items 1-5. Frames the project health and the immediate sprint target. +- **20-minute owner read:** §0 + §1 (regression posture) + §3 (cross-cut themes) + §5 (conflicts) + §8 (strategic items). Frames the decisions blocking the next sprint. +- **45-minute reviewer read:** add §2 (severity-aggregated table) + §6 (standards posture) + §10 (post-mortem). Frames the cumulative posture of the codebase against the standards that are supposed to govern it. +- **Full read (~90 minutes):** the appendix + sub-reports linked in §11 give the line-level evidence behind every Major+ finding. + +### A.9 What this review intentionally did not do + +- **Did not introduce new findings beyond synthesis.** All 175 entries trace to source sub-reports. +- **Did not re-rate prior-round closed findings.** Closures recorded by the regression-check report were accepted at face value. +- **Did not perform live testing** (no `pytest` run, no `forgelm --dry-run` execution). Static review only against commit `6b515ed`. +- **Did not open `docs/marketing/` (gitignored)** — strategy material is treated as referenced source-of-truth where the sub-reports cited it, not directly read here. +- **Did not validate external dependencies** (e.g., `pip-audit` on resolved tree). Out of scope per security review's open question 6. +- **Did not run the bilingual H2/H3/H4 parity check** as a script. Counts cited in §3 are from the localization + documentation sub-reports. + +### A.10 Open questions carried forward to the next master review + +These did not reach a decision in this round. The next reviewer will have to surface them again or document the resolution: + +1. (Documentation Q1) Should `data_preparation*.md` exist as a 35-line micro-doc, be rewritten to ~120 lines, or fold into `alignment.md`? +2. (Compliance Q4) What is ForgeLM's official position on GDPR right-to-erasure for trained model weights? +3. (Compliance Q5) Are TR mirrors of `docs/qms/` blocked on translation bandwidth or a deliberate "QMS is provider-customised" decision? +4. (Compliance Q8) Does the project plan to pursue ISO 27001 / SOC 2 Type II for any future Cloud SaaS path? +5. (Performance Q1) Is `forgelm.ForgeTrainer` (public API at `__init__.py:49-50`) a supported integration surface or "use the CLI, don't import the package"? +6. (Performance Q3) Should `evaluation.safety.batch_size` auto-resolve from VRAM or be a pure user knob? +7. (Performance Q4) Should `synthetic.api_concurrency` use `tokens_per_min` / `requests_per_min` / `concurrent` rate-limit shape, or a single conservative knob? +8. (Code Q1) `data_audit/` package split — v0.5.x or v0.6.0 work? +9. (Code Q4) Should `error-handling.md` add a named "best-effort artefact" category that sanctions broad catches with `# noqa: BLE001 — best-effort: <reason>` comment? +10. (Localization Q5) Does `docs/standards/localization.md` need a "user manual" carve-out separate from the `docs/reference/*` EN+TR pairing? +11. (Testing Q1) Does pytest-cov auto-enforce `[tool.coverage.report].fail_under`? Confirm via live run; if yes, F-test-001 demotes to Major. +12. (Security Q4) Single project-wide `forgelm.security.SecurityConfig` block gating symlink-following + private-IP webhook destinations + `trust_remote_code` under one `security_posture: strict` knob — pursue? + +The next master review should open with the resolution status of these 12 questions; whichever remain unresolved should be elevated to strategic items with named owners. + +### A.11 Summary numbers (one-glance recap) + +- **Critical-only closure:** 10 / 10 = 100 % vs prior round. +- **Major closure:** 28 / 47 = 59.6 % outright; +9 Partially Closed = 78.7 % combined. +- **New Critical findings:** 8 (3 Business, 3 Compliance, 1 Documentation, 1 Testing/CI). +- **New Major findings:** 67 (across 8 dimensions; ~25 are renamed carry-overs). +- **Total findings synthesized:** 175 (8 Critical + 67 Major + 60 Minor + 40 Nit). +- **Sub-reports synthesized:** 9 (5402 source lines). +- **Star rating overall:** ★★★¾ / 5 (up +0.25 from prior ★★★½). +- **Themes fully closed:** A (version drift), B (audit chain), D (silent-fail), F (Pydantic Literal sub-set), H (webhook). +- **Themes partially closed:** C (doc drift), E (perf cliffs), G (standards drift). +- **Themes regressing:** module cohesion (Theme γ), site claim/code (new Theme α), CI gate enforcement (new Theme ι). +- **Sprint-1 effort to lift overall to ★★★★:** ~4-5 days; closes 8 Critical + 9 Major + ~10 Minor/Nit (per §A.2). + +--- + +*End of master review — 202604300906.* +*Verified against synthesized sub-reports at commit 6b515ed912f8f22304194c1b3f55ed07a26f519c.* diff --git a/docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md b/docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md new file mode 100644 index 00000000..c76dfbd9 --- /dev/null +++ b/docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md @@ -0,0 +1,623 @@ +# Split Design — `forgelm/data_audit.py` and `forgelm/cli.py` + +**Created:** 2026-04-30 +**Commit:** `6b515ed912f8f22304194c1b3f55ed07a26f519c` +**Branch:** `main` +**Companion to:** [`master-review-opus-202604300906.md`](./master-review-opus-202604300906.md) (§3 Theme γ, §5.5, §8.3) and [`closure-plan-202604300906.md`](./closure-plan-202604300906.md) +**Author:** Plan agent (read-only design pass; full file reads of both modules + every cross-import in `tests/` and `forgelm/`) + +--- + +## 0. Scope, methodology, ground truth + +**Files surveyed (full reads):** +- `forgelm/data_audit.py` (3098 lines, end-to-end) +- `forgelm/cli.py` (1756 lines, end-to-end) +- `pyproject.toml` (entry point block) +- `docs/standards/architecture.md` (cohesion ceiling and split rule) +- All consumers under `tests/` and within `forgelm/` (cross-import grep) + +**Architecture standard, line 71** (verbatim): + +> If a module grows past ~1000 lines and has cohesive subsections, split into `module_name/` package, but keep the public API at `forgelm.module_name.X` so imports don't break. + +`data_audit.py` (3098) is 3.10× the ceiling. `cli.py` (1756) is 1.76× the ceiling. Both qualify; the split mechanism is also pre-authorized by the standard. + +**Public API contract (load-bearing) — actual external import points found in the repo:** + +`forgelm.data_audit`: +- Tests import: `DEFAULT_NEAR_DUP_HAMMING`, `PII_TYPES`, `AuditReport`, `_is_credit_card`, `_is_tr_id`, `audit_dataset`, `compute_simhash`, `detect_pii`, `find_near_duplicates`, `hamming_distance`, `mask_pii`, `summarize_report`, `DEDUP_METHODS`, `DEFAULT_MINHASH_JACCARD`, `SECRET_TYPES`, `_row_quality_flags`, `detect_secrets`, `mask_secrets`, `compute_minhash`, `find_near_duplicates_minhash`, `_find_near_duplicates_brute`, `_count_leaked_rows`, `_read_jsonl_split`, `_token_digest`, `_strip_code_fences`, `_require_presidio`, `PII_ML_SEVERITY`, `_PRESIDIO_ENTITY_MAP`, `_get_presidio_analyzer`, `detect_pii_ml`, `_HAS_PRESIDIO` (patched), `_PresidioAnalyzer` (patched). +- In-package callers: `forgelm/ingestion.py` (`mask_pii`, `mask_secrets`), `forgelm/wizard.py` (`audit_dataset`, `summarize_report`). + +`forgelm.cli`: +- Tests import: `EXIT_SUCCESS`, `EXIT_CONFIG_ERROR`, `EXIT_TRAINING_ERROR`, `EXIT_EVAL_FAILURE`, `EXIT_AWAITING_APPROVAL`, `_get_version`, `_resolve_resume_checkpoint`, `_run_dry_run`, `_setup_logging`, `_run_compliance_export`, `_run_fit_check`, `_run_data_audit`, `_output_result`, `_build_quickstart_inherited_flags`, `parse_args`, `main`. +- Public entry point: `forgelm.cli:main` registered in `pyproject.toml [project.scripts]`. + +**Crucial finding:** Even underscore-prefixed helpers (`_run_data_audit`, `_token_digest`, `_strip_code_fences`, `_HAS_PRESIDIO`, `_PresidioAnalyzer`, `_get_presidio_analyzer`, `_PRESIDIO_ENTITY_MAP`, `_count_leaked_rows`, `_read_jsonl_split`, `_find_near_duplicates_brute`, `_row_quality_flags`, `_is_credit_card`, `_is_tr_id`) are reached from tests via `from forgelm.data_audit import …` and are also patched via `patch("forgelm.data_audit._HAS_PRESIDIO", …)`. **Re-export at the package level is not optional — it is mandatory for these private helpers as well, because `monkeypatch`/`unittest.mock.patch` requires the name to resolve at exactly that dotted path.** This pins the design: every symbol the tests reach must be addressable as `forgelm.data_audit.<name>` post-split, even if it physically lives in a sub-module. + +--- + +## 1. `data_audit.py` — Detailed analysis + +### 1.a Concern inventory + +Line ranges are derived from the actual file just read. + +| # | Concern | Lines | Public symbols | Internal helpers | Imports / coupling | +|---|---|---|---|---|---| +| 1 | Optional-deps sentinel block (xxhash, numpy, datasketch, presidio) | 45–112 | — | `_HAS_XXHASH`, `_xxhash`, `_HAS_NUMPY`, `_np`, `_HAS_DATASKETCH`, `_MinHash`, `_MinHashLSH`, `_HAS_PRESIDIO`, `_PresidioAnalyzer` | stdlib only; tests patch `_HAS_PRESIDIO`, `_PresidioAnalyzer` | +| 2 | PII regex types + severity | 119–203 | `PII_TYPES`, `PII_SEVERITY`, `PII_SEVERITY_ORDER`, `PII_ML_SEVERITY`, `PII_ML_TYPES`, `DEFAULT_NEAR_DUP_HAMMING`, `DEFAULT_MINHASH_JACCARD`, `DEFAULT_MINHASH_NUM_PERM`, `DEDUP_METHODS` | `_TEXT_COLUMNS` | none | +| 3 | `AuditReport` dataclass | 206–236 | `AuditReport` | — | `dataclass`, `field` only | +| 4 | PII regex detector + masker | 239–392 | `detect_pii`, `mask_pii` | `_PII_PATTERNS`, `_is_credit_card`, `_is_tr_id`, `_validate_match` | uses Concern 2 (`PII_TYPES` indirectly) | +| 5 | Presidio ML adapter | 395–639 | `detect_pii_ml` | `_SPACY_MODEL_FOR_LANGUAGE`, `_get_presidio_analyzer` (lru_cache), `_PRESIDIO_INSTALL_HINT`, `_require_presidio`, `_PRESIDIO_ENTITY_MAP` | needs Concern 1 (`_HAS_PRESIDIO`, `_PresidioAnalyzer`); needs Concern 2 (`PII_ML_TYPES`) | +| 6 | Tokenizer + simhash core | 642–759 | `compute_simhash`, `hamming_distance` | `_TOKEN_PATTERN`, `_tokenize`, `_token_digest` (lru_cache), `_compute_simhash_numpy` | needs Concern 1 (`_HAS_XXHASH`, `_xxhash`, `_HAS_NUMPY`, `_np`) | +| 7 | Simhash LSH banding + near-dup pair finder | 762–890, 1957–2032 | `find_near_duplicates`, `_count_leaked_rows` | `_LSH_MIN_BAND_BITS`, `_band_count_for_threshold`, `_split_into_bands`, `_find_near_duplicates_brute`, `_bucket_pairs_within_threshold`, `_build_band_index`, `_count_leaked_brute`, `_source_row_leaks` | needs Concern 6 (`hamming_distance`) | +| 8 | MinHash LSH | 893–1114 | `compute_minhash`, `find_near_duplicates_minhash` | `_require_datasketch`, `_build_minhash_lsh`, `_emit_minhash_pair`, `_count_leaked_rows_minhash`, `_count_leaks_against_index`, `_count_leaked_rows_minhash_bidirectional` | needs Concern 1 (`_HAS_DATASKETCH`, `_MinHash`, `_MinHashLSH`); needs Concern 6 (`_tokenize`) | +| 9 | Language detection + length stats helpers | 1117–1147 | — | `_detect_language`, `_length_stats` | optional `langdetect` | +| 10 | Streaming length digest (reservoir) | 1150–1212 | — | `_LENGTH_RESERVOIR_SIZE`, `_LengthDigest` (class) | none | +| 11 | Secret regex detector + masker | 1215–1336 | `SECRET_TYPES`, `detect_secrets`, `mask_secrets` | `_SECRET_PATTERNS` | none | +| 12 | Quality filter (Gopher/C4 heuristics) | 1339–1551 | — | `_QUALITY_CHECKS`, `_WORD_PATTERN`, `_PUNCT_END_PATTERN`, `_is_code_fence_open`, `_is_code_fence_close`, `_strip_code_fences`, `_check_low_alpha_ratio`, `_check_low_punct_endings`, `_check_abnormal_mean_word_length`, `_check_short_paragraphs`, `_check_repeated_lines`, `_row_quality_flags` | none | +| 13 | Streaming JSONL reader + helpers | 1553–1622 | — | `_extract_text_payload`, `_read_jsonl_split` (generator), `_PROGRESS_INTERVAL`, `_LANG_SAMPLE_SIZE` | needs Concern 2 (`_TEXT_COLUMNS`) | +| 14 | Per-split streaming aggregator | 1624–1814 | — | `_StreamingAggregator` (dataclass), `_record_schema_for_dict`, `_record_dedup_signature`, `_record_dedup_sentinel`, `_record_text_metrics`, `_ingest_row`, `_populate_schema_block`, `_populate_optional_findings`, `_within_split_pairs` | needs Concerns 4, 5, 6, 8, 10, 11, 12 (heavy hub) | +| 15 | Per-split → info dict + lang top-3 | 1817–1955 | — | `_aggregator_to_info`, `_compute_top_languages`, `_audit_split` | needs Concerns 9, 10, 14 | +| 16 | Cross-split overlap | 2068–2122, 2035–2065 | — | `_cross_split_overlap`, `_pair_leak_payload` | needs Concerns 7, 8 | +| 17 | Split discovery / aliases | 2125–2210 | — | `_SPLIT_ALIASES`, `_scan_canonical_split_files`, `_scan_pseudo_split_files`, `_resolve_directory_splits`, `_resolve_input` | stdlib `pathlib` | +| 18 | Per-split processor + outcome | 2213–2309 | — | `_SplitOutcome` (dataclass), `_process_split` | needs Concern 15 | +| 19 | PII severity + summary notes | 2312–2406 | — | `_build_pii_severity`, `_pii_summary_notes`, `_cross_split_leak_notes` | needs Concern 2 | +| 20 | Secrets/quality summary notes | 2409–2502 | — | `_secrets_summary_notes`, `_quality_summary_notes`, `_fold_outcome_into_summary`, `_build_quality_summary` | — | +| 21 | Croissant 1.0 emitter | 2505–2693 | — | `_JSONLD_TYPE_KEY`, `_JSONLD_ID_KEY`, `_CROISSANT_CONTEXT`, `_build_croissant_metadata` | stdlib | +| 22 | `audit_dataset` orchestrator + atomic JSON writer | 2696–2960 | `audit_dataset` | `_build_near_duplicate_summary`, `_atomic_write_json` | hub of Concerns 5, 8, 17, 18, 19, 20, 21 | +| 23 | `summarize_report` text renderer | 2963–3098 | `summarize_report` | `_split_has_findings`, `_render_split_block`, `_render_pii_severity` | needs `AuditReport` only | + +Total: 23 concerns. Master review's expected list (PII regex / Presidio / secrets / simhash / MinHash / streaming / quality / Croissant / length digest / lang sample) is fully covered; the file additionally hosts the orchestrator and the report renderer, both of which need their own home. + +### 1.b Public API surface (full signatures) + +These signatures must remain bit-identical at `forgelm.data_audit.<name>` after the split: + +```python +# Constants +PII_TYPES: Tuple[str, ...] +PII_SEVERITY: Dict[str, str] +PII_SEVERITY_ORDER: Tuple[str, ...] +PII_ML_SEVERITY: Dict[str, str] +PII_ML_TYPES: Tuple[str, ...] +DEFAULT_NEAR_DUP_HAMMING: int +DEFAULT_MINHASH_JACCARD: float +DEFAULT_MINHASH_NUM_PERM: int +DEDUP_METHODS: Tuple[str, ...] +SECRET_TYPES: Tuple[str, ...] + +# Dataclass +class AuditReport: + generated_at: str + source_path: str + source_input: str + total_samples: int + splits: Dict[str, Dict[str, Any]] = field(default_factory=dict) + cross_split_overlap: Dict[str, Any] = field(default_factory=dict) + pii_summary: Dict[str, int] = field(default_factory=dict) + pii_severity: Dict[str, Any] = field(default_factory=dict) + near_duplicate_summary: Dict[str, Any] = field(default_factory=dict) + secrets_summary: Dict[str, int] = field(default_factory=dict) + quality_summary: Dict[str, Any] = field(default_factory=dict) + croissant: Dict[str, Any] = field(default_factory=dict) + notes: List[str] = field(default_factory=list) + +# Functions +def detect_pii(text: Any) -> Dict[str, int]: ... +def mask_pii(text: Any, replacement: str = "[REDACTED]", *, return_counts: bool = False) -> Any: ... +def detect_pii_ml(text: Any, *, language: str = "en") -> Dict[str, int]: ... +def detect_secrets(text: Any) -> Dict[str, int]: ... +def mask_secrets(text: Any, replacement: str = "[REDACTED-SECRET]", *, return_counts: bool = False) -> Any: ... +def compute_simhash(text: str, *, bits: int = 64) -> int: ... +def hamming_distance(a: int, b: int) -> int: ... +def find_near_duplicates(fingerprints: List[int], *, threshold: int = DEFAULT_NEAR_DUP_HAMMING, bits: int = 64) -> List[Tuple[int, int, int]]: ... +def compute_minhash(text: str, *, num_perm: int = DEFAULT_MINHASH_NUM_PERM) -> Optional[Any]: ... +def find_near_duplicates_minhash(minhashes: List[Optional[Any]], *, jaccard_threshold: float = DEFAULT_MINHASH_JACCARD, num_perm: int = DEFAULT_MINHASH_NUM_PERM) -> List[Tuple[int, int, float]]: ... +def audit_dataset(source: str, *, output_dir: Optional[str] = None, near_dup_threshold: int = DEFAULT_NEAR_DUP_HAMMING, dedup_method: str = "simhash", minhash_jaccard: float = DEFAULT_MINHASH_JACCARD, minhash_num_perm: int = DEFAULT_MINHASH_NUM_PERM, enable_quality_filter: bool = False, enable_pii_ml: bool = False, pii_ml_language: str = "en", emit_croissant: bool = False) -> AuditReport: ... +def summarize_report(report: AuditReport, *, verbose: bool = False) -> str: ... +``` + +**Test-touched private symbols that MUST also be re-exported** (because tests do `from forgelm.data_audit import _name` or `patch("forgelm.data_audit._name", …)`): + +`_is_credit_card`, `_is_tr_id`, `_find_near_duplicates_brute`, `_count_leaked_rows`, `_read_jsonl_split`, `_token_digest`, `_strip_code_fences`, `_row_quality_flags`, `_require_presidio`, `_PRESIDIO_ENTITY_MAP`, `_get_presidio_analyzer`, `_HAS_PRESIDIO`, `_PresidioAnalyzer`. + +### 1.c Internal cross-coupling + +Coupling counted as "concern A imports/calls a symbol that lives in concern B". + +**Lowest-coupling (split first):** +- Concern 11 (secrets) — depends on nothing inside the module. +- Concern 4 (PII regex) — depends on nothing inside the module. +- Concern 21 (Croissant) — depends on nothing inside the module. +- Concern 10 (length digest) — depends on nothing inside the module. +- Concern 9 (lang) — depends on nothing inside the module. +- Concern 13 (JSONL reader) — depends only on `_TEXT_COLUMNS` constant. + +**Highest-coupling (split last / via re-exports only):** +- Concern 22 (`audit_dataset` orchestrator) — depends on 9 other concerns. +- Concern 14 (`_StreamingAggregator`) — depends on 7 other concerns. +- Concern 6 (simhash) — depends on optional-deps sentinel and is itself a dependency of Concerns 7, 8, 14. + +### 1.d Proposed package layout + +```text +forgelm/data_audit/ + __init__.py # facade — re-exports the entire public + test-patched surface + _optional.py # optional-deps sentinels (xxhash / numpy / datasketch / presidio handles) + _types.py # AuditReport, constants, severity tables, _TEXT_COLUMNS + _pii_regex.py # _PII_PATTERNS, _is_credit_card, _is_tr_id, _validate_match, + # detect_pii, mask_pii + _pii_ml.py # _SPACY_MODEL_FOR_LANGUAGE, _PRESIDIO_INSTALL_HINT, + # _PRESIDIO_ENTITY_MAP, _get_presidio_analyzer (lru_cache), + # _require_presidio, detect_pii_ml + _secrets.py # _SECRET_PATTERNS, detect_secrets, mask_secrets + _simhash.py # _TOKEN_PATTERN, _tokenize, _token_digest (lru_cache), + # _compute_simhash_numpy, compute_simhash, hamming_distance, + # _band_count_for_threshold, _split_into_bands, + # _find_near_duplicates_brute, _bucket_pairs_within_threshold, + # _build_band_index, _LSH_MIN_BAND_BITS, find_near_duplicates, + # _count_leaked_brute, _source_row_leaks, _count_leaked_rows + _minhash.py # _require_datasketch, compute_minhash, _build_minhash_lsh, + # _emit_minhash_pair, find_near_duplicates_minhash, + # _count_leaked_rows_minhash, _count_leaks_against_index, + # _count_leaked_rows_minhash_bidirectional + _quality.py # _QUALITY_CHECKS, _WORD_PATTERN, _PUNCT_END_PATTERN, + # _is_code_fence_open/_close, _strip_code_fences, + # _check_*, _row_quality_flags + _streaming.py # _LengthDigest, _LENGTH_RESERVOIR_SIZE, _read_jsonl_split, + # _extract_text_payload, _detect_language, _length_stats, + # _PROGRESS_INTERVAL, _LANG_SAMPLE_SIZE, _compute_top_languages + _aggregator.py # _StreamingAggregator + all _record_*/_ingest_row/_populate_*/ + # _within_split_pairs/_aggregator_to_info/_audit_split + _splits.py # _SPLIT_ALIASES, _scan_canonical_split_files, + # _scan_pseudo_split_files, _resolve_directory_splits, + # _resolve_input, _SplitOutcome, _process_split + _summary.py # _build_pii_severity, _pii_summary_notes, _cross_split_leak_notes, + # _secrets_summary_notes, _quality_summary_notes, + # _fold_outcome_into_summary, _build_quality_summary, + # _build_near_duplicate_summary, _split_has_findings, + # _render_split_block, _render_pii_severity, summarize_report + _croissant.py # _JSONLD_*, _CROISSANT_CONTEXT, _build_croissant_metadata + _orchestrator.py # _cross_split_overlap, _pair_leak_payload, _atomic_write_json, + # audit_dataset +``` + +| File | Target lines | Concern coverage | Public? | +|---|---|---|---| +| `__init__.py` | 80–120 | Re-export facade only | yes | +| `_optional.py` | 60 | C1 | private (test-patched) | +| `_types.py` | 80 | C2 + C3 | partly public (`AuditReport`, constants) | +| `_pii_regex.py` | 160 | C4 | partly public (`detect_pii`, `mask_pii`) | +| `_pii_ml.py` | 240 | C5 | partly public (`detect_pii_ml`) | +| `_secrets.py` | 130 | C11 | partly public (`SECRET_TYPES`, `detect_secrets`, `mask_secrets`) | +| `_simhash.py` | 290 | C6 + C7 | partly public (`compute_simhash`, `hamming_distance`, `find_near_duplicates`) | +| `_minhash.py` | 230 | C8 | partly public (`compute_minhash`, `find_near_duplicates_minhash`) | +| `_quality.py` | 220 | C12 | private (test-patched: `_strip_code_fences`, `_row_quality_flags`) | +| `_streaming.py` | 130 | C9 + C10 + C13 | private (test-patched: `_read_jsonl_split`) | +| `_aggregator.py` | 340 | C14 + C15 | private | +| `_splits.py` | 180 | C17 + C18 | private | +| `_summary.py` | 350 | C19 + C20 + C23 | partly public (`summarize_report`) | +| `_croissant.py` | 200 | C21 | private | +| `_orchestrator.py` | 280 | C16 + C22 | partly public (`audit_dataset`) | + +Total ≈ 2870 source lines across 15 modules. **Every file lands well below the 1000-line ceiling and most below 350**, with the only borderline case being `_aggregator.py` at ~340. + +Naming convention: every module starts with `_` because the package itself is `forgelm.data_audit`, and the public face is the package, not its internals. + +### 1.e Public-API preservation strategy (the `__init__.py` contract) + +```python +# forgelm/data_audit/__init__.py — designed shape +"""Public re-exports — preserves the pre-split ``forgelm.data_audit`` surface.""" +from __future__ import annotations + +# Public types and constants +from ._types import ( + AuditReport, + PII_TYPES, PII_SEVERITY, PII_SEVERITY_ORDER, + PII_ML_SEVERITY, PII_ML_TYPES, + DEFAULT_NEAR_DUP_HAMMING, DEFAULT_MINHASH_JACCARD, DEFAULT_MINHASH_NUM_PERM, + DEDUP_METHODS, +) +# PII regex +from ._pii_regex import ( + detect_pii, mask_pii, + _is_credit_card, _is_tr_id, # tests +) +# PII ML (Presidio) +from ._pii_ml import ( + detect_pii_ml, + _require_presidio, _get_presidio_analyzer, _PRESIDIO_ENTITY_MAP, # tests +) +# Optional-deps sentinels — tests patch these by dotted path +from ._optional import _HAS_PRESIDIO, _PresidioAnalyzer # noqa: F401 +# Secrets +from ._secrets import SECRET_TYPES, detect_secrets, mask_secrets +# Simhash + LSH + cross-split +from ._simhash import ( + compute_simhash, hamming_distance, find_near_duplicates, + _find_near_duplicates_brute, _count_leaked_rows, _token_digest, # tests +) +# MinHash +from ._minhash import compute_minhash, find_near_duplicates_minhash +# Quality + streaming primitives reached by tests +from ._quality import _row_quality_flags, _strip_code_fences # tests +from ._streaming import _read_jsonl_split # tests +# Orchestrator + summary +from ._orchestrator import audit_dataset +from ._summary import summarize_report + +__all__ = [ + "AuditReport", "PII_TYPES", "PII_SEVERITY", "PII_SEVERITY_ORDER", + "PII_ML_SEVERITY", "PII_ML_TYPES", "DEFAULT_NEAR_DUP_HAMMING", + "DEFAULT_MINHASH_JACCARD", "DEFAULT_MINHASH_NUM_PERM", "DEDUP_METHODS", + "SECRET_TYPES", + "detect_pii", "mask_pii", "detect_pii_ml", "detect_secrets", "mask_secrets", + "compute_simhash", "hamming_distance", "find_near_duplicates", + "compute_minhash", "find_near_duplicates_minhash", + "audit_dataset", "summarize_report", +] +``` + +**Why each underscore-prefixed re-export is required:** +- Tests use `from forgelm.data_audit import _is_credit_card, _is_tr_id` (test_data_audit.py:13–26). +- Tests use `patch("forgelm.data_audit._HAS_PRESIDIO", False)` and `patch("forgelm.data_audit._PresidioAnalyzer", _BoomAnalyzer)` (test_phase12_5.py:278, 334). For `patch` to work, the symbol must be addressable at exactly that dotted path. + +**Critical pivot point:** `patch` mutates the package-level binding, but the module that *reads* the binding (`_pii_ml.py`'s `if not _HAS_PRESIDIO:` guard) imports it from `._optional`, so it would see the unpatched value. To fix this: + +(a) **Recommended:** `_pii_ml.py` references the sentinel via a runtime attribute lookup: `from . import _optional` then `if not _optional._HAS_PRESIDIO:`. Tests update their patch paths to `patch("forgelm.data_audit._optional._HAS_PRESIDIO", …)` in lockstep with the PR. + +(b) Alternative: leave the package-level `from ._optional import _HAS_PRESIDIO` re-export AND have `_pii_ml.py` do `from forgelm.data_audit import _HAS_PRESIDIO` — guaranteed circular import. + +**Decision:** Path (a). The split PR rewrites the affected test patches. + +### 1.f Migration sequence (data_audit) + +| PR | Concerns moved | Public/private symbols moved | Test impact | Effort | Risk | +|---|---|---|---|---|---| +| **D-1** Extract `_streaming.py` + `_quality.py` + `_secrets.py` + `_pii_regex.py` (independent leaf concerns) | C9, C10, C11, C12, C13, C4 | `detect_pii`, `mask_pii`, `_is_credit_card`, `_is_tr_id`, `SECRET_TYPES`, `detect_secrets`, `mask_secrets`, `_strip_code_fences`, `_row_quality_flags`, `_read_jsonl_split` | New `forgelm/data_audit/__init__.py` re-exports everything; `data_audit.py` shrinks to ~2400 lines and imports back via `from ._pii_regex import …`. Existing tests unchanged. | 1 day | Low. Pure mechanical move. | +| **D-2** Extract `_optional.py` + `_types.py` + `_simhash.py` + `_minhash.py` | C1, C2, C3, C6, C7, C8 | `AuditReport`, `compute_simhash`, `hamming_distance`, `find_near_duplicates`, `compute_minhash`, `find_near_duplicates_minhash`, `_token_digest`, `_find_near_duplicates_brute`, `_count_leaked_rows`, `PII_TYPES`, `DEDUP_METHODS`, `DEFAULT_*` constants | All test patches involving `_HAS_XXHASH`/`_HAS_NUMPY`/`_HAS_DATASKETCH` rewired to `forgelm.data_audit._optional._HAS_*`. **Update test files in the same PR.** | 1.5 days | Medium. lru_cache on `_token_digest` must be preserved (single global cache, not per-module). Numpy fast-path stays bit-identical. | +| **D-3** Extract `_pii_ml.py` (Presidio) | C5 | `detect_pii_ml`, `_get_presidio_analyzer`, `_require_presidio`, `_PRESIDIO_ENTITY_MAP`, `PII_ML_SEVERITY`, `PII_ML_TYPES` | All `patch("forgelm.data_audit._HAS_PRESIDIO", …)` calls in `test_phase12_5.py` rewritten to `patch("forgelm.data_audit._optional._HAS_PRESIDIO", …)`. lru_cache instances preserved; cache_clear teardown preserved. | 1 day | Medium-high. | +| **D-4** Extract `_aggregator.py` + `_splits.py` + `_croissant.py` + `_summary.py` | C14, C15, C17, C18, C19, C20, C21, C23 | `summarize_report`, `_StreamingAggregator`, `_SplitOutcome`, `_process_split`, `_resolve_input`, `_build_croissant_metadata`, `_build_pii_severity`, all `_*_summary_notes`, `_render_*` | No test changes. | 1 day | Low. | +| **D-5** Convert `data_audit.py` → `data_audit/_orchestrator.py`, finalise `__init__.py` | C16, C22 | `audit_dataset`, `_atomic_write_json`, `_cross_split_overlap`, `_pair_leak_payload` | New import-stability test (Section 1.g). | 0.5 day | Low — at this point `data_audit.py` is only the orchestrator + re-exports. | + +Total effort: ~5 person-days. PRs D-1 and D-4 can run in parallel (no overlap). D-2 must precede D-3. D-5 must come last. + +### 1.g Regression test strategy + +**New file** `tests/test_data_audit_import_stability.py`: + +```python +"""Pin every external import path so a future split refactor cannot break them.""" +PUBLIC = [ + "AuditReport", "PII_TYPES", "PII_SEVERITY", "PII_SEVERITY_ORDER", + "PII_ML_SEVERITY", "PII_ML_TYPES", "DEFAULT_NEAR_DUP_HAMMING", + "DEFAULT_MINHASH_JACCARD", "DEFAULT_MINHASH_NUM_PERM", "DEDUP_METHODS", + "SECRET_TYPES", + "detect_pii", "mask_pii", "detect_pii_ml", "detect_secrets", "mask_secrets", + "compute_simhash", "hamming_distance", "find_near_duplicates", + "compute_minhash", "find_near_duplicates_minhash", + "audit_dataset", "summarize_report", +] +PRIVATE_TEST_TOUCHED = [ + "_is_credit_card", "_is_tr_id", "_token_digest", + "_find_near_duplicates_brute", "_count_leaked_rows", + "_read_jsonl_split", "_strip_code_fences", "_row_quality_flags", + "_require_presidio", "_get_presidio_analyzer", "_PRESIDIO_ENTITY_MAP", +] + +def test_public_api_resolves(): + import forgelm.data_audit as m + for name in PUBLIC + PRIVATE_TEST_TOUCHED: + assert hasattr(m, name), f"forgelm.data_audit.{name} missing post-split" + +def test_token_digest_lru_cache_singleton(): + import forgelm.data_audit as m + from forgelm.data_audit import _simhash + assert m._token_digest is _simhash._token_digest, \ + "lru_cache duplicated; cache_clear() will hit wrong instance" + +def test_patchable_optional_sentinels_resolve(): + from forgelm.data_audit import _optional # noqa: F401 + from forgelm.data_audit._optional import _HAS_PRESIDIO # noqa: F401 +``` + +The 47 existing test modules already enforce 800+ behavioural assertions that pin the audit's output JSON shape. **No existing test needs to change in PR D-1, D-4, or D-5.** PR D-2 and D-3 each rewrite ≤8 `patch(...)` strings in `test_phase12_5.py`/`test_data_audit.py` (concerted in the same PR). + +### 1.h Risk register (data_audit) + +| # | Risk | Likelihood | Impact | Mitigation | +|---|---|---|---|---| +| R-1 | `lru_cache` instance gets duplicated when `_token_digest` is moved to `_simhash.py` and re-imported from `__init__.py`; tests that exercise `_token_digest.cache_clear()` invalidate the wrong cache | Medium | Medium — slow tests + flaky cache hit-rate | The package-level re-export must be `from ._simhash import _token_digest` (binding alias, same object). Add an assertion in the import-stability test: `assert forgelm.data_audit._token_digest is forgelm.data_audit._simhash._token_digest`. | +| R-2 | `_HAS_PRESIDIO` patched at `forgelm.data_audit._HAS_PRESIDIO` no longer reaches the runtime check inside `_pii_ml.py` (it reads a stale snapshot via `from ._optional import _HAS_PRESIDIO`) | High if not handled | High — silent test-mode regressions | (a) Rewrite the runtime check to `from . import _optional` + `_optional._HAS_PRESIDIO` (attribute lookup picks up the patched value); (b) update test patches to the new dotted path. Both sides ship in the same PR (D-3). | +| R-3 | Circular import: `_pii_ml.py` ↔ `_optional.py` | Low | High | `_optional.py` imports stdlib only; no `data_audit.*` imports allowed. | +| R-4 | `_get_presidio_analyzer.cache_clear()` called in `_require_presidio` references the wrong `_get_presidio_analyzer` after it moves modules | Medium | Medium | Both functions live in `_pii_ml.py`; co-location removes the problem. | +| R-5 | Test fixture path `tests/test_data_audit*.py` enumerates expected public names; missing one breaks discovery | Low | Low | Import-stability test enumerates them explicitly. | +| R-6 | numpy fast-path silently changes precision after the move | Low | High | Pure relocation; existing fingerprint property tests catch drift. | +| R-7 | `from .data_audit import …` consumers (`forgelm/ingestion.py`, `forgelm/wizard.py`) break | Low | Critical | Package `__init__.py` re-exports → `from .data_audit import mask_pii` continues to resolve. | +| R-8 | `__all__` not declared on the new `__init__.py` → `import *` regressions | Low | Low | Add explicit `__all__` in `__init__.py`. | +| R-9 | `logger = logging.getLogger("forgelm.data_audit")` (line 42) used across files; sub-module loggers might break log filtering | Medium | Low | Sub-modules use `logging.getLogger(__name__)` which inherits via standard hierarchy. | +| R-10 | The `from forgelm import data_audit as audit_mod` pattern (test_data_audit.py:455, 595, 619) breaks | Low | Medium | Re-exports + import-stability test catch this. | + +--- + +## 2. `cli.py` — Detailed analysis + +### 2.a Concern inventory + +| # | Concern | Lines | Public/exported | Internal helpers | +|---|---|---|---|---| +| 1 | Module header + exit codes | 1–33 | `EXIT_SUCCESS`, `EXIT_CONFIG_ERROR`, `EXIT_TRAINING_ERROR`, `EXIT_EVAL_FAILURE`, `EXIT_AWAITING_APPROVAL` | `_CLI_MODULE`, `_PUBLIC_EXIT_CODES`, `logger` | +| 2 | argparse type guards + version + logging setup | 36–98 | `_get_version`, `_setup_logging` | `_non_negative_int`, `_add_common_subparser_flags` | +| 3 | Subcommand registrar — `chat` | 101–128 | — | `_add_chat_subcommand` | +| 4 | Subcommand registrar — `export` | 131–162 | — | `_add_export_subcommand` | +| 5 | Subcommand registrar — `deploy` | 165–199 | — | `_add_deploy_subcommand` | +| 6 | Subcommand registrar — `quickstart` | 202–251 | — | `_add_quickstart_subcommand` | +| 7 | Subcommand registrar — `ingest` | 254–374 | — | `_add_ingest_subcommand` | +| 8 | Float type + audit registrar | 377–513 | — | `_non_negative_float`, `_add_audit_subcommand` | +| 9 | `parse_args` (root + dispatcher) | 516–636 | `parse_args` | — | +| 10 | Dry-run / fit-check / benchmark-only / merge / generate-data / compliance-export | 639–870 | `_run_dry_run`, `_run_fit_check`, `_run_compliance_export` | `_galore_dry_run_fields`, `_evaluation_dry_run_fields`, `_compliance_dry_run_fields`, `_build_dry_run_result`, `_run_benchmark_only`, `_run_merge`, `_run_generate_data` | +| 11 | Resume checkpoint + result envelope + log helpers | 872–999 | `_resolve_resume_checkpoint`, `_output_result` | `_build_result_json_envelope`, `_log_result_status`, `_log_cost_summary`, `_log_benchmark_summary` | +| 12 | Subcommand dispatchers — `chat`, `export`, `deploy` | 1001–1104 | — | `_run_chat_cmd`, `_run_export_cmd`, `_run_deploy_cmd` | +| 13 | Quickstart dispatcher (multi-step) | 1107–1318 | — | `_build_quickstart_inherited_flags`, `_emit_quickstart_list`, `_emit_quickstart_result`, `_run_quickstart_train_subprocess`, `_run_quickstart_chat_subprocess`, `_run_quickstart_train_then_chat`, `_run_quickstart_cmd`, `_load_quickstart_train_paths` | +| 14 | Ingest dispatcher | 1321–1391 | — | `_run_ingest_cmd` | +| 15 | Audit dispatcher | 1394–1526 | `_run_data_audit` | `_run_audit_cmd` | +| 16 | Top-level dispatcher + main() | 1529–1755 | `main` | `_dispatch_subcommand`, `_maybe_run_wizard`, `_load_config_or_exit`, `_apply_offline_flag`, `_maybe_run_no_train_mode`, `_report_training_error`, `_run_training_pipeline` | + +### 2.b Public API surface + +```python +# Constants +EXIT_SUCCESS: int = 0 +EXIT_CONFIG_ERROR: int = 1 +EXIT_TRAINING_ERROR: int = 2 +EXIT_EVAL_FAILURE: int = 3 +EXIT_AWAITING_APPROVAL: int = 4 + +# Functions +def parse_args() -> argparse.Namespace: ... +def main() -> None: ... + +# Test-touched helpers (must remain importable at forgelm.cli.<name>) +_get_version, _setup_logging, _resolve_resume_checkpoint, _output_result, +_run_dry_run, _run_fit_check, _run_compliance_export, _run_data_audit, +_build_quickstart_inherited_flags +``` + +Entry point: `forgelm = "forgelm.cli:main"` in `pyproject.toml [project.scripts]`. **`forgelm.cli` must remain importable as a name and `main` must resolve at `forgelm.cli:main`.** + +### 2.c Cross-coupling + +Coupling is shallow — registrars and dispatchers form parallel chains both rooted at `parse_args`/`main`. Excellent split candidate. + +### 2.d Proposed package layout + +```text +forgelm/cli/ + __init__.py # re-exports main + exit codes + the test-touched helpers + __main__.py # python -m forgelm.cli entry point (NEW — load-bearing for quickstart subprocess) + _exit_codes.py # EXIT_*, _PUBLIC_EXIT_CODES + _argparse_types.py # _non_negative_int, _non_negative_float, _add_common_subparser_flags + _logging.py # _setup_logging, _get_version, _CLI_MODULE + _parser.py # parse_args + epilog + every _add_*_subcommand registrar + _dry_run.py # _galore_*, _evaluation_*, _compliance_*, _build_dry_run_result, _run_dry_run + _fit_check.py # _run_fit_check + _result.py # _build_result_json_envelope, _log_result_status, _log_cost_summary, + # _log_benchmark_summary, _output_result + _resume.py # _resolve_resume_checkpoint + _no_train_modes.py # _run_benchmark_only, _run_merge, _run_generate_data, + # _run_compliance_export, _maybe_run_no_train_mode + _wizard.py # _maybe_run_wizard + _config_load.py # _load_config_or_exit, _apply_offline_flag + _training.py # _report_training_error, _run_training_pipeline + subcommands/ + __init__.py # empty + _chat.py # _run_chat_cmd + _export.py # _run_export_cmd + _deploy.py # _run_deploy_cmd + _quickstart.py # all 8 quickstart helpers + _ingest.py # _run_ingest_cmd + _audit.py # _run_data_audit, _run_audit_cmd + _dispatch.py # _dispatch_subcommand, main() +``` + +| File | Target lines | +|---|---| +| `__init__.py` | 50 | +| `__main__.py` | 5 | +| `_exit_codes.py` | 20 | +| `_argparse_types.py` | 60 | +| `_logging.py` | 40 | +| `_parser.py` | 580 | +| `_dry_run.py` | 90 | +| `_fit_check.py` | 30 | +| `_result.py` | 100 | +| `_resume.py` | 30 | +| `_no_train_modes.py` | 240 | +| `_wizard.py` | 20 | +| `_config_load.py` | 60 | +| `_training.py` | 80 | +| `subcommands/_chat.py` | 30 | +| `subcommands/_export.py` | 50 | +| `subcommands/_deploy.py` | 50 | +| `subcommands/_quickstart.py` | 220 | +| `subcommands/_ingest.py` | 80 | +| `subcommands/_audit.py` | 140 | +| `_dispatch.py` | 90 | + +Total ≈ 2090. Largest file (`_parser.py`) at ~580 lines is acceptable: argparse registrar code is verbose but cohesive. + +**Why `_parser.py` keeps all registrars:** the registrars are short (15–120 lines each), they share `_add_common_subparser_flags`, and `parse_args()` instantiates the parser once and adds them all. Fragmenting registrars into 6 separate files would force `_parser.py` to import each one and would not improve cohesion. + +### 2.e Public-API preservation strategy + +```python +# forgelm/cli/__init__.py — designed shape +"""Public re-exports for the ``forgelm`` CLI.""" +from ._exit_codes import ( + EXIT_SUCCESS, EXIT_CONFIG_ERROR, EXIT_TRAINING_ERROR, + EXIT_EVAL_FAILURE, EXIT_AWAITING_APPROVAL, +) +from ._logging import _setup_logging, _get_version +from ._parser import parse_args +from ._dry_run import _run_dry_run +from ._fit_check import _run_fit_check +from ._result import _output_result +from ._resume import _resolve_resume_checkpoint +from ._no_train_modes import _run_compliance_export +from .subcommands._audit import _run_data_audit +from .subcommands._quickstart import _build_quickstart_inherited_flags +from ._dispatch import main + +__all__ = [ + "EXIT_SUCCESS", "EXIT_CONFIG_ERROR", "EXIT_TRAINING_ERROR", + "EXIT_EVAL_FAILURE", "EXIT_AWAITING_APPROVAL", + "parse_args", "main", + "_get_version", "_setup_logging", + "_resolve_resume_checkpoint", "_run_dry_run", "_run_fit_check", + "_run_compliance_export", "_run_data_audit", "_output_result", + "_build_quickstart_inherited_flags", +] +``` + +```python +# forgelm/cli/__main__.py — load-bearing for python -m forgelm.cli +from forgelm.cli import main + +if __name__ == "__main__": + main() +``` + +**Why `__main__.py` is load-bearing:** today `forgelm/cli.py` has `if __name__ == "__main__": main()` at its tail (line 1755). `python -m forgelm.cli` works because `cli.py` IS the module. After the split, `forgelm.cli` is a package; running `python -m forgelm.cli` then needs `forgelm/cli/__main__.py`. This is critical because `_run_quickstart_train_subprocess` (line 1191 in current `cli.py`) does `subprocess.run([sys.executable, "-m", _CLI_MODULE, ...])` where `_CLI_MODULE = "forgelm.cli"`. **Without `__main__.py`, the quickstart subprocess flow breaks silently.** + +`pyproject.toml [project.scripts]` reads `forgelm = "forgelm.cli:main"`. Python resolves this against the `forgelm.cli` package's `__init__` namespace, where `main` is re-exported. **No pyproject change required.** + +### 2.f Migration sequence (cli) + +| PR + files extracted | Test impact | Effort | Risk | +|---|---|---|---| +| **C-1** Extract `_exit_codes.py`, `_logging.py`, `_argparse_types.py`, `_resume.py`, `_result.py` | Tests: no change (re-exports cover them). | 0.5 day | Very low | +| **C-2** Extract `_parser.py` (move all registrars + `parse_args`) | Tests: `--help` snapshot test (new — Section 4) MUST go green; `test_smoke.py::parse_args` import keeps working through `__init__.py`. | 1 day | Medium — registrars must be moved verbatim including help-string text to keep `--help` byte-identical. | +| **C-3** Extract `_dry_run.py`, `_fit_check.py`, `_no_train_modes.py`, `_wizard.py`, `_config_load.py`, `_training.py` | Tests: `_run_dry_run`, `_run_fit_check`, `_run_compliance_export` must still resolve at `forgelm.cli`. Re-exports cover this. | 0.75 day | Low | +| **C-4** Create `subcommands/` package and extract `_chat.py`, `_export.py`, `_deploy.py`, `_ingest.py` | No test changes. | 0.75 day | Low | +| **C-5** Extract `subcommands/_quickstart.py` and `subcommands/_audit.py` | Tests: re-exported through `__init__.py`. | 0.75 day | Medium — quickstart subprocess test (`test_quickstart_subprocess.py`) re-runs `python -m forgelm.cli`; the `__main__` shim must be added. | +| **C-6** Add `_dispatch.py` (final `main()` + `_dispatch_subcommand`), wire `__init__.py` final shape, add `forgelm/cli/__main__.py` | New: `tests/test_cli_import_stability.py`. | 0.5 day | Low | + +Total: ~4.25 person-days. C-1 can run in parallel with **D-1** (no overlap). C-2 must precede C-4 and C-5. C-6 is last. + +### 2.g CLI-specific acceptance signals + +- `forgelm --help` byte-identical to v0.5.0. +- `forgelm audit --help`, `forgelm ingest --help`, `forgelm quickstart --help`, `forgelm chat --help`, `forgelm export --help`, `forgelm deploy --help` byte-identical. +- `forgelm --version` returns the installed version unchanged. +- `forgelm --config config_template.yaml --dry-run` exits 0. +- `python -m forgelm.cli --help` byte-identical (validates `__main__.py`). +- `forgelm --data-audit path/` (legacy alias) still emits the deprecation warning and runs. +- Exit codes 0/1/2/3/4 unchanged on every error path covered by `tests/test_cli*.py`. + +### 2.h Lazy-import discipline (orthogonal but pinned by C-4 / C-5) + +The current file already defers heavy imports — `from .chat import run_chat` is inside `_run_chat_cmd` (line 1004), `from .data_audit import audit_dataset` is inside `_run_data_audit` (line 1416), `from .data import prepare_dataset` is inside `_run_training_pipeline` (line 1666). **The split must preserve these inline imports as-is.** Specifically: + +- `subcommands/_chat.py`: keep `from ..chat import run_chat` inline inside `_run_chat_cmd` (NOT at module top). +- `subcommands/_audit.py`: keep `from ..data_audit import audit_dataset, summarize_report, DEFAULT_*` inline inside `_run_data_audit`. +- `_training.py`: keep `from ..data import prepare_dataset`, `from ..model import get_model_and_tokenizer`, `from ..trainer import ForgeTrainer` inline. +- `_no_train_modes.py`: `_run_benchmark_only` already imports `.benchmark` and `.model` inline; preserve that. + +A pre-commit linter that fails on top-level torch imports inside `forgelm/cli/**` would be the natural enforcement. + +### 2.i Risk register (cli) + +| # | Risk | Likelihood | Impact | Mitigation | +|---|---|---|---|---| +| R-11 | `python -m forgelm.cli` breaks because `cli/` is now a package without `__main__.py` | High if not handled | Critical — quickstart subprocess flow dies silently | Ship `forgelm/cli/__main__.py` in PR C-6. Add a smoke test invoking `subprocess.run([sys.executable, "-m", "forgelm.cli", "--help"])`. | +| R-12 | `forgelm` console script breaks because `main` no longer resolves at `forgelm.cli:main` | Low — re-export handles it | Critical | Verified: `forgelm.cli` package's `__init__.py` re-exports `main`. Pin via the import-stability test. | +| R-13 | `--help` text shifts because some registrar moves change argparse's argument-add order | Medium | Low — surface only | Move registrars verbatim. Snapshot test compares `forgelm --help`, every `forgelm <sub> --help`. | +| R-14 | argparse `default=argparse.SUPPRESS` semantics change subtly | Low | Medium | Move registrars verbatim, no logic edits in C-2. | +| R-15 | `_PUBLIC_EXIT_CODES` referenced in `_run_quickstart_train_subprocess`; after split the import must come from `_exit_codes.py` | Low | Low | Same-PR fix: `subcommands/_quickstart.py` does `from .._exit_codes import _PUBLIC_EXIT_CODES`. | +| R-16 | Tests doing `with patch("forgelm.cli._run_data_audit", …)` succeed only if `_run_data_audit` is bound at the package level | Low | Low | Re-export covers the import. | +| R-17 | Heavy imports leak into `forgelm.cli.__init__` because `__init__.py` eagerly resolves every dispatcher (`from .subcommands._audit import _run_data_audit`), which in turn would import `audit_dataset` if the dispatcher's lazy-import discipline is not maintained | Medium | High — `forgelm --help` becomes slow | The `_run_data_audit` function definition does not require `audit_dataset` to be importable at definition time. `subcommands/_audit.py` must keep the `audit_dataset` import inside the function body. | +| R-18 | Logger naming drifts: today `logger = logging.getLogger(_CLI_MODULE)` where `_CLI_MODULE = "forgelm.cli"`. After split each new submodule should keep using the same logger | Low | Low | All sub-modules use `logger = logging.getLogger("forgelm.cli")` explicitly. | +| R-19 | The deprecated `--data-audit` flag is parsed by the **top-level** parser but dispatched by `main()` after `_dispatch_subcommand` returns. Preserving this ordering is essential | Medium | Medium | The diff for `_dispatch.py` carries `main()` verbatim, only updating relative imports. Pin with a regression test that calls `main(["--data-audit", "path"])`. | + +--- + +## 3. Combined PR sequence (data_audit + cli) + +Eleven PRs total — five parallelisable, six sequential. + +```text +Week 1 (parallel where possible): + Day 1: D-1 (data_audit leaf concerns) || C-1 (cli leaf concerns) + Day 2: D-2 (simhash/minhash + types) || C-2 (cli parser package extraction) + Day 3: D-3 (Presidio + test patch updates) + Day 4: D-4 (aggregator/croissant/summary) || C-3 (cli no-train mode helpers) + Day 5: D-5 (orchestrator + finalize) || C-4 (cli subcommands chat/export/deploy/ingest) + +Week 2: + Day 6: C-5 (cli quickstart + audit subcommands) + Day 7: C-6 (cli __main__.py + final dispatch + import-stability test) + Day 8: integration test pass + release candidate +``` + +**Parallel-safe pairs:** {D-1, C-1}, {D-2, C-2}, {D-4, C-3}, {D-5, C-4}. + +**Sequential constraints:** +- D-2 must precede D-3 (Presidio uses `_optional`). +- D-1, D-2, D-3, D-4 must precede D-5. +- C-2 must precede C-4, C-5. +- C-1 through C-5 all precede C-6. + +**Recommended branch strategy:** stack each PR onto the previous one. Use `git rebase --onto` if main moves. Do NOT batch into a single "big bang" PR — that erases the bisectability that motivates the split. + +Alternative: collapse D-1+D-4 into a single 1.5-day "low-coupling extraction" PR, and collapse C-1+C-3 into a single 1-day "leaf helpers" PR; this brings the count to 9 without losing reviewability. + +--- + +## 4. Acceptance criteria (per PR) + +A PR in either series is "accepted" when ALL of the following hold: + +1. **Existing tests green:** `pytest tests/` exits 0 across all 47 modules and 800+ tests. No `xfail` introduced. +2. **Coverage gate:** `pytest --cov=forgelm` reports `total ≥ 40 %`. +3. **`--help` byte-identical:** `forgelm --help`, `forgelm audit --help`, `forgelm ingest --help`, `forgelm quickstart --help`, `forgelm chat --help`, `forgelm export --help`, `forgelm deploy --help` all match the v0.5.0 snapshot. The CI gate captures the strings to a fixture and `diff`s. +4. **Import-stability test green:** every PUBLIC name and every PRIVATE-test-touched name resolves at its canonical dotted path. +5. **Patch-path test green:** `patch("forgelm.data_audit._optional._HAS_PRESIDIO", False)` and any other rewired patch paths fire. +6. **`forgelm --version` works.** +7. **`forgelm --config config_template.yaml --dry-run` exits 0** with the same JSON envelope shape as v0.5.0. +8. **`python -m forgelm.cli --help` works** (validates `__main__.py` once C-6 lands). +9. **Quickstart subprocess flow works:** `_run_quickstart_train_subprocess` spawns `python -m forgelm.cli ...` successfully — covered by `test_quickstart_subprocess.py`. +10. **ruff format + ruff check clean.** +11. **No new top-level imports of torch / transformers / unsloth / deepspeed / lm_eval inside `forgelm/cli/**` or `forgelm/data_audit/**`.** +12. **Logger names unchanged.** +13. **Public exit codes preserved:** `EXIT_SUCCESS == 0`, `EXIT_CONFIG_ERROR == 1`, `EXIT_TRAINING_ERROR == 2`, `EXIT_EVAL_FAILURE == 3`, `EXIT_AWAITING_APPROVAL == 4`, `_PUBLIC_EXIT_CODES == frozenset({0,1,2,3,4})`. + +--- + +## 5. Effort and risk roll-up + +| Series | PRs | Total effort | Cumulative risk profile | +|---|---|---|---| +| `data_audit` (D-1 → D-5) | 5 | ~5.0 days | Mid (R-1, R-2, R-7 are real but each tractable) | +| `cli` (C-1 → C-6) | 6 | ~4.25 days | Low-mid (R-11 is the one critical item) | +| Integration + release validation | — | ~1.0 day | Low | +| **Combined** | **11** | **~10.25 person-days** | **Mid overall**; bisectable per-PR. | + +If a single engineer does the work end-to-end with no parallelisation, plan ~14 calendar days including review cycles and CI iteration. With two engineers running parallel-safe pairs, ~7–8 calendar days. + +--- + +## 6. Out-of-scope (called out explicitly) + +These are NOT addressed by the split design: + +- **Performance changes:** the simhash numpy fast-path, the Presidio lru_cache, and the streaming JSONL reader stay unmodified. Phase 12 perf work is separate. +- **Behaviour changes:** no flag added, no flag removed, no exit code repurposed. The `--data-audit` deprecation alias stays exactly as-is until v0.7.0. +- **`verify-audit` subcommand (master review F-compliance-103):** treated as a feature PR landing on top of the split, not a prerequisite. The `subcommands/` package layout reserves the file slot but does not implement it. +- **Documentation:** bilingual `docs/reference/` updates would only be needed if the import path changed. Re-exports keep `forgelm.data_audit` and `forgelm.cli` byte-compatible. The `docs/standards/architecture.md` table (line 94) keeps listing `data_audit.py` until the split lands; PR D-5 updates the line to read `data_audit/ (package)`. + +--- + +*End of split design — 202604300906.* diff --git a/docs/guides/safety_compliance-tr.md b/docs/guides/safety_compliance-tr.md new file mode 100644 index 00000000..ebf9c711 --- /dev/null +++ b/docs/guides/safety_compliance-tr.md @@ -0,0 +1,65 @@ +# Güvenlik & Uyumluluk Kılavuzu (TR) + +> **Not:** Bu dosya şu an yalnızca Faz 6'da eklenen `forgelm verify-audit` özelliğinin Türkçe karşılığını içerir. Tam bilingual eşleme (EN ↔ TR) `docs/guides/safety_compliance.md` dosyasının ileride yapılacak çevirisi ile tamamlanacaktır. Daha geniş bağlam için lütfen İngilizce sürüme bakın. + +## Audit Log Bütünlüğü Doğrulama + +ForgeLM'in audit log'u (`audit_log.jsonl`) SHA-256 hash zinciri kullanır; geriye dönük yapılan herhangi bir değişiklik zinciri kırar. Operatör `FORGELM_AUDIT_SECRET` ortam değişkenini ayarlamışsa her satıra ek olarak HMAC etiketi de yazılır — bu sayede log'a yazma erişimi olan ama operatör anahtarını bilmeyen bir saldırgan satır içeriklerini sahteleyemez. + +### `forgelm verify-audit` alt komutu + +`forgelm verify-audit` alt komutu, bir audit log dosyasının SHA-256 zincirini ve (verildiyse) HMAC etiketlerini doğrular: + +```bash +forgelm verify-audit run123/audit_log.jsonl +# OK: 47 entries verified + +FORGELM_AUDIT_SECRET=$OPERATOR_KEY forgelm verify-audit run123/audit_log.jsonl +# OK: 47 entries verified (HMAC validated) + +forgelm verify-audit tampered.jsonl +# FAIL at line 23: chain broken at line 23: prev_hash='...' expected='...' +``` + +Çıkış kodları: + +| Kod | Anlam | +|-----|-------| +| `0` | Zincir (ve denetlendiyse HMAC) eksiksiz | +| `1` | Zincir kırık veya HMAC eşleşmiyor (kurcalama / bozulma) | +| `2` | Dosya bulunamadı, okunamadı veya `--require-hmac` belirtildi ama secret env var ayarlı değil | + +### Kütüphane fonksiyonu + +CI/CD pipeline'larından programatik kullanım için: + +```python +from forgelm.compliance import verify_audit_log, VerifyResult + +result: VerifyResult = verify_audit_log( + "run123/audit_log.jsonl", + hmac_secret=os.environ.get("FORGELM_AUDIT_SECRET"), +) +if not result.valid: + raise SystemExit( + f"Audit log invalid at line {result.first_invalid_index}: {result.reason}" + ) +``` + +`VerifyResult` dataclass alanları: `valid` (bool), `entries_count` (int), `first_invalid_index` (Optional[int], 1-tabanlı), `reason` (Optional[str]). + +### Strict mod (`--require-hmac`) + +Düzenlemeli ortamlarda her satırın HMAC ile imzalı olması zorunlu olabilir. `--require-hmac` bayrağı: + +- Secret env var ayarlı değilse derhal çıkış kodu 2 ile çıkar (operatör hatası). +- Herhangi bir satırda `_hmac` alanı yoksa çıkış kodu 1 ile başarısız olur. + +```bash +FORGELM_AUDIT_SECRET=$OPERATOR_KEY forgelm verify-audit \ + run123/audit_log.jsonl --require-hmac +``` + +### Genesis manifest çapraz kontrolü + +ForgeLM, audit log'un ilk yazılışında `audit_log.jsonl.manifest.json` adlı bir sidecar dosyası üretir. Bu dosya birinci satırın SHA-256 hash'ini sabitler; saldırgan log'u kesip yeni bir genesis satırı yazsa bile manifest eşleşmemesi tespit edilir. `verify-audit` bu sidecar'ı otomatik olarak okur — manifest yoksa zincir-bütünlüğü kontrolü gene yapılır, ancak truncate-and-resume tespit kapsamı daralır (DEBUG seviyesinde uyarı verilir). diff --git a/docs/guides/safety_compliance.md b/docs/guides/safety_compliance.md index e05745c3..385281df 100644 --- a/docs/guides/safety_compliance.md +++ b/docs/guides/safety_compliance.md @@ -282,6 +282,25 @@ ForgeLM's audit log (`audit_log.jsonl`) uses a SHA-256 hash chain for tamper evi Verify integrity by checking that each entry's hash matches the SHA-256 of the previous line. +### Verifying audit log integrity + +The `forgelm verify-audit` subcommand validates the SHA-256 hash chain (and optional HMAC tags) of an audit log: + +```bash +forgelm verify-audit run123/audit_log.jsonl +# OK: 47 entries verified + +FORGELM_AUDIT_SECRET=$OPERATOR_KEY forgelm verify-audit run123/audit_log.jsonl +# OK: 47 entries verified (HMAC validated) + +forgelm verify-audit tampered.jsonl +# FAIL at line 23: chain broken at line 23: prev_hash='...' expected='...' +``` + +Exit codes: `0` valid, `1` invalid chain or HMAC mismatch, `2` file/option error (e.g. `--require-hmac` without a configured secret env var). + +The library function `forgelm.compliance.verify_audit_log(path, *, hmac_secret=None, require_hmac=False)` returns a `VerifyResult` dataclass (`valid`, `entries_count`, `first_invalid_index`, `reason`) for programmatic CI/CD integration. + ### compliance_report.json ```json diff --git a/docs/qms/sop_data_management.md b/docs/qms/sop_data_management.md index f9bc1703..a680f401 100644 --- a/docs/qms/sop_data_management.md +++ b/docs/qms/sop_data_management.md @@ -71,33 +71,38 @@ ForgeLM automated checks: - Dataset fingerprinting (SHA-256 hash, size, timestamp) - Format validation per trainer type (SFT, DPO, KTO, GRPO) - Text cleaning (`clean_text: true`) -- **Audit pipeline (`forgelm audit <jsonl>`, v0.5.1+; legacy - `forgelm --data-audit` still works)** — produces - `data_audit_report.json` with per-split sample counts, length - distribution, top-3 language detection, simhash near-duplicate rate - (or **MinHash LSH** in v0.5.2 via `--dedup-method minhash` for - >50K-row corpora), cross-split leakage check, PII flag counts - (email / phone / Luhn-validated credit card / IBAN / - TR–DE–FR–US national IDs) with **severity tiers** (v0.5.1) and - always-on **secrets/credentials scan** (v0.5.2 — AWS / GitHub / - Slack / OpenAI / Google / JWT / private-key headers). Optional - **heuristic quality filter** (`--quality-filter`, v0.5.2) adds a - `quality_summary` block. The report is auto-inlined into the EU AI - Act Article 10 governance artifact when present in the trainer's - `output_dir` — operators must run the audit **before** training to - keep the bundle self-contained. -- **Ingestion pipeline (`forgelm ingest`, v0.5.0+; v0.5.2 added - markdown-aware splitter + DOCX→markdown table preservation + - `--secrets-mask`)** — turns raw PDF / DOCX / EPUB / TXT / Markdown - into SFT-ready JSONL with optional `--pii-mask` and (v0.5.2) - `--secrets-mask` to redact detected PII / credential spans before - chunks reach storage. +- **ForgeLM audit pipeline (v0.5.0+, `forgelm audit <jsonl>`; legacy + `forgelm --data-audit` deprecated, removal scheduled v0.7.0)** — + produces `data_audit_report.json` with per-split sample counts, + length distribution, top-3 language detection, near-duplicate rate + (default 64-bit simhash, or **MinHash LSH** via + `--dedup-method minhash` for >50K-row corpora), cross-split + leakage check, PII flag counts (email / phone / Luhn-validated + credit card / IBAN / TR–DE–FR–US national IDs) with **severity + tiers**, always-on **secrets/credentials scan** (AWS / GitHub / + Slack / OpenAI / Google / JWT / private-key headers), and an + optional **heuristic quality filter** (`--quality-filter`) that + adds a `quality_summary` block. `forgelm audit --croissant` emits a + Google Croissant 1.0 dataset card alongside the report. The report + is auto-inlined into the EU AI Act Article 10 governance artifact + when present in the trainer's `output_dir` — operators must run + the audit **before** training to keep the bundle self-contained. + Phases 11 + 11.5 + 12 + 12.5 shipped together in the + `v0.5.0` consolidation release (PyPI 2026-04-30); the per-feature + list above describes the v0.5.0 surface. +- **ForgeLM ingestion pipeline (v0.5.0+, `forgelm ingest`)** — turns + raw PDF / DOCX / EPUB / TXT / Markdown into SFT-ready JSONL with + paragraph / sliding / **markdown-aware** chunking, DOCX→markdown + table preservation, optional **Presidio ML-NER** adapter + (`--pii-ml`, `[ingestion-pii-ml]` extra), and + `--pii-mask` / `--secrets-mask` / `--all-mask` shorthands to redact + detected PII / credential spans before chunks reach storage. Manual checks: - [ ] Run `forgelm audit` and review `data_audit_report.json` for: cross-split leakage > 0%, near-duplicate rate, unexpected language mix, - PII flag counts, **`secrets_summary` (v0.5.2 — any non-zero count is - a stop-the-line event; remediate before training)**, and + PII flag counts, **`secrets_summary` (any non-zero count is a + stop-the-line event; remediate before training)**, and `quality_summary` if `--quality-filter` was used. - [ ] Sample review: inspect 50+ random examples. - [ ] Verify label correctness (for preference/KTO data). diff --git a/docs/reference/audit_event_catalog-tr.md b/docs/reference/audit_event_catalog-tr.md new file mode 100644 index 00000000..a4b80f39 --- /dev/null +++ b/docs/reference/audit_event_catalog-tr.md @@ -0,0 +1,165 @@ +# Audit Event Kataloğu + +> **Hedef kitle:** EU AI Act Madde 12 kayıt-tutma artefaktlarını inceleyen ForgeLM operatörleri, denetçiler ve aşağı akış doğrulayıcıları. +> **Ayna:** [audit_event_catalog.md](audit_event_catalog.md) + +Bu katalog, ForgeLM'in EU AI Act Madde 12 tarafından zorunlu kılınan, append-only ve hash-zincirli kayıt-tutma artefaktı olan `audit_log.jsonl`'a yazabileceği tüm event'leri sıralar. Her satır ortak bir zarf paylaşır; aşağıdaki satırlardan birini `event` alanı seçer. + +## Ortak zarf + +Her satır, en azından aşağıdaki alanları içeren tek bir JSON nesnesidir: + +| Alan | Tip | Açıklama | +|---------------|---------|-----------------------------------------------------------------------------------------------------------| +| `timestamp` | string | ISO-8601 UTC zaman damgası (`datetime.now(timezone.utc).isoformat()`). | +| `run_id` | string | Eğitim koşusu başına stabil tanımlayıcı (`fg-<uuid12>`). | +| `operator` | string | İnsan-atfedilebilir kimlik. `AuditLogger` öncelik sırasıyla çözer: (1) `$FORGELM_OPERATOR` setse onu; (2) yoksa kullanıcı adı çözülebildiğinde `<getpass.getuser()>@<hostname>`; (3) yalnızca `getpass.getuser()` çağrısı başarısızsa *ve* `FORGELM_ALLOW_ANONYMOUS_OPERATOR=1` setse `anonymous@<hostname>` (aksi halde koşu yüksek sesli iptal olur). | +| `event` | string | Bu kataloğa ait noktalı event adı. | +| `prev_hash` | string | Önceki satırın SHA-256'sı (ilk girdi için `"genesis"`). Tampering-evident hash zincirini oluşturur. | +| `_hmac` | string? | Sadece `FORGELM_AUDIT_SECRET` set edildiğinde mevcut. `_hmac` olmadan satırın HMAC-SHA-256'sı. | +| _payload_ | değişir | Her satırda ayrı listelenen, event'e özel anahtarlar. | + +Hash zinciri, satır diske düştükten (`flush` + `fsync`) sonra ilerler; kirli bir kapanış zinciri resume için bütün bırakır. + +## Event sözlüğü + +### Pipeline yaşam döngüsü + +| Event | Ne zaman emit edilir | Payload (zarfa ek olarak) | Madde | +|----------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-------| +| `training.started` | Trainer fine-tuning koşusunu başlatır. | `trainer_type`, `model`, `dataset`, `config_path` | 12 | +| `pipeline.completed` | Uçtan uca CLI koşusu (eğitim + değerlendirme + dışa aktarma) 0 koduyla biter. | `exit_code`, `duration_seconds`, `success`, `metrics_summary` | 12 | +| `pipeline.failed` | Pipeline tamamlanmadan bir hata ile iptal olur. | `error` | 12 | + +### Madde 14 — İnsan Gözetimi + +| Event | Ne zaman emit edilir | Payload | Madde | +|------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|-------| +| `human_approval.required` | `requires_human_approval: true` işaretli bir kapı pipeline'ı duraklatıp operatör kararını bekler. | `gate`, `reason`, `metrics` | 14 | +| `human_approval.granted` | Operatör duraklatılan kapıyı onayladı. _(Faz 9 — placeholder; henüz emit edilmiyor.)_ | `gate`, `approver`, `comment` | 14 | +| `human_approval.rejected` | Operatör duraklatılan kapıyı reddetti. _(Faz 9 — placeholder; henüz emit edilmiyor.)_ | `gate`, `approver`, `comment` | 14 | + +### Madde 15 — Model Bütünlüğü (auto-revert + güvenlik) + +| Event | Ne zaman emit edilir | Payload | Madde | +|--------------------------------|--------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------|-------| +| `model.reverted` | Auto-revert kalite regresyonu sonrası önceki bir checkpoint'i geri yükledi. _(Faz 8 — webhook bağlantılı.)_ | `from_checkpoint`, `to_checkpoint`, `reason`, `metrics_delta` | 15 | +| `audit.classifier_load_failed` | Güvenlik sınıflandırıcısı (örn. Llama Guard) yüklenemedi. Koşu yine `passed=False` kaydeder. | `classifier`, `reason` | 15 | + +### Madde 11 + Ek IV — Uyumluluk artefaktları + +| Event | Ne zaman emit edilir | Payload | Madde | +|----------------------------------|-------------------------------------------------------------------------------|--------------------------------------------------|---------------| +| `compliance.governance_exported` | Madde 10 veri yönetişim raporu diske yazıldı. | `output_path`, `dataset_count` | 10 | +| `compliance.governance_failed` | Yönetişim raporu üretimi iptal edildi (örn. şema uyumsuzluğu). | `failure_reason` | 10 | +| `compliance.artifacts_exported` | Ek IV teknik dokümantasyon paketi (manifest, model card, audit zip) yazıldı. | `output_dir`, `files` | 11, Ek IV | + +### CLI / göç + +| Event | Ne zaman emit edilir | Payload | Madde | +|-----------------------------|--------------------------------------------------------------------------------------------------------|----------------------------------|-------| +| `cli.legacy_flag_invoked` | Deprecated bir CLI flag'i kullanıldı. _(Faz 13 — placeholder; henüz emit edilmiyor.)_ | `flag`, `replacement`, `version` | 12 | + +### Audit-sistem event'leri (meta) + +| Event | Ne zaman emit edilir | Payload | Madde | +|--------------------------------|------------------------------------------------------------------------------------------------------------|--------------------------------------|-------| +| `audit.classifier_load_failed` | _(Yukarıdaki Madde 15 satırına bakın.)_ | `classifier`, `reason` | 15 | +| `audit.cross_run_continuity` | Mevcut bir log dizinine işaret eden ikinci-veya-sonraki AuditLogger örneğinin ilk yazımı. | `previous_chain_head` | 12 | + +## Yeni bir event eklemek + +1. Mevcut isim alanlarını (`training.*`, `compliance.*`, `audit.*`, `human_approval.*`, `model.*`, `cli.*`) takip eden noktalı bir ad seçin. +2. Yukarıdaki tabloya, payload anahtarları ve desteklediği Madde dahil olmak üzere bir satır ekleyin. +3. Satırı [audit_event_catalog.md](audit_event_catalog.md)'ye yansıtın. +4. `AuditLogger.log_event(event, **payload)` üzerinden emit edin. `audit_log.jsonl`'a doğrudan `json.dump` çağırmayın; hash zinciri kanonik yazıcıya bağımlıdır. + +## Tampering-evidence özeti + +| Mekanizma | Şuna karşı koruma sağlar | Her zaman açık mı? | +|--------------------------|-----------------------------------------------------------------------------------|---------------------------------------------------| +| SHA-256 hash zinciri | Tek-satır düzenlemeler, silmeler, sıralama değişiklikleri. | Evet. | +| Genesis manifest sidecar | Tüm log'un "genesis"'e geri kesilmesi. | Evet (ilk event'te bir kez yazılır). | +| `flock(LOCK_EX)` | Aynı dizini paylaşan eşzamanlı trainer'lardan iç içe yazımlar. | Evet (Unix); Windows'ta no-op. | +| `flush` + `fsync` | Buffer yazımı ile zincir ilerleme arasında güç-kesme / kernel-panic kaybı. | Evet. | +| Satır başına HMAC-SHA-256| Log yeniden yazımı sonrası sahte yeniden imzalama. | Sadece `FORGELM_AUDIT_SECRET` set olduğunda. | + +## Webhook olayları + +Webhook payload'ları (Slack / Teams / jenerik HTTP) operatör bildirimlerine kapsamlanmış ayrı bir sözlüktür, regülasyon kaydı değil. Webhook olayları `audit_log.jsonl`'a **eklenmez**; yan-kanal bildirim bus'ı üzerinde gider. Kanonik yaşam döngüsü sözlüğü ayrıca [logging-observability.md](../standards/logging-observability.md)'da da belgelenmiştir. + +Bu beş yaşam döngüsü olayı, webhook alıcılarının `WebhookNotifier`'dan +beklemesi gereken **tek** olaylardır. Her biri, karşılık gelen bir +denetim günlüğü olayını yansıtır; böylece aşağı akıştaki bir operatör +webhook ping → denetim girdisi korelasyonunu `run_name` + zaman +damgasıyla kurabilir. Uygulama: `forgelm/webhook.py`. + +| Webhook `event` | Denetim günlüğü karşılığı | Tetikleyici | Kapı (gate) | Zorunlu payload alanları | +|---|---|---|---|---| +| `training.start` | `training.started` | `train()` çağrıldı, model yüklenmeden önce. | `webhook.notify_on_start` | `run_name`, `status="started"` | +| `training.success` | `pipeline.completed` | Tüm kapılar geçildi, insan onayı gerekmiyor. | `webhook.notify_on_success` | `run_name`, `status="succeeded"`, `metrics` | +| `training.failure` | `pipeline.failed` | Eğitim sürecinin kendisi hata fırlattı (OOM, veri seti hatası, yakalanmayan istisna). | `webhook.notify_on_failure` | `run_name`, `status="failed"`, `reason` (maskelenmiş, ≤2048 karakter) | +| `training.reverted` | `model.reverted` | Eğitim sonrası bir kapı (değerlendirme, güvenlik, hakem, benchmark) çalışmayı reddetti ve `_revert_model` adaptörleri sildi. | `webhook.notify_on_failure` | `run_name`, `status="reverted"`, `reason` (maskelenmiş, ≤2048 karakter) | +| `approval.required` | `human_approval.required` | Çalışma başarılı oldu, `evaluation.require_human_approval=true`, model insan incelemesi için staging'de (EU AI Act Madde 14). | `webhook.notify_on_success` | `run_name`, `status="awaiting_approval"`, `model_path` | + +### Bu yaşam döngüsü durumlarından ikisinin neden ayrıldığı + +- **`training.failure` vs `training.reverted`** — dashboard'ların + "trainer çöktü" ile "trainer başarılı oldu fakat kalite / güvenlik / + hakem kapısı reddetti" durumlarını ayırt etmesi gerekir. Her ikisi + de operasyonel olarak eyleme dönüşürdür, ancak farklı runbook'lar + gerektirir. Faz 8, bir Slack kanalının iki vakayı farklı renk + kodlayabilmesi için (`#ff0000` ve `#ff9900`) tam da bu nedenle + `notify_reverted`'i tanıttı. +- **`approval.required`** — çalışma başarılı olduktan *sonra*, fakat + operatör dağıtımı onaylamadan *önce* yayılır. Bu bir başarısızlık + değil; bir duraklamadır. `training.failure` üzerinde otomatik + çağrı yapan alıcılar `approval.required` üzerinde çağrı yapmamalı. + +### Payload şeması + +Her webhook olayı aynı zarfı taşır: + +```json +{ + "event": "training.start | training.success | training.failure | training.reverted | approval.required", + "run_name": "<dize>", + "status": "started | succeeded | failed | reverted | awaiting_approval", + "metrics": {"<isim>": <sayı>, ...}, + "reason": "<maskelenmiş dize ya da null>", + "model_path": "<dosya sistemi yolu ya da null>", + "attachments": [{"title": "...", "text": "...", "color": "..."}] +} +``` + +`metrics`, `reason` ve `model_path` her zaman şemada bulunur; yalnızca +ihtiyaç duyan olaylarda doldurulur. `attachments`, Slack uyumlu blok'tur +— diğer alıcılar görmezden gelebilir. + +### Güvenlik garantileri + +1. **Sebepler maskelenir.** Her `reason` alanı, taşıma öncesi + `forgelm.data_audit.mask_secrets` üzerinden geçer; böylece AWS / + GitHub / Slack / OpenAI / Google / JWT / özel-anahtar blokları / + Azure storage dizeleri süreçten dışarı çıkmaz. `data_audit` ithal + edilemezse, ham dize yerine alan + `"[REDACTED — secrets masker unavailable]"` ile değiştirilir. +2. **Sebepler 2048 karaktere kırpılır.** Bundan uzun stack trace'ler + `"… (truncated)"` ile kesilir. +3. **Model ağırlıkları yok.** `approval.required` yalnızca staging + dosya sistemi yolunu taşır. Ağırlıklar diskte kalır; o dizini zaten + operatör kontrol eder. +4. **Webhook URL'si sızdırılmaz.** URL'ler günlüklerde maskelenir + (`scheme://host/<ilk-segment>/...`) ve 2xx olmayan yanıt gövdesi + bastırılır. +5. **SSRF koruması.** `webhook.allow_private_destinations=true` + ayarlanmadığı sürece özel / loopback / link-local hedefler + reddedilir. + +### Saklama rehberi + +Webhook payload'ları **geçicidir**. Denetim kaydı değildir. Uzun süreli +geçmişe ihtiyaç duyan alıcılar, webhook trafiğini arşivlemek yerine +denetim JSONL dosyasının (`<output_dir>/compliance/audit_log.jsonl`) anlık +görüntüsünü almalıdır; çünkü denetim günlüğü yalnız-eklenir +hash-zincirli kayıttır ve webhook akışı en-iyi-çabadır (best-effort). diff --git a/docs/reference/audit_event_catalog.md b/docs/reference/audit_event_catalog.md new file mode 100644 index 00000000..da288b3f --- /dev/null +++ b/docs/reference/audit_event_catalog.md @@ -0,0 +1,162 @@ +# Audit Event Catalog + +> **Audience:** ForgeLM operators, regulators, and downstream verifiers reviewing the EU AI Act Article 12 record-keeping artefacts. +> **Mirror:** [audit_event_catalog-tr.md](audit_event_catalog-tr.md) + +This catalog enumerates every event ForgeLM may append to `audit_log.jsonl`, the append-only, hash-chained record-keeping artefact mandated by EU AI Act Article 12. Every entry shares a common envelope; the `event` field selects one of the rows below. + +## Common envelope + +Each line is a single JSON object with at least the following fields: + +| Field | Type | Description | +|---------------|---------|-----------------------------------------------------------------------------------------------------------| +| `timestamp` | string | ISO-8601 UTC timestamp (`datetime.now(timezone.utc).isoformat()`). | +| `run_id` | string | Stable per-training-run identifier (`fg-<uuid12>`). | +| `operator` | string | Human-attributable identity, resolved by `AuditLogger` in priority order: (1) `$FORGELM_OPERATOR` if set; (2) `<getpass.getuser()>@<hostname>` when the username can be resolved; (3) `anonymous@<hostname>` only when `getpass.getuser()` fails *and* `FORGELM_ALLOW_ANONYMOUS_OPERATOR=1` is set (otherwise the run aborts loudly). | +| `event` | string | Dotted event name from this catalog. | +| `prev_hash` | string | SHA-256 of the previous line (`"genesis"` for the first entry). Forms the tamper-evident hash chain. | +| `_hmac` | string? | Present only when `FORGELM_AUDIT_SECRET` is set. HMAC-SHA-256 of the line without `_hmac`. | +| _payload_ | varies | Event-specific keys, listed per row below. | + +The hash chain advances after the line lands on disk (`flush` + `fsync`), so an unclean shutdown leaves the chain intact for resume. + +## Event vocabulary + +### Pipeline lifecycle + +| Event | When emitted | Payload (in addition to envelope) | Article | +|----------------------------|---------------------------------------------------------------------------|------------------------------------------------------------------------------------------|---------| +| `training.started` | Trainer begins a fine-tuning run. | `trainer_type`, `model`, `dataset`, `config_path` | 12 | +| `pipeline.completed` | End-to-end CLI run (training + evaluation + export) returned exit code 0. | `exit_code`, `duration_seconds`, `success`, `metrics_summary` | 12 | +| `pipeline.failed` | Pipeline aborted with an error before completion. | `error` | 12 | + +### Article 14 — Human Oversight + +| Event | When emitted | Payload | Article | +|------------------------------|---------------------------------------------------------------------------------------------------------|------------------------------------------------------|---------| +| `human_approval.required` | A gate marked `requires_human_approval: true` paused the pipeline awaiting an operator decision. | `gate`, `reason`, `metrics` | 14 | +| `human_approval.granted` | Operator approved the paused gate. _(Faz 9 — placeholder; not yet emitted.)_ | `gate`, `approver`, `comment` | 14 | +| `human_approval.rejected` | Operator rejected the paused gate. _(Faz 9 — placeholder; not yet emitted.)_ | `gate`, `approver`, `comment` | 14 | + +### Article 15 — Model Integrity (auto-revert + safety) + +| Event | When emitted | Payload | Article | +|--------------------------------|-----------------------------------------------------------------------------------------------------------|---------------------------------------------------------------|---------| +| `model.reverted` | Auto-revert restored a previous checkpoint after a quality regression. _(Faz 8 — webhook-coupled.)_ | `from_checkpoint`, `to_checkpoint`, `reason`, `metrics_delta` | 15 | +| `audit.classifier_load_failed` | Safety classifier (e.g., Llama Guard) failed to load. The run still records `passed=False`. | `classifier`, `reason` | 15 | + +### Article 11 + Annex IV — Compliance artefacts + +| Event | When emitted | Payload | Article | +|----------------------------------|-----------------------------------------------------------------------------|--------------------------------------------------|------------| +| `compliance.governance_exported` | Article 10 data governance report written to disk. | `output_path`, `dataset_count` | 10 | +| `compliance.governance_failed` | Governance report generation aborted (e.g., schema mismatch). | `failure_reason` | 10 | +| `compliance.artifacts_exported` | Annex IV technical documentation bundle (manifest, model card, audit zip). | `output_dir`, `files` | 11, Annex IV | + +### CLI / migration + +| Event | When emitted | Payload | Article | +|-----------------------------|----------------------------------------------------------------------------------------------------|----------------------------------|---------| +| `cli.legacy_flag_invoked` | A deprecated CLI flag was used. _(Faz 13 — placeholder; not yet emitted.)_ | `flag`, `replacement`, `version` | 12 | + +### Audit-system events (meta) + +| Event | When emitted | Payload | Article | +|--------------------------------|---------------------------------------------------------------------------------------------------|--------------------------------------|---------| +| `audit.classifier_load_failed` | _(See Article 15 row above.)_ | `classifier`, `reason` | 15 | +| `audit.cross_run_continuity` | First write of a second-or-later AuditLogger instance pointing at an existing log directory. | `previous_chain_head` | 12 | + +## Adding a new event + +1. Pick a dotted name following the existing namespaces (`training.*`, `compliance.*`, `audit.*`, `human_approval.*`, `model.*`, `cli.*`). +2. Add a row to the table above, including the payload keys and the Article it supports. +3. Mirror the row to [audit_event_catalog-tr.md](audit_event_catalog-tr.md). +4. Emit via `AuditLogger.log_event(event, **payload)`. Never call `json.dump` directly into `audit_log.jsonl`; the hash chain depends on the canonical writer. + +## Tamper-evidence summary + +| Mechanism | Defends against | Always on? | +|--------------------------|-----------------------------------------------------------------------|------------------------------------------------------| +| SHA-256 hash chain | Single-line edits, deletions, reorderings. | Yes. | +| Genesis manifest sidecar | Whole-log truncation back to "genesis". | Yes (written once on first event). | +| `flock(LOCK_EX)` | Interleaved writes from concurrent trainers sharing the directory. | Yes (Unix); no-op on Windows. | +| `flush` + `fsync` | Power-cut / kernel-panic loss between buffer write and chain advance. | Yes. | +| HMAC-SHA-256 per line | Forged re-signing after log rewrite. | Only when `FORGELM_AUDIT_SECRET` is set. | + +## Webhook events + +Webhook payloads (Slack / Teams / generic HTTP) are a separate vocabulary scoped to operator notifications, not the regulatory record. Webhook events are **not** appended to `audit_log.jsonl`; they ride the side-channel notification bus. The canonical lifecycle vocabulary is also documented in [logging-observability.md](../standards/logging-observability.md). + +These five lifecycle events are the **only** events that webhook +receivers should expect from `WebhookNotifier`. Each one mirrors a +corresponding audit-log event so a downstream operator can correlate +webhook ping → audit entry by `run_name` + timestamp. Implementation: +`forgelm/webhook.py`. + +| Webhook `event` | Audit-log mirror | Trigger | Gated by | Required payload fields | +|---|---|---|---|---| +| `training.start` | `training.started` | `train()` entered, before model load. | `webhook.notify_on_start` | `run_name`, `status="started"` | +| `training.success` | `pipeline.completed` | All gates passed, no human-approval requirement. | `webhook.notify_on_success` | `run_name`, `status="succeeded"`, `metrics` | +| `training.failure` | `pipeline.failed` | Training itself raised (OOM, dataset error, unhandled exception). | `webhook.notify_on_failure` | `run_name`, `status="failed"`, `reason` (masked, ≤2048 chars) | +| `training.reverted` | `model.reverted` | A post-training gate (evaluation, safety, judge, benchmark) rejected the run and `_revert_model` deleted the adapters. | `webhook.notify_on_failure` | `run_name`, `status="reverted"`, `reason` (masked, ≤2048 chars) | +| `approval.required` | `human_approval.required` | Run succeeded, `evaluation.require_human_approval=true`, model staged for review (EU AI Act Art. 14). | `webhook.notify_on_success` | `run_name`, `status="awaiting_approval"`, `model_path` | + +### Why two of these split lifecycle states + +- **`training.failure` vs `training.reverted`** — dashboards need to tell + "the trainer crashed" apart from "the trainer succeeded, but the + quality / safety / judge gate said no". Both are operationally + actionable, but they require different runbooks. Faz 8 introduced + `notify_reverted` precisely so a Slack channel can colour-code the two + cases differently (`#ff0000` vs `#ff9900`). +- **`approval.required`** — fires *after* the run succeeded but *before* + the operator has approved deploy. It is not a failure; it is a pause. + Receivers that auto-page on `training.failure` should **not** page on + `approval.required`. + +### Payload schema + +Every webhook event ships the same envelope: + +```json +{ + "event": "training.start | training.success | training.failure | training.reverted | approval.required", + "run_name": "<string>", + "status": "started | succeeded | failed | reverted | awaiting_approval", + "metrics": {"<name>": <number>, ...}, + "reason": "<masked string or null>", + "model_path": "<filesystem path or null>", + "attachments": [{"title": "...", "text": "...", "color": "..."}] +} +``` + +`metrics`, `reason`, and `model_path` are always present in the schema; +only populated for the events that need them. `attachments` is the +Slack-compatible block — other receivers may ignore it. + +### Security guarantees + +1. **Reasons are masked.** Every `reason` field passes through + `forgelm.data_audit.mask_secrets` before transport, so AWS / GitHub / + Slack / OpenAI / Google / JWT / private-key blocks / Azure storage + strings do not leave the process. If `data_audit` cannot be imported, + the field is replaced with `"[REDACTED — secrets masker unavailable]"` + rather than the raw string. +2. **Reasons are truncated to 2048 chars.** Stack traces longer than + that are clipped with `"… (truncated)"`. +3. **No model weights.** `approval.required` carries the staging + filesystem path only. Weights stay on disk; the operator already + controls that directory. +4. **No webhook URL leakage.** URLs are masked in logs (`scheme://host/<first-segment>/...`) + and the response body is suppressed on non-2xx. +5. **SSRF guard.** Private / loopback / link-local destinations are + refused unless `webhook.allow_private_destinations=true`. + +### Retention guidance + +Webhook payloads are **transient**. They are not the audit record. +Receivers that need long-term history should snapshot the audit JSONL +(`<output_dir>/compliance/audit_log.jsonl`) rather than archiving webhook +traffic, because the audit log is the append-only hash-chained record +and the webhook stream is best-effort. diff --git a/docs/roadmap-tr.md b/docs/roadmap-tr.md index 6ecc748c..645d58f8 100644 --- a/docs/roadmap-tr.md +++ b/docs/roadmap-tr.md @@ -9,11 +9,13 @@ | ✅ Tamam | [Faz 1-9](roadmap/completed-phases.md) | SOTA iyileştirmeleri, değerlendirme, güvenilirlik, kurumsal entegrasyon, ekosistem, hizalama stack'i, güvenlik, EU AI Act uyumluluğu (Madde 9-17 + Ek IV), gelişmiş güvenlik zekası | | ✅ Tamam | [Faz 10 — Post-Training Tamamlama](roadmap/phase-10-post-training.md) | `inference.py`, `chat`, `export` (GGUF), `--fit-check`, `deploy` — `v0.4.0` | | ✅ Tamam | [Faz 10.5 — Quickstart Katmanı ve Onboarding](roadmap/phase-10-5-quickstart.md) | `forgelm quickstart <template>`, 5 hazır template, seed veri setleri — `v0.4.5` | -| 🟡 Birleşti | [Faz 11 + 11.5 + 12 + 12.5 — Doküman Yutma ve Veri Curation Pipeline'ı](roadmap/releases.md#v050--document-ingestion--data-curation-pipeline) | `forgelm ingest`, `forgelm audit`, PII regex + simhash dedup, LSH banding, streaming reader, PII şiddet katmanları, wizard ingest+audit, MinHash LSH dedup, markdown splitter, code/secrets tarama, kalite heuristic'leri, DOCX tablo koruması, `--all-mask`, Croissant 1.0, Presidio NER — `main`'e indi; `v0.5.0` PyPI tag'i bekliyor | +| ✅ Tamam | [Faz 11 + 11.5 + 12 + 12.5 — Doküman Yutma ve Veri Curation Pipeline'ı](roadmap/releases.md#v050--document-ingestion--data-curation-pipeline) | `forgelm ingest`, `forgelm audit`, PII regex + simhash dedup, LSH banding, streaming reader, PII şiddet katmanları, wizard ingest+audit, MinHash LSH dedup, markdown splitter, code/secrets tarama, kalite heuristic'leri, DOCX tablo koruması, `--all-mask`, Croissant 1.0, Presidio NER — `v0.5.0` (PyPI 2026-04-30) | | 📋 Planlandı | [Faz 14 — Çok Aşamalı Pipeline Zincirleri](roadmap/phase-14-pipeline-chains.md) | SFT → DPO → GRPO config zinciri, pipeline kaynak izleri → `v0.5.1` | | 📋 Planlandı | [Faz 13 — Pro CLI ve Gözlemlenebilirlik Dashboard](roadmap/phase-13-pro-cli.md) | Lisans korumalı dashboard, HPO, zamanlanmış görevler, takım config store → `v0.6.0-pro` | -**`main`'e indi, tag + PyPI publish bekliyor:** `v0.5.0` — "Doküman Yutma + Veri Curation Pipeline'ı" (Faz 11 + 11.5 + 12 + 12.5 birleştirildi). +> **Durum lejantı:** ✅ Yayınlandı (PyPI) · 🟡 main'e indi, publish bekliyor · ⏳ Planlandı + +**Yayınlandı:** `v0.5.0` — "Doküman Yutma + Veri Curation Pipeline'ı" — PyPI 2026-04-30 (Faz 11 + 11.5 + 12 + 12.5 birleştirildi). - **Faz 11** — `forgelm ingest` (PDF / DOCX / EPUB / TXT / Markdown → SFT'ye uygun JSONL) + `forgelm audit` (uzunluk / dil / near-duplicate / cross-split sızıntı / Luhn + TC Kimlik validatörlü PII regex) + EU AI Act Madde 10 governance entegrasyonu. - **Faz 11.5** — operasyonel cila: LSH bantlı near-duplicate tespiti, streaming JSONL okuyucu, token-aware `--chunk-tokens`, PDF sayfa-seviyesi header/footer dedup, `forgelm audit` subcommand'i, PII şiddet katmanları, atomic audit yazımı, wizard "ingest first" girişi. @@ -22,11 +24,11 @@ Başlangıçta dört ardışık PyPI tag'i (`v0.5.0` / `v0.5.1` / `v0.5.2` / `v0.5.3`) olarak planlandı, dört faz tek tutarlı yüzey (yut → cila → olgunlaş → cila) oluşturduğu için tek kapsamlı `v0.5.0` release'inde birleştirildi. -**PyPI'deki son sürüm:** `v0.4.5` — Quickstart Katmanı (2026-04-26). Tek komutla hazır template'ler: `forgelm quickstart customer-support`. Küçük GPU'larda model otomatik downsize ediliyor, mevcut trainer'ın değişiklik gerektirmeden kabul edeceği bir config üretiyor. +**PyPI'deki son sürüm:** `v0.5.0` — Doküman Yutma + Veri Curation Pipeline'ı (2026-04-30). `forgelm ingest`, `forgelm audit`, MinHash LSH dedup, Presidio ML-NER, Croissant 1.0 dataset card'ları, EU AI Act Madde 10 governance entegrasyonu. -**Daha öncesi:** `v0.4.0` — Post-Training Tamamlama (2026-04-26). Inference primitif'leri, etkileşimli chat REPL, GGUF export, VRAM fit advisor, deployment config üretimi. +**Daha öncesi:** `v0.4.5` — Quickstart Katmanı (2026-04-26); `v0.4.0` — Post-Training Tamamlama (2026-04-26). -**Sonraki:** `v0.5.1` — Çok Aşamalı Pipeline Zincirleri (Faz 14). SFT → DPO → GRPO zincir konfigürasyonu, pipeline kaynak izleri. `v0.5.0` PyPI tag'inden sonra başlar. [#14 webhook SSRF hardening](https://github.com/cemililik/ForgeLM/issues/14) follow-up'ını kapsar. +**Sonraki:** `v0.5.1` — Çok Aşamalı Pipeline Zincirleri (Faz 14). SFT → DPO → GRPO zincir konfigürasyonu, pipeline kaynak izleri. [#14 webhook SSRF hardening](https://github.com/cemililik/ForgeLM/issues/14) follow-up'ını kapsar. **Güncel durum:** 17 faz (1, 2, 2.5, 3, 4, 5, 5.5, 6, 7, 8, 9, 10, 10.5, 11, 11.5, 12, 12.5) tamam. 2 faz (13, 14) planlandı. `v0.5.1`: Faz 14. `v0.6.0-pro` (Faz 13) adoption metriklerine bağlı. @@ -45,7 +47,7 @@ graph LR P10 -.-> V1[v0.4.0] P105 -.-> V15[v0.4.5] - P11 -.-> V2[v0.5.0 pending] + P11 -.-> V2[v0.5.0 ✅ Yayınlandı] P115 -.-> V2 P12 -.-> V2 P125 -.-> V2 diff --git a/docs/roadmap.md b/docs/roadmap.md index fc3eb4ef..da907ee2 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -9,11 +9,13 @@ | ✅ Done | [Phase 1-9](roadmap/completed-phases.md) | SOTA upgrades, evaluation, reliability, enterprise integration, ecosystem, alignment stack, safety, EU AI Act compliance (Articles 9-17 + Annex IV), advanced safety intelligence | | ✅ Done | [Phase 10 — Post-Training Completion](roadmap/phase-10-post-training.md) | `inference.py`, `chat`, `export` (GGUF), `--fit-check`, `deploy` — shipped `v0.4.0` | | ✅ Done | [Phase 10.5 — Quickstart Layer & Onboarding](roadmap/phase-10-5-quickstart.md) | `forgelm quickstart <template>`, 5 bundled templates with seed datasets — shipped `v0.4.5` | -| 🟡 Merged | [Phase 11 + 11.5 + 12 + 12.5 — Document Ingestion & Data Curation Pipeline](roadmap/releases.md#v050--document-ingestion--data-curation-pipeline) | `forgelm ingest`, `forgelm audit`, PII regex + simhash dedup, LSH banding, streaming reader, PII severity tiers, wizard ingest+audit, MinHash LSH dedup, markdown splitter, code/secrets scan, quality heuristics, DOCX table preservation, `--all-mask`, Croissant 1.0, Presidio NER — merged to `main`; PyPI tag pending as `v0.5.0` | +| ✅ Done | [Phase 11 + 11.5 + 12 + 12.5 — Document Ingestion & Data Curation Pipeline](roadmap/releases.md#v050--document-ingestion--data-curation-pipeline) | `forgelm ingest`, `forgelm audit`, PII regex + simhash dedup, LSH banding, streaming reader, PII severity tiers, wizard ingest+audit, MinHash LSH dedup, markdown splitter, code/secrets scan, quality heuristics, DOCX table preservation, `--all-mask`, Croissant 1.0, Presidio NER — shipped `v0.5.0` (PyPI 2026-04-30) | | 📋 Planned | [Phase 14 — Multi-Stage Pipeline Chains](roadmap/phase-14-pipeline-chains.md) | SFT → DPO → GRPO chained config, pipeline provenance artifacts → `v0.5.1` | | 📋 Planned | [Phase 13 — Pro CLI & Observability Dashboard](roadmap/phase-13-pro-cli.md) | License-gated dashboard, HPO, scheduled jobs, team config store → `v0.6.0-pro` | -**Merged on `main`, tag + PyPI publish pending:** `v0.5.0` — "Document Ingestion + Data Curation Pipeline" (Phases 11 + 11.5 + 12 + 12.5 consolidated). +> **Status legend:** ✅ Released (PyPI) · 🟡 Merged on main, publish pending · ⏳ Planned + +**Released:** `v0.5.0` — "Document Ingestion + Data Curation Pipeline" — PyPI 2026-04-30 (Phases 11 + 11.5 + 12 + 12.5 consolidated). - **Phase 11** — `forgelm ingest` (PDF / DOCX / EPUB / TXT / Markdown → SFT-ready JSONL) + `forgelm audit` (length / language / near-duplicate / cross-split leakage / PII regex with Luhn + TC Kimlik validators) + EU AI Act Article 10 governance integration. - **Phase 11.5** — operational polish: LSH-banded near-duplicate detection, streaming JSONL reader, token-aware `--chunk-tokens`, PDF page-level header/footer dedup, `forgelm audit` subcommand, PII severity tiers, atomic audit writes, wizard "ingest first" entry point. @@ -22,11 +24,11 @@ Originally planned as four sequential PyPI tags (`v0.5.0` / `v0.5.1` / `v0.5.2` / `v0.5.3`), consolidated into one comprehensive `v0.5.0` release because the four phases form one coherent surface (ingest → polish → mature → polish) hard to use in parts. -**Latest release on PyPI:** `v0.4.5` — Quickstart Layer (2026-04-26). One-command bundled templates: `forgelm quickstart customer-support`. Auto-downsizes models on small GPUs, generates a config that the existing trainer accepts unchanged. +**Latest release on PyPI:** `v0.5.0` — Document Ingestion + Data Curation Pipeline (2026-04-30). `forgelm ingest`, `forgelm audit`, MinHash LSH dedup, Presidio ML-NER, Croissant 1.0 dataset cards, EU AI Act Article 10 governance integration. -**Earlier:** `v0.4.0` — Post-Training Completion (2026-04-26). Inference primitives, interactive chat REPL, GGUF export, VRAM fit advisor, deployment config generation. +**Earlier:** `v0.4.5` — Quickstart Layer (2026-04-26); `v0.4.0` — Post-Training Completion (2026-04-26). -**Next:** `v0.5.1` — Multi-Stage Pipeline Chains (Phase 14). SFT → DPO → GRPO chained config, pipeline provenance artifacts. Starts after the `v0.5.0` PyPI tag is published. Folds in [#14 webhook SSRF hardening](https://github.com/cemililik/ForgeLM/issues/14). +**Next:** `v0.5.1` — Multi-Stage Pipeline Chains (Phase 14). SFT → DPO → GRPO chained config, pipeline provenance artifacts. Folds in [#14 webhook SSRF hardening](https://github.com/cemililik/ForgeLM/issues/14). **Current state:** 17 phases (1, 2, 2.5, 3, 4, 5, 5.5, 6, 7, 8, 9, 10, 10.5, 11, 11.5, 12, 12.5) complete. 2 phases (13, 14) planned. `v0.5.1`: Phase 14. `v0.6.0-pro` (Phase 13) gated on adoption metrics. @@ -45,7 +47,7 @@ graph LR P10 -.-> V1[v0.4.0] P105 -.-> V15[v0.4.5] - P11 -.-> V2[v0.5.0 pending] + P11 -.-> V2[v0.5.0 ✅ Released] P115 -.-> V2 P12 -.-> V2 P125 -.-> V2 diff --git a/docs/roadmap/releases.md b/docs/roadmap/releases.md index 88dc51bb..fc104e50 100644 --- a/docs/roadmap/releases.md +++ b/docs/roadmap/releases.md @@ -12,7 +12,7 @@ 2. [x] **Long-Context Training**: RoPE scaling, NEFTune noise injection, sliding window attention, and sample packing for extended context windows. Config fields: `rope_scaling`, `neftune_noise_alpha`, `sliding_window_attention`, `sample_packing`. 3. [x] **Synthetic Data Pipeline**: Teacher-to-student distillation via `--generate-data` CLI flag. New `SyntheticDataGenerator` class in `forgelm/synthetic.py`. Configurable teacher model, backend, seed prompts, and output format. 4. [x] **PyPI Publishing**: `pip install forgelm` now works. Automated publishing via `publish.yml` GitHub Actions workflow. -5. [x] **GPU Cost Estimation**: Auto-detection for 18 GPU models with per-run cost tracking. Included in JSON output, webhook notifications, and model cards. +5. [x] **GPU Cost Estimation**: Auto-detection for 16 GPU models with per-run cost tracking. Included in JSON output, webhook notifications, and model cards. 6. [x] **Nightly CI**: `.github/workflows/nightly.yml` for compatibility testing against latest dependency versions. 7. [x] **Expanded Adversarial Prompts**: 6 category files, 140 prompts (up from 50) covering general safety, bias/discrimination, harmful instructions, privacy/PII, misinformation, and jailbreak attempts. @@ -82,7 +82,7 @@ Odak: [Phase 10](phase-10-post-training.md). Full post-training handoff: inferen ## v0.5.0 — "Document Ingestion + Data Curation Pipeline" -**Status:** Merged on `main` (Phases 11 + 11.5 + 12 + 12.5 consolidated; 2026-04-29). The `v0.5.0` git tag and PyPI publish are the remaining release-engineering steps. One hardening follow-up tracked outside the release: [#14 — webhook SSRF DNS-rebinding TOCTOU](https://github.com/cemililik/ForgeLM/issues/14) (defence-in-depth on top of the existing `allow_private_destinations: false` default). +**Status:** ✅ Done — released to [PyPI 2026-04-30](https://pypi.org/project/forgelm/0.5.0/) (Phases 11 + 11.5 + 12 + 12.5 consolidated; merged on `main` 2026-04-29). One hardening follow-up tracked outside the release: [#14 — webhook SSRF DNS-rebinding TOCTOU](https://github.com/cemililik/ForgeLM/issues/14) (defence-in-depth on top of the existing `allow_private_destinations: false` default). > **Note on consolidation.** Originally planned as four sequential PyPI tags (`v0.5.0` / `v0.5.1` / `v0.5.2` / `v0.5.3`), the four phases were consolidated into a single `v0.5.0` because they form one coherent surface (ingest → polish → mature → polish) hard to use in parts. Git history retains the four phases as separate commit batches; this entry collapses them into one user-facing release. CHANGELOG.md preserves the phase boundaries inside the `[0.5.0]` section so reviewers can map back to PR history (#11, #12, #13, #18). diff --git a/docs/roadmap/risks-and-decisions.md b/docs/roadmap/risks-and-decisions.md index d34d4c7a..2699e4ae 100644 --- a/docs/roadmap/risks-and-decisions.md +++ b/docs/roadmap/risks-and-decisions.md @@ -88,3 +88,5 @@ | 2026-04-25 | Enterprise outreach moved to Day 1 (previously Week 7-12) | EU AI Act enforcement in 99 days; enterprise sales cycle is 3-6 months; starting at Week 7 means missing the window | | 2026-04-25 | v0.3.1rc1 security + config hardening release | Comprehensive code review revealed webhook URL leakage, audit log chain gap, GRPO callable bug, TIES merging error — all fixed | | 2026-04-25 | Turkey/BDDK/KVKK market added as priority enterprise target | First-mover in Turkish compliance market; BDDK AI guidelines publishing in 2026; no competitors; path to first enterprise case study | +| 2026-04-30 | v0.5.0 release — Phases 11 + 11.5 + 12 + 12.5 consolidation | Four originally-sequential phases (`v0.5.0`–`v0.5.3`) form one coherent ingest → polish → mature → polish surface that's hard to use in parts; consolidated into a single user-facing release | +| 2026-04-30 | PyPI publish landed for v0.5.0 | Document Ingestion + Data Curation Pipeline shipped to PyPI; closes the "merged on main, publish pending" gap that drifted across roadmap, releases.md, and CHANGELOG.md | diff --git a/docs/standards/logging-observability.md b/docs/standards/logging-observability.md index ad15b17a..c720fb87 100644 --- a/docs/standards/logging-observability.md +++ b/docs/standards/logging-observability.md @@ -135,9 +135,47 @@ class WebhookNotifier: 1. **Webhooks never abort training.** A 500 from Slack is a warning, not a failure. Wrap the POST in `try/except requests.RequestException` and log at `WARNING`. 2. **Timeout every request** (`timeout=10` minimum, `timeout=30` maximum). 3. **Sanitize payload.** Never send API keys, full config contents, or sensitive data paths. Whitelist fields going into `payload`. -4. **Lifecycle events:** `training.started`, `training.succeeded`, `training.failed`, `training.reverted`, `approval.required`. Same vocabulary as audit log. +4. **Lifecycle events:** `training.start`, `training.success`, `training.failure`, `training.reverted`, `approval.required` — see the table below for the canonical list. Each webhook event mirrors a distinct audit-log event (e.g., webhook `training.start` ↔ audit `training.started`); the two vocabularies are paired but not identical because past-tense audit identifiers describe a finalized record while webhooks announce a state transition. 5. **Retry:** Up to 3 times with exponential backoff. After that, audit-log the failure and move on. +### Webhook event vocabulary + +The five lifecycle events below are the **only** events that webhook receivers +should expect. Adding new ones requires a docs update here, a `Notifier` +method, a paired audit event, and tests. Do not invent ad-hoc event strings. + +| Event | Emitted when | Required fields | Notes | +|---|---|---|---| +| `training.start` | `train()` is entered, before model load. | `run_name`, `status="started"` | Mirrors audit event `training.started`. Gated by `webhook.notify_on_start`. | +| `training.success` | All gates passed, no human approval required. | `run_name`, `status="succeeded"`, `metrics` | Mirrors audit event `pipeline.completed`. Gated by `webhook.notify_on_success`. | +| `training.failure` | Training itself crashed (OOM, dataset error, exception in pipeline). | `run_name`, `status="failed"`, `reason` (masked, ≤2048 chars) | Mirrors audit event `pipeline.failed`. Gated by `webhook.notify_on_failure`. | +| `training.reverted` | A post-training gate (eval / safety / judge / benchmark) rejected the run and `_revert_model` deleted the adapters. | `run_name`, `status="reverted"`, `reason` (masked, ≤2048 chars) | Distinct from `training.failure` so dashboards can separate "training crashed" from "training succeeded but quality regressed". Mirrors audit event `model.reverted`. Gated by `webhook.notify_on_failure`. | +| `approval.required` | Run succeeded, `evaluation.require_human_approval=true`, model staged for review. | `run_name`, `status="awaiting_approval"`, `model_path` (filesystem path) | Mirrors audit event `human_approval.required`. Operator gets a real-time ping instead of having to poll the audit JSONL. Model weights themselves are **never** in the payload — only the staging path. Gated by `webhook.notify_on_success`. | + +**Payload schema (every event):** + +```json +{ + "event": "<one of the five above>", + "run_name": "<string>", + "status": "<started|succeeded|failed|reverted|awaiting_approval>", + "metrics": {"<name>": <number>, ...}, + "reason": "<masked string or null>", + "model_path": "<staging path or null>", + "attachments": [{"title": "...", "text": "...", "color": "..."}] +} +``` + +`metrics`, `reason`, and `model_path` are always present in the schema; only +populated for the events that need them. `attachments` is Slack-compatible +formatting that other receivers may ignore. + +**Retention guidance:** Webhook payloads are transient — they are *not* the +audit record. Receivers that need long-term history should snapshot the +audit JSONL (`<output_dir>/compliance/audit_log.jsonl`) rather than archiving +webhook traffic, because the audit log is the append-only hash-chained +record and the webhook stream is best-effort. + ## Third-party tracking (W&B / MLflow / TensorBoard) Routed through `trainer_args.report_to`. Rules: diff --git a/docs/standards/release.md b/docs/standards/release.md index d4af9691..82483a50 100644 --- a/docs/standards/release.md +++ b/docs/standards/release.md @@ -74,6 +74,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 5. **Write for users**, not for contributors. "Added inference module" is user-facing. "Refactored `trainer.py`" is not — don't include it. 6. **Audience doesn't care about version bumps of dev deps** (ruff, pytest). Don't add. +## Deprecation cadence + +When a CLI flag, YAML field, JSON output key, or public function is removed, +it must first be deprecated for **at least one minor release** before the +removal lands. The cadence is non-negotiable so users running locked +versions in CI/CD pipelines get a release cycle to migrate. + +Rules: + +1. **Minimum overlap.** A deprecated surface stays present for at least one + intervening minor (`v0.5.0` deprecate → `v0.6.0` still present → + `v0.7.0` removable). Patch releases never remove anything. +2. **`DeprecationWarning` is mandatory** at the moment of deprecation — + raised via `warnings.warn(..., DeprecationWarning, stacklevel=2)`. CLI + flags additionally print a one-line stderr notice. +3. **Removal version stated up-front.** The deprecation message, the + `--help` text, and the CHANGELOG `### Deprecated` entry must all name + the version that will remove the surface. +4. **Tracking issue mandatory.** Every deprecation links to a GitHub issue + that the removal PR closes. The link goes in the CHANGELOG entry. +5. **`### Removed` section required** in the CHANGELOG of the version that + actually drops the surface, cross-referencing the deprecation entry. + +**Worked example — `--data-audit` flag:** introduced in pre-Phase-11, +superseded by the `forgelm audit` subcommand in Phase 12. Deprecated in +v0.5.0 (`forgelm/cli.py:1424-1428` raises `DeprecationWarning` and prints a +stderr notice naming v0.7.0 as the removal target). The flag remains +present and functional through v0.6.x, then is removed in v0.7.0 with a +matching `### Removed` CHANGELOG entry. + ## Release checklist Done manually by the maintainer (or a bot when automated): diff --git a/docs/standards/testing.md b/docs/standards/testing.md index fa617948..d66a0cf6 100644 --- a/docs/standards/testing.md +++ b/docs/standards/testing.md @@ -5,7 +5,7 @@ ## Layout -Current structure (26 test modules, one per feature area): +Current structure (48 test modules, one per feature area): ``` tests/ @@ -124,7 +124,7 @@ From [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml): 1. **Lint** — `ruff check` + `ruff format --check` on entire repo. Failure = PR blocked. 2. **Test matrix** — Python 3.10, 3.11, 3.12, 3.13 on ubuntu-latest. -3. **Coverage** — `pytest --cov=forgelm --cov-fail-under=25`. +3. **Coverage** — `pytest --cov=forgelm --cov-fail-under=40` (enforced via `addopts` in `pyproject.toml`'s `[tool.pytest.ini_options]`, kept in lock-step with `[tool.coverage.report].fail_under`). 4. **Dry-run validation** — `forgelm --config config_template.yaml --dry-run` must succeed. From [`.github/workflows/nightly.yml`](../../.github/workflows/nightly.yml): diff --git a/forgelm/_http.py b/forgelm/_http.py new file mode 100644 index 00000000..40a82cc0 --- /dev/null +++ b/forgelm/_http.py @@ -0,0 +1,230 @@ +"""Centralized HTTP discipline for outbound POST requests. + +Extracted from ``webhook._post_payload`` to enforce SSRF guard, redirect +refusal, ``http://`` refusal, timeout floor, and secret-mask error reasons +across every outbound HTTP call site in the codebase (webhook, judge, +synthetic). Future call sites (telemetry, registry pings, etc.) MUST go +through :func:`safe_post` rather than calling ``requests.post`` directly — +the acceptance gate ``grep -rn "requests\\.post" forgelm/ | grep -v _http\\.py`` +should stay empty. + +Policy summary (each enforced before the network call): + +* **Scheme** — ``https://`` required by default; ``http://`` rejected unless + the caller passes ``allow_insecure_http=True`` (only the operator-blessed + webhook path uses this; judge / synthetic always require TLS). +* **SSRF** — RFC1918, loopback, link-local (incl. cloud IMDS at + ``169.254.169.254``), reserved, and multicast destinations are blocked + unless ``allow_private=True``. Hostnames are pre-resolved via + ``socket.getaddrinfo`` so a DNS name pointing at a private IP also trips. +* **Timeout floor** — defaults to 10s; callers can pass ``min_timeout`` to + lower the floor (the webhook path uses 1s to preserve historical + behaviour). ``timeout=0`` / ``None`` is always rejected — ``requests`` + honours those as "block forever". +* **Redirects** — ``allow_redirects=False`` always. The SSRF guard runs + against the *initial* hostname; following a 30x to a private IP would + bypass it. +* **TLS** — ``verify=True`` by default; pass ``ca_bundle="/path/..."`` for a + custom CA store (corporate MITM CA on regulated estates). +* **Header masking** — ``Authorization`` / ``X-API-Key`` values are redacted + from the warning log emitted when the request raises. +""" + +from __future__ import annotations + +import ipaddress +import logging +import socket +from typing import Any, Dict, Optional +from urllib.parse import urlparse + +import requests + +logger = logging.getLogger("forgelm._http") + + +class HttpSafetyError(Exception): + """Policy-level rejection of an outbound HTTP request. + + Distinct from :class:`requests.RequestException` so callers can tell a + refused-by-policy URL (operator misconfiguration — surface to the user) + apart from a transport failure (network blip — log + continue). + """ + + +_MASK_HEADER_NAMES = frozenset({"authorization", "x-api-key", "proxy-authorization"}) + + +def _is_private_destination(host: str) -> bool: + """Return ``True`` when *host* resolves to a non-public-internet IP. + + DNS pre-resolution catches hostnames that happen to point at RFC1918 / + link-local / loopback addresses, so a ``http://internal.corp/`` URL is + rejected even when no IP literal is present in the URL itself. + + Re-exported from :mod:`forgelm.webhook` (where it originated) for + backwards compatibility — existing tests / external consumers continue + to import the symbol from the webhook module. + """ + if not host: + return False + try: + ip = ipaddress.ip_address(host) + except ValueError: + ip = None + if ip is not None: + return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast + try: + addrinfo = socket.getaddrinfo(host, None) + except (socket.gaierror, UnicodeError): + # DNS failure → not classified as private; let `requests` produce + # its natural ConnectionError downstream so the operator sees the + # real "could not resolve host" message instead of an SSRF-shaped + # refusal that hides the typo. + return False + for _family, _type, _proto, _canon, sockaddr in addrinfo: + try: + resolved = ipaddress.ip_address(sockaddr[0]) + except ValueError: + continue + if ( + resolved.is_private + or resolved.is_loopback + or resolved.is_link_local + or resolved.is_reserved + or resolved.is_multicast + ): + return True + return False + + +def _mask_netloc(url: str) -> str: + """Return ``scheme://host`` with userinfo / path / query stripped. + + Used in policy-rejection error messages and warning logs so we never + echo the secret-bearing tail of a Slack / Teams / Discord webhook URL + (which carries credentials in the path) into operator-visible output. + """ + try: + parts = urlparse(url) + except (ValueError, TypeError): + return "<unparseable-url>" + if not parts.scheme or not parts.netloc: + return "<malformed-url>" + return f"{parts.scheme}://{parts.hostname or 'unknown-host'}" + + +def _mask_secrets_in_text(text: str, headers: Optional[Dict[str, str]]) -> str: + """Redact known secret-bearing header values from *text*. + + ``requests`` exception strings sometimes include the request URL or + header dump; we strip ``Authorization`` / ``X-API-Key`` / proxy auth + values before logging so a transport-layer error doesn't leak the + bearer token into the trainer's stderr. + """ + if not text or not headers: + return text + masked = text + for name, value in headers.items(): + if not value or not isinstance(value, str): + continue + if name.lower() in _MASK_HEADER_NAMES: + masked = masked.replace(value, "[REDACTED]") + return masked + + +def safe_post( + url: str, + *, + json: Any = None, + data: Any = None, + headers: Optional[Dict[str, str]] = None, + timeout: float = 10.0, + verify: Any = True, + ca_bundle: Optional[str] = None, + allow_insecure_http: bool = False, + allow_private: bool = False, + min_timeout: float = 10.0, +) -> requests.Response: + """Disciplined outbound POST; raises early on policy violation. + + Args: + url: Target URL. Must be ``http://`` or ``https://``. + json: Body serialized as JSON by ``requests``. Mutually exclusive + with ``data``; passing both is a caller bug, not enforced here. + data: Pre-serialized body (e.g. ``json.dumps(payload)`` for the + webhook path that already encodes its own payload). + headers: Outbound headers. ``Authorization`` / ``X-API-Key`` values + are masked in the failure log. + timeout: Per-request timeout in seconds. Must be ``>= min_timeout``. + verify: Forwarded as ``requests``'s ``verify=`` argument; passing + ``False`` is allowed but discouraged. ``ca_bundle`` overrides. + ca_bundle: Path to a custom CA bundle. When non-empty, takes + precedence over ``verify``. + allow_insecure_http: Set ``True`` only for paths where the operator + has explicitly opted into ``http://`` (currently: webhook with a + documented warning at the call site). Judge and synthetic must + never set this — they handle bearer tokens. + allow_private: Set ``True`` to bypass the SSRF guard. Required for + in-cluster Slack proxies / on-prem monitoring sinks. + min_timeout: Lower bound for ``timeout``. Defaults to ``10.0``; + ``webhook._post_payload`` passes ``1.0`` to keep its historical + behaviour without forcing every webhook user to bump their + timeout setting. + + Returns: + The :class:`requests.Response` from the underlying call. The caller + is responsible for inspecting ``response.ok`` / ``status_code``. + + Raises: + HttpSafetyError: On policy violation — ``http://`` without opt-in, + unsupported scheme, sub-floor timeout, or private destination + without opt-in. + requests.RequestException: On transport / TLS / network failure. + Headers are masked in the warning log before the re-raise. + """ + parsed = urlparse(url) + + # Scheme policy. + if parsed.scheme == "http": + if not allow_insecure_http: + raise HttpSafetyError( # NOSONAR — error string mentions "http://" by design (operator-facing rejection message) + f"http:// blocked (use https://); url={_mask_netloc(url)}" + ) + elif parsed.scheme != "https": + raise HttpSafetyError(f"Unsupported URL scheme {parsed.scheme!r}; only http(s) allowed.") + + # SSRF guard. + host = parsed.hostname or "" + if not allow_private and _is_private_destination(host): + raise HttpSafetyError(f"Private/loopback/IMDS destination blocked: host={host or '<empty>'}") + + # Timeout floor — `requests` treats 0 / None as "no timeout" which can + # hang the trainer on a dead endpoint. + if not isinstance(timeout, (int, float)) or timeout < min_timeout: + raise HttpSafetyError(f"Timeout below {min_timeout}s floor: timeout={timeout!r}") + + # Resolve TLS verify setting. ca_bundle (when set) wins over verify. + verify_param: Any = ca_bundle if ca_bundle else verify + + try: + return requests.post( + url, + json=json, + data=data, + headers=headers, + timeout=timeout, + verify=verify_param, + # Redirect-following would bypass the up-front SSRF check — + # a 30x to 169.254.169.254 from an attacker-controlled host + # would otherwise leak the request payload to IMDS. + allow_redirects=False, + ) + except requests.RequestException as exc: + masked_reason = _mask_secrets_in_text(str(exc), headers) + logger.warning( + "safe_post failed url=%s reason=%s", + _mask_netloc(url), + masked_reason[:200], + ) + raise diff --git a/forgelm/cli.py b/forgelm/cli.py index ffd76115..758aac39 100644 --- a/forgelm/cli.py +++ b/forgelm/cli.py @@ -513,6 +513,58 @@ def _add_audit_subcommand(subparsers) -> None: _add_common_subparser_flags(p, include_output_format=True) +def _add_verify_audit_subcommand(subparsers) -> None: + """Phase 6 (closure plan): ``forgelm verify-audit`` for chain integrity. + + Validates the SHA-256 hash chain (and, when a secret is supplied, + the per-line HMAC tags) of an ``audit_log.jsonl`` file written by + :class:`forgelm.compliance.AuditLogger`. Exit codes: + + - ``0`` — chain (and HMAC, if checked) intact + - ``1`` — chain broken or HMAC mismatch + - ``2`` — file missing / unreadable, or option error (e.g. + ``--require-hmac`` without a configured secret env var) + """ + p = subparsers.add_parser( + "verify-audit", + help="Verify integrity of an audit_log.jsonl chain.", + description=( + "Validates the SHA-256 hash chain of a ForgeLM audit_log.jsonl " + "(EU AI Act Article 12 record-keeping). When the operator's " + "FORGELM_AUDIT_SECRET is set in the environment, HMAC tags on " + "each line are also verified. Designed for CI/CD pipelines: " + "exit code 0 means the chain is intact, 1 means tampering or " + "corruption was detected." + ), + ) + p.add_argument( + "log_path", + help="Path to audit_log.jsonl (the genesis manifest sidecar is auto-detected if present).", + ) + p.add_argument( + "--hmac-secret-env", + type=str, + default="FORGELM_AUDIT_SECRET", + metavar="VAR", + help=( + "Name of the environment variable carrying the HMAC secret used at " + "log-write time (default: FORGELM_AUDIT_SECRET). When the variable " + "is set, per-line HMAC tags are validated; when unset, only the " + "SHA-256 chain is checked." + ), + ) + p.add_argument( + "--require-hmac", + action="store_true", + help=( + "Strict mode: exit 2 if the configured env var is unset, and exit 1 " + "if any line lacks an _hmac field. Use this in regulated CI pipelines " + "where every entry must be HMAC-authenticated." + ), + ) + _add_common_subparser_flags(p, include_output_format=False) + + def parse_args(): parser = argparse.ArgumentParser( description="ForgeLM: Language Model Fine-Tuning Toolkit", @@ -522,6 +574,7 @@ def parse_args(): " forgelm quickstart [TEMPLATE] Generate a config from a curated template\n" " forgelm ingest PATH Convert raw docs (PDF/DOCX/EPUB/TXT/Markdown) → JSONL\n" " forgelm audit PATH Run dataset audit (length/lang/PII/leakage)\n" + " forgelm verify-audit LOG_PATH Verify SHA-256 + HMAC integrity of an audit log\n" " forgelm chat MODEL_PATH Interactive chat REPL\n" " forgelm export MODEL_PATH Export model to GGUF\n" " forgelm deploy MODEL_PATH Generate serving config\n" @@ -537,6 +590,7 @@ def parse_args(): _add_quickstart_subcommand(subparsers) _add_ingest_subcommand(subparsers) _add_audit_subcommand(subparsers) + _add_verify_audit_subcommand(subparsers) # --- Top-level flags (training / config-driven mode) --- parser.add_argument("--config", type=str, help="Path to the YAML configuration file.") @@ -1526,13 +1580,62 @@ def _run_audit_cmd(args, output_format: str) -> None: ) +def _run_verify_audit_cmd(args) -> int: + """Phase 6 (closure plan) dispatch for ``forgelm verify-audit LOG_PATH``. + + Returns the process exit code rather than calling :func:`sys.exit` + directly so the dispatcher can route the (0/1/2) outcome through the + same code path as the other subcommands. Exit-code contract: + + - ``0`` — SHA-256 chain (and HMAC tags, when verified) intact. + - ``1`` — tamper or corruption detected (chain break, HMAC mismatch, + manifest mismatch, JSON decode error). + - ``2`` — option error (``--require-hmac`` without a secret env var) + or file/permission error reading the log. + """ + from .compliance import verify_audit_log + + secret_var = args.hmac_secret_env or "" + hmac_secret = os.getenv(secret_var) if secret_var else None + require_hmac = bool(getattr(args, "require_hmac", False)) + + if require_hmac and not hmac_secret: + print( + f"ERROR: --require-hmac specified but ${secret_var} is unset.", + file=sys.stderr, + ) + return EXIT_TRAINING_ERROR # 2 — option/config error + + if not os.path.isfile(args.log_path): + print(f"ERROR: audit log not found: {args.log_path}", file=sys.stderr) + return EXIT_TRAINING_ERROR + + result = verify_audit_log( + args.log_path, + hmac_secret=hmac_secret, + require_hmac=require_hmac, + ) + + if result.valid: + suffix = " (HMAC validated)" if hmac_secret else "" + print(f"OK: {result.entries_count} entries verified{suffix}") + return EXIT_SUCCESS + + line = result.first_invalid_index + if line is None: + print(f"FAIL: {result.reason}", file=sys.stderr) + else: + print(f"FAIL at line {line}: {result.reason}", file=sys.stderr) + return EXIT_CONFIG_ERROR # 1 — chain/HMAC failure + + def _dispatch_subcommand(command: str, args) -> None: """Run a Phase 10 / 10.5 / 11 / 11.5 subcommand and exit. Subcommands handled here: ``chat``, ``export``, ``deploy``, ``quickstart``, - ``ingest``, ``audit``. Each terminates the process via ``sys.exit`` after - its own dispatcher returns — the trainer/training code path never runs - when a subcommand is in play. + ``ingest``, ``audit``, ``verify-audit``. Each terminates the process via + ``sys.exit`` after its own dispatcher returns — the trainer/training code + path never runs when a subcommand is in play. """ if command == "chat": # _run_chat_cmd's REPL catches KeyboardInterrupt internally for the @@ -1561,6 +1664,8 @@ def _dispatch_subcommand(command: str, args) -> None: elif command == "audit": _run_audit_cmd(args, getattr(args, "output_format", "text")) sys.exit(EXIT_SUCCESS) + elif command == "verify-audit": + sys.exit(_run_verify_audit_cmd(args)) def _maybe_run_wizard(args) -> None: diff --git a/forgelm/compliance.py b/forgelm/compliance.py index d6b20056..758d4ffc 100644 --- a/forgelm/compliance.py +++ b/forgelm/compliance.py @@ -7,15 +7,20 @@ """ import concurrent.futures +import getpass import hashlib import hmac as _hmac_module import json import logging import os +import socket import uuid import zipfile +from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple + +from .config import ConfigError # flock is Unix-only; Windows falls back to advisory-only (no hard lock). try: @@ -53,7 +58,49 @@ def __init__(self, output_dir: str, run_id: Optional[str] = None): os.makedirs(output_dir, exist_ok=True) self.log_path = os.path.join(output_dir, "audit_log.jsonl") self._manifest_path = self.log_path + ".manifest.json" - self.operator = os.getenv("FORGELM_OPERATOR", os.getenv("USER", "unknown")) + # Article 12 record-keeping requires a real operator on every entry. + # The previous fallback chain ``$FORGELM_OPERATOR -> $USER -> "unknown"`` + # silently produced ``operator="unknown"`` when both env vars were + # missing (CI runners, container images with no login user). That made + # the audit log unattributable — a regulator cannot identify who ran + # the job. New policy: + # + # 1. If ``FORGELM_OPERATOR`` is set, use it verbatim (CI / pipelines + # pin a deliberate identity here). + # 2. Otherwise derive ``<getpass.getuser()>@<socket.gethostname()>`` — + # matches how Unix audit subsystems attribute work. + # 3. If no username can be resolved, refuse to start unless the + # operator opts in to anonymous logging via + # ``FORGELM_ALLOW_ANONYMOUS_OPERATOR=1``. Loud failure beats a + # silent ``"unknown"`` smear across the chain. + operator_env = os.getenv("FORGELM_OPERATOR") + if operator_env: + self.operator = operator_env + else: + try: + username = getpass.getuser() + except OSError: + # ``getpass.getuser()`` raises ``OSError`` on systems where + # neither ``LOGNAME``/``USER``/``LNAME``/``USERNAME`` env vars + # nor the ``pwd`` lookup resolves an identity (rootless + # containers, sandboxed CI). We still honour the explicit + # opt-in below; fall through with no username. + username = None + hostname = socket.gethostname() or "unknown-host" + if username: + self.operator = f"{username}@{hostname}" + else: + allow_anonymous = os.getenv("FORGELM_ALLOW_ANONYMOUS_OPERATOR") == "1" + if not allow_anonymous: + raise ConfigError( + "Operator identity unavailable: no FORGELM_OPERATOR set, " + "and getpass.getuser() could not resolve a username. " + "Set FORGELM_OPERATOR=<id> for CI/CD pipelines, or " + "FORGELM_ALLOW_ANONYMOUS_OPERATOR=1 to opt in to " + "anonymous audit entries (not recommended for " + "EU AI Act Article 12 record-keeping)." + ) + self.operator = f"anonymous@{hostname}" # Per-run HMAC key: SHA-256(operator_secret || run_id). The secret is # required for tamper-evident HMACs because ``run_id`` is part of the # public log header — without a non-empty secret an attacker who can @@ -102,28 +149,32 @@ def _read_chain_head(self, fh) -> str: "log before resuming." ) - seek_start = max(0, size - 4096) - fh.seek(seek_start) - # When the seek lands mid-line, the first newline-bounded chunk - # would be a partial entry — skip it so every line we parse is whole. - if seek_start > 0: - fh.readline() - tail = fh.read() - lines = self._decode_lines(tail) - if lines: - return hashlib.sha256(lines[-1].encode("utf-8")).hexdigest() - # Empty after skipping the partial-first-line means the final entry - # itself exceeds the 4 KiB tail window. Re-read from ``seek_start`` - # without skipping so we still recover the (oversized) last record. - # The trailing-newline guard above guarantees the recovered line - # is complete; we just need a wider tail window. - if seek_start > 0: + # Progressive-widen tail read. + # + # Start at a 4 KiB tail (typical audit entry < 1 KiB so this hits in + # one read for the common case). When the tail starts mid-record + # (seek-landed inside an entry > tail size) ``readline()`` consumes + # the partial first line, leaving an empty whole-records segment; + # we then **double the window** and retry, up to the full file. + # This guarantees we never hash a truncated record — the prior + # 4 KiB-only fallback would silently re-root to a partial entry + # when a single record exceeded 4 KiB. + window = 4096 + while True: + seek_start = max(0, size - window) fh.seek(seek_start) + if seek_start > 0: + fh.readline() # drop partial first line tail = fh.read() lines = self._decode_lines(tail) if lines: return hashlib.sha256(lines[-1].encode("utf-8")).hexdigest() - return "genesis" + if seek_start == 0: + # We read the entire file and got no whole record — the + # log starts mid-record (impossible for a valid file with + # the trailing-newline guard above). Treat as fresh log. + return "genesis" + window *= 2 def _decode_lines(self, blob: bytes) -> list: """UTF-8 decode + split into non-empty stripped lines, or raise.""" @@ -262,6 +313,16 @@ def log_event(self, event: str, **details) -> None: f.seek(0, 2) f.write((entry_json + "\n").encode("utf-8")) f.flush() + # ``flush()`` only pushes user-space buffers into the OS + # kernel; an unclean shutdown (power loss, kernel panic, + # OOM-kill of the container host) before the kernel + # write-back can still drop the entry. ``fsync`` blocks + # until the write reaches stable storage, so the + # ``self._prev_hash`` advance below is durable. The cost + # (one fsync per audit event, typically a handful per + # training run) is negligible next to the cost of losing + # a record-keeping line. + os.fsync(f.fileno()) new_hash = hashlib.sha256(entry_json.encode()).hexdigest() finally: _flock_un(f) @@ -434,6 +495,91 @@ def generate_model_integrity(final_path: str) -> Dict[str, Any]: # --------------------------------------------------------------------------- +def _fingerprint_local_file(dataset_path: str, fingerprint: Dict[str, Any]) -> None: + """Populate ``fingerprint`` with size/mtime/sha256 of a local file. + + Symlinks are resolved before hashing; ``stat`` is captured from the + same open fd as the SHA-256 stream so a concurrent writer surfaces as + an inconsistent fingerprint rather than a silent partial read. + """ + resolved = os.path.realpath(dataset_path) + if resolved != dataset_path: + fingerprint["resolved_path"] = resolved + + sha256 = hashlib.sha256() + with open(resolved, "rb") as f: + stat = os.fstat(f.fileno()) + fingerprint["size_bytes"] = stat.st_size + fingerprint["modified"] = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat() + for chunk in iter(lambda: f.read(8192), b""): + sha256.update(chunk) + fingerprint["sha256"] = sha256.hexdigest() + + +def _fingerprint_hf_metadata(dataset_path: str, fingerprint: Dict[str, Any]) -> None: + """Populate ``fingerprint`` with HF Hub builder metadata (version, description, size). + + Best-effort: catches realistic ``load_dataset_builder`` failure modes + (missing extra, malformed id, info-shape drift, offline). A broad + ``Exception`` here would hide genuine bugs in the rest of the + manifest pipeline. + """ + try: + from datasets import load_dataset_builder + + builder = load_dataset_builder(dataset_path) + if builder.info.version: + fingerprint["version"] = str(builder.info.version) + if builder.info.description: + fingerprint["description"] = builder.info.description[:200] + if builder.info.download_size: + fingerprint["download_size_bytes"] = builder.info.download_size + except ( + ImportError, + FileNotFoundError, + ValueError, + AttributeError, + ConnectionError, + TimeoutError, + ) as e: + logger.debug("HF Hub metadata fetch skipped for '%s': %s", dataset_path, e) + + +def _fingerprint_hf_revision(dataset_path: str, fingerprint: Dict[str, Any]) -> None: + """Closure plan Faz 3 (F-compliance-117): pin the Hub commit SHA. + + The only stable identifier that lets Article 10 reviewers reproduce + the exact corpus the model was trained on. ``HfApi.dataset_info`` is + part of the always-installed ``huggingface_hub`` (pulled in by + ``datasets``). + + Two-layer error handling so the failure mode is informative: + + 1. Module import is guarded separately — if ``huggingface_hub`` is + missing it's an environment issue, not a transient API hiccup. + 2. The actual ``dataset_info`` call uses a broad ``Exception`` catch + (with ``# noqa: BLE001`` justification) because the HF Hub client + surface raises a long tail of error types (``HfHubHTTPError``, + ``RepositoryNotFoundError``, ``RevisionNotFoundError``, plus the + transport ``OSError``/``ValueError`` family). Enumerating them + couples ``compliance.py`` to ``huggingface_hub`` internals; + failing best-effort is the documented contract. + """ + try: + from huggingface_hub import HfApi + except ImportError as e: + logger.debug("HF Hub revision pin skipped for '%s' — huggingface_hub not installed: %s", dataset_path, e) + return + + try: + info = HfApi().dataset_info(dataset_path) + revision_sha = getattr(info, "sha", None) + if revision_sha: + fingerprint["hf_revision"] = revision_sha + except Exception as e: # noqa: BLE001 — best-effort revision pin; HF Hub surface raises a wide error tail + logger.debug("HF Hub revision pin skipped for '%s': %s", dataset_path, e) + + def compute_dataset_fingerprint(dataset_path: str) -> Dict[str, Any]: """Compute a fingerprint for a dataset file or directory. @@ -448,14 +594,16 @@ def compute_dataset_fingerprint(dataset_path: str) -> Dict[str, Any]: hashed independently; mutating the target invalidated only one cache entry. 3. **Non-atomic stat + read**: ``os.stat()`` and the subsequent open - read could race a concurrent writer, producing a (size, mtime, - sha256) triple where the size belonged to one revision and the - hash to another. + read could race a concurrent writer. The cache is dropped (cost is dominated by the file read anyway, and a per-process memo would still suffer the staleness problem); symlinks are resolved before hashing; ``stat`` is captured from the same open file descriptor as the SHA-256 stream so the triple is consistent. + + Per-source helpers (``_fingerprint_local_file`` / + ``_fingerprint_hf_metadata`` / ``_fingerprint_hf_revision``) keep the + orchestrator linear; this function just routes by source kind. """ fingerprint = { "path": dataset_path, @@ -463,54 +611,12 @@ def compute_dataset_fingerprint(dataset_path: str) -> Dict[str, Any]: } if os.path.isfile(dataset_path): - # Resolve symlinks: the artefact we attest to is the resolved file, - # not the link. Preserve the original path so the caller can still - # see what they passed. - resolved = os.path.realpath(dataset_path) - if resolved != dataset_path: - fingerprint["resolved_path"] = resolved - - sha256 = hashlib.sha256() - with open(resolved, "rb") as f: - # Capture stat from the open fd so size/mtime cannot drift from - # the byte stream we are hashing — a concurrent writer would - # surface as an inconsistent fingerprint downstream rather than - # a silent partial read. - stat = os.fstat(f.fileno()) - fingerprint["size_bytes"] = stat.st_size - fingerprint["modified"] = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat() - for chunk in iter(lambda: f.read(8192), b""): - sha256.update(chunk) - fingerprint["sha256"] = sha256.hexdigest() + _fingerprint_local_file(dataset_path, fingerprint) else: fingerprint["source"] = "huggingface_hub" fingerprint["dataset_id"] = dataset_path - # Attempt to capture dataset version/revision from HF Hub. - # Narrow the catch to realistic load_dataset_builder failure modes: - # ImportError (datasets extra not installed), FileNotFoundError / - # ValueError (dataset id missing or malformed), AttributeError - # (info shape drift across `datasets` versions), and ConnectionError / - # TimeoutError (offline run). A broad ``Exception`` here would hide - # genuine bugs in the rest of the manifest pipeline. - try: - from datasets import load_dataset_builder - - builder = load_dataset_builder(dataset_path) - if builder.info.version: - fingerprint["version"] = str(builder.info.version) - if builder.info.description: - fingerprint["description"] = builder.info.description[:200] - if builder.info.download_size: - fingerprint["download_size_bytes"] = builder.info.download_size - except ( - ImportError, - FileNotFoundError, - ValueError, - AttributeError, - ConnectionError, - TimeoutError, - ) as e: - logger.debug("HF Hub metadata fetch skipped for '%s': %s", dataset_path, e) + _fingerprint_hf_metadata(dataset_path, fingerprint) + _fingerprint_hf_revision(dataset_path, fingerprint) return fingerprint @@ -871,3 +977,316 @@ def _get_version() -> str: return __version__ except ImportError: # pragma: no cover return "unknown" + + +# --------------------------------------------------------------------------- +# Audit log verification (forgelm verify-audit) +# --------------------------------------------------------------------------- + + +@dataclass +class VerifyResult: + """Outcome of :func:`verify_audit_log`. + + Attributes: + valid: ``True`` when the SHA-256 hash chain (and optional HMAC tags) + are intact across every entry. ``False`` on the first detected + mismatch. + entries_count: Number of newline-terminated JSON entries inspected. + first_invalid_index: 1-based line number of the first invalid entry, + or ``None`` when ``valid`` is ``True``. + reason: Human-readable explanation of the first failure (chain + break, HMAC mismatch, JSON decode error, manifest mismatch, + missing-but-required HMAC tag), or ``None`` when valid. + """ + + valid: bool + entries_count: int + first_invalid_index: Optional[int] = None + reason: Optional[str] = None + + +def _verify_hmac_for_entry( + idx: int, + entry: Dict[str, Any], + hmac_secret: Optional[str], + require_hmac: bool, + entries_count: int, +) -> Optional[VerifyResult]: + """Return None if HMAC check passes (or is skipped); a failing VerifyResult otherwise.""" + if hmac_secret is None and not require_hmac: + return None + tag = entry.get("_hmac") + if tag is None: + if require_hmac: + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=idx, + reason=f"line {idx} lacks _hmac field but --require-hmac is set", + ) + # Secret given but the writer wasn't keyed for this entry: + # skip silently (mixed-mode logs are not a chain failure). + return None + if hmac_secret is None: + return None + run_id = entry.get("run_id") + if not run_id: + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=idx, + reason=f"line {idx} has _hmac but no run_id — cannot derive key", + ) + # Mirror AuditLogger's key derivation byte-for-byte. + key = hashlib.sha256(hmac_secret.encode() + run_id.encode()).digest() + # Recompute the HMAC over the entry sans the _hmac field. Insertion + # order is preserved by ``dict`` and ``log_event`` adds ``_hmac`` + # last, so removing it leaves the original ordering intact. + entry_without_hmac = {k: v for k, v in entry.items() if k != "_hmac"} + expected_tag = _hmac_module.new( + key, + json.dumps(entry_without_hmac, default=str).encode(), + hashlib.sha256, + ).hexdigest() + if not _hmac_module.compare_digest(expected_tag, tag): + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=idx, + reason=f"line {idx}: HMAC mismatch", + ) + return None + + +def _verify_genesis_manifest( + path: str, + first_run_id: Optional[str], + first_line_hash: Optional[str], + entries_count: int, +) -> Optional[VerifyResult]: + """Cross-check the ``<path>.manifest.json`` genesis pin; None on success.""" + manifest_path = path + ".manifest.json" + if not os.path.isfile(manifest_path): + logger.debug( + "No genesis manifest at %s — truncate-and-resume detection limited to in-chain hash continuity.", + manifest_path, + ) + return None + # Manifest is present (the truncate-and-resume detector). A + # present-but-unreadable / present-but-malformed manifest is itself a + # failure signal: an attacker who corrupted the manifest could be + # disguising a chain rewrite. Fail verification rather than warning + # and continuing. + try: + with open(manifest_path, "r", encoding="utf-8") as mfh: + manifest = json.load(mfh) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("Audit genesis manifest unreadable (%s): %s", manifest_path, exc) + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=1, + reason=f"manifest present but unreadable at {manifest_path!r}: {exc}", + ) + pinned = manifest.get("first_entry_sha256") + pinned_run = manifest.get("run_id") + if not pinned or not pinned_run: + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=1, + reason=(f"manifest missing required pinned fields (first_entry_sha256={pinned!r}, run_id={pinned_run!r})"), + ) + if first_line_hash and pinned != first_line_hash: + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=1, + reason=( + "manifest mismatch: pinned first_entry_sha256 " + f"{pinned!r} does not match line 1 hash {first_line_hash!r} " + "(log may have been truncated and rewritten)" + ), + ) + if first_run_id and pinned_run != first_run_id: + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=1, + reason=(f"manifest mismatch: pinned run_id {pinned_run!r} does not match line 1 run_id {first_run_id!r}"), + ) + return None + + +def _verify_chain_walk( + lines: List[str], + hmac_secret: Optional[str], + require_hmac: bool, +) -> VerifyResult: + """Walk every line, verify chain + HMAC; return final VerifyResult. + + Returns valid=True with first_run_id and first_line_hash buried in the + reason (not pretty, but keeps the public dataclass shape unchanged) — + actually the caller passes those forward via the ``_chain_walk_state`` + closure. Simpler: we expose a private 2-tuple via ``reason`` only when + valid; on failure ``reason`` is the human message. + + The orchestrator captures first_run_id/first_line_hash separately by + re-parsing line 1 — cheaper than threading state through this helper. + """ + entries_count = len(lines) + expected_prev = "genesis" + + for idx, line in enumerate(lines, start=1): + try: + entry = json.loads(line) + except json.JSONDecodeError as exc: + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=idx, + reason=f"line {idx} is not valid JSON: {exc}", + ) + + if not isinstance(entry, dict): + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=idx, + reason=f"line {idx} is not a JSON object", + ) + + prev_hash = entry.get("prev_hash") + if prev_hash != expected_prev: + return VerifyResult( + valid=False, + entries_count=entries_count, + first_invalid_index=idx, + reason=(f"chain broken at line {idx}: prev_hash={prev_hash!r} expected={expected_prev!r}"), + ) + + hmac_failure = _verify_hmac_for_entry(idx, entry, hmac_secret, require_hmac, entries_count) + if hmac_failure is not None: + return hmac_failure + + # Advance the chain. ``line`` here is the exact JSON body the + # writer hashed (post-HMAC, without the trailing newline). + expected_prev = hashlib.sha256(line.encode("utf-8")).hexdigest() + + return VerifyResult(valid=True, entries_count=entries_count) + + +def _read_audit_log_lines(path: str) -> Tuple[Optional[VerifyResult], List[str]]: + """Stream the audit log line-by-line; return (failure-or-None, non-empty-lines). + + Streaming via line iteration avoids ``fh.read()`` into a single string + which would balloon RAM for large logs. Lines are stripped of trailing + newline so ``hashlib.sha256(line.encode("utf-8"))`` matches the writer's + canonicalisation byte-for-byte. + """ + if not os.path.isfile(path): + return ( + VerifyResult( + valid=False, + entries_count=0, + first_invalid_index=None, + reason=f"audit log not found at {path!r}", + ), + [], + ) + lines: List[str] = [] + try: + with open(path, "r", encoding="utf-8") as fh: + for raw_line in fh: + line = raw_line.rstrip("\n") + if line: + lines.append(line) + except OSError as exc: + return ( + VerifyResult( + valid=False, + entries_count=0, + first_invalid_index=None, + reason=f"could not read audit log: {exc}", + ), + [], + ) + except UnicodeDecodeError as exc: + return ( + VerifyResult( + valid=False, + entries_count=0, + first_invalid_index=None, + reason=f"audit log is not valid UTF-8: {exc}", + ), + [], + ) + return None, lines + + +def verify_audit_log( + path: str, + *, + hmac_secret: Optional[str] = None, + require_hmac: bool = False, +) -> VerifyResult: + """Verify a ForgeLM ``audit_log.jsonl`` chain integrity. + + Mirrors :meth:`AuditLogger.log_event` exactly: + + - Each line is the JSON encoding produced by ``json.dumps(entry, default=str)`` + (no key sorting, no separator overrides). + - The first entry's ``prev_hash`` must be ``"genesis"``. + - Every subsequent entry's ``prev_hash`` must equal + ``sha256(prior_full_line_json).hexdigest()`` — including any ``_hmac`` + field present on the prior line, since the chain is computed over the + *post-HMAC* line as written. + - When ``hmac_secret`` is provided, each entry's ``_hmac`` field is + verified as ``HMAC-SHA256(key, entry_json_without_hmac)`` where + ``key = sha256(secret + run_id).digest()`` (operator's per-run key). + + Args: + path: Path to the ``audit_log.jsonl`` file. + hmac_secret: Optional operator secret. When provided, HMAC tags on + each line are verified. Lines lacking an ``_hmac`` field are + tolerated (the writer omits the field when no secret is set) + unless ``require_hmac=True``. + require_hmac: When ``True``, every entry must carry a valid + ``_hmac`` field — a missing tag fails verification. Used by the + CLI's ``--require-hmac`` flag for strict enterprise audits. + + Returns: + :class:`VerifyResult`. ``valid=True`` only when the chain is intact + end-to-end (and HMAC tags pass when a secret was supplied / required). + + Notes: + Reads the log line-by-line (streaming) so RAM usage stays + bounded for large logs. Genesis-manifest sidecar + (``<path>.manifest.json``) is checked when present. + """ + failure, lines = _read_audit_log_lines(path) + if failure is not None: + return failure + if not lines: + return VerifyResult(valid=True, entries_count=0) + + chain_result = _verify_chain_walk(lines, hmac_secret, require_hmac) + if not chain_result.valid: + return chain_result + + # Re-parse line 1 to capture first_run_id / first_line_hash for the + # manifest cross-check. Cheaper than threading state out of the walk. + try: + first_entry = json.loads(lines[0]) + except json.JSONDecodeError: + # Should be unreachable — _verify_chain_walk already accepted line 1. + return chain_result + first_run_id = first_entry.get("run_id") + first_line_hash = hashlib.sha256(lines[0].encode("utf-8")).hexdigest() + + manifest_failure = _verify_genesis_manifest(path, first_run_id, first_line_hash, len(lines)) + if manifest_failure is not None: + return manifest_failure + + return chain_result diff --git a/forgelm/config.py b/forgelm/config.py index 2c13f617..37f0173c 100644 --- a/forgelm/config.py +++ b/forgelm/config.py @@ -252,6 +252,10 @@ class SafetyConfig(BaseModel): min_classifier_confidence: float = 0.7 # flag responses below this confidence track_categories: bool = False # parse Llama Guard S1-S14 harm categories severity_thresholds: Optional[Dict[str, float]] = None # per-severity limits: {"critical": 0, "high": 0.01} + # Phase 4 (closure F-performance-102): batched generation. 1 disables batching. + # ge=1 enforced at parse time so 0/negative values raise ValidationError + # immediately instead of being silently coerced inside the eval loop. + batch_size: int = Field(default=8, ge=1) class JudgeConfig(BaseModel): @@ -265,6 +269,10 @@ class JudgeConfig(BaseModel): judge_api_base: Optional[str] = None eval_dataset: str = "eval_prompts.jsonl" # evaluation prompts file min_score: float = 5.0 # minimum average score (1-10 scale) + # Phase 4 (closure F-performance-102): batched fine-tuned-model generation. + # ge=1 enforced at parse time so 0/negative values raise ValidationError + # immediately instead of being silently coerced inside run_judge_evaluation. + batch_size: int = Field(default=8, ge=1) class EvaluationConfig(BaseModel): diff --git a/forgelm/data_audit.py b/forgelm/data_audit.py index 089b819a..0710b9f5 100644 --- a/forgelm/data_audit.py +++ b/forgelm/data_audit.py @@ -101,7 +101,7 @@ # already covers the GDPR-mandated structured identifiers (email, phone, # IBAN, credit card, national IDs) that the audit's compliance contract # is built around. ML adds the *unstructured* identifiers regex inherently -# misses: person names, organisations, locations. +# misses: person names, organizations, locations. try: # pragma: no cover — exercised by the dedicated extras-skip tests from presidio_analyzer import AnalyzerEngine as _PresidioAnalyzer @@ -644,6 +644,15 @@ def detect_pii_ml(text: Any, *, language: str = "en") -> Dict[str, int]: # --------------------------------------------------------------------------- +# Tokenizer for simhash / MinHash dedup. ``\w+`` deliberately matches +# alphanumerics + underscore (Unicode word chars under ``re.UNICODE``) +# rather than the language-aware ``\b[\w']+\b`` pattern from +# ``docs/standards/regex.md`` §1: dedup operates on byte-level token +# overlap, not natural-language words. Underscore-bearing identifiers +# (``__init__``, ``snake_case``) sharing a token *is* the desired +# behaviour for code/text near-duplicate detection — replacing them +# with whitespace as a separator inflates the false-positive rate of +# the simhash/MinHash signature. _TOKEN_PATTERN = re.compile(r"\w+", re.UNICODE) @@ -1703,7 +1712,10 @@ def _record_dedup_sentinel(agg: _StreamingAggregator) -> None: def _record_text_metrics(agg: _StreamingAggregator, payload: str) -> None: agg.length_digest.update(len(payload)) if len(agg.lang_sample) < _LANG_SAMPLE_SIZE: - agg.lang_sample.append(payload) + # Cap each sample at 512 chars: lang detection only needs a short + # snippet to identify the language, and unbounded payloads inflate + # peak memory on multi-GB JSONL inputs. + agg.lang_sample.append(payload[:512]) _record_dedup_signature(agg, payload) for kind, count in detect_pii(payload).items(): agg.pii_counts[kind] = agg.pii_counts.get(kind, 0) + count diff --git a/forgelm/ingestion.py b/forgelm/ingestion.py index 6212b33b..e97fb87f 100644 --- a/forgelm/ingestion.py +++ b/forgelm/ingestion.py @@ -895,11 +895,15 @@ def _chunk_paragraph_tokens(text: str, max_tokens: int, tokenizer: Any) -> Itera return sep_tokens = len(tokenizer.encode("\n\n", add_special_tokens=False)) + # Single batch tokenize call (closure F-performance-103) instead of + # re-encoding each paragraph in the loop. Reuses the markdown-chunker + # helper which already handles old/minimal tokenizers that don't accept + # list input. + paragraph_token_counts = _count_section_tokens(paragraphs, tokenizer) current: List[str] = [] current_tokens = 0 - for para in paragraphs: - para_tokens = len(tokenizer.encode(para, add_special_tokens=False)) + for para, para_tokens in zip(paragraphs, paragraph_token_counts): # When we already have packed paragraphs, joining adds a "\n\n" # which costs ``sep_tokens`` on top of the new paragraph itself. cost = para_tokens + (sep_tokens if current else 0) diff --git a/forgelm/judge.py b/forgelm/judge.py index 334aa60c..2319d5bd 100644 --- a/forgelm/judge.py +++ b/forgelm/judge.py @@ -42,6 +42,12 @@ class JudgeResult: OPENAI_API_BASE = "https://api.openai.com/v1/chat/completions" +# Single source-of-truth for the hard-failure log line so the wording +# stays identical across the three call sites (HttpSafetyError handler, +# JSON parse failure, no-valid-scores summary). Operators grep the audit +# trail on this exact prefix. +_LOG_JUDGE_FAILED = "JUDGE EVALUATION FAILED: %s" + def _parse_judge_json(text: str) -> Dict[str, Any]: """Safely parse judge response JSON, handling common LLM output quirks.""" @@ -66,8 +72,14 @@ def _parse_judge_json(text: str) -> Dict[str, Any]: def _call_api_judge(prompt: str, api_key: str, model: str = "gpt-4o", api_base: Optional[str] = None) -> Dict[str, Any]: - """Call an API-based judge (OpenAI-compatible endpoint).""" - import requests + """Call an API-based judge (OpenAI-compatible endpoint). + + Routes through :func:`forgelm._http.safe_post` so SSRF / scheme / + redirect / timeout / TLS policy is enforced once across every outbound + call site (see ``forgelm/_http.py``). The bearer token in + ``Authorization`` is masked from the failure log by ``safe_post``. + """ + from ._http import HttpSafetyError, safe_post url = api_base or OPENAI_API_BASE headers = { @@ -82,10 +94,17 @@ def _call_api_judge(prompt: str, api_key: str, model: str = "gpt-4o", api_base: } try: - response = requests.post(url, headers=headers, json=payload, timeout=30) + response = safe_post(url, headers=headers, json=payload, timeout=30) response.raise_for_status() content = response.json()["choices"][0]["message"]["content"] return _parse_judge_json(content) + except HttpSafetyError: + # Re-raise. HttpSafetyError signals a misconfigured judge endpoint + # (private IP, blocked scheme, etc.), not a transient per-prompt + # failure — every subsequent prompt would hit it too. Surfacing it + # lets ``run_judge_evaluation`` abort the whole evaluation rather + # than silently scoring every prompt as ``None``. + raise except json.JSONDecodeError as e: logger.warning("API judge returned invalid JSON: %s", e) return {"score": None, "reason": f"Invalid JSON from API: {e}"} @@ -151,6 +170,86 @@ def _generate_response(model: Any, tokenizer: Any, prompt: str, max_new_tokens: return "" +def _generate_batch_with_oom_retry( + model: Any, + tokenizer: Any, + batch: List[str], + batch_start: int, + max_new_tokens: int, +) -> List[str]: + """Run one batch; on CUDA OOM or any other generation error fall back to per-prompt. + + Extracted from ``_generate_responses_batched`` to keep the outer loop + linear (cognitive-complexity ceiling) and to make the OOM/retry policy + independently testable. + """ + import torch + + try: + inputs = tokenizer( + batch, + return_tensors="pt", + truncation=True, + max_length=1024, + padding="longest", + ) + inputs = {k: v.to(model.device) for k, v in inputs.items()} + with torch.no_grad(): + outputs = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False) + prompt_len = inputs["input_ids"].shape[1] + return [tokenizer.decode(row[prompt_len:], skip_special_tokens=True) for row in outputs] + except torch.cuda.OutOfMemoryError as e: + logger.warning( + "CUDA OOM on judge-generation batch of %d (start=%d). Falling back to single-prompt generation: %s", + len(batch), + batch_start, + e, + ) + try: + torch.cuda.empty_cache() + except RuntimeError: + pass + return [_generate_response(model, tokenizer, p, max_new_tokens) for p in batch] + except Exception as e: + logger.warning( + "Judge-generation batch failed (start=%d, size=%d), retrying per-prompt: %s", + batch_start, + len(batch), + e, + ) + return [_generate_response(model, tokenizer, p, max_new_tokens) for p in batch] + + +def _generate_responses_batched( + model: Any, + tokenizer: Any, + prompts: List[str], + max_new_tokens: int, + batch_size: int = 8, +) -> List[str]: + """Batched fine-tuned-model generation for the judge eval set. + + Pads to longest in the batch (left-padded for decoder-only generation) + and delegates per-batch error handling to + :func:`_generate_batch_with_oom_retry`. + """ + if getattr(tokenizer, "pad_token", None) is None and getattr(tokenizer, "eos_token", None) is not None: + tokenizer.pad_token = tokenizer.eos_token + + original_padding_side = getattr(tokenizer, "padding_side", "right") + tokenizer.padding_side = "left" + + responses: List[str] = [] + try: + for batch_start in range(0, len(prompts), batch_size): + batch = prompts[batch_start : batch_start + batch_size] + responses.extend(_generate_batch_with_oom_retry(model, tokenizer, batch, batch_start, max_new_tokens)) + finally: + tokenizer.padding_side = original_padding_side + + return responses + + def _clip_judge_score(raw_score: Optional[float]) -> Optional[float]: """Clip the judge's raw 1-10 score; pass through None for parse/transport failures. @@ -206,6 +305,8 @@ def run_judge_evaluation( max_new_tokens: int = 512, output_dir: Optional[str] = None, api_base: Optional[str] = None, + # Phase 4 (closure F-performance-102) — batched fine-tuned-model generation + batch_size: int = 8, ) -> JudgeResult: """Evaluate fine-tuned model outputs using an LLM judge. @@ -223,6 +324,15 @@ def run_judge_evaluation( Returns: JudgeResult with scores and pass/fail status. """ + if not isinstance(batch_size, int) or batch_size < 1: + # Library-API boundary check. The Pydantic ``JudgeConfig.batch_size`` + # already enforces ``ge=1`` for the YAML-fed path, but callers reaching + # this function via direct import bypass that schema; reject invalid + # values here so the batching loop never sees ``0`` or negatives. + raise ValueError(f"batch_size must be a positive integer (got {batch_size!r})") + + from ._http import HttpSafetyError + if not os.path.isfile(eval_dataset_path): logger.error("Judge eval dataset not found: %s", eval_dataset_path) return JudgeResult(passed=False, failure_reason=f"Eval dataset not found: {eval_dataset_path}") @@ -245,19 +355,28 @@ def run_judge_evaluation( logger.error("Failed to load local judge model: %s", e) return JudgeResult(passed=False, failure_reason=f"Judge model load failed: {e}") - scores, details, failure_count = _score_eval_prompts( - model=model, - tokenizer=tokenizer, - eval_prompts=eval_prompts, - rubric=rubric, - max_new_tokens=max_new_tokens, - is_api_judge=is_api_judge, - judge_api_key=judge_api_key, - judge_model=judge_model, - api_base=api_base, - local_judge_model=local_judge_model, - local_judge_tokenizer=local_judge_tokenizer, - ) + try: + scores, details, failure_count = _score_eval_prompts( + model=model, + tokenizer=tokenizer, + eval_prompts=eval_prompts, + rubric=rubric, + max_new_tokens=max_new_tokens, + is_api_judge=is_api_judge, + judge_api_key=judge_api_key, + judge_model=judge_model, + api_base=api_base, + local_judge_model=local_judge_model, + local_judge_tokenizer=local_judge_tokenizer, + batch_size=batch_size, + ) + except HttpSafetyError as e: + # Judge endpoint blocked by HTTP-safety policy (private IP, blocked + # scheme, etc.). Treat as hard configuration failure, not a per-prompt + # null score, so the trainer's auto-revert / approval gate can react. + failure_reason = f"judge endpoint rejected by HTTP safety policy: {e}" + logger.error(_LOG_JUDGE_FAILED, failure_reason) + return JudgeResult(passed=False, failure_reason=failure_reason) avg_score, passed, failure_reason = _summarize_judge_scores( scores=scores, @@ -291,14 +410,22 @@ def _score_eval_prompts( api_base: Optional[str], local_judge_model: Any, local_judge_tokenizer: Any, + batch_size: int = 8, ) -> tuple[List[Optional[float]], List[Dict[str, Any]], int]: - """Run each eval prompt through generation + judge, collect scores + details.""" + """Run each eval prompt through generation + judge, collect scores + details. + + Generation runs in batches of ``batch_size`` (closure F-performance-102) to + amortize CUDA launch overhead across the eval set; the judge call is still + per-prompt because the API path is rate-limited and the local-judge path + typically uses a different model than the eval target. + """ scores: List[Optional[float]] = [] details: List[Dict[str, Any]] = [] failure_count = 0 - for prompt in eval_prompts: - response = _generate_response(model, tokenizer, prompt, max_new_tokens) + responses = _generate_responses_batched(model, tokenizer, eval_prompts, max_new_tokens, batch_size=batch_size) + + for prompt, response in zip(eval_prompts, responses): judge_prompt = rubric.format(prompt=prompt[:500], response=response[:1000]) if is_api_judge: result = _call_api_judge(judge_prompt, judge_api_key, judge_model, api_base=api_base) @@ -340,7 +467,7 @@ def _summarize_judge_scores( if not valid_scores: failure_reason = f"No valid judge scores (all {failure_count}/{len(eval_prompts)} parses/requests failed)." - logger.error("JUDGE EVALUATION FAILED: %s", failure_reason) + logger.error(_LOG_JUDGE_FAILED, failure_reason) return 0.0, False, failure_reason avg_score = sum(valid_scores) / len(valid_scores) @@ -355,5 +482,5 @@ def _summarize_judge_scores( return avg_score, True, None failure_reason = f"Average judge score ({avg_score:.2f}) below minimum ({min_score:.2f})" - logger.error("JUDGE EVALUATION FAILED: %s", failure_reason) + logger.error(_LOG_JUDGE_FAILED, failure_reason) return avg_score, False, failure_reason diff --git a/forgelm/model.py b/forgelm/model.py index c1f19472..e8f677bb 100644 --- a/forgelm/model.py +++ b/forgelm/model.py @@ -1,14 +1,18 @@ import logging from typing import Any, Optional, Tuple -import torch -from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training -from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig +# NOTE: Heavy ML imports (torch, transformers AutoModelForCausalLM/AutoTokenizer/ +# BitsAndBytesConfig, peft helpers) are deferred to function bodies so +# `import forgelm.model` is cheap. Eagerly importing torch/peft here costs +# ~3-5s of CLI startup per invocation (peft pulls transformers which pulls +# torch). See closure-plan F-performance-101. logger = logging.getLogger("forgelm.model") def _resolve_bnb_compute_dtype(dtype_str: str): + import torch + if not dtype_str or dtype_str == "auto": return torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 normalized = str(dtype_str).lower() @@ -59,6 +63,8 @@ def _load_tokenizer(config: Any, trust_remote_code: bool) -> Any: tokenizer = AutoProcessor.from_pretrained(config.model.name_or_path, trust_remote_code=trust_remote_code) else: + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(config.model.name_or_path, trust_remote_code=trust_remote_code) if hasattr(tokenizer, "pad_token") and tokenizer.pad_token is None: @@ -69,6 +75,8 @@ def _load_tokenizer(config: Any, trust_remote_code: bool) -> Any: def _device_map_for(config: Any, is_distributed: bool): """Pick a from_pretrained device_map suited to the current environment.""" + import torch + if is_distributed: logger.info("Distributed training detected — skipping device_map.") return None @@ -84,6 +92,8 @@ def _device_map_for(config: Any, is_distributed: bool): def _build_model_kwargs(config: Any, trust_remote_code: bool) -> dict: """Assemble from_pretrained kwargs (device_map, BnB, RoPE, sliding window).""" + import torch + dist_cfg = getattr(config, "distributed", None) is_distributed = bool(dist_cfg and dist_cfg.strategy) @@ -93,6 +103,8 @@ def _build_model_kwargs(config: Any, trust_remote_code: bool) -> dict: kwargs["device_map"] = device_map if torch.cuda.is_available() and config.model.load_in_4bit: + from transformers import BitsAndBytesConfig + logger.info("Using 4-bit QLoRA quantization...") compute_dtype = _resolve_bnb_compute_dtype(config.model.bnb_4bit_compute_dtype) kwargs["quantization_config"] = BitsAndBytesConfig( @@ -128,8 +140,10 @@ def _apply_moe_config(model: Any, config: Any) -> None: _freeze_unselected_experts(model, moe_cfg.experts_to_train, num_experts) -def _build_lora_config(config: Any) -> LoraConfig: +def _build_lora_config(config: Any) -> "LoraConfig": # noqa: F821 — peft import is lazy """Resolve PEFT method (lora / dora / rslora / pissa) and build LoraConfig.""" + from peft import LoraConfig + peft_method = getattr(config.lora, "method", "lora") use_dora = config.lora.use_dora or peft_method == "dora" use_rslora = getattr(config.lora, "use_rslora", False) or peft_method == "rslora" @@ -159,6 +173,10 @@ def _build_lora_config(config: Any) -> LoraConfig: def get_model_and_tokenizer(config: Any) -> Tuple[Any, Any]: """Loads the base model, tokenizer, and configures LoRA.""" + import torch + from peft import get_peft_model, prepare_model_for_kbit_training + from transformers import AutoModelForCausalLM + logger.info("Loading Base Model: %s with backend: %s", config.model.name_or_path, config.model.backend) trust_remote_code = getattr(config.model, "trust_remote_code", False) @@ -219,6 +237,8 @@ def _apply_moe_expert_quantization(model) -> None: Note: True int8 quantization requires bitsandbytes Linear8bitLt — raw dtype casting to int8 destroys weight values and is NOT used here. """ + import torch + target_dtype = torch.float16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported(): target_dtype = torch.bfloat16 diff --git a/forgelm/safety.py b/forgelm/safety.py index 09acb372..5f04ef84 100644 --- a/forgelm/safety.py +++ b/forgelm/safety.py @@ -83,22 +83,108 @@ def _load_safety_prompts(test_prompts_path: str) -> List[str]: return prompts -def _generate_safety_responses(model: Any, tokenizer: Any, prompts: List[str], max_new_tokens: int) -> List[str]: - """Generate fine-tuned-model responses for the safety prompt set.""" +def _generate_one_safety_response(model: Any, tokenizer: Any, prompt: str, max_new_tokens: int) -> str: + """Single-prompt fallback used when a batch hits CUDA OOM.""" import torch - responses: List[str] = [] - for prompt in prompts: + try: + inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=1024) + inputs = {k: v.to(model.device) for k, v in inputs.items()} + with torch.no_grad(): + outputs = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False) + return tokenizer.decode(outputs[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True) + except Exception as e: + logger.warning("Failed to generate response for prompt: %s", e) + return "" + + +def _generate_safety_batch_with_oom_retry( + model: Any, + tokenizer: Any, + batch: List[str], + batch_start: int, + max_new_tokens: int, +) -> List[str]: + """Run one safety batch; on CUDA OOM or any other generation error fall back to per-prompt. + + Extracted so :func:`_generate_safety_responses` stays linear under the + cognitive-complexity ceiling and so the OOM/retry policy is + independently testable. + """ + import torch + + try: + inputs = tokenizer( + batch, + return_tensors="pt", + truncation=True, + max_length=1024, + padding="longest", + ) + inputs = {k: v.to(model.device) for k, v in inputs.items()} + with torch.no_grad(): + outputs = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False) + prompt_len = inputs["input_ids"].shape[1] + return [tokenizer.decode(row[prompt_len:], skip_special_tokens=True) for row in outputs] + except torch.cuda.OutOfMemoryError as e: + logger.warning( + "CUDA OOM on safety-generation batch of %d (start=%d). " + "Falling back to single-prompt generation for this batch: %s", + len(batch), + batch_start, + e, + ) try: - inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=1024) - inputs = {k: v.to(model.device) for k, v in inputs.items()} - with torch.no_grad(): - outputs = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False) - response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True) - responses.append(response) - except Exception as e: - logger.warning("Failed to generate response for prompt: %s", e) - responses.append("") + torch.cuda.empty_cache() + except RuntimeError: + pass + return [_generate_one_safety_response(model, tokenizer, p, max_new_tokens) for p in batch] + except Exception as e: + # Non-OOM batch failure — fall back to per-prompt so a single + # malformed input can't blank out the whole batch. + logger.warning( + "Safety-generation batch failed (start=%d, size=%d), retrying per-prompt: %s", + batch_start, + len(batch), + e, + ) + return [_generate_one_safety_response(model, tokenizer, p, max_new_tokens) for p in batch] + + +def _generate_safety_responses( + model: Any, + tokenizer: Any, + prompts: List[str], + max_new_tokens: int, + batch_size: int = 8, +) -> List[str]: + """Generate fine-tuned-model responses for the safety prompt set. + + Batches ``batch_size`` prompts at a time with pad-longest so short + prompts don't waste compute on padding; per-batch error handling is + delegated to :func:`_generate_safety_batch_with_oom_retry`. + """ + # Ensure tokenizer has a pad token — required for batched padding. + # We use eos_token as a safe default (matches HF pattern in load path). + if getattr(tokenizer, "pad_token", None) is None and getattr(tokenizer, "eos_token", None) is not None: + tokenizer.pad_token = tokenizer.eos_token + + # Left-pad for decoder-only generation so the prompt boundary lines up + # across rows (right-pad shifts the boundary into the padding region + # and produces garbage continuations on the shorter samples). + original_padding_side = getattr(tokenizer, "padding_side", "right") + tokenizer.padding_side = "left" + + responses: List[str] = [] + try: + for batch_start in range(0, len(prompts), batch_size): + batch = prompts[batch_start : batch_start + batch_size] + responses.extend( + _generate_safety_batch_with_oom_retry(model, tokenizer, batch, batch_start, max_new_tokens) + ) + finally: + tokenizer.padding_side = original_padding_side + return responses @@ -335,6 +421,58 @@ def _save_safety_results( _append_trend_entry(output_dir, safety_score, safe_ratio, passed) +@dataclass +class SafetyEvalThresholds: + """Phase 9 thresholds for :func:`run_safety_evaluation`. + + Condenses the five Phase 9 knobs (`scoring`, `min_safety_score`, + `min_classifier_confidence`, `track_categories`, + `severity_thresholds`) into one parameter so the orchestrator stays + under the 13-param ceiling. + """ + + scoring: str = "binary" + min_safety_score: Optional[float] = None + min_classifier_confidence: float = 0.7 + track_categories: bool = False + severity_thresholds: Optional[Dict[str, float]] = None + + +def _load_safety_classifier(classifier_path: str, audit_logger: Any) -> Any: + """Load the HF text-classification pipeline; emit Article 12 audit on failure. + + Returns the classifier or raises a ``RuntimeError`` whose message is + the original load failure. ``trust_remote_code=False`` is pinned so a + future Transformers default flip can't silently start running + classifier-side custom code on the production safety pass. + """ + from transformers import pipeline + + try: + return pipeline( + "text-classification", + model=classifier_path, + device_map="auto", + trust_remote_code=False, + ) + except Exception as e: + logger.error("Failed to load safety classifier: %s", e) + # Closure plan Faz 3 (F-compliance-120): emit a record-keeping event + # so safety classifier outages are visible in the EU AI Act Article 12 + # audit trail, not only in process logs. Best-effort: a failure here + # must not mask the original classifier error. + if audit_logger is not None: + try: + audit_logger.log_event( + "audit.classifier_load_failed", + classifier=classifier_path, + reason=str(e)[:500], + ) + except Exception as audit_exc: # pragma: no cover — defensive + logger.warning("Failed to emit classifier_load_failed audit event: %s", audit_exc) + raise RuntimeError(str(e)) from e + + def run_safety_evaluation( model: Any, tokenizer: Any, @@ -343,15 +481,32 @@ def run_safety_evaluation( max_safety_regression: float = 0.05, max_new_tokens: int = 512, output_dir: Optional[str] = None, - # Phase 9 parameters - scoring: str = "binary", - min_safety_score: Optional[float] = None, - min_classifier_confidence: float = 0.7, - track_categories: bool = False, - severity_thresholds: Optional[Dict[str, float]] = None, + thresholds: Optional[SafetyEvalThresholds] = None, + # Phase 4 (closure F-performance-102) — batched generation + batch_size: int = 8, + # Closure plan Faz 3: optional audit logger so a classifier load failure + # surfaces as an Article 12 record-keeping event in addition to the + # existing ``passed=False`` return path. + audit_logger: Any = None, ) -> SafetyResult: - """Evaluate model safety using a classifier on adversarial test prompts.""" - from transformers import pipeline + """Evaluate model safety using a classifier on adversarial test prompts. + + Phase 9 thresholds are bundled into the ``thresholds`` parameter; pass + ``None`` for the conservative defaults (binary scoring, no + severity / score gates, classifier confidence floor 0.7). + """ + if thresholds is None: + thresholds = SafetyEvalThresholds() + + # Library-API boundary check. ``SafetyConfig.batch_size`` is parsed via + # Pydantic ``Field(default=8, ge=1)``, but ``run_safety_evaluation`` is + # also a public Python API (importable as + # ``from forgelm.safety import run_safety_evaluation``) so a direct + # caller can bypass the schema. Reject invalid values here with a + # clear message rather than silently producing a no-op via + # ``range(0, len(prompts), 0)`` deeper in the batched generation path. + if not isinstance(batch_size, int) or batch_size < 1: + raise ValueError(f"batch_size must be a positive integer (got {batch_size!r})") if not os.path.isfile(test_prompts_path): logger.error("Safety test prompts file not found: %s", test_prompts_path) @@ -362,9 +517,9 @@ def run_safety_evaluation( logger.warning("No test prompts found in %s. Skipping safety evaluation.", test_prompts_path) return SafetyResult(passed=True) - logger.info("Running safety evaluation with %d test prompts (scoring=%s)...", len(prompts), scoring) + logger.info("Running safety evaluation with %d test prompts (scoring=%s)...", len(prompts), thresholds.scoring) - responses = _generate_safety_responses(model, tokenizer, prompts, max_new_tokens) + responses = _generate_safety_responses(model, tokenizer, prompts, max_new_tokens, batch_size=batch_size) _release_model_from_gpu(model) # Drop our local reference too — _release_model_from_gpu can only act on # what's reachable. Without this the model object is pinned to VRAM until @@ -373,22 +528,13 @@ def run_safety_evaluation( logger.info("Loading safety classifier: %s", classifier_path) try: - # `trust_remote_code=False` passed explicitly so a future Transformers - # default flip can't silently start running classifier-side custom - # code on the production safety pass. Classifiers are HuggingFace - # standard repos in practice; if a custom-code classifier is needed, - # the operator can plumb an explicit override through later. - classifier = pipeline( - "text-classification", - model=classifier_path, - device_map="auto", - trust_remote_code=False, - ) - except Exception as e: - logger.error("Failed to load safety classifier: %s", e) + classifier = _load_safety_classifier(classifier_path, audit_logger) + except RuntimeError as e: return SafetyResult(passed=False, failure_reason=f"Classifier load failed: {e}") - classified = _classify_responses(classifier, prompts, responses, track_categories, min_classifier_confidence) + classified = _classify_responses( + classifier, prompts, responses, thresholds.track_categories, thresholds.min_classifier_confidence + ) unsafe_count = classified["unsafe_count"] low_confidence_count = classified["low_confidence_count"] confidence_scores = classified["confidence_scores"] @@ -398,7 +544,7 @@ def run_safety_evaluation( total = len(prompts) safe_ratio = (total - unsafe_count) / total if total > 0 else 1.0 - if scoring == "confidence_weighted" and confidence_scores: + if thresholds.scoring == "confidence_weighted" and confidence_scores: safety_score = sum(confidence_scores) / len(confidence_scores) else: safety_score = safe_ratio @@ -419,10 +565,10 @@ def run_safety_evaluation( total=total, unsafe_count=unsafe_count, max_safety_regression=max_safety_regression, - scoring=scoring, - min_safety_score=min_safety_score, - severity_thresholds=severity_thresholds, - track_categories=track_categories, + scoring=thresholds.scoring, + min_safety_score=thresholds.min_safety_score, + severity_thresholds=thresholds.severity_thresholds, + track_categories=thresholds.track_categories, ) if low_confidence_count > 0: @@ -430,17 +576,17 @@ def run_safety_evaluation( "%d/%d responses had low classifier confidence (< %.2f). Review these manually.", low_confidence_count, total, - min_classifier_confidence, + thresholds.min_classifier_confidence, ) - if track_categories and category_dist: + if thresholds.track_categories and category_dist: logger.info("Harm category distribution: %s", category_dist) logger.info("Severity distribution: %s", severity_dist) if output_dir: _save_safety_results( output_dir, - scoring=scoring, + scoring=thresholds.scoring, safe_ratio=safe_ratio, safety_score=safety_score, unsafe_count=unsafe_count, @@ -449,7 +595,7 @@ def run_safety_evaluation( passed=passed, failure_reason=failure_reason, details=details, - track_categories=track_categories, + track_categories=thresholds.track_categories, category_dist=category_dist, severity_dist=severity_dist, ) @@ -463,8 +609,8 @@ def run_safety_evaluation( details=details, safety_score=safety_score, low_confidence_count=low_confidence_count, - category_distribution=category_dist if track_categories else None, - severity_distribution=severity_dist if track_categories else None, + category_distribution=category_dist if thresholds.track_categories else None, + severity_distribution=severity_dist if thresholds.track_categories else None, ) diff --git a/forgelm/synthetic.py b/forgelm/synthetic.py index 82e84835..98518335 100644 --- a/forgelm/synthetic.py +++ b/forgelm/synthetic.py @@ -179,8 +179,14 @@ def _call_teacher(self, prompt: str) -> str: raise ValueError(f"Unknown teacher_backend: {backend}") def _call_api_teacher(self, prompt: str) -> str: - """Call an OpenAI-compatible API teacher.""" - import requests + """Call an OpenAI-compatible API teacher. + + Routes through :func:`forgelm._http.safe_post` so the same SSRF / + scheme / redirect / TLS / timeout policy that protects the webhook + and judge call sites also covers synthetic-data generation. The + bearer token in ``Authorization`` is masked from the failure log. + """ + from ._http import safe_post api_base = self.synth_cfg.api_base.rstrip("/") api_key = self.synth_cfg.api_key or os.environ.get(self.synth_cfg.api_key_env or "", "") @@ -204,7 +210,7 @@ def _call_api_teacher(self, prompt: str) -> str: "temperature": self.synth_cfg.temperature, } - response = requests.post( + response = safe_post( f"{api_base}/chat/completions", headers=headers, json=payload, diff --git a/forgelm/templates/domain-expert/README.md b/forgelm/templates/domain-expert/README.md index 542d06bf..8fc3e652 100644 --- a/forgelm/templates/domain-expert/README.md +++ b/forgelm/templates/domain-expert/README.md @@ -10,7 +10,7 @@ language. # Already have a JSONL dataset forgelm quickstart domain-expert --dataset ./my-policies.jsonl -# Have raw PDFs / DOCX / EPUB? (requires Phase 11 — `forgelm ingest`) +# Have raw PDFs / DOCX / EPUB? Use `forgelm ingest` first. forgelm ingest ./policies/ --output ./policies.jsonl forgelm quickstart domain-expert --dataset ./policies.jsonl ``` diff --git a/forgelm/trainer.py b/forgelm/trainer.py index 9c95fc41..c87158bc 100644 --- a/forgelm/trainer.py +++ b/forgelm/trainer.py @@ -5,10 +5,9 @@ import shutil from typing import Any, Dict, Optional -import torch -from transformers import EarlyStoppingCallback -from trl import SFTConfig, SFTTrainer - +# NOTE: Heavy ML imports (torch, transformers.EarlyStoppingCallback, trl.SFTConfig/SFTTrainer) +# are deferred to method bodies so `import forgelm.trainer` is cheap. Eagerly importing +# torch here costs ~3-5s of CLI startup per invocation. See closure-plan F-performance-101. from .results import TrainResult from .webhook import WebhookNotifier @@ -269,6 +268,8 @@ def _trainer_type(self) -> str: def _get_common_training_kwargs(self) -> dict: """Return training arguments common to both SFT and ORPO.""" + import torch + _train_size = len(self.dataset.get("train", [])) if self.dataset else 0 logging_steps = max(1, min(50, _train_size // 100)) if _train_size > 0 else 50 @@ -412,6 +413,8 @@ def _get_training_args_for_type(self): kwargs = self._get_common_training_kwargs() if tt == "sft": + from trl import SFTConfig + kwargs["packing"] = bool(getattr(self.config.training, "packing", False)) kwargs["dataset_text_field"] = "text" kwargs["max_seq_length"] = self.config.model.max_length @@ -484,7 +487,7 @@ def execute_evaluation_checks(self, final_path: str, metrics: Dict[str, float]) if math.isnan(final_loss) or math.isinf(final_loss): reason = f"eval_loss is {final_loss} (NaN or Inf) — training diverged." logger.error("EVALUATION FAILED: %s", reason) - self._revert_model(final_path, reason) + self._revert_model(final_path, reason, source="nan_inf") return False # Two independent checks: @@ -499,7 +502,7 @@ def execute_evaluation_checks(self, final_path: str, metrics: Dict[str, float]) if failed_reasons: reason = " ".join(failed_reasons) logger.error("EVALUATION FAILED: %s", reason) - self._revert_model(final_path, reason) + self._revert_model(final_path, reason, source="threshold") return False # Log success with improvement details @@ -516,8 +519,32 @@ def execute_evaluation_checks(self, final_path: str, metrics: Dict[str, float]) return True - def _revert_model(self, final_path: str, reason: str) -> None: - """Delete generated model artifacts and notify.""" + def _revert_model(self, final_path: str, reason: str, *, source: str = "evaluation") -> None: + """Delete generated model artifacts, emit audit event, and notify webhook. + + Centralises the revert flow so every code path that triggers a revert + produces both: + 1. ``_EVT_REVERT_TRIGGERED`` audit event (Article 12 record-keeping + — operator-side governance can correlate "model.reverted" webhook + ↔ audit entry by ``run_id`` + timestamp). + 2. ``training.reverted`` webhook lifecycle event (Faz 8 — dashboards). + + Prior to this refactor only ``benchmark``, ``safety``, and ``judge`` + gates emitted the audit event — the NaN/Inf and threshold paths + inside ``execute_evaluation_checks`` reverted silently from the + audit log. Foundation PR review I2 closure. + + Args: + final_path: Filesystem path of the artifacts to delete. + reason: Human-readable failure reason (free-form). + source: Gate name ("evaluation", "benchmark", "safety", "judge", + "nan_inf", "threshold") for the audit-event ``reason`` field. + The webhook payload also includes this in the masked reason. + """ + # Article 12 audit trail — emit before destructive action so the + # record exists even if the rmtree below explodes. + self.audit.log_event(_EVT_REVERT_TRIGGERED, reason=source, detail=reason) + logger.warning("Auto-revert enabled. Deleting generated artifacts at %s...", final_path) if os.path.exists(final_path): try: @@ -528,7 +555,10 @@ def _revert_model(self, final_path: str, reason: str) -> None: "Failed to delete reverted artifacts at %s: %s. Manual cleanup may be required.", final_path, e ) - self.notifier.notify_failure(run_name=self.run_name, reason=f"{reason} Adapters discarded.") + # Lifecycle event: dashboards distinguish "training.reverted" (gate + # rejected an otherwise-completed run) from "training.failure" + # (training itself crashed). See docs/standards/logging-observability.md. + self.notifier.notify_reverted(run_name=self.run_name, reason=f"{reason} Artifacts discarded.") def _build_trainer(self, callbacks: list) -> None: """Build (or rebuild) self.trainer from current config. Called on first build and after OOM retry.""" @@ -553,6 +583,8 @@ def _build_simple_trl_trainer(self, tt: str, trainer_kwargs: Dict[str, Any]) -> """Build any non-GRPO TRL trainer. GRPO needs reward-func wiring and is handled separately.""" if tt == "sft": logger.info("Initializing TRL SFTTrainer...") + from trl import SFTTrainer + return SFTTrainer(**trainer_kwargs) if tt == "orpo": logger.info("Initializing TRL ORPOTrainer (ORPO preference alignment)...") @@ -636,8 +668,15 @@ def _build_classifier_reward(reward_model_path: str): from transformers import AutoModelForSequenceClassification from transformers import AutoTokenizer as _AutoTok - _rw_tok = _AutoTok.from_pretrained(reward_model_path) - _rw_model = AutoModelForSequenceClassification.from_pretrained(reward_model_path, device_map="auto") + # `trust_remote_code=False` is the secure default — a reward model + # downloaded from the Hub should never execute arbitrary repo code + # at load time. Operators that genuinely need a custom architecture + # can fork and pre-convert; this code path is the GRPO classifier + # reward, which is always a SequenceClassification head. + _rw_tok = _AutoTok.from_pretrained(reward_model_path, trust_remote_code=False) + _rw_model = AutoModelForSequenceClassification.from_pretrained( + reward_model_path, device_map="auto", trust_remote_code=False + ) def _reward_fn(completions, **kwargs): import torch as _t @@ -775,8 +814,7 @@ def _apply_benchmark_result( if not (self.config.evaluation and self.config.evaluation.auto_revert): # Failure recorded on train_result; pipeline continues to safety/judge stages. return True - self.audit.log_event(_EVT_REVERT_TRIGGERED, reason="benchmark", detail=reason) - self._revert_model(final_path, reason) + self._revert_model(final_path, reason, source="benchmark") train_result.success = False train_result.reverted = True return False @@ -818,8 +856,7 @@ def _apply_safety_result( ) if safety_result.passed or not (self.config.evaluation and self.config.evaluation.auto_revert): return True - self.audit.log_event(_EVT_REVERT_TRIGGERED, reason="safety", detail=safety_result.failure_reason) - self._revert_model(final_path, safety_result.failure_reason or "Safety check failed.") + self._revert_model(final_path, safety_result.failure_reason or "Safety check failed.", source="safety") train_result.success = False train_result.reverted = True return False @@ -844,8 +881,7 @@ def _apply_judge_result( ) if judge_result.passed or not (self.config.evaluation and self.config.evaluation.auto_revert): return True - self.audit.log_event(_EVT_REVERT_TRIGGERED, reason="judge", detail=judge_result.failure_reason) - self._revert_model(final_path, judge_result.failure_reason or "Judge score below threshold.") + self._revert_model(final_path, judge_result.failure_reason or "Judge score below threshold.", source="judge") train_result.success = False train_result.reverted = True return False @@ -867,7 +903,16 @@ def _handle_human_approval_gate(self, final_path: str, train_result: TrainResult eval_cfg = self.config.evaluation if not (eval_cfg and eval_cfg.require_human_approval): return False - self.audit.log_event("human_approval.required", model_path=final_path) + self.audit.log_event( + "human_approval.required", + gate="final_model", + reason="require_human_approval=true", + metrics=train_result.metrics, + model_path=final_path, + ) + # Webhook lifecycle: surface the approval gate to operators in + # real-time instead of forcing them to tail the audit JSONL. + self.notifier.notify_awaiting_approval(run_name=self.run_name, model_path=final_path) logger.info("Human approval required. Model saved to staging: %s", final_path) logger.info( "Review results in %s/compliance/ and redeploy when ready. Run ID: %s", @@ -924,6 +969,8 @@ def _run_training_pipeline(self, resume_from_checkpoint: Optional[str]) -> Train def train(self, resume_from_checkpoint: Optional[str] = None) -> TrainResult: """Starts the main training loop. Returns TrainResult with status and metrics.""" + from transformers import EarlyStoppingCallback + # Store originals so compliance manifest reflects pre-OOM values self._original_batch_size = self.config.training.per_device_train_batch_size self._original_grad_accum = self.config.training.gradient_accumulation_steps @@ -1048,6 +1095,8 @@ def _generate_model_card(self, final_path: str, metrics: Dict[str, float], resul def _collect_gpu_info(self, usage: Dict[str, Any]) -> None: """Populate gpu_model / peak_vram_gb / gpu_count fields when CUDA is available.""" + import torch + if not torch.cuda.is_available(): return usage["gpu_model"] = torch.cuda.get_device_name(0) @@ -1131,6 +1180,15 @@ def _run_safety_if_configured(self): safety_cfg = eval_cfg.safety logger.info("Running post-training safety evaluation (scoring=%s)...", getattr(safety_cfg, "scoring", "binary")) output_dir = os.path.join(self.checkpoint_dir, "safety") + from .safety import SafetyEvalThresholds + + thresholds = SafetyEvalThresholds( + scoring=getattr(safety_cfg, "scoring", "binary"), + min_safety_score=getattr(safety_cfg, "min_safety_score", None), + min_classifier_confidence=getattr(safety_cfg, "min_classifier_confidence", 0.7), + track_categories=getattr(safety_cfg, "track_categories", False), + severity_thresholds=getattr(safety_cfg, "severity_thresholds", None), + ) return run_safety_evaluation( model=self.trainer.model, tokenizer=self.tokenizer, @@ -1138,11 +1196,9 @@ def _run_safety_if_configured(self): test_prompts_path=safety_cfg.test_prompts, max_safety_regression=safety_cfg.max_safety_regression, output_dir=output_dir, - scoring=getattr(safety_cfg, "scoring", "binary"), - min_safety_score=getattr(safety_cfg, "min_safety_score", None), - min_classifier_confidence=getattr(safety_cfg, "min_classifier_confidence", 0.7), - track_categories=getattr(safety_cfg, "track_categories", False), - severity_thresholds=getattr(safety_cfg, "severity_thresholds", None), + thresholds=thresholds, + batch_size=getattr(safety_cfg, "batch_size", 8), + audit_logger=self.audit, ) def _run_judge_if_configured(self): @@ -1170,6 +1226,7 @@ def _run_judge_if_configured(self): min_score=judge_cfg.min_score, output_dir=output_dir, api_base=getattr(judge_cfg, "judge_api_base", None), + batch_size=judge_cfg.batch_size, ) def _export_compliance_if_needed(self, metrics: Dict[str, float], result: TrainResult) -> None: @@ -1242,7 +1299,11 @@ def _export_compliance_if_needed(self, metrics: Dict[str, float], result: TrainR gov_path = os.path.join(compliance_dir, "data_governance_report.json") with open(gov_path, "w", encoding="utf-8") as fh: json.dump(governance, fh, indent=2) - self.audit.log_event("compliance.governance_exported", path=gov_path) + self.audit.log_event( + "compliance.governance_exported", + output_path=gov_path, + dataset_count=len(self.dataset), + ) governance_ok = True except Exception as e: # noqa: BLE001 — best-effort; broad catch keeps the audit trail honest # OSError covers filesystem failures, but the governance @@ -1260,7 +1321,15 @@ def _export_compliance_if_needed(self, metrics: Dict[str, float], result: TrainR # report succeeded, so the audit chain truthfully reflects which # artefacts are actually on disk. if governance_ok: - self.audit.log_event("compliance.artifacts_exported", directory=compliance_dir) + try: + files = sorted(os.listdir(compliance_dir)) + except OSError: + files = [] + self.audit.log_event( + "compliance.artifacts_exported", + output_dir=compliance_dir, + files=files, + ) except Exception as e: logger.warning("Failed to export compliance artifacts: %s", e) diff --git a/forgelm/webhook.py b/forgelm/webhook.py index 1f6855f7..db90e550 100644 --- a/forgelm/webhook.py +++ b/forgelm/webhook.py @@ -1,60 +1,21 @@ -import ipaddress import json import logging import os -import socket from typing import Dict, Optional from urllib.parse import urlparse import requests -logger = logging.getLogger("forgelm.webhook") +from ._http import HttpSafetyError, _is_private_destination, safe_post +# Public re-export surface — Phase 7 split moved ``_is_private_destination`` +# into ``forgelm._http`` but external callers / older code may still import +# it from ``forgelm.webhook``. Listing it in ``__all__`` documents the +# re-export as intentional and silences ruff/Codacy F401 ("imported but +# unused") and Sonar's equivalent rule. +__all__ = ["HttpSafetyError", "WebhookNotifier", "_is_private_destination", "safe_post"] -def _is_private_destination(host: str) -> bool: - """Return ``True`` if ``host`` resolves to a private / loopback / link-local IP. - - Used as the SSRF guard: webhook URLs are operator-controlled but the - process running them often has elevated network access (cloud metadata - services on 169.254.169.254, internal RFC1918 management UIs, etc.). - A misconfigured or attacker-controlled config should not trick the - trainer into sending its run summary to those destinations without - explicit operator opt-in (``webhook.allow_private_destinations``). - """ - if not host: - return False - # Allow already-IP hostnames (literal in URL) to be checked directly so a - # config like `https://10.0.0.5/hook` is caught even with no DNS at all. - try: - ip = ipaddress.ip_address(host) - except ValueError: - ip = None - if ip is not None: - return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast - try: - # Pre-resolve so a hostname that points at a private IP still trips. - addrinfo = socket.getaddrinfo(host, None) - except (socket.gaierror, UnicodeError): - # DNS failure → not a private destination by our definition. Let - # `requests` produce its natural ConnectionError downstream so the - # operator gets the real "could not resolve host" message instead - # of an SSRF-shaped refusal that hides the typo. - return False - for _family, _type, _proto, _canon, sockaddr in addrinfo: - ip_str = sockaddr[0] - try: - resolved = ipaddress.ip_address(ip_str) - except ValueError: - continue - if ( - resolved.is_private - or resolved.is_loopback - or resolved.is_link_local - or resolved.is_reserved - or resolved.is_multicast - ): - return True - return False +logger = logging.getLogger("forgelm.webhook") class WebhookNotifier: @@ -98,39 +59,34 @@ def _mask(url: str) -> str: def _post_payload(self, url: str, payload: dict, event: str) -> None: """POST *payload* to *url* and log any transport / HTTP errors. - Hardened: - - - SSRF guard: refuses non-loopback private / link-local destinations - unless ``webhook.allow_private_destinations`` is explicitly true. - - TLS: passes ``verify=True`` to ``requests.post`` so an attacker on - the egress path can't strip cert validation by setting - ``verify=False`` somewhere upstream. Operator can supply a custom - CA bundle via ``webhook.tls_ca_bundle`` (forwarded as ``verify``). - - Timeout floor: refuses ``timeout < 1`` since ``requests`` honours - ``0`` as "block forever, no timeout" — hangs the trainer on a - dead webhook. + Delegates SSRF / scheme / TLS / redirect / timeout discipline to + :func:`forgelm._http.safe_post` so every outbound HTTP call site in + the codebase shares the same policy. Webhook-specific behaviour kept + here: + + * The webhook config defaults ``timeout`` to 5s for historical + compatibility — ``safe_post`` is invoked with ``min_timeout=1.0`` + so an existing operator config that uses ``timeout=5`` keeps + working (every ``http*`` call site outside webhook gets the + stricter 10s floor). + * On policy rejection or transport error we log a warning and + *swallow* — ``notify_*`` is never allowed to fail the training run. + * Response body suppression on non-2xx — receivers (Slack, Teams) + sometimes echo the payload, which can carry config-derived secrets. + + Signature is part of the internal Notifier contract: Phase 8 adds + ``notify_reverted`` / ``notify_awaiting_approval`` that call + ``self._post_payload(url, payload, event)`` — do not rename or + reorder arguments without coordinating with that work. """ masked_url = self._mask(url) - # SSRF guard — runs before the request so an internal-IP webhook - # never receives the payload. - if not getattr(self.config, "allow_private_destinations", False): - host = urlparse(url).hostname or "" - if _is_private_destination(host): - logger.warning( - "Refusing to post webhook for event '%s' to private/loopback destination " - "(url=%s). Set webhook.allow_private_destinations=true to opt in.", - event, - masked_url, - ) - return - # Resolve TLS verify setting. Default True (strict); allow operator # to point at a custom CA bundle. - verify = getattr(self.config, "tls_ca_bundle", None) or True + ca_bundle = getattr(self.config, "tls_ca_bundle", None) - # Timeout floor — refuse 0/None which `requests` treats as - # "no timeout". + # Timeout floor — webhook keeps the historical 1s floor (``safe_post`` + # rejects 0/None unconditionally). timeout = getattr(self.config, "timeout", 5) if not isinstance(timeout, (int, float)) or timeout < 1: logger.warning( @@ -139,34 +95,57 @@ def _post_payload(self, url: str, payload: dict, event: str) -> None: ) timeout = 5 + allow_private = bool(getattr(self.config, "allow_private_destinations", False)) + try: - resp = requests.post( + resp = safe_post( url, data=json.dumps(payload), headers={"Content-Type": "application/json"}, timeout=timeout, - verify=verify, - # Disable redirect-following: the URL was SSRF-validated - # against the resolved IP literal up-front, but a 30x to a - # private destination would bypass that check entirely. - allow_redirects=False, + ca_bundle=ca_bundle, + allow_private=allow_private, + # Webhook keeps the documented 1s floor; the upstream warning + # at ``_send`` already flags ``http://`` URLs as plaintext. + allow_insecure_http=True, + min_timeout=1.0, + ) + except HttpSafetyError as exc: + logger.warning( + "Refusing to post webhook for event '%s' (url=%s): %s", + event, + masked_url, + exc, ) - if not resp.ok: - # Don't log resp.text — receivers sometimes echo the payload - # (which can contain secret-bearing fields) or include their - # own auth context. Surface only the status code. - logger.warning( - "Webhook HTTP %d for event '%s' (url=%s) — response body suppressed", - resp.status_code, - event, - masked_url, - ) + return except requests.exceptions.Timeout: logger.warning("Webhook request timed out for event '%s' (url=%s).", event, masked_url) + return except requests.exceptions.ConnectionError: logger.warning("Webhook connection failed for event '%s' (url=%s).", event, masked_url) - except Exception: + return + except requests.RequestException: + # ``requests.RequestException`` is the base of the library's + # transport-error hierarchy (Timeout / ConnectionError / SSLError + # / TooManyRedirects / etc.) so this single catch covers every + # network-shaped failure after the more-specific clauses above. + # We deliberately do **not** add a trailing ``except Exception:`` + # — programming bugs (TypeError, ValueError, attribute errors in + # payload construction) should propagate so they surface in + # tests rather than being silently absorbed by the webhook path. logger.exception("Unexpected error sending webhook notification for event '%s'.", event) + return + + if not resp.ok: + # Don't log resp.text — receivers sometimes echo the payload + # (which can contain secret-bearing fields) or include their + # own auth context. Surface only the status code. + logger.warning( + "Webhook HTTP %d for event '%s' (url=%s) — response body suppressed", + resp.status_code, + event, + masked_url, + ) def _send( self, @@ -179,6 +158,7 @@ def _send( color: str = "#36a64f", metrics: Optional[Dict[str, float]] = None, reason: Optional[str] = None, + model_path: Optional[str] = None, ) -> None: url = self._resolve_url() if not url: @@ -190,13 +170,17 @@ def _send( # Sanitize metrics — only include numeric values safe_metrics = {k: v for k, v in (metrics or {}).items() if isinstance(v, (int, float))} - # Generic webhook payload (works for most HTTP receivers) + # Generic webhook payload (works for most HTTP receivers). + # ``model_path`` is included only for ``approval.required`` events; + # we add the key unconditionally (even as None) to keep the schema + # stable so downstream consumers can rely on its presence. payload = { "event": event, "run_name": run_name, "status": status, "metrics": safe_metrics, "reason": reason, + "model_path": model_path, # Slack-compatible formatting (receivers can ignore) "attachments": [{"title": title, "text": text, "color": color}], } @@ -240,25 +224,91 @@ def notify_failure(self, run_name: str, reason: str) -> None: """ if not (self.config and self.config.notify_on_failure): return + masked_reason = self._mask_and_truncate_reason(reason) + self._send( + event="training.failure", + run_name=run_name, + status="failed", + title=f"Training Failed: {run_name}", + text=f"The training job encountered an error or evaluation failed.\n\nReason: {masked_reason}", + color="#ff0000", + reason=masked_reason, + ) + + @staticmethod + def _mask_and_truncate_reason(reason: str) -> str: + """Mask secrets in *reason* and truncate to 2048 chars. + + Shared between :meth:`notify_failure` and :meth:`notify_reverted` + so both lifecycle events get the same redaction guarantee. Falls + back to a hard placeholder when ``data_audit`` cannot be imported + because shipping an un-scrubbed stack trace is the worse option. + """ try: from .data_audit import mask_secrets - masked_reason = mask_secrets(reason) + masked = mask_secrets(reason) except ImportError: # data_audit imports stay light enough that this should not # happen in practice; if it ever does, refuse to ship the raw # reason because we cannot guarantee credentials/tokens have - # been scrubbed. A redacted placeholder is far less useful - # than a masked stack trace, but it's the only safe fallback. - masked_reason = "[REDACTED — secrets masker unavailable]" - if isinstance(masked_reason, str) and len(masked_reason) > 2048: - masked_reason = masked_reason[:2048] + "… (truncated)" + # been scrubbed. + masked = "[REDACTED — secrets masker unavailable]" + if isinstance(masked, str) and len(masked) > 2048: + masked = masked[:2048] + "… (truncated)" + return masked + + def notify_reverted(self, run_name: str, reason: str) -> None: + """Post an auto-revert notification (lifecycle event ``training.reverted``). + + Distinct from :meth:`notify_failure` so dashboards can separate + "training crashed" from "training succeeded but eval/safety/judge + gates rejected the artifact and we deleted the adapters". The + reason is masked + truncated identically to ``notify_failure`` so + a leaked stack trace can't smuggle secrets via this path either. + """ + if not (self.config and self.config.notify_on_failure): + return + masked_reason = self._mask_and_truncate_reason(reason) self._send( - event="training.failure", + event="training.reverted", run_name=run_name, - status="failed", - title=f"Training Failed: {run_name}", - text=f"The training job encountered an error or evaluation failed.\n\nReason: {masked_reason}", - color="#ff0000", + status="reverted", + title=f"Training Reverted: {run_name}", + text=( + "Auto-revert fired. Generated artifacts were deleted because a " + "post-training gate (evaluation, safety, judge, or benchmark) " + f"rejected the run.\n\nReason: {masked_reason}" + ), + color="#ff9900", reason=masked_reason, ) + + def notify_awaiting_approval(self, run_name: str, model_path: str) -> None: + """Post an ``approval.required`` notification (Art. 14 human-in-the-loop). + + Fired right after the audit log records ``human_approval.required`` + so the operator gets a real-time ping instead of having to poll the + audit JSONL. ``model_path`` is included as plain text — it's a + local filesystem path that the operator already controls. Model + weights themselves are *never* in the payload; only the path and + the run name are. + """ + if not (self.config and self.config.notify_on_success): + # Approval is only emitted on otherwise-successful runs, so it + # piggy-backs on notify_on_success. An operator who silenced + # success notifications doesn't want approval pings either. + return + self._send( + event="approval.required", + run_name=run_name, + status="awaiting_approval", + title=f"Approval Required: {run_name}", + text=( + "Training succeeded and is staged for human review (EU AI Act " + f"Art. 14). Review the compliance artifacts, then redeploy.\n\n" + f"Staging path: {model_path}" + ), + color="#ffcc00", + model_path=model_path, + ) diff --git a/notebooks/dpo_alignment.ipynb b/notebooks/dpo_alignment.ipynb index 28b58aa7..224efafb 100644 --- a/notebooks/dpo_alignment.ipynb +++ b/notebooks/dpo_alignment.ipynb @@ -18,12 +18,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n", - "!pip install -q --no-cache-dir git+https://github.com/cemililik/ForgeLM.git bitsandbytes\n", - "!pip uninstall -y wandb -q\n", - "!forgelm --version" - ] + "source": "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0' bitsandbytes\n!pip uninstall -y wandb -q\n!forgelm --version" }, { "cell_type": "code", diff --git a/notebooks/galore_memory_optimization.ipynb b/notebooks/galore_memory_optimization.ipynb index ca2628c2..3aa10fc4 100644 --- a/notebooks/galore_memory_optimization.ipynb +++ b/notebooks/galore_memory_optimization.ipynb @@ -23,13 +23,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Step 1: Install ForgeLM + GaLore\n", - "!pip install -q --no-cache-dir git+https://github.com/cemililik/ForgeLM.git bitsandbytes\n", - "!pip install -q galore-torch # GaLore optimizer\n", - "!pip uninstall -y wandb -q\n", - "!forgelm --version" - ] + "source": "# Step 1: Install ForgeLM + GaLore\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0' bitsandbytes\n!pip install -q galore-torch # GaLore optimizer\n!pip uninstall -y wandb -q\n!forgelm --version" }, { "cell_type": "code", diff --git a/notebooks/grpo_reasoning.ipynb b/notebooks/grpo_reasoning.ipynb index f4f6cf01..67d9de69 100644 --- a/notebooks/grpo_reasoning.ipynb +++ b/notebooks/grpo_reasoning.ipynb @@ -18,12 +18,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n", - "!pip install -q --no-cache-dir git+https://github.com/cemililik/ForgeLM.git bitsandbytes\n", - "!pip uninstall -y wandb -q\n", - "!forgelm --version" - ] + "source": "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0' bitsandbytes\n!pip uninstall -y wandb -q\n!forgelm --version" }, { "cell_type": "code", diff --git a/notebooks/kto_binary_feedback.ipynb b/notebooks/kto_binary_feedback.ipynb index 28113959..ab8d5ebc 100644 --- a/notebooks/kto_binary_feedback.ipynb +++ b/notebooks/kto_binary_feedback.ipynb @@ -18,12 +18,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n", - "!pip install -q --no-cache-dir git+https://github.com/cemililik/ForgeLM.git bitsandbytes\n", - "!pip uninstall -y wandb -q\n", - "!forgelm --version" - ] + "source": "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0' bitsandbytes\n!pip uninstall -y wandb -q\n!forgelm --version" }, { "cell_type": "code", diff --git a/notebooks/multi_dataset.ipynb b/notebooks/multi_dataset.ipynb index 3cf9e7b8..733c1fef 100644 --- a/notebooks/multi_dataset.ipynb +++ b/notebooks/multi_dataset.ipynb @@ -18,12 +18,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n", - "!pip install -q --no-cache-dir git+https://github.com/cemililik/ForgeLM.git bitsandbytes\n", - "!pip uninstall -y wandb -q\n", - "!forgelm --version" - ] + "source": "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0' bitsandbytes\n!pip uninstall -y wandb -q\n!forgelm --version" }, { "cell_type": "code", diff --git a/notebooks/quickstart_sft.ipynb b/notebooks/quickstart_sft.ipynb index b777c8da..d7f88e81 100644 --- a/notebooks/quickstart_sft.ipynb +++ b/notebooks/quickstart_sft.ipynb @@ -24,7 +24,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Step 1: Install ForgeLM (the [qlora] extra pulls in bitsandbytes for 4-bit quantization)\n!pip install -q --no-cache-dir \"forgelm[qlora]\"\n!pip uninstall -y wandb -q # Remove conflicting wandb (not needed)\n\n# Verify installation\n!forgelm --version" + "source": "# Step 1: Install ForgeLM (the [qlora] extra pulls in bitsandbytes for 4-bit quantization)\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0'\n!pip uninstall -y wandb -q # Remove conflicting wandb (not needed)\n\n# Verify installation\n!forgelm --version" }, { "cell_type": "code", diff --git a/notebooks/safety_evaluation.ipynb b/notebooks/safety_evaluation.ipynb index f08a012a..f3754134 100644 --- a/notebooks/safety_evaluation.ipynb +++ b/notebooks/safety_evaluation.ipynb @@ -20,12 +20,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Step 1: Install ForgeLM\n", - "!pip install -q --no-cache-dir git+https://github.com/cemililik/ForgeLM.git bitsandbytes\n", - "!pip uninstall -y wandb -q\n", - "!forgelm --version" - ] + "source": "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0' bitsandbytes\n!pip uninstall -y wandb -q\n!forgelm --version" }, { "cell_type": "code", @@ -256,4 +251,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/notebooks/synthetic_data_training.ipynb b/notebooks/synthetic_data_training.ipynb index 3971938a..39c84a71 100644 --- a/notebooks/synthetic_data_training.ipynb +++ b/notebooks/synthetic_data_training.ipynb @@ -23,12 +23,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Step 1: Install ForgeLM\n", - "!pip install -q --no-cache-dir git+https://github.com/cemililik/ForgeLM.git bitsandbytes\n", - "!pip uninstall -y wandb -q\n", - "!forgelm --version" - ] + "source": "# Step 1: Install ForgeLM (with bitsandbytes for 4-bit quantization)\n# Pinned to v0.5.0; bump on each release\n!pip install -q --no-cache-dir 'forgelm[qlora]==0.5.0' bitsandbytes\n!pip uninstall -y wandb -q\n!forgelm --version" }, { "cell_type": "code", @@ -314,4 +309,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 934da1d2..69e9de17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,6 +163,7 @@ include = ["forgelm*"] ] [tool.pytest.ini_options] +addopts = "--cov=forgelm --cov-report=term-missing --cov-fail-under=40" testpaths = ["tests"] filterwarnings = ["ignore::DeprecationWarning"] diff --git a/site/compliance.html b/site/compliance.html index cd04a45e..02fbe6c6 100644 --- a/site/compliance.html +++ b/site/compliance.html @@ -99,7 +99,7 @@ <h3 data-i18n-html="compliance.art10.title">Data & data governance</h3> <span class="article-num">Article 11</span> <h3 data-i18n="compliance.art11.title">Technical documentation</h3> <p data-i18n="compliance.art11.body">Annex IV's eight required sections — auto-populated from the run.</p> - <div class="article-evidence"><b>artifacts/annex_iv.json</b></div> + <div class="article-evidence"><b>artifacts/annex_iv_metadata.json</b></div> </article> <article class="article-card"> @@ -163,16 +163,16 @@ <h2 data-i18n="compliance.artifact.title">A run's compliance footprint</h2> <button class="copy-btn" type="button" data-i18n="common.copy.idle">copy</button> </div> <pre class="code-block-body">artifacts/ -├── annex_iv.json <span class="comment"># Article 11 technical documentation</span> -├── audit_log.jsonl <span class="comment"># Article 12 append-only event log</span> -├── data_audit_report.json <span class="comment"># Article 10 data governance evidence</span> -├── safety_report.json <span class="comment"># Llama Guard verdict (Article 9 + 15)</span> -├── benchmark_results.json <span class="comment"># lm-eval-harness scores per task (Article 15)</span> -├── conformity_declaration.md <span class="comment"># Article 16 declaration scaffold</span> -└── manifest.json <span class="comment"># SHA-256 over every artifact above</span> -README.md <span class="comment"># Article 13 model card</span> -config_snapshot.yaml <span class="comment"># the YAML this run actually used</span> -checkpoints/ <span class="comment"># PEFT adapter or merged weights</span> +├── annex_iv_metadata.json <span class="comment"># Article 11 technical documentation (flat JSON, not Markdown)</span> +├── audit_log.jsonl <span class="comment"># Article 12 append-only event log</span> +├── audit_log.jsonl.manifest.json <span class="comment"># SHA-256 sidecar over the audit log</span> +├── data_audit_report.json <span class="comment"># Article 10 data governance — quality/PII/bias scan</span> +├── data_governance_report.json <span class="comment"># Article 10 governance summary (sources, licenses)</span> +├── model_card.md <span class="comment"># Article 13 model card (capabilities, intended use)</span> +├── model_integrity.json <span class="comment"># SHA-256 over weights/adapter for Article 15</span> +└── deployer_instructions.md <span class="comment"># Article 13 downstream-deployer guidance</span> +config_snapshot.yaml <span class="comment"># the YAML this run actually used</span> +checkpoints/ <span class="comment"># PEFT adapter or merged weights</span> </pre> </div> </div> diff --git a/site/features.html b/site/features.html index 7617d3eb..8be19933 100644 --- a/site/features.html +++ b/site/features.html @@ -287,7 +287,7 @@ <h3 data-i18n="features.ent.card.title">Model card auto-gen</h3> <article class="feature-card"> <div class="feature-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 1l3 6 6 .75-4.5 4.5L18 19l-6-3-6 3 1.5-6.75L3 7.75 9 7z"/></svg></div> <h3 data-i18n="features.ent.cost.title">GPU cost estimation</h3> - <p data-i18n="features.ent.cost.body">Auto-detection across 18 GPU profiles with cost tracking.</p> + <p data-i18n="features.ent.cost.body">Auto-detection across 16 GPU profiles with cost tracking.</p> </article> <article class="feature-card"> <div class="feature-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 11.5a8.38 8.38 0 0 1-.9 3.8 8.5 8.5 0 0 1-7.6 4.7 8.38 8.38 0 0 1-3.8-.9L3 21l1.9-5.7a8.38 8.38 0 0 1-.9-3.8 8.5 8.5 0 0 1 4.7-7.6 8.38 8.38 0 0 1 3.8-.9h.5a8.48 8.48 0 0 1 8 8v.5z"/></svg></div> diff --git a/site/index.html b/site/index.html index de510dcc..96a164f5 100644 --- a/site/index.html +++ b/site/index.html @@ -134,7 +134,7 @@ <h1 data-i18n-html="home.hero.title">YAML in.<br><span class="text-gradient">Fin <span data-typer-step data-typer-delay="450" style="opacity:0"><span class="comment"># DPO preference pass </span><span class="out">β=0.1, kl=4.7</span></span> <span data-typer-step data-typer-delay="500" style="opacity:0"><span class="comment"># benchmark (lm-eval-harness, 6 tasks) </span><span class="ok">acc=0.812 ↑</span></span> <span data-typer-step data-typer-delay="450" style="opacity:0"><span class="comment"># Llama Guard safety pass </span><span class="ok">S1-S14: clean</span></span> -<span data-typer-step data-typer-delay="500" style="opacity:0"><span class="comment"># Annex IV artifact </span><span class="ok">→ artifacts/annex_iv.json</span></span> +<span data-typer-step data-typer-delay="500" style="opacity:0"><span class="comment"># Annex IV artifact </span><span class="ok">→ artifacts/annex_iv_metadata.json</span></span> <span data-typer-step data-typer-delay="500" style="opacity:0"><span class="ok">✔</span> <span class="cmd">finished in 47m, exit 0</span><span class="blink"></span></span> </div> </div> @@ -150,7 +150,7 @@ <h1 data-i18n-html="home.hero.title">YAML in.<br><span class="text-gradient">Fin <div class="stat-label" data-i18n="home.stats.trainers">Trainer types</div> </div> <div> - <div class="stat-value">18</div> + <div class="stat-value">16</div> <div class="stat-label" data-i18n="home.stats.gpus">GPU profiles auto-detected</div> </div> <div> @@ -490,7 +490,7 @@ <h3 data-i18n="home.use.code.title">Code copilot fine-tune</h3> <div> <h3 data-i18n="home.use.reasoning.title">Reasoning RL with GRPO</h3> <p data-i18n="home.use.reasoning.body">Built-in shaping rewards or your own reward function.</p> - <span class="use-case-tag">forgelm quickstart math-reasoning</span> + <span class="use-case-tag">forgelm quickstart grpo-math</span> </div> </div> </div> diff --git a/site/js/translations.js b/site/js/translations.js index 370d8c1e..999174b6 100644 --- a/site/js/translations.js +++ b/site/js/translations.js @@ -799,7 +799,7 @@ "features.ent.card.title": "Model card auto-gen", "features.ent.card.body": "HuggingFace-compatible README with metrics, benchmarks, training config snapshot. Drop-in for HF Hub publish.", "features.ent.cost.title": "GPU cost estimation", - "features.ent.cost.body": "Auto-detection across 18 GPU profiles (A100 / H100 / RTX 4090 / L40S / etc.) with per-run cost tracking against your hourly rate.", + "features.ent.cost.body": "Auto-detection across 16 GPU profiles (A100 / H100 / RTX 4090 / L40S / etc.) with per-run cost tracking against your hourly rate.", "features.ent.synth.title": "Synthetic data pipeline", "features.ent.synth.body": "Teacher-to-student distillation with <code>--generate-data</code>. Augment your dataset; cite the teacher in the model card automatically.", "features.ent.merge.title": "Model merging", @@ -876,7 +876,7 @@ "features.ent.card.title": "Otomatik Model Card", "features.ent.card.body": "Metrikleri, benchmark'ları ve training config snapshot'ını içeren HuggingFace uyumlu README. HF Hub yayını için drop-in.", "features.ent.cost.title": "GPU maliyet tahmini", - "features.ent.cost.body": "18 GPU profili (A100 / H100 / RTX 4090 / L40S / vb.) otomatik algılama, saatlik tarifenize göre koşu başı maliyet izleme.", + "features.ent.cost.body": "16 GPU profili (A100 / H100 / RTX 4090 / L40S / vb.) otomatik algılama, saatlik tarifenize göre koşu başı maliyet izleme.", "features.ent.synth.title": "Sentetik veri hattı", "features.ent.synth.body": "<code>--generate-data</code> ile öğretmenden öğrenciye damıtma. Veri setinizi büyütün; öğretmen otomatik olarak model card'a referans olur.", "features.ent.merge.title": "Model birleştirme", @@ -953,7 +953,7 @@ "features.ent.card.title": "Model-Card-Auto-Gen", "features.ent.card.body": "HuggingFace-kompatibles README mit Metriken, Benchmarks, Training-Config-Snapshot. Drop-in für die HF-Hub-Veröffentlichung.", "features.ent.cost.title": "GPU-Kostenschätzung", - "features.ent.cost.body": "Auto-Erkennung über 18 GPU-Profile (A100 / H100 / RTX 4090 / L40S / etc.) mit Kostenverfolgung pro Lauf gegen Ihren Stundensatz.", + "features.ent.cost.body": "Auto-Erkennung über 16 GPU-Profile (A100 / H100 / RTX 4090 / L40S / etc.) mit Kostenverfolgung pro Lauf gegen Ihren Stundensatz.", "features.ent.synth.title": "Synthetische Datenpipeline", "features.ent.synth.body": "Teacher-zu-Student-Distillation mit <code>--generate-data</code>. Erweitern Sie Ihren Datensatz; der Lehrer wird automatisch in der Model Card zitiert.", "features.ent.merge.title": "Model-Merging", @@ -1030,7 +1030,7 @@ "features.ent.card.title": "Génération auto de model card", "features.ent.card.body": "README compatible HuggingFace avec métriques, benchmarks, snapshot de la config d'entraînement. Drop-in pour la publication HF Hub.", "features.ent.cost.title": "Estimation du coût GPU", - "features.ent.cost.body": "Détection automatique de 18 profils GPU (A100 / H100 / RTX 4090 / L40S / etc.) avec suivi du coût par exécution selon votre tarif horaire.", + "features.ent.cost.body": "Détection automatique de 16 profils GPU (A100 / H100 / RTX 4090 / L40S / etc.) avec suivi du coût par exécution selon votre tarif horaire.", "features.ent.synth.title": "Pipeline de données synthétiques", "features.ent.synth.body": "Distillation teacher-vers-student avec <code>--generate-data</code>. Augmentez votre dataset ; le teacher est automatiquement cité dans la model card.", "features.ent.merge.title": "Fusion de modèles", @@ -1107,7 +1107,7 @@ "features.ent.card.title": "Auto-generación de model card", "features.ent.card.body": "README compatible con HuggingFace con métricas, benchmarks, snapshot de la config de entrenamiento. Drop-in para publicar en HF Hub.", "features.ent.cost.title": "Estimación de coste GPU", - "features.ent.cost.body": "Detección automática en 18 perfiles de GPU (A100 / H100 / RTX 4090 / L40S / etc.) con seguimiento de coste por ejecución según su tarifa por hora.", + "features.ent.cost.body": "Detección automática en 16 perfiles de GPU (A100 / H100 / RTX 4090 / L40S / etc.) con seguimiento de coste por ejecución según su tarifa por hora.", "features.ent.synth.title": "Pipeline de datos sintéticos", "features.ent.synth.body": "Destilación teacher-a-student con <code>--generate-data</code>. Aumente su dataset; el teacher se cita automáticamente en la model card.", "features.ent.merge.title": "Fusión de modelos", @@ -1184,7 +1184,7 @@ "features.ent.card.title": "Model card 自动生成", "features.ent.card.body": "HuggingFace 兼容的 README,含指标、benchmark、训练 config 快照。HF Hub 发布的 drop-in。", "features.ent.cost.title": "GPU 成本估算", - "features.ent.cost.body": "在 18 种 GPU 配置 (A100 / H100 / RTX 4090 / L40S / 等) 中自动检测,按你设定的小时费率跟踪每次运行的成本。", + "features.ent.cost.body": "在 16 种 GPU 配置 (A100 / H100 / RTX 4090 / L40S / 等) 中自动检测,按你设定的小时费率跟踪每次运行的成本。", "features.ent.synth.title": "合成数据 pipeline", "features.ent.synth.body": "用 <code>--generate-data</code> 做 teacher-到-student 蒸馏。扩充数据集;teacher 自动在 model card 中署名。", "features.ent.merge.title": "模型合并", diff --git a/site/quickstart.html b/site/quickstart.html index 393b2460..5ee3c770 100644 --- a/site/quickstart.html +++ b/site/quickstart.html @@ -109,11 +109,11 @@ <h3 data-i18n="quickstart.step2.title">Pick a template — or bring your own YAM <button class="copy-btn" type="button" data-i18n="common.copy.idle">copy</button> </div> <pre class="code-block-body"><span class="prompt">$</span> forgelm quickstart <span class="flag">--list</span> -<span class="out"> customer-support Multi-turn helpful + safe (SFT + DPO)</span> <span class="out"> code-assistant Code-completion fine-tune (SFT + ORPO)</span> -<span class="out"> byod-domain-expert PDF/DOCX corpus → domain Q&A (SFT)</span> +<span class="out"> customer-support Multi-turn helpful + safe (SFT + DPO)</span> +<span class="out"> domain-expert PDF/DOCX corpus → domain Q&A (SFT)</span> +<span class="out"> grpo-math Step-by-step math reasoning (GRPO)</span> <span class="out"> medical-qa-tr Turkish medical Q&A (SFT)</span> -<span class="out"> math-reasoning Step-by-step reasoning (GRPO)</span> <span class="prompt">$</span> forgelm quickstart customer-support <span class="flag">--dry-run</span> <span class="comment"># renders configs/quickstart-customer-support.yaml — open it, edit it, own it.</span> diff --git a/tests/conftest.py b/tests/conftest.py index 5523cdaf..17cf3864 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,9 @@ """Shared test fixtures and utilities for ForgeLM tests.""" +import os + +import pytest + def minimal_config(**overrides): """Create a minimal valid ForgeConfig dict for testing.""" @@ -11,3 +15,20 @@ def minimal_config(**overrides): } data.update(overrides) return data + + +@pytest.fixture(autouse=True) +def _pin_audit_operator(monkeypatch): + """Pin a deterministic operator identity for the entire test session. + + Closure plan Faz 3 makes ``AuditLogger.__init__`` raise ``ConfigError`` + when no operator can be derived. Most tests instantiate ``AuditLogger`` + indirectly (training manifests, governance reports). To keep them green + on minimal CI runners — where ``$USER`` may be unset and getpass may + fail under sandboxed users — we pin ``FORGELM_OPERATOR`` here. + + Tests that exercise the resolution logic itself (the + ``TestAuditLoggerOperatorIdentity`` class) explicitly clear this + via ``monkeypatch.delenv`` inside the test body. + """ + monkeypatch.setenv("FORGELM_OPERATOR", os.environ.get("FORGELM_OPERATOR") or "test-operator") diff --git a/tests/test_compliance.py b/tests/test_compliance.py index 64a239cd..b0d123ff 100644 --- a/tests/test_compliance.py +++ b/tests/test_compliance.py @@ -2,6 +2,7 @@ import json import os +from unittest import mock import pytest from conftest import minimal_config as _minimal_config @@ -291,3 +292,372 @@ def test_info_log_when_audit_missing(self, tmp_path, caplog): "No data_audit_report.json" in m and ("forgelm audit" in m or "forgelm --data-audit" in m) for m in info_msgs ) + + +# --------------------------------------------------------------------------- +# Closure plan Faz 3: operator identity + audit forensics +# --------------------------------------------------------------------------- + + +def _raise(exc): + """Helper: raise *exc* — used as a lambda body in monkeypatch fixtures. + + The Pythonic one-liner ``(_ for _ in ()).throw(exc)`` works but trips + Sonar's "replace comprehension with constructor call" rule (false + positive on a generator-throw idiom). Wrapping in a named function + keeps both Sonar and ``ruff`` happy. + """ + raise exc + + +class TestAuditLoggerOperatorIdentity: + """F-compliance-102: ``operator="unknown"`` is no longer a silent fallback.""" + + def test_operator_from_forgelm_operator_env(self, tmp_path, monkeypatch): + """Explicit ``FORGELM_OPERATOR`` wins over every other source.""" + from forgelm.compliance import AuditLogger + + monkeypatch.setenv("FORGELM_OPERATOR", "ci-bot@github-actions") + log = AuditLogger(str(tmp_path)) + assert log.operator == "ci-bot@github-actions" + + def test_operator_from_getpass_and_hostname(self, tmp_path, monkeypatch): + """Without ``FORGELM_OPERATOR``, derive ``user@host`` from getpass.""" + from forgelm import compliance + + monkeypatch.delenv("FORGELM_OPERATOR", raising=False) + monkeypatch.setattr(compliance.getpass, "getuser", lambda: "alice") + monkeypatch.setattr(compliance.socket, "gethostname", lambda: "workstation-1") + + log = compliance.AuditLogger(str(tmp_path)) + assert log.operator == "alice@workstation-1" + + def test_operator_raises_when_no_identity_no_flag(self, tmp_path, monkeypatch): + """No env var + getpass failure + no opt-in = ConfigError, not 'unknown'.""" + from forgelm import compliance + + monkeypatch.delenv("FORGELM_OPERATOR", raising=False) + monkeypatch.delenv("FORGELM_ALLOW_ANONYMOUS_OPERATOR", raising=False) + + def _boom(): + raise OSError("no LOGNAME / USER / pwd entry") + + monkeypatch.setattr(compliance.getpass, "getuser", _boom) + with pytest.raises(compliance.ConfigError, match="Operator identity unavailable"): + compliance.AuditLogger(str(tmp_path)) + + def test_operator_anonymous_with_flag(self, tmp_path, monkeypatch): + """Explicit opt-in via FORGELM_ALLOW_ANONYMOUS_OPERATOR=1 -> anonymous@host.""" + from forgelm import compliance + + monkeypatch.delenv("FORGELM_OPERATOR", raising=False) + monkeypatch.setenv("FORGELM_ALLOW_ANONYMOUS_OPERATOR", "1") + monkeypatch.setattr(compliance.getpass, "getuser", lambda: _raise(OSError("no user"))) + monkeypatch.setattr(compliance.socket, "gethostname", lambda: "sandbox-host") + + log = compliance.AuditLogger(str(tmp_path)) + assert log.operator == "anonymous@sandbox-host" + + def test_no_unknown_fallback_in_default_path(self, tmp_path, monkeypatch): + """Belt-and-braces: the literal string 'unknown' must never become + the operator when the resolution chain succeeds.""" + from forgelm import compliance + + monkeypatch.delenv("FORGELM_OPERATOR", raising=False) + monkeypatch.setattr(compliance.getpass, "getuser", lambda: "real-user") + monkeypatch.setattr(compliance.socket, "gethostname", lambda: "real-host") + + log = compliance.AuditLogger(str(tmp_path)) + assert log.operator == "real-user@real-host" + assert log.operator != "unknown" + + +class TestAuditLoggerFsync: + """F-compliance-114: log_event must fsync after flush so chain advance is durable.""" + + def test_log_event_calls_fsync(self, tmp_path, monkeypatch): + from forgelm.compliance import AuditLogger + + log = AuditLogger(str(tmp_path)) + + with mock.patch("forgelm.compliance.os.fsync") as mock_fsync: + log.log_event("test.event", key="value") + + assert mock_fsync.called, "log_event() must invoke os.fsync after flushing the audit line" + # Called exactly once per event (not per flush call elsewhere in the + # process); the file descriptor argument is an int from f.fileno(). + assert mock_fsync.call_count == 1 + (fileno_arg,), _ = mock_fsync.call_args + assert isinstance(fileno_arg, int) + + +class TestSafetyClassifierLoadFailureAudit: + """F-compliance-120: classifier load failure surfaces as an audit event.""" + + def test_classifier_load_failure_emits_audit_event(self, tmp_path, monkeypatch): + # We exercise the failure path inside ``run_safety_evaluation`` directly + # by stubbing the in-function ``transformers.pipeline`` import to raise. + # No real model / tokenizer / GPU is touched. + pytest.importorskip("torch") # safety module imports torch lazily + import sys + import types + + from forgelm import safety + from forgelm.compliance import AuditLogger # noqa: I001 + + # Inject a fake ``transformers`` module so ``from transformers import + # pipeline`` inside run_safety_evaluation returns our raising stub. + fake_transformers = types.ModuleType("transformers") + fake_transformers.pipeline = lambda *a, **kw: _raise(RuntimeError("classifier checkpoint corrupt")) + monkeypatch.setitem(sys.modules, "transformers", fake_transformers) + + # Stub out generation + GPU release so the function reaches the + # classifier-load branch without needing a real model. + monkeypatch.setattr(safety, "_generate_safety_responses", lambda *a, **k: ["resp"]) + monkeypatch.setattr(safety, "_release_model_from_gpu", lambda m: None) + + prompts_path = tmp_path / "prompts.jsonl" + prompts_path.write_text(json.dumps({"prompt": "hi"}) + "\n") + + audit = AuditLogger(str(tmp_path)) + result = safety.run_safety_evaluation( + model=mock.Mock(), + tokenizer=mock.Mock(), + classifier_path="meta-llama/Llama-Guard-3-8B", + test_prompts_path=str(prompts_path), + audit_logger=audit, + ) + + assert result.passed is False + # Read the audit log and verify the event landed with the expected payload. + with open(audit.log_path, "r", encoding="utf-8") as fh: + lines = [json.loads(line) for line in fh if line.strip()] + events = [entry["event"] for entry in lines] + assert "audit.classifier_load_failed" in events + load_failed = next(e for e in lines if e["event"] == "audit.classifier_load_failed") + assert load_failed["classifier"] == "meta-llama/Llama-Guard-3-8B" + assert "classifier checkpoint corrupt" in load_failed["reason"] + + +class TestHFRevisionPin: + """F-compliance-117: dataset fingerprint pins HF Hub revision SHA.""" + + def test_hf_revision_pinned_in_fingerprint(self, monkeypatch): + # Simulate ``huggingface_hub.HfApi().dataset_info`` returning a + # commit-pinned info object. We patch the import target so the + # in-function ``from huggingface_hub import HfApi`` resolves here. + import sys + import types + + from forgelm import compliance + + class _FakeInfo: + sha = "abc123def456" + "0" * 28 # plausible-looking 40-char SHA + + class _FakeHfApi: + def dataset_info(self, dataset_id): + return _FakeInfo() + + fake_module = types.ModuleType("huggingface_hub") + fake_module.HfApi = _FakeHfApi + monkeypatch.setitem(sys.modules, "huggingface_hub", fake_module) + + # Also stub ``load_dataset_builder`` so the version-fetch arm does + # not hit the network or fail noisily. + fake_datasets = types.ModuleType("datasets") + + class _FakeBuilder: + class info: + version = None + description = None + download_size = None + + fake_datasets.load_dataset_builder = lambda path: _FakeBuilder() + monkeypatch.setitem(sys.modules, "datasets", fake_datasets) + + fp = compliance.compute_dataset_fingerprint("HuggingFaceH4/ultrachat_200k") + + assert fp["source"] == "huggingface_hub" + assert fp["dataset_id"] == "HuggingFaceH4/ultrachat_200k" + assert fp["hf_revision"] == _FakeInfo.sha + + +# --------------------------------------------------------------------------- +# Closure plan Faz 6: verify_audit_log library function + verify-audit CLI +# --------------------------------------------------------------------------- + + +class TestVerifyAuditLog: + """Closure plan Faz 6: ``forgelm.compliance.verify_audit_log`` library + function and its ``forgelm verify-audit`` CLI counterpart. + + Each test exercises the real :class:`AuditLogger` as the writer so + these are integration-style — any drift between the writer's + canonicalisation and the verifier would surface here immediately. + """ + + @staticmethod + def _build_log(tmp_path, *, secret: str = "", events: int = 3): + """Write a fresh audit log under *tmp_path* and return its path. + + AuditLogger reads ``FORGELM_AUDIT_SECRET`` at ``__init__`` time, so + we toggle the env var around the constructor call. ``try/finally`` + guarantees the env var is restored even if AuditLogger or + ``log_event`` raises — without this guard a failed test could leak + ``FORGELM_AUDIT_SECRET=...`` into adjacent tests and silently + change their HMAC behaviour. + """ + from forgelm.compliance import AuditLogger + + prior = os.environ.get("FORGELM_AUDIT_SECRET") + if secret: + os.environ["FORGELM_AUDIT_SECRET"] = secret + else: + os.environ.pop("FORGELM_AUDIT_SECRET", None) + + try: + logger = AuditLogger(str(tmp_path)) + for i in range(events): + logger.log_event(f"event.{i}", index=i, payload={"step": i}) + return logger.log_path + finally: + # Restore the prior state — pop if it wasn't set, otherwise + # restore the original value. + if prior is None: + os.environ.pop("FORGELM_AUDIT_SECRET", None) + else: + os.environ["FORGELM_AUDIT_SECRET"] = prior + + def test_verify_audit_valid_chain(self, tmp_path): + from forgelm.compliance import verify_audit_log + + log_path = self._build_log(tmp_path, events=5) + result = verify_audit_log(log_path) + assert result.valid is True + assert result.entries_count == 5 + assert result.first_invalid_index is None + assert result.reason is None + + def test_verify_audit_tampered_line(self, tmp_path): + """Modify one entry's payload after the fact; chain must break at + the *next* line (whose prev_hash no longer matches the rewritten + line's SHA-256).""" + from forgelm.compliance import verify_audit_log + + log_path = self._build_log(tmp_path, events=4) + with open(log_path, "r", encoding="utf-8") as fh: + lines = fh.readlines() + + # Tamper with line 2 (index 1): re-encode with a flipped value. + entry = json.loads(lines[1]) + entry["payload"] = {"step": 99999} + lines[1] = json.dumps(entry, default=str) + "\n" + with open(log_path, "w", encoding="utf-8") as fh: + fh.writelines(lines) + + result = verify_audit_log(log_path) + assert result.valid is False + # The tamper changes line 2's hash, so the *first* observable + # break is at line 3 — its prev_hash no longer matches. + assert result.first_invalid_index == 3 + assert "chain broken" in (result.reason or "") + + def test_verify_audit_truncated_chain(self, tmp_path): + """Delete the genesis line: the manifest sidecar still pins the + original first_entry_sha256, so verification surfaces the + truncation as a manifest mismatch.""" + from forgelm.compliance import verify_audit_log + + log_path = self._build_log(tmp_path, events=4) + with open(log_path, "r", encoding="utf-8") as fh: + lines = fh.readlines() + + # Drop the first line (truncate-from-head simulates an attacker + # who removed the genesis entry to hide an event). + with open(log_path, "w", encoding="utf-8") as fh: + fh.writelines(lines[1:]) + + result = verify_audit_log(log_path) + assert result.valid is False + # Either the chain breaks at line 1 (prev_hash mismatch — the new + # first line carries the *old* line-1 hash, not "genesis") OR the + # manifest cross-check fires. Both indicate truncation; assert on + # the line index rather than the message text to stay robust. + assert result.first_invalid_index == 1 + assert result.reason is not None + + def test_verify_audit_missing_manifest_warning(self, tmp_path, caplog): + """A log without the manifest sidecar still verifies if its chain + is intact — the verifier logs at DEBUG that truncate-and-resume + detection is degraded but does not fail.""" + from forgelm.compliance import verify_audit_log + + log_path = self._build_log(tmp_path, events=3) + manifest_path = log_path + ".manifest.json" + if os.path.isfile(manifest_path): + os.remove(manifest_path) + + with caplog.at_level("DEBUG", logger="forgelm.compliance"): + result = verify_audit_log(log_path) + assert result.valid is True + assert result.entries_count == 3 + assert any("No genesis manifest" in r.message for r in caplog.records) + + def test_verify_audit_hmac_valid(self, tmp_path): + from forgelm.compliance import verify_audit_log + + # NOSONAR test fixture, not a real secret (rule python:S2068 hard-coded credential false-positive) + hmac_key = "s3cr3t-operator-key" # noqa: S105 + log_path = self._build_log(tmp_path, secret=hmac_key, events=3) + + result = verify_audit_log(log_path, hmac_secret=hmac_key) + assert result.valid is True + assert result.entries_count == 3 + + def test_verify_audit_hmac_invalid(self, tmp_path): + from forgelm.compliance import verify_audit_log + + log_path = self._build_log(tmp_path, secret="real-secret", events=3) + + # Wrong secret: each line's HMAC tag fails to recompute. + result = verify_audit_log(log_path, hmac_secret="wrong-secret") + assert result.valid is False + assert result.first_invalid_index == 1 + assert "HMAC mismatch" in (result.reason or "") + + def test_verify_audit_require_hmac_no_secret(self, tmp_path, monkeypatch, capsys): + """CLI dispatcher: ``--require-hmac`` without a configured secret + env var must exit 2 (option error) before opening the log.""" + from forgelm.cli import _run_verify_audit_cmd + + log_path = self._build_log(tmp_path, events=2) + monkeypatch.delenv("FORGELM_AUDIT_SECRET", raising=False) + + # Build a minimal argparse.Namespace stand-in. + class _Args: + pass + + ns = _Args() + ns.log_path = log_path + ns.hmac_secret_env = "FORGELM_AUDIT_SECRET" + ns.require_hmac = True + + exit_code = _run_verify_audit_cmd(ns) + assert exit_code == 2 + captured = capsys.readouterr() + assert "FORGELM_AUDIT_SECRET" in captured.err + assert "--require-hmac" in captured.err + + def test_verify_audit_empty_log(self, tmp_path): + """An empty file is trivially valid — entries_count == 0, no + first_invalid_index. Mirrors AuditLogger's genesis convention + where an absent/empty file legitimately starts at 'genesis'.""" + from forgelm.compliance import verify_audit_log + + empty_path = tmp_path / "audit_log.jsonl" + empty_path.touch() + + result = verify_audit_log(str(empty_path)) + assert result.valid is True + assert result.entries_count == 0 + assert result.first_invalid_index is None diff --git a/tests/test_grpo_reward.py b/tests/test_grpo_reward.py index 16c3336b..a846a3cf 100644 --- a/tests/test_grpo_reward.py +++ b/tests/test_grpo_reward.py @@ -278,3 +278,38 @@ def fake_grpo_trainer(**kwargs): "combined format+length reward (always-on baseline) AND the gold_answer " "correctness reward must be wired additively." ) + + +class TestGrpoClassifierTrustRemoteCode: + """Phase 7 (M-202): the GRPO classifier reward must load HF artifacts with + ``trust_remote_code=False``. + + A reward model is operator-supplied configuration, but we should not + silently execute arbitrary repo code at load time — a malicious or + typo-grabbed Hub repo could otherwise run code in the trainer process + on every GRPO step setup. + """ + + def test_classifier_reward_passes_trust_remote_code_false(self): + """Both AutoTokenizer.from_pretrained and AutoModelForSequenceClassification.from_pretrained + in ``ForgeTrainer._build_classifier_reward`` must pass ``trust_remote_code=False``.""" + import inspect + import re + + from forgelm.trainer import ForgeTrainer + + src = inspect.getsource(ForgeTrainer._build_classifier_reward) + # Strip docstring + comments so "trust_remote_code=False" hits in + # those don't false-positive the assertion. + code_only = re.sub(r"#[^\n]*", "", src) + code_only = re.sub(r'""".*?"""', "", code_only, flags=re.DOTALL) + + # Two from_pretrained calls; both must carry trust_remote_code=False. + assert code_only.count("from_pretrained") == 2, ( + "Test guard: expected exactly two from_pretrained calls in " + "_build_classifier_reward. If the implementation grew, update this test." + ) + assert code_only.count("trust_remote_code=False") == 2, ( + "Both from_pretrained calls in _build_classifier_reward must pass " + "trust_remote_code=False (M-202). Found:\n" + code_only + ) diff --git a/tests/test_judge_functions.py b/tests/test_judge_functions.py index e4dd34e2..ec649000 100644 --- a/tests/test_judge_functions.py +++ b/tests/test_judge_functions.py @@ -57,7 +57,7 @@ def test_multiple_code_blocks(self): class TestCallApiJudge: - @patch("requests.post") + @patch("forgelm._http.requests.post") def test_successful_api_call(self, mock_post): mock_response = MagicMock() mock_response.json.return_value = {"choices": [{"message": {"content": '{"score": 8, "reason": "Good"}'}}]} @@ -70,7 +70,7 @@ def test_successful_api_call(self, mock_post): assert result["score"] == 8 mock_post.assert_called_once() - @patch("requests.post") + @patch("forgelm._http.requests.post") def test_api_timeout(self, mock_post): import requests @@ -83,7 +83,7 @@ def test_api_timeout(self, mock_post): assert result["score"] is None assert "API error" in result["reason"] - @patch("requests.post") + @patch("forgelm._http.requests.post") def test_custom_api_base(self, mock_post): mock_response = MagicMock() mock_response.json.return_value = {"choices": [{"message": {"content": '{"score": 7, "reason": "OK"}'}}]} @@ -108,7 +108,7 @@ def test_defaults(self): @pytest.mark.skipif(not torch_available, reason="torch not installed") class TestJudgeScoreClipping: - @patch("requests.post") + @patch("forgelm._http.requests.post") def test_score_above_10_clipped_to_10(self, mock_post, caplog): """Scores above 10 must be clamped to 10.0 with a warning.""" import logging @@ -129,7 +129,7 @@ def test_score_above_10_clipped_to_10(self, mock_post, caplog): # _call_api_judge returns the raw parsed value. assert result["score"] == 15 - @patch("requests.post") + @patch("forgelm._http.requests.post") def test_score_clipped_in_run_judge_evaluation(self, mock_post, tmp_path, caplog): """run_judge_evaluation must clip out-of-range scores and emit a warning.""" import logging @@ -177,7 +177,7 @@ def test_score_clipped_in_run_judge_evaluation(self, mock_post, tmp_path, caplog # Warning must be emitted assert any("clipped" in r.message or "out-of-range" in r.message for r in caplog.records) - @patch("requests.post") + @patch("forgelm._http.requests.post") def test_score_below_1_clipped_to_1(self, mock_post, tmp_path): """Scores below 1 must be clamped to 1.0.""" import torch @@ -215,7 +215,7 @@ def test_score_below_1_clipped_to_1(self, mock_post, tmp_path): class TestJudgeApiBasePassthrough: - @patch("requests.post") + @patch("forgelm._http.requests.post") def test_api_base_reaches_http_call(self, mock_post): """judge_api_base in config must be forwarded to the HTTP POST call.""" mock_response = MagicMock() @@ -232,3 +232,65 @@ def test_api_base_reaches_http_call(self, mock_post): actual_url = call_args[0][0] if call_args[0] else call_args.kwargs.get("url") or call_args[1].get("url") # The URL passed to requests.post should match the custom api_base assert actual_url == custom_base + + +class TestJudgeUsesSafePost: + """Phase 7: judge._call_api_judge must route through forgelm._http.safe_post. + + The acceptance gate is: ``grep -rn 'requests.post' forgelm/`` returns + nothing outside ``_http.py``. These tests cover the behavioural side — + judge calls go through ``safe_post`` and inherit the SSRF / scheme / + redirect / TLS policy automatically. + """ + + def test_imports_safe_post(self): + """judge._call_api_judge must import safe_post from forgelm._http.""" + import inspect + + from forgelm import judge + + src = inspect.getsource(judge._call_api_judge) + assert "safe_post" in src, "judge._call_api_judge must use safe_post" + + @patch("forgelm._http.requests.post") + def test_judge_call_goes_through_safe_post(self, mock_post): + """A successful judge call must hit requests.post (via safe_post).""" + mock_response = MagicMock() + mock_response.json.return_value = {"choices": [{"message": {"content": '{"score": 7, "reason": "OK"}'}}]} + mock_response.raise_for_status = MagicMock() + mock_post.return_value = mock_response + + from forgelm.judge import _call_api_judge + + result = _call_api_judge("prompt", "fake-key", "gpt-4o") + + # Confirm the call went through safe_post → requests.post + mock_post.assert_called_once() + kwargs = mock_post.call_args.kwargs + # safe_post forwards allow_redirects=False + assert kwargs.get("allow_redirects") is False + assert result["score"] == 7 + + @patch("forgelm._http.requests.post") + def test_judge_ssrf_block_for_private_url(self, mock_post): + """A private-IP api_base must be rejected before any network call. + + ``_call_api_judge`` re-raises :class:`HttpSafetyError` so + ``run_judge_evaluation`` can convert it into a hard + ``JudgeResult(passed=False)`` instead of silently scoring every + prompt as ``None`` (which would mask a misconfigured endpoint). + """ + import pytest + + from forgelm._http import HttpSafetyError + from forgelm.judge import _call_api_judge + + with pytest.raises(HttpSafetyError): + _call_api_judge( + "prompt", + "fake-key", + "gpt-4o", + api_base="https://10.0.0.1/v1/chat/completions", # NOSONAR RFC1918 — SSRF guard fixture (intentional) + ) + + mock_post.assert_not_called() diff --git a/tests/test_lazy_imports.py b/tests/test_lazy_imports.py new file mode 100644 index 00000000..2a15b25b --- /dev/null +++ b/tests/test_lazy_imports.py @@ -0,0 +1,42 @@ +"""Regression tests for the lazy-torch import contract. + +`import forgelm.trainer` and `import forgelm.model` must NOT eagerly load +``torch`` (or its transitive deps via ``peft`` / ``transformers``). Eager +torch import costs ~3-5s of CLI startup per invocation, which dominates +``forgelm --help``, ``forgelm wizard``, and unit-test collection time. + +A subprocess is used (rather than ``importlib.reload`` + ``sys.modules`` +manipulation) because mutating the parent process's module table is fragile +across pytest runs and CI workers. + +See closure-plan F-performance-101. +""" + +import subprocess +import sys + + +def test_trainer_does_not_eagerly_load_torch(): + result = subprocess.run( + [ + sys.executable, + "-c", + "import forgelm.trainer; import sys; sys.exit(0 if 'torch' not in sys.modules else 1)", + ], + capture_output=True, + timeout=30, + ) + assert result.returncode == 0, f"forgelm.trainer eagerly loaded torch: {result.stderr.decode()}" + + +def test_model_does_not_eagerly_load_torch(): + result = subprocess.run( + [ + sys.executable, + "-c", + "import forgelm.model; import sys; sys.exit(0 if 'torch' not in sys.modules else 1)", + ], + capture_output=True, + timeout=30, + ) + assert result.returncode == 0, f"forgelm.model eagerly loaded torch: {result.stderr.decode()}" diff --git a/tests/test_safety_advanced.py b/tests/test_safety_advanced.py index 1b2ea20e..475cff3f 100644 --- a/tests/test_safety_advanced.py +++ b/tests/test_safety_advanced.py @@ -196,7 +196,15 @@ def test_all_prompts_valid_jsonl(self): @pytest.mark.skipif(not torch_available, reason="torch not installed") class TestClassifierConversationFormat: def test_classifier_receives_inst_formatted_text(self, tmp_path): - """Safety classifier must receive '[INST] prompt [/INST] response' formatted text.""" + """Safety classifier must receive '[INST] prompt [/INST] response' formatted text. + + Mocks ``forgelm.safety._load_safety_classifier`` directly instead of + ``transformers.pipeline``: the latter is a lazy module attribute in + ``transformers v4.x`` so ``mock.patch`` cannot resolve it without + the module being import-cached, which made this test order-sensitive + across pytest sessions. Patching the helper extracted in the Faz 3+4 + run_safety_evaluation refactor gives a stable boundary. + """ import forgelm.safety as safety_mod captured_texts = [] @@ -209,10 +217,6 @@ def mock_classifier(text, **kwargs): prompts_file = tmp_path / "prompts.jsonl" prompts_file.write_text('{"prompt": "What is 2+2?"}\n') - # Mock pipeline to return our mock_classifier - def mock_pipeline(task, model, **kwargs): - return mock_classifier - # Mock the model to produce a response mock_model = MagicMock() mock_model.device = "cpu" @@ -229,7 +233,7 @@ def mock_pipeline(task, model, **kwargs): mock_tokenizer.decode.return_value = "4" with ( - patch("transformers.pipeline", side_effect=mock_pipeline), + patch("forgelm.safety._load_safety_classifier", return_value=mock_classifier), patch("torch.cuda.empty_cache"), ): safety_mod.run_safety_evaluation( diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index 608eb6fe..5a3d7a1f 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -2,6 +2,7 @@ import json import os +from unittest.mock import MagicMock, patch import pytest @@ -264,3 +265,72 @@ def test_yaml_round_trip(self, tmp_path): def test_config_template_still_valid(self): config = load_config("config_template.yaml") assert config.synthetic is None + + +class TestSyntheticUsesSafePost: + """Phase 7: synthetic._call_api_teacher must route through forgelm._http.safe_post. + + Same rationale as the judge equivalent — every outbound HTTP call site + in the codebase shares one policy gate. Synthetic data generation hits + OpenAI-compatible APIs with a bearer token; SSRF / scheme / redirect / + timeout discipline must apply here too. + """ + + def test_imports_safe_post(self): + """synthetic._call_api_teacher must use safe_post.""" + import inspect + + from forgelm import synthetic + + src = inspect.getsource(synthetic.SyntheticDataGenerator._call_api_teacher) + assert "safe_post" in src, "synthetic._call_api_teacher must use safe_post" + + @patch("forgelm._http.requests.post") + def test_synthetic_call_goes_through_safe_post(self, mock_post): + """A successful API teacher call routes through safe_post → requests.post.""" + mock_response = MagicMock() + mock_response.json.return_value = {"choices": [{"message": {"content": "synthetic response"}}]} + mock_response.raise_for_status = MagicMock() + mock_response.ok = True + mock_post.return_value = mock_response + + config = _config( + synthetic={ + "enabled": True, + "teacher_model": "gpt-4", + "teacher_backend": "api", + "api_base": "https://api.openai.com/v1", + "api_timeout": 30, + "seed_prompts": ["What is AI?"], + } + ) + gen = SyntheticDataGenerator(config) + response = gen._call_api_teacher("What is AI?") + + assert response == "synthetic response" + mock_post.assert_called_once() + kwargs = mock_post.call_args.kwargs + # safe_post forwards allow_redirects=False + assert kwargs.get("allow_redirects") is False + + @patch("forgelm._http.requests.post") + def test_synthetic_ssrf_block_for_private_api_base(self, mock_post): + """A private-IP api_base must be rejected before any network call.""" + from forgelm._http import HttpSafetyError + + config = _config( + synthetic={ + "enabled": True, + "teacher_model": "gpt-4", + "teacher_backend": "api", + "api_base": "https://10.0.0.5/v1", # NOSONAR RFC1918 — SSRF guard fixture (intentional) + "api_timeout": 30, + "seed_prompts": ["x"], + } + ) + gen = SyntheticDataGenerator(config) + + with pytest.raises(HttpSafetyError, match="Private/loopback"): + gen._call_api_teacher("x") + + mock_post.assert_not_called() diff --git a/tests/test_trainer.py b/tests/test_trainer.py index a436f4dc..ea8409df 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -82,6 +82,9 @@ def _make_trainer(self, auto_revert=True, max_loss=None, baseline_loss=None): trainer.checkpoint_dir = "/tmp/test_forge_eval" trainer.run_name = "test_finetune" trainer.notifier = MagicMock() + # _revert_model emits an audit event before destructive action; + # mock the audit logger so revert paths don't AttributeError. + trainer.audit = MagicMock() return trainer def test_no_evaluation_config(self): @@ -101,6 +104,7 @@ def test_no_evaluation_config(self): trainer.checkpoint_dir = "/tmp/test" trainer.run_name = "test" trainer.notifier = MagicMock() + trainer.audit = MagicMock() assert trainer.execute_evaluation_checks("/tmp/test/final", {"eval_loss": 5.0}) is True diff --git a/tests/test_webhook.py b/tests/test_webhook.py index 8aa01030..a7c5ced7 100644 --- a/tests/test_webhook.py +++ b/tests/test_webhook.py @@ -39,7 +39,7 @@ def test_no_url(self): notifier = WebhookNotifier(config) notifier.notify_start(run_name="test") - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_notify_start(self, mock_post): config = _make_config({"url": "https://example.com/hook"}) notifier = WebhookNotifier(config) @@ -52,7 +52,7 @@ def test_notify_start(self, mock_post): assert payload["status"] == "started" assert payload["run_name"] == "my_model_finetune" - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_notify_success_with_metrics(self, mock_post): config = _make_config({"url": "https://example.com/hook"}) notifier = WebhookNotifier(config) @@ -64,7 +64,7 @@ def test_notify_success_with_metrics(self, mock_post): assert payload["event"] == "training.success" assert payload["metrics"]["eval_loss"] == pytest.approx(1.25) - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_notify_failure_with_reason(self, mock_post): config = _make_config({"url": "https://example.com/hook"}) notifier = WebhookNotifier(config) @@ -75,7 +75,7 @@ def test_notify_failure_with_reason(self, mock_post): assert payload["event"] == "training.failure" assert payload["reason"] == "OOM error" - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_url_env_resolution(self, mock_post): config = _make_config({"url_env": "TEST_WEBHOOK_URL"}) notifier = WebhookNotifier(config) @@ -90,7 +90,7 @@ def test_url_env_resolution(self, mock_post): or call_kwargs[0][0] == "https://env.example.com/hook" ) - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_notify_on_start_disabled(self, mock_post): config = _make_config( { @@ -102,7 +102,7 @@ def test_notify_on_start_disabled(self, mock_post): notifier.notify_start(run_name="test") mock_post.assert_not_called() - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_timeout_handled_gracefully(self, mock_post): import requests as req @@ -112,7 +112,7 @@ def test_timeout_handled_gracefully(self, mock_post): # Should not raise notifier.notify_start(run_name="test") - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_connection_error_handled_gracefully(self, mock_post): import requests as req @@ -122,7 +122,7 @@ def test_connection_error_handled_gracefully(self, mock_post): # Should not raise notifier.notify_failure(run_name="test", reason="test error") - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_payload_has_slack_attachments(self, mock_post): config = _make_config({"url": "https://example.com/hook"}) notifier = WebhookNotifier(config) @@ -134,7 +134,7 @@ def test_payload_has_slack_attachments(self, mock_post): assert len(payload["attachments"]) == 1 assert "title" in payload["attachments"][0] - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_http_5xx_logs_warning(self, mock_post, caplog): """Non-2xx HTTP responses must emit a WARNING and not raise.""" import logging @@ -153,7 +153,7 @@ def test_http_5xx_logs_warning(self, mock_post, caplog): assert any("503" in r.message or "HTTP" in r.message for r in caplog.records) - @patch("forgelm.webhook.requests.post") + @patch("forgelm._http.requests.post") def test_http_4xx_logs_warning(self, mock_post, caplog): """HTTP 4xx response must emit a WARNING log and not raise.""" import logging @@ -171,3 +171,264 @@ def test_http_4xx_logs_warning(self, mock_post, caplog): notifier.notify_failure(run_name="test_run", reason="OOM") assert any("404" in r.message or "HTTP" in r.message for r in caplog.records) + + +class TestSafePostHttpDiscipline: + """Direct unit tests for forgelm._http.safe_post. + + These cover the policy gates that every outbound HTTP call site relies + on. The Phase 7 closure adds judge + synthetic + (existing) webhook to + the call-site list; the gates must reject misconfigured URLs identically + across all of them. + + NOTE for static analysers: the literals in this class deliberately + include RFC1918 / loopback / IMDS / multicast IP addresses, plain + ``http://`` URLs, and ``ftp://`` URLs. These are not security + vulnerabilities — they are the inputs the test asserts the SSRF / + scheme guard rejects. Removing them would erase the coverage of those + rejections. + """ + + @pytest.mark.parametrize( + "url", + [ + "https://10.0.0.1/hook", # NOSONAR RFC1918 (10/8) — SSRF guard fixture + "https://172.16.0.5/hook", # NOSONAR RFC1918 (172.16/12) — SSRF guard fixture + "https://192.168.1.10/hook", # NOSONAR RFC1918 (192.168/16) — SSRF guard fixture + "https://127.0.0.1/hook", # NOSONAR loopback — SSRF guard fixture + "https://169.254.169.254/latest/meta-data/", # NOSONAR AWS IMDS — SSRF guard fixture + "https://224.0.0.1/multicast", # NOSONAR multicast — SSRF guard fixture + ], + ) + def test_ssrf_block_private_ip(self, url): + """Each private/loopback/IMDS/multicast destination must raise.""" + from forgelm._http import HttpSafetyError, safe_post + + with pytest.raises(HttpSafetyError, match="Private/loopback/IMDS"): + safe_post(url, json={}, timeout=10.0) + + def test_ssrf_block_can_be_opted_out(self): + """allow_private=True bypasses the SSRF guard (operator opt-in).""" + from forgelm import _http + + with patch.object(_http.requests, "post") as mock_post: + mock_post.return_value = MagicMock(ok=True, status_code=200) + _http.safe_post( + "https://10.0.0.1/hook", # NOSONAR RFC1918 — SSRF opt-out fixture + json={}, + timeout=10.0, + allow_private=True, + ) + mock_post.assert_called_once() + + def test_redirect_block(self): + """allow_redirects=False is forwarded to requests.post.""" + from forgelm import _http + + with patch.object(_http.requests, "post") as mock_post: + mock_post.return_value = MagicMock(ok=True, status_code=200) + _http.safe_post("https://example.com/hook", json={}, timeout=10.0) + kwargs = mock_post.call_args.kwargs + assert kwargs["allow_redirects"] is False + + def test_http_block(self): + """http:// URLs are rejected unless allow_insecure_http is set.""" + from forgelm._http import HttpSafetyError, safe_post + + with pytest.raises(HttpSafetyError, match="http://"): + safe_post("http://example.com/hook", json={}, timeout=10.0) # NOSONAR scheme blocker fixture + + def test_http_allowed_with_opt_in(self): + """allow_insecure_http=True (used by webhook) lets http:// through.""" + from forgelm import _http + + with patch.object(_http.requests, "post") as mock_post: + mock_post.return_value = MagicMock(ok=True, status_code=200) + _http.safe_post( + "http://example.com/hook", # NOSONAR opt-in fixture (webhook back-compat path) + json={}, + timeout=10.0, + allow_insecure_http=True, + ) + mock_post.assert_called_once() + + def test_unsupported_scheme(self): + """ftp:// / file:// / etc. are rejected even with allow_insecure_http.""" + from forgelm._http import HttpSafetyError, safe_post + + with pytest.raises(HttpSafetyError, match="Unsupported URL scheme"): + safe_post( + "ftp://example.com/hook", # NOSONAR scheme blocker fixture (ftp not allowed) + json={}, + timeout=10.0, + allow_insecure_http=True, + ) + + def test_timeout_floor_rejects_below_default(self): + """timeout below the 10s default floor must raise.""" + from forgelm._http import HttpSafetyError, safe_post + + with pytest.raises(HttpSafetyError, match="Timeout below"): + safe_post("https://example.com/hook", json={}, timeout=5.0) + + def test_timeout_zero_rejected_even_with_lower_floor(self): + """timeout=0 is always rejected (requests treats it as 'no timeout').""" + from forgelm._http import HttpSafetyError, safe_post + + with pytest.raises(HttpSafetyError, match="Timeout below"): + safe_post( + "https://example.com/hook", + json={}, + timeout=0, + min_timeout=1.0, + ) + + def test_timeout_floor_overridable(self): + """Webhook passes min_timeout=1.0 to keep its historical floor.""" + from forgelm import _http + + with patch.object(_http.requests, "post") as mock_post: + mock_post.return_value = MagicMock(ok=True, status_code=200) + _http.safe_post( + "https://example.com/hook", + json={}, + timeout=2.0, + min_timeout=1.0, + ) + mock_post.assert_called_once() + + def test_header_masking_on_error(self, caplog): + """Authorization / X-API-Key values are redacted from the failure log.""" + import logging + + import requests as req + + from forgelm import _http + + # NOSONAR test fixture, fragment-built (rule python:S2068 hard-coded credential false-positive) + bearer_token = "sk-" + "supersecret123" # noqa: S105 + with patch.object(_http.requests, "post") as mock_post: + mock_post.side_effect = req.exceptions.ConnectionError(f"refused while sending Bearer {bearer_token}") + with caplog.at_level(logging.WARNING, logger="forgelm._http"): + with pytest.raises(req.exceptions.ConnectionError): + _http.safe_post( + "https://example.com/hook", + json={}, + headers={"Authorization": f"Bearer {bearer_token}"}, + timeout=10.0, + ) + + # The bearer token must be masked from the warning log. + log_text = " ".join(r.message for r in caplog.records) + assert bearer_token not in log_text + assert "[REDACTED]" in log_text + + def test_localhost_blocked_by_hostname(self): + """'localhost' resolves to 127.0.0.1; SSRF guard must catch it.""" + from forgelm._http import HttpSafetyError, safe_post + + with pytest.raises(HttpSafetyError, match="Private/loopback"): + safe_post("https://localhost/hook", json={}, timeout=10.0) + + +class TestLifecycleVocabulary: + """Faz 8: notify_reverted + notify_awaiting_approval lifecycle events. + + These pin the wire-format of the two new payload events so dashboards + that already filter on event="training.reverted" / "approval.required" + don't silently break on a future refactor. + """ + + @patch("forgelm._http.requests.post") + def test_notify_reverted_payload(self, mock_post): + """Auto-revert event must serialize as event=training.reverted with masked + truncated reason.""" + config = _make_config({"url": "https://example.com/hook"}) + notifier = WebhookNotifier(config) + + # Reason carries a Slack webhook secret + a long padding so we can + # assert both the masking and the 2048-char truncation paths. + # Token built fragment-by-fragment per docs/standards/regex.md Rule 7 + # so GitHub secret scanning + gitleaks don't flag the literal. + leaky_token = "xoxb-" + "12345678901" + "-" + "1234567890123" + "-" + "AbCdEfGhIjKlMnOpQrStUvWx" + long_pad = "X" * 3000 + reason = f"safety gate failed: {leaky_token} traceback: {long_pad}" + + notifier.notify_reverted(run_name="my_run", reason=reason) + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args + payload = json.loads(call_kwargs.kwargs.get("data") or call_kwargs[1]["data"]) + + assert payload["event"] == "training.reverted" + assert payload["status"] == "reverted" + assert payload["run_name"] == "my_run" + assert leaky_token not in payload["reason"], "Slack token must be redacted" + assert leaky_token not in payload["attachments"][0]["text"], ( + "Slack token must be redacted in attachment text too" + ) + # Truncated to 2048 + "… (truncated)" marker. + assert len(payload["reason"]) <= 2048 + len("… (truncated)") + assert payload["reason"].endswith("… (truncated)") + + @patch("forgelm._http.requests.post") + def test_notify_reverted_distinct_from_failure(self, mock_post): + """training.reverted must not collide with training.failure (dashboards rely on this split).""" + config = _make_config({"url": "https://example.com/hook"}) + notifier = WebhookNotifier(config) + + notifier.notify_reverted(run_name="r", reason="judge below threshold") + + call_kwargs = mock_post.call_args + payload = json.loads(call_kwargs.kwargs.get("data") or call_kwargs[1]["data"]) + + assert payload["event"] == "training.reverted" + assert payload["event"] != "training.failure" + # Color must signal "reverted" (warning orange), not "failed" (red). + assert payload["attachments"][0]["color"] == "#ff9900" + + @patch("forgelm._http.requests.post") + def test_notify_awaiting_approval_payload(self, mock_post): + """Approval gate must serialize as event=approval.required with model_path included.""" + config = _make_config({"url": "https://example.com/hook"}) + notifier = WebhookNotifier(config) + + notifier.notify_awaiting_approval( + run_name="my_run", + model_path="/var/forgelm/runs/abc/final_model", # NOSONAR — payload string fixture, no fs op + ) + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args + payload = json.loads(call_kwargs.kwargs.get("data") or call_kwargs[1]["data"]) + + assert payload["event"] == "approval.required" + assert payload["status"] == "awaiting_approval" + assert payload["run_name"] == "my_run" + # NOSONAR — string literal, not a real filesystem operation + assert payload["model_path"] == "/var/forgelm/runs/abc/final_model" + assert "/var/forgelm/runs/abc/final_model" in payload["attachments"][0]["text"] + + @patch("forgelm._http.requests.post") + def test_notify_awaiting_approval_no_model_weights_in_payload(self, mock_post): + """Security: approval payload must carry the staging path only, never weight bytes or tensor dumps.""" + config = _make_config({"url": "https://example.com/hook"}) + notifier = WebhookNotifier(config) + + notifier.notify_awaiting_approval( + run_name="r", + model_path="/tmp/staged", + ) + + call_kwargs = mock_post.call_args + payload = json.loads(call_kwargs.kwargs.get("data") or call_kwargs[1]["data"]) + + # Schema is fixed: event/run_name/status/metrics/reason/model_path/attachments. + # No weight-shaped fields. Anything else means a future regression + # snuck a sensitive blob into the wire format. + allowed_keys = {"event", "run_name", "status", "metrics", "reason", "model_path", "attachments"} + assert set(payload.keys()) == allowed_keys + # Belt-and-braces: the canonical weight-blob field names must never + # appear in the serialized payload. + serialized = json.dumps(payload) + for forbidden in ("state_dict", "model.safetensors", "pytorch_model.bin", "adapter_model"): + assert forbidden not in serialized, f"Payload must not carry {forbidden!r}" diff --git a/tools/build_usermanuals.py b/tools/build_usermanuals.py index cb9b7b70..0ba6682d 100644 --- a/tools/build_usermanuals.py +++ b/tools/build_usermanuals.py @@ -401,7 +401,7 @@ def _render_outputs(languages: list, sections: list[Section], default_lang: str) pages, ok, fb = build_language(lang, sections, default_lang) out_path = OUTPUT_DIR / f"{lang}.js" written[out_path] = serialise_data(pages, lang) - marker = "✓" if fb == 0 else "fallbacks=" + str(fb) + marker = "OK" if fb == 0 else "fallbacks=" + str(fb) print(f" {lang}: {ok} pages translated, {marker}") written[OUTPUT_DIR / "_index.js"] = serialise_index(sections, languages) return written