From 61d74fa01c28f2988760f59dbf2885e432983dc3 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 17:21:15 +0300 Subject: [PATCH 01/11] feat(wave-3): Faz 24 + 28 + 38 consolidated integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three closure-plan phases on a single integration branch. Suite 1298 → 1333 (+35 net); all green; ruff clean; dry-run green. PHASE 38 — `forgelm reverse-pii` (GDPR Article 15 right-of-access) - forgelm/cli/subcommands/_reverse_pii.py (new): * `forgelm reverse-pii --query VALUE [--type X] [--salt-source X] JSONL_GLOB...` — walks JSONL corpora, reports every line where the supplied identifier appears. * Two scan modes: plaintext residual (mask-leak detection) + hash-mask (SHA256(salt + identifier) via the same per-output-dir salt forgelm purge uses for target_id hashing). * Identifier types: email, phone, tr_id, us_ssn, iban, credit_card, custom regex. Glob expansion (recursive ** honoured). * Centre-truncated snippets (160 chars max) defend log spam. - New audit event `data.access_request_query` with the identifier *hashed* (Article 15 access requests must not themselves leak the subject's data into the audit log). Catalogued bilingually. - forgelm/cli/_parser.py + _dispatch.py + __init__.py: registrar + dispatch row + facade re-exports. - 18 regression tests in tests/test_reverse_pii.py. PHASE 24 — Bilingual TR mirror sweep + parity CI guard - tools/check_bilingual_parity.py (new): replaces the inline H2-only check at ci.yml:197-222 with an extended H2 + H3 + H4 structural diff. AST-free; runs in the lint job; ``--strict`` mode for CI gating. - 4 doc-pair drift fixes: * ingestion-tr.md +2 H3 (Markdown-aware splitter, DOCX table preservation — Phase 12 features). * architecture-tr.md +4 H3 (results.py, benchmark.py, judge.py, model_card.py — were missing). * distributed_training-tr.md +3 H3 (Custom DeepSpeed Config, "When to choose FSDP over DeepSpeed", LoRA + Distributed) + Multi-Node ↔ Docker reorder. * configuration-tr.md +1 H4 (model.multimodal) + evaluation.benchmark/safety order swap. - 4 user-manual H2 drift fixes: tr/training/sft.md ("Diskte ne elde edersiniz"), tr/training/simpo.md ("Veri formatı"), tr/compliance/overview.md ("Annex IV neyi içerir"), tr/concepts/data-formats.md ("Verinizi doğrulama"). - alignment.md:230 "v0.5.1 (Phase 14)" → phase-number reference (no version anchor — pipeline chains slated for v0.6.0+). - 16 regression tests in tests/test_check_bilingual_parity.py including a live-repo smoke that pins the canonical pair set passes ``--strict``. PHASE 28 — Curated cleanup (tier 1+2 absorbed, large items deferred) Code: - forgelm/config.py F-compliance-110 (**breaking**): high-risk + safety.enabled=false now raises `ConfigError`. Was a warning; EU AI Act Article 9 risk-management evidence cannot be derived from a disabled safety eval. - forgelm/config.py F-compliance-106: WebhookConfig.timeout default raised 5s → 10s (Slack/Teams gateway latency spikes). - forgelm/compliance.py F-compliance-111: missing data_audit_report.json escalated INFO → WARNING (Article 10 compliance gap should be visible in operator log dashboards). - forgelm/compliance.py M-204: added `_sanitize_md_list` helper + migrated foreseeable_misuse bullet build. - forgelm/deploy.py M-205: ollama Modelfile SYSTEM line escapes newlines (\n / \r) so multi-line operator-supplied prompts don't break the parser. - forgelm/webhook.py C-54: dropped `_is_private_destination` re-export from `__all__` (Phase 7 split moved the helper to forgelm._http; no downstream importer found). - forgelm/trainer.py C-57: GRPO reward token list carries explicit GSM8K + MATH-tuned docstring caveat. - tests/test_integration_smoke.py → tests/test_integration.py (F-test-011): rename — file is integration test, not smoke. Tests: - tests/test_eu_ai_act.py: updated 5 tests for the F-compliance-110 behaviour change + added regression `test_high_risk_safety_disabled_raises_config_error`. - tests/test_compliance.py: renamed `test_info_log_when_audit_missing` → `test_warning_log_when_audit_missing` (F-compliance-111). - tests/test_data_edge_cases.py: webhook timeout default 5 → 10 (F-compliance-106). - tests/test_integration.py: high-risk-without-safety test now asserts ConfigError raise (was warn). Tier 3 deferred to v0.6.0: - F-code-108 (Pydantic _meta.yaml schema), F-business-006 (compliance-export populated bundle), F-business-008 (QMS README rewrite), M-203 (FORGELM_GGUF_CONVERTER manifest), C-52 (_chunk_semantic implementation), F-test-009 (pytest-xdist), F-test-002 (runtime_smoke deletion). Each is >1 PR scope. DOCS - CHANGELOG.md `[Unreleased]`: Wave 3 section with detailed sub-bullets. - docs/guides/gdpr_erasure.md + -tr.md: new "Article 15 right-of-access" section pointing operators at reverse-pii. - docs/reference/audit_event_catalog.md + -tr.md: new `data.access_request_query` event row. - docs/usermanuals/{en,tr}/reference/cli.md: reverse-pii row added. - docs/usermanuals/{en,tr}/reference/json-output.md: reverse-pii envelope schema + key table + exit-code mapping. - docs/analysis/code_reviews/closure-plan-202604300906.md: 3 phases marked Wave 3 ✅, kapanış özeti added. - forgelm/cli/_parser.py epilog: reverse-pii row. - .github/workflows/ci.yml: bilingual H2-only check replaced with `tools/check_bilingual_parity.py --strict`. VALIDATION - ruff format + ruff check clean - pytest 1333 passed / 14 skipped (was 1298 → +35 net) - forgelm --config config_template.yaml --dry-run green - forgelm reverse-pii --help round-trip via main CLI - tools/check_bilingual_parity.py --strict: 8/8 doc pairs at parity Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 36 +- CHANGELOG.md | 93 ++++ .../code_reviews/closure-plan-202604300906.md | 17 +- docs/guides/alignment.md | 2 +- docs/guides/gdpr_erasure-tr.md | 39 ++ docs/guides/gdpr_erasure.md | 37 ++ docs/guides/ingestion-tr.md | 37 ++ docs/reference/architecture-tr.md | 12 + docs/reference/audit_event_catalog-tr.md | 6 + docs/reference/audit_event_catalog.md | 6 + docs/reference/configuration-tr.md | 24 +- docs/reference/distributed_training-tr.md | 63 ++- docs/usermanuals/en/reference/cli.md | 1 + docs/usermanuals/en/reference/json-output.md | 37 ++ docs/usermanuals/tr/compliance/overview.md | 17 +- docs/usermanuals/tr/concepts/data-formats.md | 14 + docs/usermanuals/tr/reference/cli.md | 1 + docs/usermanuals/tr/reference/json-output.md | 37 ++ docs/usermanuals/tr/training/sft.md | 12 + docs/usermanuals/tr/training/simpo.md | 4 + forgelm/cli/__init__.py | 14 + forgelm/cli/_dispatch.py | 1 + forgelm/cli/_parser.py | 77 +++ forgelm/cli/subcommands/_reverse_pii.py | 384 +++++++++++++ forgelm/compliance.py | 34 +- forgelm/config.py | 31 +- forgelm/deploy.py | 11 +- forgelm/trainer.py | 12 + forgelm/webhook.py | 17 +- tests/test_check_bilingual_parity.py | 208 +++++++ tests/test_compliance.py | 17 +- tests/test_data_edge_cases.py | 7 +- tests/test_eu_ai_act.py | 41 +- ...tegration_smoke.py => test_integration.py} | 12 +- tests/test_reverse_pii.py | 517 ++++++++++++++++++ tools/check_bilingual_parity.py | 356 ++++++++++++ 36 files changed, 2150 insertions(+), 84 deletions(-) create mode 100644 forgelm/cli/subcommands/_reverse_pii.py create mode 100644 tests/test_check_bilingual_parity.py rename tests/{test_integration_smoke.py => test_integration.py} (96%) create mode 100644 tests/test_reverse_pii.py create mode 100644 tools/check_bilingual_parity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d303872..03e4dc96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,32 +194,16 @@ jobs: # tools/check_site_claims.py for what's checked and why. run: python3 tools/check_site_claims.py --strict - - name: Bilingual doc H2 parity check - run: | - python -c " - import re, sys - pairs = [ - ('docs/reference/configuration.md', 'docs/reference/configuration-tr.md'), - ('docs/reference/usage.md', 'docs/reference/usage-tr.md'), - ('docs/reference/distributed_training.md','docs/reference/distributed_training-tr.md'), - ('docs/reference/data_preparation.md', 'docs/reference/data_preparation-tr.md'), - ('docs/reference/architecture.md', 'docs/reference/architecture-tr.md'), - ('docs/guides/ingestion.md', 'docs/guides/ingestion-tr.md'), - ('docs/guides/data_audit.md', 'docs/guides/data_audit-tr.md'), - ] - failed = False - for en, tr in pairs: - with open(en) as f: - en_count = sum(1 for line in f if re.match(r'^## ', line)) - with open(tr) as f: - tr_count = sum(1 for line in f if re.match(r'^## ', line)) - status = 'OK' if en_count == tr_count else 'FAIL' - print(f'{status}: {en} ({en_count} H2) vs {tr} ({tr_count} H2)') - if en_count != tr_count: - failed = True - if failed: - sys.exit(1) - " + - name: Bilingual doc H2/H3/H4 parity check (strict) + # Phase 24: extended structural parity guard. Where the prior + # inline check counted H2 only and missed reordered / demoted + # sections, ``tools/check_bilingual_parity.py`` walks every + # registered EN/TR pair and compares the full H2 + H3 + H4 + # spine. Translated text differs by definition; structural + # depth + ordering must not. Add a new pair: register it in + # ``tools/check_bilingual_parity.py::_PAIRS``; CI picks it up + # automatically the next run. + run: python3 tools/check_bilingual_parity.py --strict - name: License check run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 52143aea..2ba361e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,99 @@ All notable changes to ForgeLM are documented here. ## [Unreleased] +### Wave 3 — Faz 24 + 28 + 38 (`closure/wave3-integration`) + +Single integration branch covering three closure-plan phases: + +**Faz 38 — `forgelm reverse-pii` (GDPR Article 15 right-of-access)** + +- New CLI subcommand: `forgelm reverse-pii --query VALUE [--type + email|phone|tr_id|us_ssn|iban|credit_card|custom] [--salt-source + per_dir|env_var] JSONL_GLOB...`. Walks JSONL corpora, reports + every line where the supplied identifier appears. Two scan modes: + *plaintext residual* (mask-leak detection) and *hash-mask* + (reuses `forgelm purge`'s per-output-dir salt to re-derive the + digest). Centre-truncated snippets cap log spam. +- New audit event `data.access_request_query` (catalogued bilingually). + The identifier is **SHA-256-hashed before audit emission** — + Article 15 access requests must not themselves leak the subject's + data into the audit log. +- 18 regression tests; library re-exports through `forgelm.cli` + facade; help epilog + dispatcher row added. + +**Faz 24 — Bilingual TR mirror sweep + parity CI guard** + +- `tools/check_bilingual_parity.py` (new): replaces the inline + H2-only check in `ci.yml` with an extended H2 + H3 + H4 structural + diff. Detects missing sections, depth changes, and reorders. + AST-free; runs in the lint job. 16 regression tests; live-repo + smoke test pins the canonical pair set passes `--strict`. +- 4 doc pairs brought to parity with their EN originals: + - `docs/guides/ingestion-tr.md` — added "Markdown-aware splitter" + + "DOCX table preservation" H3 sections (Phase 12 features). + - `docs/reference/architecture-tr.md` — added 4 missing module H3s + (`results.py`, `benchmark.py`, `judge.py`, `model_card.py`). + - `docs/reference/distributed_training-tr.md` — added 3 missing H3s + (Custom DeepSpeed Config, "When to choose FSDP over DeepSpeed", + LoRA + Distributed) plus reordering Multi-Node ↔ Docker. + - `docs/reference/configuration-tr.md` — added missing + `model.multimodal` H4 block; reordered `evaluation.benchmark` + before `evaluation.safety` to match EN. +- 4 user-manual H2 drift fixes: + - `tr/training/sft.md` "Diskte ne elde edersiniz" added. + - `tr/training/simpo.md` "Veri formatı" added. + - `tr/compliance/overview.md` "Annex IV neyi içerir" added. + - `tr/concepts/data-formats.md` "Verinizi doğrulama" added. +- `docs/guides/alignment.md:230` "v0.5.1 (Phase 14)" → phase-number + reference (no version anchor — pipeline chains slated for v0.6.0+). +- CI integration: `tools/check_bilingual_parity.py --strict` replaces + the inline H2 check in `ci.yml` validate job. + +**Faz 28 — Curated cleanup** + +- `forgelm/config.py` (F-compliance-110 — **breaking**): high-risk / + unacceptable risk classification now **raises `ConfigError`** when + `evaluation.safety.enabled: false`. Was a warning; EU AI Act + Article 9 risk-management evidence cannot be derived from a + disabled safety eval. Operators with sandboxed runs must lower + the risk_classification or enable safety. +- `forgelm/config.py` (F-compliance-106): `WebhookConfig.timeout` + default raised 5s → 10s. Slack/Teams gateway latency spikes + regularly cross 5s; webhook failure is best-effort but a timeout + silently degrades the audit chain. +- `forgelm/compliance.py` (F-compliance-111): `_maybe_inline_audit_report` + missing-file branch escalated `INFO → WARNING`. A missing + `data_audit_report.json` is a real Article 10 compliance gap + (governance bundle ships without its data-quality section); the + signal must be visible in operator log dashboards. +- `forgelm/compliance.py` (M-204): added `_sanitize_md_list` helper + + migrated the `foreseeable_misuse` bullet build to use it. +- `forgelm/deploy.py::_ollama_modelfile` (M-205): SYSTEM line now + escapes newlines (`\n`/`\r`) so multi-line operator-supplied system + prompts don't break the Modelfile parser. +- `forgelm/webhook.py` (C-54): dropped `_is_private_destination` + re-export from `__all__`. The Phase 7 split moved the helper to + `forgelm._http`; no downstream importer of the webhook-side + re-export was found at the time of removal (clean drop). +- `forgelm/trainer.py` (C-57): GRPO reward token list now carries + an explicit "GSM8K + MATH-tuned" docstring caveat. Operators + training other math domains should write a custom reward callable + via `training.grpo_reward_model` rather than expecting this + stripper to generalise. +- `tests/test_integration_smoke.py` → `tests/test_integration.py` + (F-test-011): rename — the file is an integration test, not a + smoke test. + +**Validation:** + +- `ruff format` + `ruff check` clean +- `pytest`: 1333 passed / 14 skipped (was 1298 → **+35 net**: +18 + reverse-pii, +16 parity tool, +1 high-risk-raise regression). +- `forgelm --config config_template.yaml --dry-run` green +- `forgelm reverse-pii --help` + dispatch round-trip via main CLI +- `tools/check_bilingual_parity.py --strict`: 8 / 8 doc pairs at + parity. + ### Wave 2b inline review absorption (round 2) A second inline review pass surfaced 6 valid defects + 3 actionable diff --git a/docs/analysis/code_reviews/closure-plan-202604300906.md b/docs/analysis/code_reviews/closure-plan-202604300906.md index a12cac5e..e873ef81 100644 --- a/docs/analysis/code_reviews/closure-plan-202604300906.md +++ b/docs/analysis/code_reviews/closure-plan-202604300906.md @@ -93,11 +93,11 @@ Her faz bir veya birden çok PR'la kapanır. Sıralama dependency-driven; parale | **21** | **GDPR right-to-erasure — Implementation** | Tek PR | 1 | Faz 20 | ✅ Wave 2b (PR #30) | | **22** | **ISO 27001 / SOC 2 Type II — Analysis & gap assessment** | Tek PR | 1 | Faz 3, 6, 7, 8, 9, 21 (security + compliance baseline) | ⏳ PENDING | | **23** | **ISO 27001 / SOC 2 Type II — Implementation** | Tek PR (büyük) | 1 | Faz 22 | ⏳ PENDING | -| **24** | Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN/TR user-manual H2 drift) | Tek PR | 1 | Faz 16 (description= migration ile config docs auto-gen) | ⏳ PENDING | +| **24** | Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN/TR user-manual H2 drift) | Tek PR | 1 | Faz 16 (description= migration ile config docs auto-gen) | ✅ Wave 3 (PR pending) | | **25** | Site picker honesty + site-as-tested-surface CI guard | Tek PR | 1 | Faz 1 (site honesty fixes önce) | ✅ DONE | | **26** | QMS bilingual EN+TR mirror + compliance docs cleanup | Tek PR | 1 | Faz 23 (ISO/SOC 2 dokümanları QMS ile lockstep) | ⏳ PENDING | | **27** | Silent `except Exception:` sweep (Theme D residual) | Tek PR | 1 | — | ✅ DONE | -| **28** | Remaining Major + Minor cleanup | Tek PR | 1 | Hepsi öncesindekilere bağlı | ⏳ PENDING | +| **28** | Remaining Major + Minor cleanup | Tek PR | 1 | Hepsi öncesindekilere bağlı | ✅ Wave 3 (PR pending; tier 1+2 absorbed) | | **29** | Nit sweep (40 madde mass-edit) | Tek PR | 1 | — | ✅ DONE | | **30** | **Final documentation + site finalization (EN+TR only)** — tüm `docs/` + `docs/usermanuals/{en,tr}/` + `site/*` v0.5.5 nihai durumuna senkron | Tek PR | 1 | **Faz 1-29 + 34-38** (tüm fonksiyonel iş bitmiş olmalı) | ⏳ PENDING | | **31** | **Cross-OS release-tag matrix workflow** | Tek PR | 1 | — (release öncesi her şeyden) | ✅ DONE | @@ -106,7 +106,7 @@ Her faz bir veya birden çok PR'la kapanır. Sıralama dependency-driven; parale | **35** | **Air-gap pre-cache subcommands** (`cache-models` + `cache-tasks`) (GH-002, GH-003) | Tek PR | 1 | Faz 15 + Faz 34 (doctor `--offline` paydaş) | ✅ Wave 2b (PR #30) | | **36** | **Compliance verification toolbelt** (`safety-eval` standalone, `verify-annex-iv`, `verify-gguf`) (GH-004, GH-005, GH-009) | Tek PR | 1 | Faz 6 (verify-audit pattern), Faz 9 (compliance baseline) | ✅ Wave 2b (PR #30) | | **37** | **`forgelm approvals` listing subcommand** — Faz 9 follow-up (GH-007) | Tek PR | 1 | Faz 9 (`approve` + `reject` + staging directory) | ✅ Wave 2a (PR #28) | -| **38** | **`forgelm reverse-pii` GDPR Article 15 subcommand** (GH-014) | Tek PR | 1 | Faz 21 (GDPR erasure baseline) | ⏳ PENDING | +| **38** | **`forgelm reverse-pii` GDPR Article 15 subcommand** (GH-014) | Tek PR | 1 | Faz 21 (GDPR erasure baseline) | ✅ Wave 3 (PR pending) | | **33** | **v0.5.5 RELEASE** (CHANGELOG, version bump, tag, PyPI) | Tek PR | 1 | **Tüm önceki fazlar (1-32, 34-38)** | ⏳ POST-WAVE | **Toplam:** 38 faz, ~52 PR (multi-PR seri'ler dahil). @@ -115,7 +115,8 @@ Her faz bir veya birden çok PR'la kapanır. Sıralama dependency-driven; parale - ✅ DONE — Foundation / Wave 1 (PR #19 → main, 2026-04-30; PR #21 → development, 2026-05-02); faz tamamlandı. - ✅ Wave 2a (PR #28) — `closure/wave2a-integration` → development, 2026-05-04; faz tamamlandı. - ✅ Wave 2b (PR #30) — `closure/wave2b-integration` → development, 2026-05-05 (merge commit `b05edb5`); faz tamamlandı. -- ⏳ PENDING — henüz başlanmadı; bağımlılıkları sağlanan ilk batch sıradaki Wave 3+'da açılır. +- ✅ Wave 3 (PR pending) — `closure/wave3-integration` → development, 2026-05-05; Faz 24 + Faz 28 (curated tier 1+2) + Faz 38 absorption tamamlandı; PR review aşamasında. +- ⏳ PENDING — henüz başlanmadı; bağımlılıkları sağlanan ilk batch sıradaki Wave 4+'da açılır. - ⏳ POST-WAVE — release fazı (33), tüm fonksiyonel iş bittikten sonra son. **Wave 1 kapanış özeti (post-merge):** 12 faz tamamlandı (Faz 9-15, 25, 27, 29, 31, 32). Foundation (Faz 1-8) Wave 0'da PR #19 ile merge edilmişti. @@ -124,6 +125,8 @@ Her faz bir veya birden çok PR'la kapanır. Sıralama dependency-driven; parale **Wave 2b kapanış özeti (post-merge, 2026-05-05):** 5 faz tamamlandı (Faz 16 Pydantic `description=` migration, Faz 19 Library API implementation, Faz 21 GDPR erasure implementation, Faz 35 air-gap pre-cache, Faz 36 compliance verification toolbelt). PR #30 → development (merge commit `b05edb5`). 6 round absorption + 4-agent final review + 4 followup absorption commits; suite 1160 → 1298 tests (+138 net). +**Wave 3 kapanış özeti (2026-05-05, PR review aşamasında):** 3 faz tamamlandı (Faz 24 bilingual TR mirror sweep + `tools/check_bilingual_parity.py` CI guard, Faz 28 curated tier 1+2 cleanup, Faz 38 `forgelm reverse-pii` GDPR Article 15 subcommand). Branch: `closure/wave3-integration`. Yeni: 1 CLI subcommand (`reverse-pii`), 1 audit event (`data.access_request_query`), 1 yeni CI guard tool. Behaviour change: high-risk + safety-disabled artık `ConfigError` raise eder (F-compliance-110). Suite 1298 → 1333 tests (+35 net). + **Kalan iş (Wave 3+):** 7 faz + 1 release. Pending: Faz 22 (ISO 27001/SOC 2 design), Faz 23 (ISO 27001/SOC 2 impl), Faz 24 (bilingual TR mirror sweep), Faz 26 (QMS bilingual mirror), Faz 28 (Major+Minor cleanup), Faz 30 (final docs + site finalization), Faz 38 (`forgelm reverse-pii` GDPR Art. 15). Release: Faz 33 (v0.5.5 PyPI publish). --- @@ -812,7 +815,7 @@ Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.m ## 9. Polish + remaining cleanup fazları (Faz 24-29) -### Faz 24 — Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN+TR user-manual H2 drift) (Tek PR) +### Faz 24 — Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN+TR user-manual H2 drift) (Tek PR) ✅ TAMAMLANDI — Wave 3 (PR pending) **Closes:** M-DOC-002, M-DOC-007, M-DOC-012, M-DOC-013, M-DOC-014, M-DOC-015, F-loc-005, F-loc-006, F-loc-007, F-loc-008 @@ -922,7 +925,7 @@ Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.m --- -### Faz 28 — Remaining Major + Minor + Nit cleanup (Tek PR) +### Faz 28 — Remaining Major + Minor + Nit cleanup (Tek PR) ✅ TAMAMLANDI — Wave 3 (PR pending; tier 1+2 absorbed, large items deferred to v0.6.0) **Closes:** Master review tablosundan Faz 1-27'ye girmeyen tüm Major + Minor + Nit (~70+ madde). @@ -1156,7 +1159,7 @@ Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.m --- -### Faz 38 — `forgelm reverse-pii` GDPR Article 15 subcommand (Tek PR) ⏳ PENDING +### Faz 38 — `forgelm reverse-pii` GDPR Article 15 subcommand (Tek PR) ✅ TAMAMLANDI — Wave 3 (PR pending) **Closes:** GH-014 (Medium — GDPR Article 15 right of access; data subject "benim verim eğitim setinde mi?" sorgusuna yanıt veren tooling). diff --git a/docs/guides/alignment.md b/docs/guides/alignment.md index e2910e30..ff4d42cd 100644 --- a/docs/guides/alignment.md +++ b/docs/guides/alignment.md @@ -227,7 +227,7 @@ forgelm --config configs/stage2_dpo.yaml forgelm --config configs/stage3_grpo.yaml ``` -> **Coming in v0.5.1 (Phase 14):** A `pipeline:` config key will define multi-stage training chains in a single YAML file, eliminating manual config juggling between stages. +> **Planned (Phase 14 — pipeline chains):** A `pipeline:` config key will define multi-stage training chains in a single YAML file, eliminating manual config juggling between stages. Tracking issue links the v0.6.0+ release window. --- diff --git a/docs/guides/gdpr_erasure-tr.md b/docs/guides/gdpr_erasure-tr.md index c6f4a2c4..1d066a06 100644 --- a/docs/guides/gdpr_erasure-tr.md +++ b/docs/guides/gdpr_erasure-tr.md @@ -122,6 +122,45 @@ Deprecate edilen field **v0.7.0**'da kaldırılır. `--check-policy` asla 3 kodu döndürmez. Başarılı bir rapor 0 ile çıkar; verilen `--config`'i yükleme başarısızlığı ise yanıltıcı bir "ihlal yok" raporuna düşmek yerine `EXIT_CONFIG_ERROR` (sıfır olmayan) ile çıkar. CI gate isteyen operatörler ihlal sayısını JSON çıktıdan kendileri hesaplar (design §10 Q5). +## Madde 15 erişim hakkı (`forgelm reverse-pii`) + +GDPR'nin *diğer* veri-sahibi hakkı — "verim corpus'ta var mı?" — +için companion subcommand `forgelm reverse-pii`. `purge` "satırımı +sil" sorusuna yanıt verirken, `reverse-pii` "kimlik bilgim hangi +satırlarda görünüyor" sorusuna yanıt verir. + +```shell +# Plaintext residual scan: mask leak'leri tespit eder (operatör +# corpus'un maskelendiğine inanıyordu ama bir residual span maskeleme +# pass'ında kaçmış). +$ forgelm reverse-pii --query "ali@example.com" --type email \ + --output-dir ./outputs data/*.jsonl + +# Hash-mask scan: corpus ForgeLM'in hash stratejisi ile maskelenmiş. +# `purge`'ün `target_id` hash'lemesinde kullandığı per-output-dir +# salt'ı yeniden kullanır — burada hesaplanan digest, corpus'a +# yazılmış digest ile eşleşir. +$ forgelm reverse-pii --query "ali@example.com" --type email \ + --salt-source per_dir --output-dir ./outputs data/*.jsonl +``` + +Audit chain `data.access_request_query`'yi identifier *hash'lenmiş* +olarak kaydeder — Madde 15 erişim talepleri kendisi de subject'in +verisini audit log'a sızdırmamalıdır. Hash'lenmiş form, compliance +reviewer'ın iki sorguyu cleartext görmeden aynı subject'e +ilişkilendirmesini sağlayan stabil per-identifier fingerprint'tir. + +**Identifier tipleri** (`--type`): `email`, `phone`, `tr_id`, +`us_ssn`, `iban`, `credit_card`, `custom`. `custom` dışındaki tüm +tipler query'i literal string olarak ele alır (regex şekil-match +yapmaz — o iş audit-time detector'ın görevi, erişim talebinin değil). +`custom` query'i Python regex olarak yorumlar. + +**Exit kodları:** `0` = scan tamamlandı (matches listesi boş +olabilir); `1` = config hatası (boş query, malformed regex, boş +glob); `2` = runtime hatası (mid-scan I/O failure). JSON envelope +şeması için bkz. [`reference/json-output.md`](#/reference/json-output). + ## Bkz. - `docs/qms/sop_data_management.md` — retention + erasure prosedürleri dahil tam data-lifecycle SOP. diff --git a/docs/guides/gdpr_erasure.md b/docs/guides/gdpr_erasure.md index b0fac463..13d5b30e 100644 --- a/docs/guides/gdpr_erasure.md +++ b/docs/guides/gdpr_erasure.md @@ -122,6 +122,43 @@ Six new events ship with `forgelm purge` (catalogued in `docs/reference/audit_ev `--check-policy` never returns code 3. A successful report exits 0; a failure to load the supplied `--config` exits with `EXIT_CONFIG_ERROR` (non-zero) rather than degrading to a misleading "no violations" report. Operators wiring a CI gate compute the violation count from JSON output themselves (per design §10 Q5). +## Article 15 right-of-access (`forgelm reverse-pii`) + +The companion subcommand for the *other* GDPR data-subject right — +"is my data in the corpus?" — is `forgelm reverse-pii`. Where +`purge` answers "delete my row," `reverse-pii` answers "find every +line where my identifier appears." + +```shell +# Plaintext residual scan: detects mask leaks (operator believed +# the corpus was masked but a residual span slipped through). +$ forgelm reverse-pii --query "alice@example.com" --type email \ + --output-dir ./outputs data/*.jsonl + +# Hash-mask scan: corpus was masked through ForgeLM's hash strategy. +# Reuses the per-output-dir salt purge uses for `target_id` hashing, +# so the digest computed here matches the digest written into the corpus. +$ forgelm reverse-pii --query "alice@example.com" --type email \ + --salt-source per_dir --output-dir ./outputs data/*.jsonl +``` + +The audit chain records `data.access_request_query` with the +identifier *hashed* — Article 15 access requests must not themselves +leak the subject's data into the audit log. The hashed form is a +stable per-identifier fingerprint a compliance reviewer can correlate +across queries without seeing the cleartext. + +**Identifier types** (`--type`): `email`, `phone`, `tr_id`, `us_ssn`, +`iban`, `credit_card`, `custom`. All non-`custom` types treat the +query as a literal string (no regex shape match — that's the +*audit-time* detector's job, not the access-request answer). +`custom` interprets the query as a Python regex. + +**Exit codes:** `0` = scan completed (matches list may be empty); +`1` = config error (empty query, malformed regex, empty glob); `2` = +runtime error (mid-scan I/O failure). See [`reference/json-output.md`](#/reference/json-output) +for the JSON envelope schema. + ## See also - `docs/qms/sop_data_management.md` — the full data-lifecycle SOP including the retention + erasure procedures. diff --git a/docs/guides/ingestion-tr.md b/docs/guides/ingestion-tr.md index 16919f22..0c1299c9 100644 --- a/docs/guides/ingestion-tr.md +++ b/docs/guides/ingestion-tr.md @@ -280,6 +280,43 @@ Audit'in `near_duplicate_pairs` metriğindeki gürültüyü azaltır. 3 sayfadan kısa belgelerde devre dışı kalır. Yapılandırılmış notlar çıktısı `pdf_header_footer_lines_stripped` alanını taşır. +### Markdown-bilen splitter — `--strategy markdown` (Faz 12) + +Girdi gerçek bir markdown yapısına sahipse (teknik wiki'ler, README +koleksiyonları, knowledge-base export'ları), heading-bilen chunking +paragraph-greedy chunking'i geçer: her chunk bir heading ile başlayan +tutarlı bir bölümdür ve **parent heading yolu chunk'ın başına +breadcrumb olarak inline edilir** — böylece SFT loss'u doküman +bağlamını görür. + +```bash +forgelm ingest ./engineering_wiki/ --recursive --output data/wiki.jsonl \ + --strategy markdown --chunk-size 4000 +``` + +Davranış notları: + +- Sınırlar markdown heading'leri (`# H1` … `###### H6`); chunker bir + bölümün ortasından asla kesmez. +- Code-fenced blok'lar (` ``` `) atomik tutulur — asla blok ortasından + bölünmez. Bu, uzun bir kod listing'i içeren tek bölümün soft-cap'i + aşabileceği anlamına gelir (paragraph stratejisinin "uzun-paragraph + tek başına" kuralının aynısı). +- Fenced blok **içindeki** heading-şekilli satırlar (`# whoami`, + `# noqa: E402`) bölüm sınırı olarak yorumlanmaz. +- Token-bilen mod ile birleşir: `--strategy markdown --chunk-tokens + 1024 --tokenizer "Qwen/Qwen2.5-7B-Instruct"`. + +### DOCX tablo koruması (Faz 12) + +DOCX tabloları artık önceki `" | "`-birleştirilmiş düz satır yerine +**markdown tablo syntax'ı** olarak çıkarılır (header satırı + `---` +ayraç + body satırları). `--strategy markdown` ile birleştirildiğinde +tablolar chunk'lar arası bütünlüğünü korur. Eşit-olmayan satırlar +boş hücrelerle sağdan paddinglenir; tüm-boş satırlar düşürülür; ilk +boş-olmayan satır header olur. Bunun fark yarattığı SFT use-case'leri: +tablo Q&A, finansal asistan, kod-ile-veri prompt'ları. + --- ## Sorun giderme diff --git a/docs/reference/architecture-tr.md b/docs/reference/architecture-tr.md index 7697bf9e..f7639d83 100644 --- a/docs/reference/architecture-tr.md +++ b/docs/reference/architecture-tr.md @@ -86,9 +86,18 @@ Transformers veya Unsloth backend ile model yükleme. QLoRA (4-bit NF4), PEFT (L ### `trainer.py` TRL trainer'larını (SFTTrainer, DPOTrainer, KTOTrainer, ORPOTrainer, CPOTrainer/SimPO, GRPOTrainer) sarar. Pipeline: baseline → eğitim → kayıp → benchmark → güvenlik → LLM-hakim → model kaydet → model kartı → uyumluluk → webhook. GaLore optimizer seviyesinde bellek optimizasyonu (tam parametre eğitimi için gradient düşük rank projeksiyonu) ve uzun bağlam özellikleri (RoPE ölçekleme, NEFTune gürültü enjeksiyonu, kayan pencere dikkat, örnek paketleme) destekler. Otomatik geri alma, insan onay kapısı, denetim loglama, kaynak takibi içerir. +### `results.py` +Hafif `TrainResult` dataclass — torch/transformers olmadan import edilebilir. Başarı durumu, metrikler, benchmark skorları, kaynak kullanımı, güvenlik pass/fail ve hakim skorlarını taşır. + +### `benchmark.py` +EleutherAI `lm-evaluation-harness`'i sarar. Yapılandırılabilir benchmark görevleri çalıştırır, accuracy metriklerini çıkarır, `min_score` eşiğini uygular ve sonuçları kaydeder. Opsiyonel bağımlılık: `pip install forgelm[eval]`. + ### `safety.py` İki puanlama modu: binary (güvenli/güvensiz oranı) ve confidence-weighted (sınıflandırıcı güven skoru). 3 katmanlı güvenlik kapısı: binary oran → confidence skoru → ciddiyet eşiği. Llama Guard S1-S14 zarar kategorileri, ciddiyet seviyeleri (kritik/yüksek/orta/düşük), düşük güven uyarıları, çalışmalar arası trend takibi (safety_trend.jsonl). +### `judge.py` +LLM-as-Judge değerlendirme: API tabanlı hakimler (OpenAI-uyumlu endpoint) ve yerel model hakimleri destekler. Markdown code block çıkarmalı sağlam JSON parse'i içerir. 1-10 ölçeğinde puanlar; yapılandırılabilir minimum eşik. + ### `compliance.py` EU AI Act uyumluluk motoru — Madde 9-17: - `AuditLogger`: Benzersiz run_id ile ekleme-yalnızca JSON Lines olay logu (Madde 12) @@ -98,6 +107,9 @@ EU AI Act uyumluluk motoru — Madde 9-17: - `generate_deployer_instructions()`: Dağıtıcı talimatları (Madde 13) - `export_evidence_bundle()`: Denetçiler için ZIP arşivi +### `model_card.py` +HuggingFace-uyumlu README.md üretir: YAML front-matter, eğitim parametreleri tablosu, metrikler, benchmark sonuçları, config snippet ve kullanım örneği. Auth token'ları export edilen config'ten dışlar. + ### `merging.py` 4 strateji: linear interpolasyon, TIES-Merging, DARE, SLERP. State dict seviyesinde — mergekit bağımlılığı gerektirmez. diff --git a/docs/reference/audit_event_catalog-tr.md b/docs/reference/audit_event_catalog-tr.md index b54444cb..9399da4a 100644 --- a/docs/reference/audit_event_catalog-tr.md +++ b/docs/reference/audit_event_catalog-tr.md @@ -65,6 +65,12 @@ Hash zinciri, satır diske düştükten (`flush` + `fsync`) sonra ilerler; kirli | `data.erasure_warning_synthetic_data_present` | Row erasure × `output_dir`'de `synthetic_data*.jsonl` mevcut. | Tüm `completed` field'ları + `synthetic_files` | 17 | | `data.erasure_warning_external_copies` | Yüklü config boş-olmayan `webhook` block'u içeriyor; downstream tüketiciler bildirim almış olabilir. | Tüm `completed` field'ları + `webhook_targets` (redact'li URL'ler) | 17 | +### Madde 15 — GDPR Erişim Hakkı (Phase 38 — `forgelm reverse-pii`) + +| Event | Ne zaman yayılır | Payload | Madde | +|--------------------------------|---------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|-------| +| `data.access_request_query` | Her `forgelm reverse-pii` çağrısında, scan tamamlandıktan sonra (veya mid-scan I/O hatası sonrası — `error_*` field'larıyla). | `query_hash` (raw identifier'ın SHA-256'sı — asla raw), `identifier_type` ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`, `scan_mode` ∈ `{plaintext, hash}`, `files_scanned` (path'ler), `match_count`, opsiyonel `error_class`/`error_message` | 15 | + ### Air-gap pre-cache (Phase 35 — `forgelm cache-models` / `cache-tasks`) | Event | Ne zaman yayılır | Payload | Madde | diff --git a/docs/reference/audit_event_catalog.md b/docs/reference/audit_event_catalog.md index 33069232..2193debd 100644 --- a/docs/reference/audit_event_catalog.md +++ b/docs/reference/audit_event_catalog.md @@ -65,6 +65,12 @@ The hash chain advances after the line lands on disk (`flush` + `fsync`), so an | `data.erasure_warning_synthetic_data_present` | Row erasure × `synthetic_data*.jsonl` exists in `output_dir`. | All `completed` fields + `synthetic_files` | 17 | | `data.erasure_warning_external_copies` | Loaded config has a non-empty `webhook` block; downstream consumers may have received notices. | All `completed` fields + `webhook_targets` (redacted URLs) | 17 | +### Article 15 — GDPR Right-of-Access (Phase 38 — `forgelm reverse-pii`) + +| Event | When emitted | Payload | Article | +|--------------------------------|----------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|---------| +| `data.access_request_query` | Every `forgelm reverse-pii` invocation, after the scan completes (or after a mid-scan I/O failure — with `error_*`). | `query_hash` (SHA-256 of raw identifier — never raw), `identifier_type` ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`, `scan_mode` ∈ `{plaintext, hash}`, `files_scanned` (paths), `match_count`, optional `error_class`/`error_message` | 15 | + ### Air-gap pre-cache (Phase 35 — `forgelm cache-models` / `cache-tasks`) | Event | When emitted | Payload | Article | diff --git a/docs/reference/configuration-tr.md b/docs/reference/configuration-tr.md index a4719f52..eb7052e3 100644 --- a/docs/reference/configuration-tr.md +++ b/docs/reference/configuration-tr.md @@ -24,6 +24,14 @@ Tam açıklamalı örnek için `config_template.yaml` dosyasına bakın. | `quantize_experts` | bool | `false` | İnaktif expert ağırlıklarını int8'e kuantize et | | `experts_to_train` | string | `"all"` | `"all"` veya virgülle ayrılmış indeksler | +#### `model.multimodal` (İsteğe bağlı — VLM modeller) + +| Alan | Tip | Varsayılan | Açıklama | +|------|-----|-----------|----------| +| `enabled` | bool | `false` | Görüntü-dil modeli (VLM) fine-tuning'i etkinleştir | +| `image_column` | string | `"image"` | Veri setinde görüntü yolu / URL'i taşıyan kolon adı | +| `text_column` | string | `"text"` | Metin / caption taşıyan kolon adı | + --- ## `lora` @@ -138,6 +146,14 @@ training: | `max_acceptable_loss` | float | `null` | eval_loss üst sınırı | | `require_human_approval` | bool | `false` | İnsan incelemesi için duraklat (çıkış kodu 4) | +#### `evaluation.benchmark` (İsteğe bağlı) + +| Alan | Tip | Varsayılan | Açıklama | +|------|-----|-----------|----------| +| `enabled` | bool | `false` | lm-eval-harness benchmark'ları | +| `tasks` | list | `[]` | Görev isimleri (ör. `["arc_easy", "hellaswag"]`) | +| `min_score` | float | `null` | Minimum ortalama doğruluk | + #### `evaluation.safety` (İsteğe bağlı) | Alan | Tip | Varsayılan | Açıklama | @@ -152,14 +168,6 @@ training: | `track_categories` | bool | `false` | Llama Guard S1-S14 zarar kategorilerini ayrıştır | | `severity_thresholds` | dict | `null` | Ciddiyet bazlı sınırlar: `{"critical": 0, "high": 0.01}` | -#### `evaluation.benchmark` (İsteğe bağlı) - -| Alan | Tip | Varsayılan | Açıklama | -|------|-----|-----------|----------| -| `enabled` | bool | `false` | lm-eval-harness benchmark'ları | -| `tasks` | list | `[]` | Görev isimleri (ör. `["arc_easy", "hellaswag"]`) | -| `min_score` | float | `null` | Minimum ortalama doğruluk | - #### `evaluation.llm_judge` (İsteğe bağlı) | Alan | Tip | Varsayılan | Açıklama | diff --git a/docs/reference/distributed_training-tr.md b/docs/reference/distributed_training-tr.md index f1c7e200..c0ccd01b 100644 --- a/docs/reference/distributed_training-tr.md +++ b/docs/reference/distributed_training-tr.md @@ -88,6 +88,26 @@ distributed: - Eğitim hızı pahasına GPU bellek tasarrufunu maksimize eder - GPU VRAM darboğaz olduğunda ama CPU RAM yeterli olduğunda faydalı +### Custom DeepSpeed Config + +Kendi JSON dosyanızı oluşturun. HuggingFace Trainer'ın `"auto"` +değerleri ForgeLM YAML config'inizden çözmesini istiyorsanız o şekilde +bırakın: + +```json +{ + "zero_optimization": { + "stage": 2, + "overlap_comm": true, + "contiguous_gradients": true + }, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto" +} +``` + --- ## FSDP Yapılandırması @@ -113,6 +133,14 @@ distributed: fsdp_offload: false ``` +### FSDP'yi DeepSpeed yerine ne zaman seçmeli + +- **FSDP**: Native PyTorch, ekstra bağımlılık yok, daha basit kurulum, + çoğu use-case için iyi. +- **DeepSpeed**: Daha çok özellik (ZeRO-Infinity, NVMe offload), çok + büyük modeller için daha iyi optimizasyon, 70B+ parametre eğitiminde + daha çok savaş tecrübesi. + --- ## Uyumluluk Notları @@ -133,23 +161,18 @@ distributed: | `transformers` | Evet | Tam DeepSpeed ve FSDP desteği | | `unsloth` | Hayır | Yalnızca tek GPU — dağıtık config ayarlanırsa ForgeLM uyarı verir | ---- - -## Docker + Çoklu GPU +### LoRA + Dağıtık -```bash -docker run --gpus all \ - -v $(pwd)/my_config.yaml:/workspace/config.yaml \ - --shm-size=16g \ - forgelm:full \ - torchrun --nproc_per_node=4 -m forgelm.cli --config /workspace/config.yaml -``` +LoRA/DoRA adapter'ları hem DeepSpeed hem FSDP ile iyi çalışır. +Adapter'lar bölümlenmenin overhead'ini önemsiz kılacak kadar küçükken, +dondurulmuş base-model parametreleri bölümlenmeden ciddi yarar görür. -> **Not**: `--shm-size=16g` çoklu GPU eğitimi için önemlidir. PyTorch, süreçler arası iletişim için paylaşılan bellek kullanır ve Docker'ın varsayılan paylaşılan belleği (64MB) yetersizdir. +--- ## Çok Düğümlü Eğitim -Birden fazla makine üzerinde eğitim için `torchrun`'ı düğüm yapılandırmasıyla kullanın: +Birden fazla makine üzerinde eğitim için `torchrun`'ı düğüm +yapılandırmasıyla kullanın: ```bash # Düğüm 0 (master) @@ -171,6 +194,22 @@ torchrun \ -m forgelm.cli --config my_config.yaml ``` +--- + +## Docker + Çoklu GPU + +```bash +docker run --gpus all \ + -v $(pwd)/my_config.yaml:/workspace/config.yaml \ + --shm-size=16g \ + forgelm:full \ + torchrun --nproc_per_node=4 -m forgelm.cli --config /workspace/config.yaml +``` + +> **Not**: `--shm-size=16g` çoklu GPU eğitimi için önemlidir. PyTorch, süreçler arası iletişim için paylaşılan bellek kullanır ve Docker'ın varsayılan paylaşılan belleği (64MB) yetersizdir. + +--- + ## Sorun Giderme ### "CUDA out of memory" diff --git a/docs/usermanuals/en/reference/cli.md b/docs/usermanuals/en/reference/cli.md index 37dbf310..5a4d53f5 100644 --- a/docs/usermanuals/en/reference/cli.md +++ b/docs/usermanuals/en/reference/cli.md @@ -26,6 +26,7 @@ ForgeLM ships a single `forgelm` binary with subcommands. This page is the canon | `forgelm reject` | Reject a human approval request and discard staging. | | `forgelm approvals` | List pending approvals (`--pending`) or inspect one (`--show RUN_ID`). | | `forgelm purge` | GDPR Article 17 erasure: row-id, run-id, or `--check-policy` retention report. | +| `forgelm reverse-pii` | GDPR Article 15 right-of-access: search masked corpora for a subject's identifier (plaintext or hash-mask scan). | | `forgelm cache-models` | Air-gap workflow: pre-populate the HuggingFace Hub cache for one or more models. | | `forgelm cache-tasks` | Air-gap workflow: pre-populate the lm-eval task dataset cache (requires `[eval]` extra). | | `forgelm safety-eval` | Standalone safety evaluation against a model checkpoint (Llama Guard by default). | diff --git a/docs/usermanuals/en/reference/json-output.md b/docs/usermanuals/en/reference/json-output.md index 77442464..c741069e 100644 --- a/docs/usermanuals/en/reference/json-output.md +++ b/docs/usermanuals/en/reference/json-output.md @@ -256,6 +256,43 @@ Three-mode dispatcher: `--row-id`, `--run-id`, or `--check-policy`. Wave 2b Phas **Exit code mapping:** `0` = success or successful policy report; `1` = config error (unknown row, missing corpus, conflicting flags, malformed `--check-policy --config`); `2` = runtime error (I/O, atomic rename failure). +## `forgelm reverse-pii` + +Wave 3 Phase 38 — GDPR Article 15 right-of-access companion to `forgelm purge`. Walks JSONL corpora and reports every line where the supplied identifier appears. + +**Success envelope:** + +```json +{ + "success": true, + "query_hash": "abc...64-hex", + "identifier_type": "email", + "scan_mode": "plaintext", + "matches": [ + { + "file": "/work/data/train.jsonl", + "line": 42, + "snippet": "Contact: alice@example.com regarding the appointment" + } + ], + "files_scanned": [ + {"path": "/work/data/train.jsonl", "match_count": 1} + ], + "match_count": 1 +} +``` + +| Key | Type | Notes | +|---|---|---| +| `query_hash` | str | 64-character lowercase hex SHA-256 digest of the operator's raw `--query`. The raw value never appears in the envelope. | +| `identifier_type` | str | Echo of `--type` ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`. | +| `scan_mode` | str | `"plaintext"` (default — verbatim search; mask-leak detection) or `"hash"` (`--salt-source` set; SHA-256 of `salt + query` searched in the corpus). | +| `matches` | list[object] | One entry per line that matched. `snippet` is centre-truncated to 160 chars (`…` separator) so unbounded log spam is impossible. | +| `files_scanned` | list[object] | Per-file match counts, in glob-resolution order. The list is the source of truth for which paths the scan walked. | +| `match_count` | int | Equals `len(matches)`; convenience for CI gates that branch on "any match." | + +**Exit code mapping:** `0` = scan completed (matches list may be empty); `1` = config error (empty `--query`, malformed regex on `--type custom`, empty glob expansion, `--salt-source env_var` without `FORGELM_AUDIT_SECRET`); `2` = runtime error (mid-scan I/O failure). + ## `forgelm cache-models` Wave 2b Phase 35 — air-gap workflow blocker. Pre-populates the HuggingFace Hub cache. diff --git a/docs/usermanuals/tr/compliance/overview.md b/docs/usermanuals/tr/compliance/overview.md index 3fdd7ae7..86af325f 100644 --- a/docs/usermanuals/tr/compliance/overview.md +++ b/docs/usermanuals/tr/compliance/overview.md @@ -88,7 +88,22 @@ compliance: responsible_party: "Acme Corp " ``` -`compliance:` bloğundan her alan `annex_iv.json`'a akar. Gerekli alanlar config yüklenirken doğrulanır. +`compliance:` bloğundan her alan `annex_iv.json`'a akar. Gerekli alanlar config yüklenirken doğrulanır — eksik bir `intended_purpose` `--dry-run`'ı fail eder. + +## Annex IV neyi içerir + +Annex IV artefact'ının sekiz bölümü vardır; tamamı otomatik doldurulur: + +1. **Genel açıklama** — model adı, kullanım amacı, dağıtım coğrafyası. +2. **Detaylı sistem açıklaması** — base model, eğitim paradigması, veri seti özeti. +3. **İzleme** — eval eşikleri, otomatik geri alma tetikleyicileri, trend takibi. +4. **Risk yönetimi** — risk sınıflandırması, azaltıcı önlemler, kalan riskler. +5. **Yaşam döngüsü** — eğitim tarihi, sürüm, kaynak verisine referanslar. +6. **Standartlar** — listelenen uyumluluk çerçeveleri (EU AI Act, GDPR, ISO 27001). +7. **Uyumluluk beyanı** — iskelet; nihai beyan insan imzası gerektirir. +8. **Pazar-sonrası izleme planı** — dağıtılmış gözetim config'ine işaretçi. + +Tam şema için bkz. [Annex IV](#/compliance/annex-iv). ## Operasyonel sorumluluklar (siz, ForgeLM değil) diff --git a/docs/usermanuals/tr/concepts/data-formats.md b/docs/usermanuals/tr/concepts/data-formats.md index b129d6fa..9ed2f048 100644 --- a/docs/usermanuals/tr/concepts/data-formats.md +++ b/docs/usermanuals/tr/concepts/data-formats.md @@ -135,6 +135,20 @@ Ağırlıklar 1.0'a tamamlanır; her batch bu olasılıklara göre örneklenir. Otomatik algılama dosya başına bir kez. JSONL'iniz formatları karıştırırsa loader yanlış yönlendirir. Ayrı dosyalar kullanın ve her ikisini `datasets:` ile referans verin. ::: +## Verinizi doğrulama + +Eğitimden önce her zaman `forgelm audit` çalıştırın: + +```shell +$ forgelm audit data/preferences.jsonl +✓ format: preference (12,400 satır, 3 split) +✓ PII tespit edilmedi (5 orta ciddiyet, rapora bakın) +⚠ 12 chosen-rejected aynı satır — muhtemelen toplama hatası +✓ split'ler arası sızıntı yok +``` + +Tam audit semantiği için bkz. [Veri Seti Denetimi](#/data/audit). + ## Bkz. - [Doküman Ingest'i](#/data/ingestion) — PDF/DOCX/EPUB/Markdown'ı bu formatlara dönüştür. diff --git a/docs/usermanuals/tr/reference/cli.md b/docs/usermanuals/tr/reference/cli.md index 35b42f18..bb1ffcca 100644 --- a/docs/usermanuals/tr/reference/cli.md +++ b/docs/usermanuals/tr/reference/cli.md @@ -26,6 +26,7 @@ ForgeLM, subcommand'larla tek bir `forgelm` binary'si yayınlar. Bu sayfa kanoni | `forgelm reject` | İnsan onay isteğini reddet ve staging'i at. | | `forgelm approvals` | Bekleyen onayları listele (`--pending`) veya tek birini incele (`--show RUN_ID`). | | `forgelm purge` | GDPR Madde 17 silme: row-id, run-id veya `--check-policy` retention raporu. | +| `forgelm reverse-pii` | GDPR Madde 15 erişim hakkı: maskelenmiş corpora'da subject kimlik bilgisini ara (plaintext veya hash-mask scan). | | `forgelm cache-models` | Air-gap workflow: bir veya birden fazla model için HuggingFace Hub cache'ini önceden doldur. | | `forgelm cache-tasks` | Air-gap workflow: lm-eval task dataset cache'ini önceden doldur (`[eval]` extra'sı gerekir). | | `forgelm safety-eval` | Bir model checkpoint'ine karşı standalone safety evaluation (varsayılan Llama Guard). | diff --git a/docs/usermanuals/tr/reference/json-output.md b/docs/usermanuals/tr/reference/json-output.md index 516703e7..0d81d097 100644 --- a/docs/usermanuals/tr/reference/json-output.md +++ b/docs/usermanuals/tr/reference/json-output.md @@ -256,6 +256,43 @@ Audit log chain bütünlüğü kontrolü. **Exit kodu:** `0` = başarı veya başarılı policy raporu; `1` = config hatası (bilinmeyen satır, eksik corpus, çelişen flag, malformed `--check-policy --config`); `2` = runtime hatası (I/O, atomic rename başarısız). +## `forgelm reverse-pii` + +Wave 3 Phase 38 — GDPR Madde 15 erişim hakkı, `forgelm purge`'ün companion'ı. JSONL corpora'larını dolaşır ve sağlanan identifier'ın görüldüğü her satırı raporlar. + +**Başarı zarfı:** + +```json +{ + "success": true, + "query_hash": "abc...64-hex", + "identifier_type": "email", + "scan_mode": "plaintext", + "matches": [ + { + "file": "/work/data/train.jsonl", + "line": 42, + "snippet": "Contact: alice@example.com regarding the appointment" + } + ], + "files_scanned": [ + {"path": "/work/data/train.jsonl", "match_count": 1} + ], + "match_count": 1 +} +``` + +| Anahtar | Tip | Notlar | +|---|---|---| +| `query_hash` | str | Operatörün raw `--query`'sinin 64-karakter küçük-harfli hex SHA-256 digest'i. Raw değer hiçbir zaman zarfta yer almaz. | +| `identifier_type` | str | `--type` echo'su ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`. | +| `scan_mode` | str | `"plaintext"` (varsayılan — verbatim arama; mask-leak detection) veya `"hash"` (`--salt-source` set; corpus'ta `salt + query`'nin SHA-256'sı aranır). | +| `matches` | list[object] | Eşleşen her satır için bir entry. `snippet` 160 karaktere centre-truncated (`…` ayraç) — sınırsız log spam imkansız. | +| `files_scanned` | list[object] | Glob-resolution sırasında per-file match sayıları. Hangi path'lerin tarandığının kaynağı budur. | +| `match_count` | int | `len(matches)`'a eşit; "match var mı yok mu" branch'leyen CI gate'leri için convenience. | + +**Exit kodu:** `0` = scan tamamlandı (matches listesi boş olabilir); `1` = config hatası (boş `--query`, `--type custom` malformed regex, empty glob expansion, `FORGELM_AUDIT_SECRET` olmadan `--salt-source env_var`); `2` = runtime hatası (mid-scan I/O failure). + ## `forgelm cache-models` Wave 2b Phase 35 — air-gap workflow blocker. HuggingFace Hub cache'ini önceden doldurur. diff --git a/docs/usermanuals/tr/training/sft.md b/docs/usermanuals/tr/training/sft.md index 147d54e3..3072f002 100644 --- a/docs/usermanuals/tr/training/sft.md +++ b/docs/usermanuals/tr/training/sft.md @@ -116,6 +116,18 @@ Her zaman göndermeden önce `--fit-check` koşturun. **Sample packing dramatik hızlandırır.** Ortalama örneğiniz `max_length`'ten çok kısaysa `packing: true` ayarlayın — kısa örnekleri tek diziye paketler, %30-50 throughput. Talimat ayarı için kalite farkı yok. ::: +## Diskte ne elde edersiniz + +Eğitim sonrası: + +```text +checkpoints/sft/ +├── adapter_model.safetensors ← LoRA ağırlıkları (LoRA kullanılmıyorsa merged checkpoint) +├── README.md ← model kartı +├── config_snapshot.yaml ← kullanılan tam config +└── artifacts/ ← uyumluluk kanıtı (compliance.annex_iv etkinse) +``` + ## Bkz. - [DPO](#/training/dpo) — SFT'den sonra olağan adım. diff --git a/docs/usermanuals/tr/training/simpo.md b/docs/usermanuals/tr/training/simpo.md index 5d497c6e..710c4e69 100644 --- a/docs/usermanuals/tr/training/simpo.md +++ b/docs/usermanuals/tr/training/simpo.md @@ -43,6 +43,10 @@ output: dir: "./checkpoints/simpo" ``` +## Veri formatı + +DPO ile aynı — `prompt`, `chosen`, `rejected` taşıyan `preference` formatı. Bkz. [Veri Formatları](#/concepts/data-formats). + ## Parametreler | Parametre | Tip | Vars. | Açıklama | diff --git a/forgelm/cli/__init__.py b/forgelm/cli/__init__.py index e163db64..9e132ba8 100644 --- a/forgelm/cli/__init__.py +++ b/forgelm/cli/__init__.py @@ -200,6 +200,20 @@ _run_quickstart_train_then_chat, # noqa: F401 — re-export for tests ) +# Reverse-PII subcommand (Phase 38 — GDPR Article 15 right-of-access). +from .subcommands._reverse_pii import ( + _build_search_pattern, # noqa: F401 — re-export for tests + _emit_reverse_pii_result, # noqa: F401 — re-export for tests + _hash_for_audit, # noqa: F401 — re-export for tests + _resolve_files, # noqa: F401 — re-export for tests + _resolve_query_form, # noqa: F401 — re-export for tests + _run_reverse_pii_cmd, # noqa: F401 — re-export for tests + _scan_file, # noqa: F401 — re-export for tests + _truncate_snippet, # noqa: F401 — re-export for tests + _validate_identifier_type, # noqa: F401 — re-export for tests + _validate_query, # noqa: F401 — re-export for tests +) + # Compliance verification toolbelt (Phase 36). from .subcommands._safety_eval import ( _resolve_probes_path, # noqa: F401 — re-export for tests diff --git a/forgelm/cli/_dispatch.py b/forgelm/cli/_dispatch.py index de4cd9fa..7bfdb00c 100644 --- a/forgelm/cli/_dispatch.py +++ b/forgelm/cli/_dispatch.py @@ -81,6 +81,7 @@ def _dispatch_subcommand(command: str, args) -> None: "reject": "_run_reject_cmd", "approvals": "_run_approvals_cmd", "purge": "_run_purge_cmd", + "reverse-pii": "_run_reverse_pii_cmd", "cache-models": "_run_cache_models_cmd", "cache-tasks": "_run_cache_tasks_cmd", "verify-annex-iv": "_run_verify_annex_iv_cmd", diff --git a/forgelm/cli/_parser.py b/forgelm/cli/_parser.py index 549f18b2..c2588893 100644 --- a/forgelm/cli/_parser.py +++ b/forgelm/cli/_parser.py @@ -879,6 +879,81 @@ def _add_purge_subcommand(subparsers) -> None: _add_common_subparser_flags(p, include_output_format=True) +def _add_reverse_pii_subcommand(subparsers) -> None: + """Phase 38 / GDPR Article 15: search masked corpora for a subject's identifier. + + Companion to ``forgelm purge`` (Article 17 erasure): where ``purge`` + answers "delete my row," ``reverse-pii`` answers "is my row in the + corpus at all, and where?" Two scan modes: + + - **plaintext residual scan** (default) — finds operator masking + misses by matching the identifier verbatim in every JSONL line. + - **hash-mask scan** (``--salt-source per_dir|env_var``) — recomputes + ``SHA256(salt + identifier)`` using the same per-output-dir salt + ``forgelm purge`` uses for ``target_id`` hashing, then matches + that digest in masked corpora. + + Audit chain: every invocation writes + ``data.access_request_query`` with the *hashed* identifier (Article + 15 access requests must not themselves leak the subject's data + into the audit log). See ``docs/guides/gdpr_erasure.md`` for the + operator how-to. + """ + p = subparsers.add_parser( + "reverse-pii", + help="GDPR Article 15 right-of-access: search masked corpora for a subject's identifier.", + description=( + "Article 15 right-of-access: walks a glob of JSONL corpora and reports " + "every line where the supplied identifier appears (plaintext residual " + "scan by default; hash-mask scan when --salt-source is set). Audit " + "event `data.access_request_query` records the hashed identifier so " + "the chain itself does not leak the subject's data." + ), + ) + p.add_argument( + "--query", + type=str, + required=True, + metavar="VALUE", + help="Identifier to search for (e-mail, phone, ID, regex pattern, or pre-hashed digest). WARNING: the value is hashed before audit emission but is read in cleartext from the corpus while scanning.", + ) + p.add_argument( + "--type", + type=str, + default="custom", + choices=["email", "phone", "tr_id", "us_ssn", "iban", "credit_card", "custom"], + help="Identifier category. `custom` (default) treats --query as an arbitrary regular expression; every other value treats --query as a literal string and matches it verbatim.", + ) + p.add_argument( + "--salt-source", + type=str, + default=None, + choices=["per_dir", "env_var"], + help="Switch to hash-mask scan: SHA256(salt + identifier) is computed via the same per-output-dir salt `forgelm purge` uses, then searched in every JSONL line. `env_var` requires FORGELM_AUDIT_SECRET to be set; `per_dir` reads the salt file at /.forgelm_audit_salt.", + ) + p.add_argument( + "files", + nargs="+", + metavar="JSONL_GLOB", + help="One or more JSONL paths or glob patterns (e.g. `data/*.jsonl`, `corpora/**/train.jsonl`). Recursive globs (`**`) honoured.", + ) + p.add_argument( + "--output-dir", + type=str, + default=None, + metavar="DIR", + help="Output directory containing the per-output-dir salt file (only consulted when --salt-source is set). Defaults to the parent of the first resolved corpus file.", + ) + p.add_argument( + "--audit-dir", + type=str, + default=None, + metavar="DIR", + help="Where to write the audit chain entries (default: same as --output-dir). Use this when scans run config-free but the audit log lives elsewhere.", + ) + _add_common_subparser_flags(p, include_output_format=True) + + def _add_reject_subcommand(subparsers) -> None: """Article 14: discard a staged model (preserves staging dir for forensics).""" p = subparsers.add_parser( @@ -931,6 +1006,7 @@ def parse_args(): " forgelm reject RUN_ID Reject a staged model (preserves staging dir for forensics)\n" " forgelm approvals --pending List runs awaiting human approval (or --show RUN_ID)\n" " forgelm purge --row-id ID GDPR Article 17 erasure (corpus row / run artefacts / --check-policy)\n" + " forgelm reverse-pii --query VAL GDPR Article 15 right-of-access (find subject identifier in masked corpora)\n" " forgelm cache-models --model M Pre-populate HF Hub cache (air-gap workflow)\n" " forgelm cache-tasks --tasks CSV Pre-populate lm-eval task datasets (requires [eval] extra)\n" " forgelm verify-annex-iv PATH Verify EU AI Act Annex IV artifact (field set + manifest hash)\n" @@ -954,6 +1030,7 @@ def parse_args(): _add_reject_subcommand(subparsers) _add_approvals_subcommand(subparsers) _add_purge_subcommand(subparsers) + _add_reverse_pii_subcommand(subparsers) _add_cache_models_subcommand(subparsers) _add_cache_tasks_subcommand(subparsers) _add_verify_annex_iv_subcommand(subparsers) diff --git a/forgelm/cli/subcommands/_reverse_pii.py b/forgelm/cli/subcommands/_reverse_pii.py new file mode 100644 index 00000000..33fd26c7 --- /dev/null +++ b/forgelm/cli/subcommands/_reverse_pii.py @@ -0,0 +1,384 @@ +"""``forgelm reverse-pii`` — GDPR Article 15 right-of-access subcommand. + +Phase 38 closure of GH-014. Companion to ``forgelm purge`` (Phase 21, +Article 17 right-to-erasure): where ``purge`` lets a data subject ask +"delete my row," ``reverse-pii`` lets the same subject ask "is my row +in the corpus at all, and if so, where?" + +The subcommand walks a glob of JSONL corpora and reports every line +where a supplied identifier (e-mail, phone, TR national ID, custom +regex, or hash-masked digest) appears. Two complementary scan modes: + +1. **Plaintext residual scan** (default). Searches for the identifier + verbatim — the path that surfaces *mask leaks* (operator believed + the corpus was masked but a residual span was missed by the + masking pass). This is the audit-time honest-PII check. +2. **Hash-mask scan** (``--salt-source per_dir`` or ``--salt-source + env_var``). Computes ``SHA256(salt + identifier)`` using the same + per-output-dir salt that ``forgelm purge`` uses for its + ``target_id`` hashing, then searches every JSONL line for that + digest. This is the path for corpora that were masked through + ForgeLM's own ``hash`` replacement strategy. + +Audit chain: every invocation writes a ``data.access_request_query`` +event with the identifier *hashed* (never raw — Article 15 access +requests must not themselves leak the subject's identifier into the +audit log) and the per-file match count. An operator who later runs +``forgelm verify-audit`` sees a forensic record that the request was +processed without the audit log itself becoming a residual leak. + +Exit codes (per ``docs/standards/error-handling.md``): + +- 0 — command ran (matches reported, may be empty). +- 1 — config error (empty/whitespace ``--query``, unparseable regex + for ``--type custom``, glob pattern resolved to zero files). +- 2 — runtime error (I/O failure walking the corpus, audit-log write + failure when an audit dir was supplied). +""" + +from __future__ import annotations + +import glob as _glob +import hashlib +import json +import os +import re +import sys +from typing import Any, Dict, List, NoReturn, Optional, Tuple + +from .._exit_codes import EXIT_CONFIG_ERROR, EXIT_SUCCESS, EXIT_TRAINING_ERROR +from .._logging import logger + +_EVT_ACCESS_REQUEST_QUERY = "data.access_request_query" + +# Snippet shape: report at most this many characters around the match +# so the operator can verify the hit but the JSON envelope does not +# itself leak unbounded surrounding context. +_SNIPPET_MAX_CHARS = 160 + +# Identifier-type keys. Mirror the categories that +# ``forgelm/data_audit/_pii_regex.py::_PII_PATTERNS`` knows so the +# operator can ask "is there a phone matching this exact number" and +# get a result that the audit's own detector would have flagged. +_IDENTIFIER_TYPES: Tuple[str, ...] = ("email", "phone", "tr_id", "us_ssn", "iban", "credit_card", "custom") + + +def _output_error_and_exit(output_format: str, msg: str, exit_code: int) -> NoReturn: + """Mirror the JSON-vs-text envelope helper used by every other + Wave 2b/3 subcommand so the contract stays uniform.""" + if output_format == "json": + print(json.dumps({"success": False, "error": msg})) + else: + logger.error(msg) + sys.exit(exit_code) + + +def _validate_query(query: Optional[str], output_format: str) -> str: + """Refuse empty / whitespace-only queries early. + + Without this guard a stray ``forgelm reverse-pii --query ""`` would + match every line in the corpus (empty-substring semantics) and + silently spam the audit log with a useless event. + """ + if not query or not query.strip(): + _output_error_and_exit( + output_format, + "--query must be a non-empty identifier (e-mail, phone, ID, regex pattern, or pre-hashed digest).", + EXIT_CONFIG_ERROR, + ) + return query.strip() + + +def _validate_identifier_type(identifier_type: str, output_format: str) -> str: + """Refuse identifier types we don't know how to scan for.""" + if identifier_type not in _IDENTIFIER_TYPES: + _output_error_and_exit( + output_format, + f"--type {identifier_type!r} is not recognised. Choose one of: {', '.join(sorted(_IDENTIFIER_TYPES))}.", + EXIT_CONFIG_ERROR, + ) + return identifier_type + + +def _build_search_pattern(query: str, identifier_type: str, output_format: str) -> re.Pattern[str]: + """Compile the regex used to scan each JSONL line. + + For ``--type custom`` the operator's ``--query`` is interpreted as + a regex (escaped only to fix anchoring / boundaries). For every + other type the query is treated as a literal string and surrounded + with ``re.escape`` — the goal is "find this exact identifier", + not "find anything matching its shape" (the audit-time detector + already handles the latter). + """ + if identifier_type == "custom": + try: + return re.compile(query) + except re.error as exc: + _output_error_and_exit( + output_format, + f"--type custom --query {query!r} is not a valid regular expression: {exc}.", + EXIT_CONFIG_ERROR, + ) + return re.compile(re.escape(query)) + + +def _resolve_files(globs: List[str], output_format: str) -> List[str]: + """Expand each ```` positional argument to a list of files. + + Recursive globs (``**``) are honoured. Refuses an empty resolved + set as a config error — the operator either typed a wrong path or + pointed at an empty directory; either way "no files scanned" is + not a successful answer to "is this identifier in the corpus." + """ + resolved: List[str] = [] + seen: set[str] = set() + for pattern in globs: + for match in sorted(_glob.glob(pattern, recursive=True)): + if not os.path.isfile(match): + continue + absolute = os.path.abspath(match) + if absolute in seen: + continue + seen.add(absolute) + resolved.append(absolute) + if not resolved: + _output_error_and_exit( + output_format, + f"No files matched the supplied glob pattern(s): {globs!r}. " + "Check the path or run with shell-expanded paths.", + EXIT_CONFIG_ERROR, + ) + return resolved + + +def _resolve_query_form(query: str, salt_source: Optional[str], output_dir: str, output_format: str) -> Tuple[str, str]: + """Map the operator-supplied identifier to the form actually + searched in the corpus. + + Returns ``(scan_query, scan_mode)`` where ``scan_mode`` is one of: + + - ``"plaintext"`` — search for ``query`` verbatim (mask-leak path). + - ``"hash"`` — compute ``SHA256(salt + query)`` and search for the + digest (the corpus was masked via ForgeLM's hash strategy). + + ``--salt-source`` triggers hash mode and reuses the per-output-dir + salt resolution from ``forgelm purge`` so the two subcommands + cannot drift on hashing semantics. + """ + if salt_source is None: + return query, "plaintext" + # Late import: ``_resolve_salt`` lives in the purge subcommand and + # pulling it eagerly would create a load-time cycle. The two + # subcommands intentionally share the salt-resolution path so a + # purge-then-reverse-pii cycle works on the same digest. + from ._purge import _hash_target_id, _resolve_salt + + try: + salt, resolved_source = _resolve_salt(output_dir) + except OSError as exc: + _output_error_and_exit( + output_format, + f"Could not resolve audit salt for {output_dir!r}: {exc}", + EXIT_TRAINING_ERROR, + ) + if salt_source != resolved_source: + # The operator asked for ``env_var`` but the env wasn't set + # (or vice versa). Refuse rather than silently scan with the + # wrong salt source — the resulting digest would never match. + _output_error_and_exit( + output_format, + f"--salt-source={salt_source!r} requested but ``_resolve_salt`` returned {resolved_source!r}. " + "Set FORGELM_AUDIT_SECRET (env_var mode) or unset it (per_dir mode) and retry.", + EXIT_CONFIG_ERROR, + ) + return _hash_target_id(query, salt), "hash" + + +def _scan_file(path: str, pattern: re.Pattern[str]) -> List[Dict[str, Any]]: + """Walk one JSONL file; return every line where ``pattern`` matches. + + Each match record carries: + + - ``file`` — absolute path of the file. + - ``line`` — 1-based line number. + - ``snippet`` — at most ``_SNIPPET_MAX_CHARS`` characters around + the match, with the matched span left intact so the operator can + eyeball the hit. We deliberately do NOT mask the snippet: the + whole point of the access request is to surface the verbatim + data the subject asked about. + + Raises :class:`OSError` on read failure; the caller handles it. + """ + matches: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as fh: + for line_no, line in enumerate(fh, start=1): + stripped = line.rstrip("\n") + if not pattern.search(stripped): + continue + matches.append( + { + "file": path, + "line": line_no, + "snippet": _truncate_snippet(stripped), + } + ) + return matches + + +def _truncate_snippet(line: str) -> str: + """Centre-truncate ``line`` to at most ``_SNIPPET_MAX_CHARS``. + + For lines longer than the budget we keep both ends with a literal + ``…`` separator so the operator can see the start and tail context + around the match. Short lines pass through unchanged. + """ + if len(line) <= _SNIPPET_MAX_CHARS: + return line + half = (_SNIPPET_MAX_CHARS - 1) // 2 + return f"{line[:half]}…{line[-half:]}" + + +def _hash_for_audit(query: str) -> str: + """SHA-256 the operator's raw query for the audit event. + + Article 15 access requests must not write the subject's identifier + into the audit log: the very thing we're letting the subject query + is precisely what would be re-introduced if we logged it raw. We + therefore log a salt-free SHA-256 (the salt's job is collision + resistance for ``target_id`` matching, not for audit obfuscation; + here we only need a stable per-identifier fingerprint so a + compliance reviewer can correlate two queries about the same + subject without seeing the subject's data). + """ + return hashlib.sha256(query.encode("utf-8")).hexdigest() + + +def _run_reverse_pii_cmd(args, output_format: str) -> None: + """Top-level dispatcher for ``forgelm reverse-pii``.""" + query = _validate_query(getattr(args, "query", None), output_format) + identifier_type = _validate_identifier_type(getattr(args, "type", "custom") or "custom", output_format) + globs = list(getattr(args, "files", None) or []) + if not globs: + _output_error_and_exit( + output_format, + "reverse-pii requires at least one positional argument " + "(e.g. `forgelm reverse-pii --query alice@example.com data/*.jsonl`).", + EXIT_CONFIG_ERROR, + ) + + files = _resolve_files(globs, output_format) + output_dir = getattr(args, "output_dir", None) or os.path.dirname(os.path.abspath(files[0])) or "." + salt_source = getattr(args, "salt_source", None) + scan_query, scan_mode = _resolve_query_form(query, salt_source, output_dir, output_format) + pattern = _build_search_pattern(scan_query, identifier_type, output_format) + + audit = _maybe_audit_logger(getattr(args, "audit_dir", None) or output_dir) + query_hash = _hash_for_audit(query) + + matches: List[Dict[str, Any]] = [] + files_scanned: List[Dict[str, Any]] = [] + try: + for path in files: + file_matches = _scan_file(path, pattern) + matches.extend(file_matches) + files_scanned.append({"path": path, "match_count": len(file_matches)}) + except OSError as exc: + if audit is not None: + audit.log_event( + _EVT_ACCESS_REQUEST_QUERY, + query_hash=query_hash, + identifier_type=identifier_type, + scan_mode=scan_mode, + files_scanned=[fs["path"] for fs in files_scanned], + match_count=len(matches), + error_class=exc.__class__.__name__, + error_message=str(exc), + ) + _output_error_and_exit( + output_format, + f"Failed reading corpus file mid-scan: {exc}. Partial matches surfaced before the failure are not emitted.", + EXIT_TRAINING_ERROR, + ) + + if audit is not None: + audit.log_event( + _EVT_ACCESS_REQUEST_QUERY, + query_hash=query_hash, + identifier_type=identifier_type, + scan_mode=scan_mode, + files_scanned=[fs["path"] for fs in files_scanned], + match_count=len(matches), + ) + + payload: Dict[str, Any] = { + "success": True, + "query_hash": query_hash, + "identifier_type": identifier_type, + "scan_mode": scan_mode, + "matches": matches, + "files_scanned": files_scanned, + "match_count": len(matches), + } + _emit_reverse_pii_result(payload, output_format) + sys.exit(EXIT_SUCCESS) + + +def _emit_reverse_pii_result(payload: Dict[str, Any], output_format: str) -> None: + """Render the result envelope as JSON or human text.""" + if output_format == "json": + print(json.dumps(payload, indent=2, default=str)) + return + match_count = payload["match_count"] + files_scanned = payload["files_scanned"] + if match_count == 0: + print( + f"No matches for identifier hash {payload['query_hash'][:16]}… " + f"({payload['identifier_type']}, {payload['scan_mode']} mode) in " + f"{len(files_scanned)} file(s)." + ) + return + print( + f"Found {match_count} match(es) for identifier hash {payload['query_hash'][:16]}… " + f"({payload['identifier_type']}, {payload['scan_mode']} mode):" + ) + for hit in payload["matches"]: + print(f" {hit['file']}:{hit['line']} {hit['snippet']}") + print() + print(f"Files scanned: {len(files_scanned)}") + for fs in files_scanned: + print(f" {fs['path']}: {fs['match_count']} match(es)") + + +def _maybe_audit_logger(audit_dir: str): + """Best-effort construct the AuditLogger. + + Mirrors :func:`forgelm.cli.subcommands._cache._maybe_audit_logger` — + a missing operator identity should not abort an Article 15 access + request (the subject still gets their answer; the chain just + skips the breadcrumb). + """ + try: + from forgelm.compliance import AuditLogger + from forgelm.config import ConfigError + + return AuditLogger(audit_dir) + except ConfigError as exc: + logger.debug("reverse-pii: AuditLogger init failed (%s); continuing without audit log.", exc) + return None + except Exception as exc: # noqa: BLE001 — best-effort: audit is optional context here. # NOSONAR + logger.debug("reverse-pii: AuditLogger init crashed (%s); continuing without audit log.", exc) + return None + + +__all__ = [ + "_run_reverse_pii_cmd", + "_validate_query", + "_validate_identifier_type", + "_build_search_pattern", + "_resolve_files", + "_resolve_query_form", + "_scan_file", + "_truncate_snippet", + "_hash_for_audit", + "_emit_reverse_pii_result", +] diff --git a/forgelm/compliance.py b/forgelm/compliance.py index 3102de20..321aafd1 100644 --- a/forgelm/compliance.py +++ b/forgelm/compliance.py @@ -408,7 +408,14 @@ def _maybe_inline_audit_report(config: Any) -> Optional[Dict[str, Any]]: return None audit_path = os.path.join(output_dir, "data_audit_report.json") if not os.path.isfile(audit_path): - logger.info( + # Wave 3 / Faz 28 (F-compliance-111): escalated from INFO to + # WARNING. A missing data_audit_report.json is a real Article + # 10 compliance gap — the governance bundle ships without its + # data-quality section, which is exactly the surface a regulator + # would inspect first. Operators reading INFO-level logs out + # of habit miss the signal; WARNING is the documented level for + # "nothing crashed but something compliance-relevant degraded." + logger.warning( "No data_audit_report.json at %s — governance report will lack the " "Article 10 data-quality section. Run " "`forgelm audit --output %s` before training to populate it.", @@ -799,6 +806,27 @@ def _sanitize_md(text: Optional[str]) -> str: return escaped.strip() +def _sanitize_md_list(items: Optional[List[Any]]) -> List[str]: + """Apply :func:`_sanitize_md` element-wise to ``items``. + + Wave 3 / Faz 28 (M-204): a small ergonomic shim used by the + deployer-instructions builder when interpolating list-shaped + config fields (foreseeable misuse list, dataset names, etc.) into + Markdown bullets / table rows. Centralises the per-element + sanitisation so a future migration to a stricter escape policy + only has to touch :func:`_sanitize_md`. + + Returns ``[]`` for ``None`` / empty inputs so callers can spread + the result directly into a join without a None guard. Non-string + elements are stringified first (mirrors :func:`_sanitize_md`'s + permissive ``Any`` shape — operators occasionally drop ints into + list fields). + """ + if not items: + return [] + return [_sanitize_md(str(item) if not isinstance(item, str) else item) for item in items] + + def generate_deployer_instructions(config: Any, metrics: Dict[str, float], final_path: str) -> str: """Generate deployer instructions document per EU AI Act Article 13.""" comp_cfg = getattr(config, "compliance", None) @@ -843,8 +871,8 @@ def generate_deployer_instructions(config: Any, metrics: Dict[str, float], final **This model should NOT be used for:** """ if risk_cfg and risk_cfg.foreseeable_misuse: - for misuse in risk_cfg.foreseeable_misuse: - content += f"- {_sanitize_md(misuse)}\n" + for misuse in _sanitize_md_list(risk_cfg.foreseeable_misuse): + content += f"- {misuse}\n" else: content += "- Use cases not covered by the intended purpose above\n" diff --git a/forgelm/config.py b/forgelm/config.py index 35cddd0b..7b3fafaa 100644 --- a/forgelm/config.py +++ b/forgelm/config.py @@ -655,7 +655,15 @@ class WebhookConfig(BaseModel): notify_on_failure: bool = Field( default=True, description="POST a `notify_failure` event when training fails (any non-zero exit)." ) - timeout: int = Field(default=5, description="HTTP request timeout in seconds. Clamped to ≥ 1s by the notifier.") + timeout: int = Field( + default=10, + description=( + "HTTP request timeout in seconds. Clamped to ≥ 1s by the notifier. " + "Default raised to 10s in v0.5.5 (was 5s) — Slack/Teams gateway latency " + "spikes regularly cross 5s in production, and a webhook timeout silently " + "degrades the audit chain (webhook failure is best-effort)." + ), + ) allow_private_destinations: bool = Field( default=False, description="SSRF opt-in. Webhooks default to public-internet destinations only; in-cluster Slack proxies / on-prem Teams gateways need this set.", @@ -868,9 +876,24 @@ def _warn_high_risk_compliance(self) -> None: ) safety = self.evaluation.safety if self.evaluation else None if not safety or not safety.enabled: - logger.warning( - "High-risk AI classification: safety evaluation is strongly recommended. " - "Set evaluation.safety.enabled: true." + # Wave 3 / Faz 28 (F-compliance-110): a high-risk / unacceptable + # classification REQUIRES an enabled safety evaluation gate to + # back the EU AI Act Article 9 risk-management claim. Earlier + # versions only emitted a warning, which let regulated runs + # ship Annex IV bundles whose risk-management section was not + # actually evidenced. v0.5.5 escalates the warning to a hard + # ``ConfigError``: operators who genuinely want a sandboxed run + # without safety eval must lower the risk_classification (e.g. + # to ``limited-risk``) or enable ``evaluation.safety``. + raise ConfigError( + "Risk classification %r requires evaluation.safety.enabled: true " + "(EU AI Act Article 9 risk-management evidence cannot be derived " + "from a disabled safety eval). Either enable safety evaluation " + "or lower the risk_classification to a non-strict tier." + % ( + (self.compliance and self.compliance.risk_classification) + or (self.risk_assessment and self.risk_assessment.risk_category), + ) ) elif not safety.track_categories: logger.warning( diff --git a/forgelm/deploy.py b/forgelm/deploy.py index 5b9819fe..4bd56305 100644 --- a/forgelm/deploy.py +++ b/forgelm/deploy.py @@ -97,8 +97,15 @@ def _ollama_modelfile( "", ] if system_prompt: - # Escape any double quotes in the system prompt - safe_prompt = system_prompt.replace('"', '\\"') + # Escape double quotes AND newlines. Ollama's single-quoted + # SYSTEM line is parsed line-wise; a literal newline inside the + # value breaks the directive and silently falls through to the + # next line as its own (malformed) directive. Round-3 / Faz + # 28 (M-205): convert \n / \r to escape sequences instead of + # bare characters so multi-line operator-supplied prompts + # round-trip correctly through ``ollama create``. + safe_prompt = system_prompt.replace("\\", "\\\\").replace('"', '\\"') + safe_prompt = safe_prompt.replace("\r\n", "\\n").replace("\n", "\\n").replace("\r", "\\n") lines.append(f'SYSTEM "{safe_prompt}"') lines.append("") diff --git a/forgelm/trainer.py b/forgelm/trainer.py index 8d01057b..d6a64bca 100644 --- a/forgelm/trainer.py +++ b/forgelm/trainer.py @@ -54,6 +54,18 @@ # answers — stripped before comparison so "Answer: $15" matches gold "15". # Order matters: longer/multi-char tokens first to avoid partial overlaps # (e.g. "km/h" must be matched before "km"). +# +# **Domain caveat (Faz 28 / C-57 honesty fix):** this token set is +# tuned for the bundled `grpo-math` template which targets the +# **GSM8K + MATH** benchmarks (US units + currency + percent + a +# narrow metric subset). Operators training GRPO on other math +# domains (SI-only physics, scientific notation, complex numbers, +# code-with-math, multilingual quantities) should not expect this +# stripper to generalise — write a custom reward callable via the +# ``training.grpo_reward_model`` config knob (see +# ``docs/guides/alignment.md`` GRPO section) instead of widening this +# list. The v0.6.0 GRPO config-driven reward plugin migration is +# tracked under the v0.6.0 backlog. _REWARD_STRIP_TOKENS: tuple[str, ...] = ( "km/h", "m/s", diff --git a/forgelm/webhook.py b/forgelm/webhook.py index 52e82c36..b80dbba5 100644 --- a/forgelm/webhook.py +++ b/forgelm/webhook.py @@ -6,14 +6,15 @@ import requests -from ._http import HttpSafetyError, _is_private_destination, safe_post - -# Public re-export surface — Phase 7 split moved ``_is_private_destination`` -# into ``forgelm._http`` but external callers / older code may still import -# it from ``forgelm.webhook``. Listing it in ``__all__`` documents the -# re-export as intentional and silences ruff/Codacy F401 ("imported but -# unused") and Sonar's equivalent rule. -__all__ = ["HttpSafetyError", "WebhookNotifier", "_is_private_destination", "safe_post"] +from ._http import HttpSafetyError, safe_post + +# Public re-export surface. Wave 3 / Faz 28 (C-54) cleanup: dropped +# the ``_is_private_destination`` re-export. The Phase 7 split moved +# the helper into ``forgelm._http``; external callers / tooling that +# need the SSRF guard import it from there directly. No downstream +# importer of the webhook-side re-export was found at the time of +# removal, so this is a clean drop (no DeprecationWarning shim). +__all__ = ["HttpSafetyError", "WebhookNotifier", "safe_post"] logger = logging.getLogger("forgelm.webhook") diff --git a/tests/test_check_bilingual_parity.py b/tests/test_check_bilingual_parity.py new file mode 100644 index 00000000..0d3fcb87 --- /dev/null +++ b/tests/test_check_bilingual_parity.py @@ -0,0 +1,208 @@ +"""Phase 24 — `tools/check_bilingual_parity.py` regression tests. + +The bilingual parity tool is a CI gate: a regression that silently +under-reports drift would let a mirror skew without anyone noticing. +We pin the recognition contracts that the CI gate's correctness rests +on: + +1. Heading extraction skips fenced code blocks. +2. Heading-shaped lines INSIDE a fenced block are NOT treated as + headings (otherwise ``# noqa: E402`` in a Python snippet would + register as an H1). +3. Identical H2/H3/H4 spines pass; any drift fails (count diff, + reorder, depth change, missing-mirror). +4. ``--strict`` flips the exit code; advisory mode reports anyway. +5. The canonical pair set in the repo passes ``--strict`` against + the live filesystem (the same gate CI runs). +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +# Load the tool from its file path without leaking ``tools/`` onto +# ``sys.path``. Same pattern as ``tests/test_check_field_descriptions.py``. +_TOOLS_DIR = Path(__file__).parent.parent / "tools" +_PARITY_PATH = _TOOLS_DIR / "check_bilingual_parity.py" +_spec = importlib.util.spec_from_file_location("check_bilingual_parity", _PARITY_PATH) +assert _spec is not None and _spec.loader is not None, f"could not load parity-tool spec from {_PARITY_PATH!r}" +check_bilingual_parity = importlib.util.module_from_spec(_spec) +sys.modules["check_bilingual_parity"] = check_bilingual_parity +_spec.loader.exec_module(check_bilingual_parity) + +extract_headings = check_bilingual_parity.extract_headings +diff_pair = check_bilingual_parity.diff_pair +scan_pairs = check_bilingual_parity.scan_pairs +main = check_bilingual_parity.main + + +def _write(path: Path, body: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# Heading extraction +# --------------------------------------------------------------------------- + + +class TestExtractHeadings: + def test_atx_headings_at_each_depth(self, tmp_path: Path) -> None: + src = _write( + tmp_path / "doc.md", + "# Title\n## H2 one\n### H3\n#### H4\n##### H5\n###### H6\nbody\n", + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [1, 2, 3, 4, 5, 6] + assert headings[0].text == "Title" + assert headings[2].text == "H3" + + def test_fenced_code_blocks_ignored(self, tmp_path: Path) -> None: + """Heading-shaped lines inside a code block are CONTENT, not + document structure. A regression that re-treated them as + headings would inflate the parity tool's counts on every + Python doc and produce false negatives for translators.""" + src = _write( + tmp_path / "doc.md", + ("## Real H2\n```python\n# noqa: E402\n## not a real heading\n```\n## Second real H2\n"), + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [2, 2] + assert [h.text for h in headings] == ["Real H2", "Second real H2"] + + def test_tilde_fence_also_recognised(self, tmp_path: Path) -> None: + src = _write( + tmp_path / "doc.md", + "## H2\n~~~yaml\n## not heading\n~~~\n## After\n", + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [2, 2] + + def test_missing_file_returns_empty(self, tmp_path: Path) -> None: + assert extract_headings(tmp_path / "missing.md") == [] + + +# --------------------------------------------------------------------------- +# Pair diff: pass / fail / missing +# --------------------------------------------------------------------------- + + +class TestDiffPair: + def test_identical_spines_pass(self, tmp_path: Path) -> None: + en = _write(tmp_path / "en.md", "## A\n### A1\n## B\n#### B1a\n") + tr = _write(tmp_path / "tr.md", "## E\n### E1\n## D\n#### D1a\n") # different text, same spine + assert diff_pair(en, tr) is None + + def test_h3_count_drift_flagged(self, tmp_path: Path) -> None: + en = _write(tmp_path / "en.md", "## A\n### A1\n### A2\n") + tr = _write(tmp_path / "tr.md", "## A\n### A1\n") + drift = diff_pair(en, tr) + assert drift is not None + assert "H3: EN=2 TR=1" in "\n".join(drift.detail) + + def test_demoted_section_flagged(self, tmp_path: Path) -> None: + """``## Foo`` in EN but ``### Foo`` in TR — both have one + heading at the index, but the level differs. Reports it as + a per-line mismatch even though counts at one level may match.""" + en = _write(tmp_path / "en.md", "## A\n## B\n") + tr = _write(tmp_path / "tr.md", "## A\n### B\n") + drift = diff_pair(en, tr) + assert drift is not None + rendered = drift.render() + assert "EN:H2" in rendered and "TR:H3" in rendered + + def test_missing_tr_mirror_diagnosed(self, tmp_path: Path) -> None: + en = _write(tmp_path / "en.md", "## A\n") + drift = diff_pair(en, tmp_path / "tr.md") + assert drift is not None + assert "TR mirror is missing" in drift.summary + + def test_missing_en_diagnosed(self, tmp_path: Path) -> None: + tr = _write(tmp_path / "tr.md", "## A\n") + drift = diff_pair(tmp_path / "en.md", tr) + assert drift is not None + assert "EN file is missing" in drift.summary + + def test_levels_filter_restricts_comparison(self, tmp_path: Path) -> None: + """``levels=(2,)`` mirrors the legacy CI behaviour — H2 only. + Drift in H3 must NOT be reported under that filter.""" + en = _write(tmp_path / "en.md", "## A\n### A1\n## B\n") + tr = _write(tmp_path / "tr.md", "## A\n## B\n") # missing H3 + # Under H2-only filter, the spines match (2 H2's each). + assert diff_pair(en, tr, levels=(2,)) is None + # Under H2+H3 (default), the H3 drift is reported. + assert diff_pair(en, tr) is not None + + +# --------------------------------------------------------------------------- +# CLI surface +# --------------------------------------------------------------------------- + + +class TestCli: + def test_strict_returns_one_on_drift(self, tmp_path: Path, monkeypatch) -> None: + # Plant a fake repo with a single drifted pair. + repo = tmp_path / "repo" + _write(repo / "docs" / "guide.md", "## A\n### A1\n") + _write(repo / "docs" / "guide-tr.md", "## A\n") + monkeypatch.setattr(check_bilingual_parity, "_PAIRS", (("docs/guide.md", "docs/guide-tr.md"),)) + rc = main(["--strict", "--repo-root", str(repo)]) + assert rc == 1 + + def test_advisory_mode_returns_zero_even_on_drift(self, tmp_path: Path, monkeypatch) -> None: + repo = tmp_path / "repo" + _write(repo / "docs" / "guide.md", "## A\n### A1\n") + _write(repo / "docs" / "guide-tr.md", "## A\n") + monkeypatch.setattr(check_bilingual_parity, "_PAIRS", (("docs/guide.md", "docs/guide-tr.md"),)) + rc = main(["--repo-root", str(repo)]) # no --strict + assert rc == 0 + + def test_clean_pair_strict_returns_zero(self, tmp_path: Path, monkeypatch) -> None: + repo = tmp_path / "repo" + _write(repo / "docs" / "guide.md", "## A\n### A1\n") + _write(repo / "docs" / "guide-tr.md", "## E\n### E1\n") + monkeypatch.setattr(check_bilingual_parity, "_PAIRS", (("docs/guide.md", "docs/guide-tr.md"),)) + rc = main(["--strict", "--repo-root", str(repo)]) + assert rc == 0 + + def test_only_filter_restricts_to_one_pair(self, tmp_path: Path, monkeypatch) -> None: + repo = tmp_path / "repo" + _write(repo / "docs" / "a.md", "## X\n") + _write(repo / "docs" / "a-tr.md", "## X\n") + # Drifted pair NOT in --only. + _write(repo / "docs" / "b.md", "## A\n## B\n") + _write(repo / "docs" / "b-tr.md", "## A\n") + monkeypatch.setattr( + check_bilingual_parity, + "_PAIRS", + (("docs/a.md", "docs/a-tr.md"), ("docs/b.md", "docs/b-tr.md")), + ) + # Run with --only on the clean pair → should pass strict. + rc = main(["--strict", "--repo-root", str(repo), "--only", "docs/a.md"]) + assert rc == 0 + + def test_invalid_levels_rejected(self, tmp_path: Path) -> None: + rc = main(["--levels", "0,7", "--repo-root", str(tmp_path)]) + assert rc == 1 + + +# --------------------------------------------------------------------------- +# Live-repo smoke: the canonical pair set must pass --strict +# --------------------------------------------------------------------------- + + +class TestCanonicalRepoPasses: + def test_repository_pairs_pass_strict(self) -> None: + """The repo's actual EN/TR pairs must pass ``--strict`` so the + CI gate stays green. This is the test that fails first if a + future PR drifts a mirror without fixing it.""" + repo_root = Path(__file__).parent.parent + rc = main(["--strict", "--repo-root", str(repo_root)]) + assert rc == 0, ( + "Live-repo strict parity check failed; run " + "`python3 tools/check_bilingual_parity.py` for the per-pair " + "drift report." + ) diff --git a/tests/test_compliance.py b/tests/test_compliance.py index 73194096..6029737d 100644 --- a/tests/test_compliance.py +++ b/tests/test_compliance.py @@ -273,23 +273,28 @@ def test_warns_when_audit_corrupt(self, tmp_path, caplog, minimal_config): assert "data_audit" not in report assert any("Could not inline" in r.message for r in caplog.records) - def test_info_log_when_audit_missing(self, tmp_path, caplog, minimal_config): + def test_warning_log_when_audit_missing(self, tmp_path, caplog, minimal_config): # The audit CLI defaults to ./audit/ but the trainer's # output_dir is typically ./checkpoints/ — without alignment - # the inlining silently no-ops. Loud INFO log surfaces the - # path mismatch with an actionable command hint. + # the inlining silently no-ops. + # + # Wave 3 / Faz 28 (F-compliance-111): escalated from INFO to + # WARNING. A missing data_audit_report.json is a real Article + # 10 compliance gap (the governance bundle ships without its + # data-quality section); INFO-level logs are easy to miss in + # production tail-grep. config = ForgeConfig(**minimal_config(training={"output_dir": str(tmp_path)})) - with caplog.at_level("INFO", logger="forgelm.compliance"): + with caplog.at_level("WARNING", logger="forgelm.compliance"): report = generate_data_governance_report(config, dataset={}) assert "data_audit" not in report - info_msgs = [r.message for r in caplog.records if r.levelname == "INFO"] + warn_msgs = [r.message for r in caplog.records if r.levelname == "WARNING"] # Phase 11.5: hint moved from `forgelm --data-audit` (legacy) to the # new `forgelm audit` subcommand. Accept either spelling so this test # survives the deprecation window, but require the actionable command # is named. assert any( "No data_audit_report.json" in m and ("forgelm audit" in m or "forgelm --data-audit" in m) - for m in info_msgs + for m in warn_msgs ) diff --git a/tests/test_data_edge_cases.py b/tests/test_data_edge_cases.py index b87dfcd3..435e01e2 100644 --- a/tests/test_data_edge_cases.py +++ b/tests/test_data_edge_cases.py @@ -97,8 +97,13 @@ class TestWebhookTimeoutConfig: def test_default_timeout(self): from forgelm.config import WebhookConfig + # Default raised from 5s → 10s in v0.5.5 (Faz 28 / F-compliance-106). + # Slack/Teams gateway latency spikes regularly cross 5s in + # production, and a webhook timeout silently degrades the audit + # chain (webhook delivery is best-effort). 10s leaves head-room + # without blocking training-pipeline forward progress. w = WebhookConfig() - assert w.timeout == 5 + assert w.timeout == 10 def test_custom_timeout(self): from forgelm.config import WebhookConfig diff --git a/tests/test_eu_ai_act.py b/tests/test_eu_ai_act.py index 874e82f2..982f62aa 100644 --- a/tests/test_eu_ai_act.py +++ b/tests/test_eu_ai_act.py @@ -3,6 +3,7 @@ import json import os +import pytest import yaml from forgelm.compliance import ( @@ -77,13 +78,18 @@ def test_full(self): class TestForgeConfigCompliance: def test_compliance_in_config(self, minimal_config): + # Wave 3 / Faz 28 (F-compliance-110): high-risk now requires + # safety eval to be enabled (was a warning prior to v0.5.5). + # Pair the risk_classification with an enabled safety block so + # the config validates. cfg = ForgeConfig( **minimal_config( compliance={ "provider_name": "Test Corp", "intended_purpose": "Testing", "risk_classification": "high-risk", - } + }, + evaluation={"safety": {"enabled": True}}, ) ) assert cfg.compliance.provider_name == "Test Corp" @@ -115,17 +121,42 @@ def test_human_approval_in_eval(self, minimal_config): assert cfg.evaluation.require_human_approval is True def test_high_risk_warnings(self, caplog, minimal_config): + """High-risk classification with safety enabled emits the + ``auto_revert`` recommendation (still a warning, not a raise — + F-compliance-110 only escalates the *safety-disabled* branch). + """ import logging with caplog.at_level(logging.WARNING, logger="forgelm.config"): ForgeConfig( **minimal_config( risk_assessment={"risk_category": "high-risk"}, + # Faz 28 F-compliance-110: safety must be enabled + # for high-risk to load at all. The auto_revert + # recommendation still fires as a warning since + # ``evaluation.auto_revert`` defaults to False. + evaluation={"safety": {"enabled": True}}, ) ) assert "high-risk" in caplog.text assert "auto_revert" in caplog.text + def test_high_risk_safety_disabled_raises_config_error(self, minimal_config): + """F-compliance-110 regression: high-risk + safety.enabled=false + must surface as ``ConfigError`` (was a warning prior to v0.5.5). + EU AI Act Article 9 risk-management evidence cannot be derived + from a disabled safety eval; operators who genuinely want a + sandboxed run must lower the risk_classification.""" + from forgelm.config import ConfigError + + with pytest.raises(ConfigError, match="evaluation.safety.enabled"): + ForgeConfig( + **minimal_config( + risk_assessment={"risk_category": "high-risk"}, + # No safety block → safety disabled by default. + ) + ) + def test_unacceptable_risk_warnings(self, caplog, minimal_config): """``unacceptable`` (Article 5) must trip the strict gate AND emit the dedicated prohibited-practices warning on top of the auto_revert @@ -137,6 +168,10 @@ def test_unacceptable_risk_warnings(self, caplog, minimal_config): ForgeConfig( **minimal_config( risk_assessment={"risk_category": "unacceptable"}, + # Faz 28 F-compliance-110: safety required for + # unacceptable too — the strict gate covers both + # high-risk and unacceptable. + evaluation={"safety": {"enabled": True}}, ) ) # Strict gate fires: same auto_revert nudge as high-risk. @@ -297,6 +332,8 @@ def test_includes_annex_iv(self, minimal_config): config = ForgeConfig( **minimal_config( compliance={"provider_name": "Corp", "system_name": "Bot", "risk_classification": "high-risk"}, + # Faz 28 F-compliance-110: high-risk requires safety enabled. + evaluation={"safety": {"enabled": True}}, ) ) manifest = generate_training_manifest(config, {"eval_loss": 0.5}) @@ -308,6 +345,8 @@ def test_includes_risk_assessment(self, minimal_config): config = ForgeConfig( **minimal_config( risk_assessment={"intended_use": "Chat", "risk_category": "high-risk"}, + # Faz 28 F-compliance-110: high-risk requires safety enabled. + evaluation={"safety": {"enabled": True}}, ) ) manifest = generate_training_manifest(config, {}) diff --git a/tests/test_integration_smoke.py b/tests/test_integration.py similarity index 96% rename from tests/test_integration_smoke.py rename to tests/test_integration.py index d4649887..117c179e 100644 --- a/tests/test_integration_smoke.py +++ b/tests/test_integration.py @@ -159,14 +159,18 @@ def test_full_config_yaml_round_trip(self, tmp_path): assert cfg.risk_assessment.foreseeable_misuse[0] == "Users may ask for medical advice" assert cfg.data.governance.collection_method == "Manual curation by domain experts" - def test_high_risk_without_safety_warns(self, caplog): - import logging + def test_high_risk_without_safety_raises_config_error(self): + """Wave 3 / Faz 28 F-compliance-110: high-risk classification with + safety eval disabled is now a hard ``ConfigError`` (was a warning + prior to v0.5.5). EU AI Act Article 9 risk-management evidence + cannot be derived from a disabled safety eval; operators who + genuinely want a sandboxed run must lower the risk_classification.""" + from forgelm.config import ConfigError data = _full_config() del data["evaluation"]["safety"] - with caplog.at_level(logging.WARNING, logger="forgelm.config"): + with pytest.raises(ConfigError, match="evaluation.safety.enabled"): ForgeConfig(**data) - assert "High-risk AI" in caplog.text class TestDryRunWithCompliance: diff --git a/tests/test_reverse_pii.py b/tests/test_reverse_pii.py new file mode 100644 index 00000000..54d37491 --- /dev/null +++ b/tests/test_reverse_pii.py @@ -0,0 +1,517 @@ +"""Phase 38 — `forgelm reverse-pii` GDPR Article 15 right-of-access. + +Tests run torch-free + extra-free. All scenarios use synthetic JSONL +fixtures so the suite stays deterministic + offline. + +Test coverage maps to closure-plan §Faz 38 acceptance: + +1. Plaintext residual scan — identifier verbatim found / not found + in glob expansion. +2. Hash-mask scan — SHA256(salt + identifier) recomputed via the + same per-output-dir salt as `forgelm purge`; matches the masked + corpus. +3. Audit-event emission — `data.access_request_query` written; + identifier hashed (NEVER raw). +4. Glob expansion — multiple files scanned in deterministic order; + per-file match count surfaced. +5. Failure paths — empty query / unparseable custom regex / empty + glob expansion → exit code 1; mid-scan I/O failure → exit 2. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import unittest.mock as _mock +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def _seed_corpus(corpus_path: Path, rows: list[dict]) -> None: + corpus_path.parent.mkdir(parents=True, exist_ok=True) + with open(corpus_path, "w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row) + "\n") + + +def _read_audit_events(audit_log_path: Path) -> list[dict]: + events: list[dict] = [] + with open(audit_log_path, "r", encoding="utf-8") as fh: + for raw in fh: + line = raw.strip() + if not line: + continue + events.append(json.loads(line)) + return events + + +def _build_args( + *, + query: str | None = None, + type: str | None = "custom", + salt_source: str | None = None, + files: list[str] | None = None, + output_dir: str | None = None, + audit_dir: str | None = None, +) -> SimpleNamespace: + """Strict argparse-shaped namespace; misspelled attrs raise.""" + return SimpleNamespace( + query=query, + type=type, + salt_source=salt_source, + files=files, + output_dir=output_dir, + audit_dir=audit_dir, + ) + + +@pytest.fixture(autouse=True) +def _set_operator_env(monkeypatch): + """Every test runs with a deterministic FORGELM_OPERATOR so AuditLogger + does not refuse to start on shared CI runners. Also clears any + inherited FORGELM_AUDIT_SECRET from a prior test in the same + invocation so per_dir / env_var modes are predictable.""" + monkeypatch.setenv("FORGELM_OPERATOR", "test-operator@reverse-pii-test") + monkeypatch.delenv("FORGELM_AUDIT_SECRET", raising=False) + + +# --------------------------------------------------------------------------- +# §1 Test — Plaintext residual scan: verbatim match found +# --------------------------------------------------------------------------- + + +class TestPlaintextResidualMatch: + def test_email_residual_found_with_line_and_snippet(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "train.jsonl" + _seed_corpus( + corpus, + [ + {"id": "row-A", "text": "Hello, my name is Alice"}, + {"id": "row-B", "text": "Contact me at alice@example.com please"}, + {"id": "row-C", "text": "Carol said hi"}, + ], + ) + args = _build_args( + query="alice@example.com", + type="email", + files=[str(corpus)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is True + assert payload["match_count"] == 1 + assert payload["scan_mode"] == "plaintext" + assert payload["identifier_type"] == "email" + # Single match is on row-B (line 2, 1-based). + match = payload["matches"][0] + assert match["file"] == os.path.abspath(str(corpus)) + assert match["line"] == 2 + assert "alice@example.com" in match["snippet"] + # files_scanned reports the per-file match count too. + assert payload["files_scanned"] == [{"path": os.path.abspath(str(corpus)), "match_count": 1}] + + def test_no_residual_found_returns_empty_match_list(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "row-A", "text": "no PII here"}]) + args = _build_args( + query="someone-else@example.com", + type="email", + files=[str(corpus)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["match_count"] == 0 + assert payload["matches"] == [] + # files_scanned still records that we walked the file. + assert len(payload["files_scanned"]) == 1 + assert payload["files_scanned"][0]["match_count"] == 0 + + +# --------------------------------------------------------------------------- +# §2 Test — Hash-mask scan: SHA256(salt + identifier) found in masked corpus +# --------------------------------------------------------------------------- + + +class TestHashMaskScan: + def test_per_dir_hash_matches_masked_row(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._purge import _hash_target_id, _resolve_salt + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + # Pre-create the salt file; compute the digest the operator + # would have written to the masked corpus. + salt, source = _resolve_salt(str(tmp_path)) + assert source == "per_dir" + masked_digest = _hash_target_id("alice@example.com", salt) + + corpus = tmp_path / "masked.jsonl" + _seed_corpus( + corpus, + [ + {"id": "row-A", "text": "first record, no PII"}, + {"id": "row-B", "text": f"hashed identifier embedded: {masked_digest}"}, + {"id": "row-C", "text": "control row"}, + ], + ) + args = _build_args( + query="alice@example.com", + type="email", + salt_source="per_dir", + files=[str(corpus)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["scan_mode"] == "hash" + assert payload["match_count"] == 1 + # The match snippet contains the hash digest, NOT the raw email. + snippet = payload["matches"][0]["snippet"] + assert masked_digest in snippet + assert "alice@example.com" not in snippet + + def test_env_var_salt_source_requires_secret_env(self, tmp_path: Path, capsys, monkeypatch) -> None: + """``--salt-source env_var`` without ``FORGELM_AUDIT_SECRET`` + must refuse rather than silently scan with the wrong salt.""" + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + # Explicitly clear (autouse fixture also does this; belt + braces). + monkeypatch.delenv("FORGELM_AUDIT_SECRET", raising=False) + corpus = tmp_path / "masked.jsonl" + _seed_corpus(corpus, [{"id": "row-A", "text": "x"}]) + args = _build_args( + query="alice@example.com", + type="email", + salt_source="env_var", + files=[str(corpus)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1, "env_var without FORGELM_AUDIT_SECRET must surface as EXIT_CONFIG_ERROR" + + +# --------------------------------------------------------------------------- +# §3 Test — Audit event emitted; identifier hashed (NEVER raw) +# --------------------------------------------------------------------------- + + +class TestAuditEventDoesNotLeakIdentifier: + def test_access_request_event_carries_hashed_query_only(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + audit_dir = tmp_path / "audit" + audit_dir.mkdir() + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "row-A", "text": "alice@example.com is here"}]) + + args = _build_args( + query="alice@example.com", + type="email", + files=[str(corpus)], + output_dir=str(tmp_path), + audit_dir=str(audit_dir), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + + events = _read_audit_events(audit_dir / "audit_log.jsonl") + access_events = [e for e in events if e["event"] == "data.access_request_query"] + assert len(access_events) == 1 + evt = access_events[0] + # Hashed identifier present; raw email NOT. + expected_hash = hashlib.sha256(b"alice@example.com").hexdigest() + assert evt["query_hash"] == expected_hash + # Critical: raw identifier appears NOWHERE in the event payload. + as_str = json.dumps(evt) + assert "alice@example.com" not in as_str, f"audit event leaked raw identifier:\n{as_str}" + assert evt["identifier_type"] == "email" + assert evt["scan_mode"] == "plaintext" + assert evt["match_count"] == 1 + + +# --------------------------------------------------------------------------- +# §4 Test — Glob expansion: multiple files, deterministic ordering +# --------------------------------------------------------------------------- + + +class TestGlobExpansion: + def test_glob_pattern_expands_to_multiple_files_with_per_file_counts(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + # Plant three corpora; identifier appears in two of them. + for name, rows in [ + ("a.jsonl", [{"id": "1", "text": "alice@example.com here"}]), + ("b.jsonl", [{"id": "1", "text": "no PII"}]), + ("c.jsonl", [{"id": "1", "text": "alice@example.com again"}]), + ]: + _seed_corpus(tmp_path / name, rows) + + args = _build_args( + query="alice@example.com", + type="email", + files=[str(tmp_path / "*.jsonl")], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + # 3 files scanned; 2 matches total (a + c each contribute 1). + assert len(payload["files_scanned"]) == 3 + assert payload["match_count"] == 2 + per_file = {fs["path"]: fs["match_count"] for fs in payload["files_scanned"]} + assert per_file[os.path.abspath(str(tmp_path / "a.jsonl"))] == 1 + assert per_file[os.path.abspath(str(tmp_path / "b.jsonl"))] == 0 + assert per_file[os.path.abspath(str(tmp_path / "c.jsonl"))] == 1 + + def test_recursive_glob_descends_into_subdirs(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + nested = tmp_path / "year-2026" / "q2" + nested.mkdir(parents=True) + _seed_corpus(nested / "train.jsonl", [{"id": "1", "text": "ali@example.com here"}]) + + args = _build_args( + query="ali@example.com", + type="email", + files=[str(tmp_path / "**" / "train.jsonl")], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["match_count"] == 1 + + +# --------------------------------------------------------------------------- +# §5 Test — Failure paths +# --------------------------------------------------------------------------- + + +class TestFailurePaths: + def test_empty_query_exits_config_error(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "1", "text": "x"}]) + args = _build_args(query=" ", files=[str(corpus)], output_dir=str(tmp_path)) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1 + + def test_unparseable_custom_regex_exits_config_error(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "1", "text": "x"}]) + # Unbalanced bracket; re.compile will raise. + args = _build_args(query="[invalid", type="custom", files=[str(corpus)], output_dir=str(tmp_path)) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1 + + def test_unknown_identifier_type_rejected(self, tmp_path: Path) -> None: + # The argparse layer also catches this, but the dispatcher + # validates as belt-and-suspenders for direct library callers. + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "1", "text": "x"}]) + args = _build_args(query="x", type="bogus_type", files=[str(corpus)], output_dir=str(tmp_path)) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1 + + def test_empty_glob_exits_config_error(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + args = _build_args( + query="alice@example.com", + type="email", + files=[str(tmp_path / "nonexistent" / "*.jsonl")], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1 + + def test_no_files_argument_exits_config_error(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + args = _build_args(query="x", type="email", files=[], output_dir=str(tmp_path)) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1 + + def test_mid_scan_io_failure_writes_failed_audit_event(self, tmp_path: Path, monkeypatch) -> None: + from forgelm.cli.subcommands import _reverse_pii + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "1", "text": "alice@example.com"}]) + audit_dir = tmp_path / "audit" + audit_dir.mkdir() + + # Spy on _scan_file → raise OSError on the second call (the + # second file gets opened and the read fails mid-line). + original_scan = _reverse_pii._scan_file + call_count = [0] + + def _flaky_scan(path: str, pattern): + call_count[0] += 1 + if call_count[0] >= 2: + raise OSError("simulated mid-scan I/O failure") + return original_scan(path, pattern) + + monkeypatch.setattr(_reverse_pii, "_scan_file", _flaky_scan) + + # Two files so the second invocation exercises the failure path. + corpus2 = tmp_path / "train2.jsonl" + _seed_corpus(corpus2, [{"id": "2", "text": "bob@example.com"}]) + + args = _build_args( + query="alice@example.com", + type="email", + files=[str(corpus), str(corpus2)], + output_dir=str(tmp_path), + audit_dir=str(audit_dir), + ) + with pytest.raises(SystemExit) as ei: + _reverse_pii._run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 2 + + events = _read_audit_events(audit_dir / "audit_log.jsonl") + access_events = [e for e in events if e["event"] == "data.access_request_query"] + assert len(access_events) == 1 + evt = access_events[0] + assert evt["error_class"] == "OSError" + assert "simulated mid-scan I/O failure" in evt["error_message"] + + +# --------------------------------------------------------------------------- +# §6 Test — Snippet truncation defends unbounded log spam +# --------------------------------------------------------------------------- + + +class TestSnippetTruncation: + def test_long_line_centre_truncated(self, tmp_path: Path, capsys) -> None: + from forgelm.cli.subcommands._reverse_pii import _SNIPPET_MAX_CHARS, _run_reverse_pii_cmd + + # Construct a line longer than the snippet budget with the + # match in the middle. + prefix = "x" * 200 + suffix = "y" * 200 + corpus = tmp_path / "train.jsonl" + with open(corpus, "w", encoding="utf-8") as fh: + fh.write(f"{prefix} alice@example.com {suffix}\n") + + args = _build_args( + query="alice@example.com", + type="email", + files=[str(corpus)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + snippet = payload["matches"][0]["snippet"] + assert len(snippet) <= _SNIPPET_MAX_CHARS + # Ellipsis marks where the middle was elided. + assert "…" in snippet + + +# --------------------------------------------------------------------------- +# §7 Test — Facade re-exports + parser wiring + dispatch +# --------------------------------------------------------------------------- + + +class TestReversePiiFacade: + def test_facade_re_exports_dispatcher_and_helpers(self) -> None: + from forgelm import cli as _cli_facade + + for name in ( + "_run_reverse_pii_cmd", + "_validate_query", + "_validate_identifier_type", + "_resolve_files", + "_scan_file", + "_truncate_snippet", + "_hash_for_audit", + "_emit_reverse_pii_result", + ): + assert hasattr(_cli_facade, name), f"forgelm.cli must re-export {name!r}" + + def test_parser_registers_reverse_pii_subcommand(self) -> None: + """`forgelm reverse-pii --help` must succeed without --config.""" + from forgelm.cli._parser import parse_args + + # parse_args reads sys.argv; mock it. + with _mock.patch.object(sys, "argv", ["forgelm", "reverse-pii", "--help"]): + with pytest.raises(SystemExit) as ei: + parse_args() + # argparse exits 0 on --help. + assert ei.value.code == 0 + + def test_parser_accepts_minimum_arguments(self, tmp_path: Path) -> None: + from forgelm.cli._parser import parse_args + + corpus = tmp_path / "train.jsonl" + corpus.write_text("{}\n") + with _mock.patch.object( + sys, + "argv", + ["forgelm", "reverse-pii", "--query", "alice@example.com", str(corpus)], + ): + args = parse_args() + assert args.command == "reverse-pii" + assert args.query == "alice@example.com" + assert args.files == [str(corpus)] + # Defaults round-trip. + assert args.type == "custom" + assert args.salt_source is None + + def test_dispatch_table_registers_reverse_pii(self) -> None: + """The dispatcher table must include the reverse-pii row so the + CLI binary actually routes the subcommand to its handler.""" + from forgelm.cli._dispatch import _dispatch_subcommand + + # Exercise the dispatch via a monkeypatch on the facade so we + # don't need to actually run the scanner. + called = {} + + def _fake_dispatch(args, output_format): + called["command"] = "reverse-pii" + called["query"] = getattr(args, "query", None) + sys.exit(0) + + from forgelm import cli as _cli_facade + + original = _cli_facade._run_reverse_pii_cmd + _cli_facade._run_reverse_pii_cmd = _fake_dispatch + try: + args = _build_args(query="x", type="email", files=["dummy"]) + args.command = "reverse-pii" + args.output_format = "json" + with pytest.raises(SystemExit) as ei: + _dispatch_subcommand("reverse-pii", args) + assert ei.value.code == 0 + assert called == {"command": "reverse-pii", "query": "x"} + finally: + _cli_facade._run_reverse_pii_cmd = original diff --git a/tools/check_bilingual_parity.py b/tools/check_bilingual_parity.py new file mode 100644 index 00000000..0f92248f --- /dev/null +++ b/tools/check_bilingual_parity.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Phase 24 — bilingual EN ↔ TR doc structural parity guard. + +The project's localisation standard +(:doc:`docs/standards/localization.md`) requires that every TR mirror +of a user-facing doc carry the same H1 / H2 / H3 / H4 spine as the EN +original — same count, same nesting depth, same sequence. Translated +text differs by definition; structural drift does not. + +Replaces the inline H2-only count check at +``.github/workflows/ci.yml:197-222`` with an extended H2 + H3 + H4 +diff that catches the more subtle drifts the previous check missed: + +- Section added in EN but not TR (or vice versa). +- H3 demoted to H4 in one mirror but not the other. +- Reordering between mirrors (operator reads them side-by-side and + loses their place). + +Usage: + + # Advisory mode — report drift, exit 0. + python3 tools/check_bilingual_parity.py + + # Strict mode (CI gate) — exit 1 on any drift. + python3 tools/check_bilingual_parity.py --strict + + # Restrict to one pair (useful while editing). + python3 tools/check_bilingual_parity.py --only docs/guides/ingestion.md + +The tool is AST-free and Pydantic-free — it only needs ``re`` and the +filesystem, so it runs in the same lint job as ``ruff`` without any +optional extra. + +Pair registry: see ``_PAIRS`` below. A new bilingual doc gets added +in two places: the registry here AND the +``check_bilingual_parity.py`` invocation in CI; the registry's +docstring documents both. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Optional, Sequence, Tuple + +# Pair registry. Tuples of (EN path, TR path) keyed off the +# repository root. When a new mirrored doc is added, register it +# here; CI will then pick it up automatically. Order is alphabetical +# for predictable reporting. +_PAIRS: Tuple[Tuple[str, str], ...] = ( + # docs/guides/ + ("docs/guides/data_audit.md", "docs/guides/data_audit-tr.md"), + ("docs/guides/gdpr_erasure.md", "docs/guides/gdpr_erasure-tr.md"), + ("docs/guides/ingestion.md", "docs/guides/ingestion-tr.md"), + # docs/reference/ + ("docs/reference/architecture.md", "docs/reference/architecture-tr.md"), + ("docs/reference/configuration.md", "docs/reference/configuration-tr.md"), + ("docs/reference/data_preparation.md", "docs/reference/data_preparation-tr.md"), + ("docs/reference/distributed_training.md", "docs/reference/distributed_training-tr.md"), + ("docs/reference/usage.md", "docs/reference/usage-tr.md"), +) + +# Match a Markdown ATX heading: 1-6 leading hashes, then a single +# space, then the heading text. We deliberately exclude setext +# headings (``===``/``---`` underlines) — the project's docs use ATX +# exclusively per ``docs/standards/documentation.md``, and a setext +# heading in a mirror would itself be a lint finding. +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$") + +# Code fence opening / closing — heading-like lines inside a code +# block are content, not document structure. We track ``open / close`` +# state via a simple boolean toggle on triple-backtick or triple-tilde. +_FENCE_RE = re.compile(r"^(```+|~~~+)") + + +@dataclass(frozen=True) +class Heading: + """One heading extracted from a markdown document.""" + + level: int + text: str + line: int + + def signature(self) -> str: + """Stable signature for parity reporting. + + We do NOT compare heading *text* across the mirror (the whole + point of localisation is that the text differs). We do + compare nesting depth — that's the structural invariant the + translator must preserve. + """ + return f"H{self.level}" + + +def extract_headings(path: Path) -> List[Heading]: + """Return every ATX heading in ``path``, ignoring code fences. + + Quietly returns ``[]`` when the file does not exist (callers + surface that as a "missing mirror" diagnostic). Other I/O errors + propagate so the operator sees the underlying cause. + """ + if not path.is_file(): + return [] + headings: List[Heading] = [] + in_fence = False + fence_marker: Optional[str] = None + with open(path, "r", encoding="utf-8") as fh: + for line_no, raw in enumerate(fh, start=1): + line = raw.rstrip("\n") + fence_match = _FENCE_RE.match(line) + if fence_match: + marker = fence_match.group(1)[:3] # collapse runs to first three chars + if not in_fence: + in_fence = True + fence_marker = marker + elif fence_marker is not None and marker == fence_marker: + in_fence = False + fence_marker = None + continue + if in_fence: + continue + heading_match = _HEADING_RE.match(line) + if heading_match is None: + continue + level = len(heading_match.group(1)) + text = heading_match.group(2).strip() + headings.append(Heading(level=level, text=text, line=line_no)) + return headings + + +@dataclass(frozen=True) +class PairDrift: + """A single drift finding for an EN/TR doc pair.""" + + en_path: str + tr_path: str + summary: str + detail: List[str] + + def render(self) -> str: + head = f"FAIL: {self.en_path} ↔ {self.tr_path}" + body = "\n".join(f" {line}" for line in self.detail) + return f"{head}\n {self.summary}\n{body}" if body else f"{head}\n {self.summary}" + + +def diff_pair(en_path: Path, tr_path: Path, *, levels: Sequence[int] = (2, 3, 4)) -> Optional[PairDrift]: + """Return a :class:`PairDrift` when EN/TR structural spines diverge. + + ``levels`` selects which heading depths to compare; defaults to H2 + + H3 + H4 because that's the granularity the translation standard + pins (H1 is the doc title and is captured by the file naming + convention, deeper than H4 is interpreted as content not + structure). + + Returns ``None`` when both mirrors have identical level sequences + at the selected depths. + """ + if not en_path.is_file(): + return PairDrift( + en_path=str(en_path), + tr_path=str(tr_path), + summary=f"EN file is missing: {en_path}", + detail=[], + ) + if not tr_path.is_file(): + return PairDrift( + en_path=str(en_path), + tr_path=str(tr_path), + summary=f"TR mirror is missing: {tr_path}", + detail=[], + ) + + en_headings = [h for h in extract_headings(en_path) if h.level in levels] + tr_headings = [h for h in extract_headings(tr_path) if h.level in levels] + + en_signature = [h.signature() for h in en_headings] + tr_signature = [h.signature() for h in tr_headings] + + if en_signature == tr_signature: + return None + + detail: List[str] = [] + # Per-level count summary first — fast scan for the operator. + for level in levels: + en_count = sum(1 for h in en_headings if h.level == level) + tr_count = sum(1 for h in tr_headings if h.level == level) + if en_count != tr_count: + detail.append(f"H{level}: EN={en_count} TR={tr_count} Δ={en_count - tr_count:+d}") + if not detail: + # Counts match but the *sequence* of levels differs. + detail.append( + "Heading counts match but ordering differs — open both files side-by-side; " + "a section was reordered or its depth changed." + ) + + # Per-line diff: cap at the first 12 mismatches to keep the report + # readable. Operators editing a single drift do not need to see + # every downstream consequence. + diff_lines = _signature_diff_lines(en_headings, tr_headings) + if diff_lines: + detail.append("First mismatched headings (EN | TR):") + detail.extend(f" {line}" for line in diff_lines[:12]) + if len(diff_lines) > 12: + detail.append(f" …({len(diff_lines) - 12} more, fix the first ones first)") + + summary = f"{len(diff_lines)} structural drift(s) across H{','.join(str(level) for level in levels)}" + return PairDrift( + en_path=str(en_path), + tr_path=str(tr_path), + summary=summary, + detail=detail, + ) + + +def _signature_diff_lines(en: List[Heading], tr: List[Heading]) -> List[str]: + """Return human-readable per-pair lines for headings whose level + differs at the same index, plus orphan tails on either side.""" + lines: List[str] = [] + max_len = max(len(en), len(tr)) + for i in range(max_len): + en_h = en[i] if i < len(en) else None + tr_h = tr[i] if i < len(tr) else None + if en_h is None: + assert tr_h is not None # max_len bound ensures at least one is non-None + lines.append(f"(none) | TR:{tr_h.signature()} L{tr_h.line} {tr_h.text!r}") + continue + if tr_h is None: + lines.append(f"EN:{en_h.signature()} L{en_h.line} {en_h.text!r} | (none)") + continue + if en_h.level == tr_h.level: + continue + lines.append( + f"EN:{en_h.signature()} L{en_h.line} {en_h.text!r} | TR:{tr_h.signature()} L{tr_h.line} {tr_h.text!r}" + ) + return lines + + +def scan_pairs( + pairs: Iterable[Tuple[str, str]], + *, + repo_root: Path, + levels: Sequence[int] = (2, 3, 4), +) -> List[PairDrift]: + """Run :func:`diff_pair` over every registered pair; return drifts only.""" + drifts: List[PairDrift] = [] + for en_rel, tr_rel in pairs: + drift = diff_pair(repo_root / en_rel, repo_root / tr_rel, levels=levels) + if drift is not None: + drifts.append(drift) + return drifts + + +def _resolve_pairs(only: Optional[str]) -> Tuple[Tuple[str, str], ...]: + """When ``--only`` is supplied, return just that pair (matched on + either side); otherwise return the full registry.""" + if only is None: + return _PAIRS + target = os.path.normpath(only) + for en, tr in _PAIRS: + if os.path.normpath(en) == target or os.path.normpath(tr) == target: + return ((en, tr),) + raise SystemExit( + f"--only {only!r} did not match any registered pair. " + f"Choose one of:\n " + "\n ".join(f"{en} ↔ {tr}" for en, tr in _PAIRS) + ) + + +def _format_summary(drifts: List[PairDrift], total_pairs: int) -> str: + if not drifts: + return f"OK: all {total_pairs} bilingual doc pair(s) carry the same H2/H3/H4 spine." + return ( + f"FAIL: {len(drifts)} of {total_pairs} bilingual doc pair(s) drifted " + "(see per-pair details above). Edit the TR mirror so the H2/H3/H4 " + "sequence matches the EN original; translated text may differ but " + "structural depth + ordering must not." + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Phase 24 — verify EN/TR bilingual doc structural parity (H2 + H3 + H4).", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit 1 on any structural drift (CI gate). Without --strict the tool reports findings but exits 0 (advisory mode).", + ) + parser.add_argument( + "--only", + type=str, + default=None, + metavar="PATH", + help="Restrict the scan to one EN or TR file (matched against the registered pairs).", + ) + parser.add_argument( + "--levels", + type=str, + default="2,3,4", + metavar="CSV", + help="Comma-separated heading levels to compare (default: `2,3,4`). Use `2` to mirror the legacy CI behaviour.", + ) + parser.add_argument( + "--repo-root", + type=str, + default=None, + metavar="DIR", + help="Repository root (default: derived from this script's path).", + ) + args = parser.parse_args(argv) + + try: + levels = tuple(int(part.strip()) for part in args.levels.split(",") if part.strip()) + except ValueError: + print( + f"--levels {args.levels!r} is not a comma-separated list of integers.", + file=sys.stderr, + ) + return 1 + if not levels or any(level < 1 or level > 6 for level in levels): + print( + f"--levels values must be 1-6 ATX heading depths; got {levels!r}.", + file=sys.stderr, + ) + return 1 + + repo_root = Path(args.repo_root).resolve() if args.repo_root else Path(__file__).resolve().parent.parent + pairs = _resolve_pairs(args.only) + drifts = scan_pairs(pairs, repo_root=repo_root, levels=levels) + + for drift in drifts: + print(drift.render()) + print() + print(_format_summary(drifts, len(pairs))) + + if args.strict and drifts: + return 1 + return 0 + + +__all__ = [ + "Heading", + "PairDrift", + "_PAIRS", + "diff_pair", + "extract_headings", + "main", + "scan_pairs", +] + + +if __name__ == "__main__": + sys.exit(main()) From 95e2bf86bb290e910975ffaf69e932e9183052a8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 17:24:13 +0300 Subject: [PATCH 02/11] docs(wave-3): multi-agent code review prompt for closure/wave3-integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-contained 4-agent review prompt at HEAD ``61d74fa``. 4 specialty agents: - Code-correctness — Phase 38 reverse-pii dispatcher defects, audit payload tracing, hash-mask round-trip with forgelm purge. - Standards-compliance — XPR-01..07 (audit vocab, exit codes, JSON envelope, regex / ReDoS, error handling, BC, bilingual parity). - Test-rigor — non-vacuous assertions, edge-case coverage, regression pinning of new contracts. - Privacy / security — reverse-pii cleartext-PII surface, audit-log leak paths, snippet truncation safety, custom-regex ReDoS exposure. Severity bar tuned: behaviour changes need migration paths; reverse-pii cleartext-leak paths are CRITICAL; F-compliance-110 (high-risk + safety-disabled raise) is the only intentional break. ``.gitignore`` allowlist extended to re-include the new prompt under the otherwise-internal ``docs/analysis/code_reviews/`` tree. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 + .../code_reviews/wave3-review-prompt.md | 362 ++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 docs/analysis/code_reviews/wave3-review-prompt.md diff --git a/.gitignore b/.gitignore index 776bf97c..f216509d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ docs/analysis/code_reviews/* !docs/analysis/code_reviews/master-review-opus-202604300906.md !docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md !docs/analysis/code_reviews/wave2b-final-merge-review-prompt.md +!docs/analysis/code_reviews/wave3-review-prompt.md # Generated by tools/build_usermanuals.py — produced from # docs/usermanuals//**/*.md during the CI deploy. Not committed; diff --git a/docs/analysis/code_reviews/wave3-review-prompt.md b/docs/analysis/code_reviews/wave3-review-prompt.md new file mode 100644 index 00000000..0a3804dc --- /dev/null +++ b/docs/analysis/code_reviews/wave3-review-prompt.md @@ -0,0 +1,362 @@ +# Wave 3 — Multi-Agent Code Review Prompt (PR pending) + +> **Use this prompt verbatim** to launch the multi-agent review on the +> consolidated Wave 3 PR. The prompt is self-contained: each reviewing +> agent can pick it up cold without context from prior conversations. +> +> Wave 3 is a single integration branch consolidating three closure-plan +> phases (Faz 24 + 28 + 38). No prior absorption rounds yet — this is +> the FIRST review pass. Treat the bar accordingly: catch the issues +> now, before any operator-visible regressions land. + +--- + +## Repo & branch under review + +- **Repo:** `cemililik/ForgeLM` +- **Branch under review:** `closure/wave3-integration` +- **HEAD SHA at prompt freeze:** `61d74fa` +- **Diff scope vs `development`:** ~32 files, ~685 insertions, ~84 deletions. + Two new modules (`forgelm/cli/subcommands/_reverse_pii.py`, + `tools/check_bilingual_parity.py`), two new test modules + (`tests/test_reverse_pii.py`, `tests/test_check_bilingual_parity.py`), + one rename (`test_integration_smoke.py` → `test_integration.py`). +- **What it consolidates:** Faz 24 (bilingual TR mirror sweep + parity + CI guard), Faz 28 (curated tier 1+2 cleanup, ~10 items), Faz 38 + (`forgelm reverse-pii` GDPR Article 15 right-of-access subcommand). + +## What ForgeLM is (60 seconds) + +A **config-driven, enterprise-grade LLM fine-tuning toolkit** — YAML +in, fine-tuned model + compliance artifacts out. Built for CI/CD +pipelines, not notebooks. Six alignment paradigms +(SFT/DPO/SimPO/KTO/ORPO/GRPO), integrated safety evaluation +(Llama Guard), EU AI Act compliance artefacts (Articles 9–17 + Annex +IV), append-only audit log, opt-in human approval gate, auto-revert +on quality regression, GDPR Article 15 + 17 subject rights tooling +(this wave adds Article 15). Read [docs/product_strategy.md](../../product_strategy.md) +for the fuller background. + +Project rulebook lives under [docs/standards/](../../standards/) — read +[coding.md](../../standards/coding.md), +[error-handling.md](../../standards/error-handling.md), +[testing.md](../../standards/testing.md), +[logging-observability.md](../../standards/logging-observability.md), +[regex.md](../../standards/regex.md), +[localization.md](../../standards/localization.md), and +[code-review.md](../../standards/code-review.md) before commenting. +Project-wide guidance for AI agents is in the root `CLAUDE.md`. + +## What this PR ships + +| Phase | Type | Headline change | +|---|---|---| +| 38 | New CLI subcommand | `forgelm reverse-pii --query VALUE [--type X] [--salt-source X] JSONL_GLOB...` — GDPR Article 15 right-of-access companion to `forgelm purge`. Two scan modes (plaintext residual + hash-mask via the same per-output-dir salt as purge). New audit event `data.access_request_query` with the identifier *hashed* (Article 15 access requests must not leak the subject's data into the audit log). 18 regression tests. | +| 24 | New CI guard tool | `tools/check_bilingual_parity.py` replaces the inline H2-only check at `ci.yml:197-222` with an extended H2 + H3 + H4 structural diff. AST-free, runs in lint job, `--strict` mode for CI gate. 16 regression tests. 8 doc pairs registered. 4 doc-pair drift fixes + 4 user-manual H2 drift fixes + alignment.md phase-reference de-anchor. | +| 28 | Curated cleanup | 10 items absorbed. **Behaviour change (F-compliance-110):** high-risk + safety.enabled=false now raises `ConfigError`. **Defaults change (F-compliance-106):** webhook timeout 5s → 10s. **Log level (F-compliance-111):** missing data_audit_report.json `INFO → WARNING`. Plus M-204 `_sanitize_md_list` helper, M-205 ollama newline guard, C-54 webhook re-export drop, C-57 GRPO reward docstring honesty, F-test-011 rename. | + +## What we want from this round + +**Goal:** decide whether `closure/wave3-integration` is mergeable to +`development`. Block on real defects; do not gate on style nits or +hypothetical futures. + +Be concrete. Every finding must include: + +1. **Severity tag** — `CRITICAL` / `HIGH` / `MEDIUM` / `LOW` / `NIT`. +2. **Finding ID** — `F-W3-NN` (e.g. `F-W3-01`, `F-W3-12`). +3. **File:line citation** — `forgelm/cli/subcommands/_reverse_pii.py:142` + or a short range `:138-150`. +4. **One-paragraph reasoning** — what is wrong, what is the user-visible + consequence, and why it qualifies for that severity tier. +5. **Suggested fix** — a code or text snippet, not a vague direction. +6. **Test that would have caught it** — name the test file + a stub + of the assertion. + +Severity bar: + +- `CRITICAL` — would corrupt data, silently lose audit events, leak + PII / secrets, break the public exit-code contract (`0/1/2/3/4`), + break the `--offline` / air-gap promise, or invalidate the + audit-log hash chain. Blocks merge. +- `HIGH` — runtime crash on a documented happy path, missing test + for a documented contract, schema mismatch with `pyproject.toml` + / `config.py`, documentation claim with no code backing, or a + behaviour change that breaks 0.5.x users without a clear migration + path. Blocks merge. +- `MEDIUM` — logic bug in an off-the-happy-path branch, observability + gap, or design-doc statement that contradicts shipped code. + Should be fixed before merge but the call is the maintainer's. +- `LOW` — defensive-coding improvement, minor inconsistency, missing + edge-case test that would not have changed PR direction. +- `NIT` — naming, line wrap, comment phrasing. Note in passing + only; do not stack these. + +## Areas to scrutinise + +These are the surfaces most likely to harbour real issues. Hit them +specifically. + +### Phase 38 — `forgelm reverse-pii` (largest new code surface) + +- **Audit-event identifier hashing.** The contract is "Article 15 + access requests must not leak the subject's data into the audit + log." Walk every audit-event field — does the raw `--query` value + ever appear in the event payload (`query_hash` only, no `query`, + no `match_snippets`, etc.)? Are mid-scan failure events also + hashed? See `_run_reverse_pii_cmd` + `_hash_for_audit`. +- **Hash-mask scan correctness.** `_resolve_query_form` reuses + `forgelm purge`'s `_resolve_salt` + `_hash_target_id`. Trace a + full purge → reverse-pii cycle: same identifier, same output_dir, + default salt source — does the reverse-pii digest match the + digest purge writes into hash-masked corpora? If `--salt-source + env_var` is requested but `FORGELM_AUDIT_SECRET` is unset, what + happens? (Should refuse with `EXIT_CONFIG_ERROR`, see test + `TestHashMaskScan::test_env_var_salt_source_requires_secret_env`.) +- **Glob expansion edge cases.** `_resolve_files` uses + `glob.glob(recursive=True)`. Walk: + - Operator passes `data/*.jsonl` but the dir is empty → exit 1 + (config error). Confirm. + - Operator passes a literal path that doesn't exist → exit 1. + - Operator passes a directory (not a file) → directories are + silently skipped (the resolver filters via `os.path.isfile`). + Is that the right policy or should it fail loudly? + - Two glob patterns with overlap → does dedupe via `seen` work + correctly (absolute path comparison)? +- **Snippet truncation.** `_truncate_snippet` centre-truncates at + `_SNIPPET_MAX_CHARS = 160`. Is 160 enough? Walk a real-world + corpus row (~500 chars typical) — does the operator see enough + context to verify the hit? Could the truncation ellipsis (`…`) + itself be an issue (UTF-8 byte count vs char count)? +- **Custom regex injection.** `--type custom` interprets the + `--query` as a Python regex. An adversarial query could trigger + a ReDoS (catastrophic backtracking). Per + [docs/standards/regex.md], compiled patterns should be reviewed + for backtracking exposure. The dispatcher does not enforce a + timeout on `pattern.search(line)` — is that acceptable? +- **Audit dir resolution.** When the operator does NOT pass + `--audit-dir`, the dispatcher uses `--output-dir`, which itself + defaults to the parent of the first resolved corpus file. Walk: + if the operator scans `/etc/passwd*.jsonl` (an absurd test case), + the audit log lands at `/etc/audit_log.jsonl` — that's a permission + failure or worse. Is the default safe? Should it require + explicit `--output-dir`? +- **Facade re-exports.** `forgelm.cli.__init__.py` re-exports 10 + names from `_reverse_pii`. Is the underscore-prefix surface + consistent with other subcommands? Does any of the re-exports + pull heavy deps at import time (lazy-import discipline per Phase + 19)? See `tests/test_library_api.py::TestLazyImportDiscipline` + for the pattern. + +### Phase 24 — `tools/check_bilingual_parity.py` (new CI gate) + +- **Heading recognition correctness.** `extract_headings` skips + fenced code blocks via a simple boolean toggle. Walk: + - Triple-tilde fences (`~~~`) — handled? + - Nested fences (a `~~~` block containing ` ``` ` content) — + does the toggle stay correct? + - A code block opened but never closed — does the rest of the + file get treated as code (false negatives) or does the parser + recover? + - Setext headings (`===` / `---` underlines) — explicitly NOT + matched. Confirm. +- **Pair registry maintenance.** `_PAIRS` is an explicit tuple. + When a new bilingual doc is added, the registry must be updated. + Is there a CI check that catches a new `*-tr.md` file added without + a registry entry? (If not, the strict mode would silently miss + drift on the new pair.) +- **Strict-mode exit code.** Returns 0 on clean, 1 on drift. Per + [docs/standards/error-handling.md], CI gate tools may use exit 1 + for "violations found" — but the script lives in `tools/`, not + the `forgelm` CLI, so the public 0/1/2/3/4 contract does not + apply. Confirm the script's exit-code contract is documented + *somewhere* (e.g., the SKILL.md or release.md). +- **Live-repo smoke test.** `tests/test_check_bilingual_parity.py:: + TestCanonicalRepoPasses::test_repository_pairs_pass_strict` runs + the tool against the live filesystem. Is that fragile (a future + PR that drifts a mirror would fail this test in addition to the + CI gate, as a redundancy)? Or does the redundancy cause double- + reporting that confuses operators? + +### Phase 28 — Behaviour change (F-compliance-110) + +- **Migration impact.** Operators with `risk_classification: + high-risk` + `safety.enabled: false` previously got a warning; + now they get a hard `ConfigError`. Walk: + - Is there a clear migration path documented? (CHANGELOG entry + + release note + the error message itself.) + - Could a sandboxed-test config legitimately need this combination? + What's the documented escape hatch (lower the risk_classification? + enable safety eval?)? + - Are existing fixtures / templates (`config_template.yaml`, + `forgelm/templates/*.yaml`) compatible with the new contract? +- **Test coverage.** The new `test_high_risk_safety_disabled_raises_config_error` + pins the breaking-change contract. Does it also cover the + `unacceptable` tier (Article 5 prohibited practices)? +- **Webhook timeout default change.** 5s → 10s could affect + performance benchmarks or long-tail Slack/Teams gateway timing. + Is the new default backward-compatible (operators who explicitly + set `timeout: 5` continue to get 5s)? +- **Audit-report log level (F-compliance-111).** INFO → WARNING. + Could this generate noise in CI runs that don't actually need + the data_audit_report (e.g. quickstart smoke tests)? Is the + warning suppressible via `--quiet`? + +### Cross-phase (XPR) checks + +- **XPR-01 Audit-event vocabulary.** `data.access_request_query` + is the new event. Verify: + - Catalogue entries (`docs/reference/audit_event_catalog.md` + + `-tr.md`) match the actual emit payload. + - Every emit site of the event uses the centralised + `_EVT_ACCESS_REQUEST_QUERY` constant. +- **XPR-02 Exit-code contract.** Every `sys.exit(N)` in + `_reverse_pii.py` resolves to 0/1/2 per + [docs/standards/error-handling.md]. Confirm no literal integer + exits. +- **XPR-03 JSON envelope schema.** The reverse-pii success envelope + documented at + `docs/usermanuals/{en,tr}/reference/json-output.md` matches the + actual emitted shape (`{"success": true, "query_hash", ...}`). +- **XPR-04 Documentation drift.** Every CLI flag of `reverse-pii` + appears in: + - `docs/usermanuals/{en,tr}/reference/cli.md` (top-level table). + - `docs/usermanuals/{en,tr}/reference/json-output.md` (envelope schema). + - `docs/guides/gdpr_erasure.md` + `-tr.md` (operator guide). + - `forgelm/cli/_parser.py` epilog. +- **XPR-05 Test rigour.** Scan the new test files for vacuous + assertions (`assert True`, `assert payload`, asserting only on + fixture's own shape). None of the new tests should pass on a + broken implementation. +- **XPR-06 Backwards compatibility.** F-compliance-110 is a hard + break. Are there other accidental BC breaks? Walk: + - Removed `_is_private_destination` re-export from + `forgelm.webhook` — confirm no test or downstream import was + missed. + - Renamed `test_integration_smoke.py` → `test_integration.py` — + confirm no CI workflow matches against the old name. +- **XPR-07 Bilingual mirror parity.** `tools/check_bilingual_parity.py + --strict` passes against the live tree. Does it report ALL drifts + (no silent passes from a missing pair entry)? Spot-check a known + pair. + +## Ground rules + +- **Read the relevant standard before commenting.** The project + [docs/standards/](../../standards/) directory is the rulebook. +- **Cite line numbers.** If you cannot point at a line, the finding + is not actionable. +- **Distinguish the tool from the contract.** A bug in + `check_bilingual_parity.py` is a tool defect. A drift the tool + caught is a doc defect. Different fixes, different urgency. +- **No phantom severity inflation.** A doc typo is `NIT`. A + user-visible CLI bug on the happy path is `HIGH` minimum. +- **No re-litigating closed scope.** "Should ForgeLM also do X" is + not a finding. Stay in the diff. +- **No suggestions to add a Web UI / GUI / inference engine / + pretraining pipeline.** See + [docs/marketing/strategy/05-yapmayacaklarimiz.md] — these are out + of scope by design (read root `CLAUDE.md` "What ForgeLM is not"). +- **Behaviour changes need migration paths.** F-compliance-110 is + the only intentional break in this PR; if you flag another, name + it explicitly. +- **Reverse-pii is a privacy-sensitive surface.** The tool reads + cleartext PII from disk and writes hashed identifiers to the + audit chain. Treat any path where the cleartext can leak into + the audit log as `CRITICAL`. + +## Required deliverable structure + +Each reviewing agent returns a single Markdown report with this +skeleton: + +```markdown +# Wave 3 — Review of `closure/wave3-integration` + +## Summary +- Verdict: [Block / Conditional / Approve] +- CRITICAL: N · HIGH: N · MEDIUM: N · LOW: N · NIT: N +- One-sentence headline of the highest-severity finding. + +## Findings + +### F-W3-NN · · +- **File:** `path/to/file.py:LINE` (or `:LINE-RANGE`) +- **What's wrong:** <1-3 sentences — concrete defect, not vague concern> +- **User-visible consequence:** +- **Suggested fix:** +- **Regression test:** + +(repeat per finding, ordered: CRITICAL → HIGH → MEDIUM → LOW → NIT) + +## Cross-phase observations +(F-XPR-NN entries, same structure) + +## Verified absorptions +List anything the PR claims to do that you confirmed is actually +done. E.g.: +- F-W3-Audit [verified] — raw `--query` does not appear in any + field of `data.access_request_query` payload (walked the emit + site at `_run_reverse_pii_cmd:182-200`). +- F-W3-Glob [verified] — empty glob expansion exits 1 (regression + at `tests/test_reverse_pii.py::TestFailurePaths::test_empty_glob_exits_config_error`). + +## What this report deliberately did not cover +(Out-of-scope items the agent looked at and declined to flag, with +one sentence each. This keeps later reviewers from re-treading the +same ground.) + +## Merge recommendation +- One paragraph: "Ship as-is" / "Ship after addressing F-W3-NN" / + "Block — see CRITICAL findings." This is the agent's call. The + maintainer arbitrates between agents. +``` + +## How to launch + +Spawn at minimum these agents in parallel: + +1. **Code-correctness agent** — focuses on Phase 38 (`_reverse_pii.py`) + defects. Read the dispatcher end-to-end. Trace the audit-event + payload. Walk every error path with paper-and-pencil. Verify + the hash-mask path round-trips against `forgelm purge`'s salt. +2. **Standards-compliance agent** — focuses on cross-phase checks + XPR-01 through XPR-07 against + [docs/standards/](../../standards/). Audit-event vocabulary, + exit codes, JSON envelope schema, regex / ReDoS exposure, + error-handling discipline. Bilingual parity (Faz 24) is the + localisation standard's domain. +3. **Test-rigor agent** — focuses on whether the tests in + `test_reverse_pii.py` (18 cases), `test_check_bilingual_parity.py` + (16 cases), and the modified `test_eu_ai_act.py` / + `test_compliance.py` actually verify what they claim to. Hunt + for vacuous assertions and missing edge-case coverage on the + new contracts. +4. **Privacy / security agent** — focuses on the + `forgelm reverse-pii` surface specifically. This is a tool that + reads cleartext PII from operator-supplied corpora. Walk every + code path where the raw query could leak to disk (audit log, + stdout, stderr, JSON envelope, log records). Walk the snippet + truncation: could a malformed UTF-8 corpus + centre-truncation + produce invalid output? Walk the custom-regex path: ReDoS + exposure? + +Each agent gets this prompt verbatim plus a one-line `Focus:` +directive naming its specialty. Agents must not coordinate; the +maintainer deduplicates after. + +## Delivery + +- Agents return their reports in + `docs/analysis/code_reviews/wave3-.md`. +- The maintainer (Cemil) merges findings, dedups, and posts the + unified list on the PR as a top-level comment. +- Block / Conditional / Approve gating decision rests with the + maintainer after consolidation, **not** with any individual agent. +- After this round, surviving findings will be addressed in + absorption commits, then the branch will be merged to + `development`. + +--- + +*Prompt frozen 2026-05-05 against `closure/wave3-integration` HEAD `61d74fa`.* From 5491572023da2c1ba08d377df3e16562ddf82c3a Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 18:05:57 +0300 Subject: [PATCH 03/11] fix(wave-3-followup): absorb 4-agent review + 6 inline PR comment findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convergent CRITICAL/HIGH absorptions across the 4 reviewer reports (code-correctness, privacy-security, standards, test-rigor) on the Phase 38 reverse-pii surface: * F-W3-01 / F-W3-PS-02 / F-W3S-01 — audit fail-closed. ConfigError still skips with a WARNING (was DEBUG); every other exception class refuses with EXIT_TRAINING_ERROR naming the audit dir. Closes the silent-drop path that let Article 15 access requests complete without a forensic-chain entry on read-only audit dirs. * F-W3-PS-01 — salted audit hash. _hash_for_audit now reuses the same per-output-dir salt purge uses for target_id, defeating wordlist attacks against the audit log AND making purge ↔ reverse-pii digests cross-tool correlatable for the same subject. * F-W3-02 — default --type changed `custom` → `literal`. Stops silently treating dots in e-mails as regex wildcards on the no-flag path documented in the operator guide. * F-W3-03 — snippet truncation centred on the match span. The matched identifier is preserved within the 160-char window; long lines no longer ellide the very thing the operator is trying to verify. * F-W3-04 / F-W3-PS-11 — UnicodeDecodeError caught on mid-scan; emits a failure-flavoured `data.access_request_query` event and exits EXIT_TRAINING_ERROR (was Python's uncontrolled exit 1 with no audit record). * F-W3-05 — docs no longer claim a "ForgeLM hash strategy" the toolkit does not ship. Hash-mode framed as "external pipeline using purge's salt"; gdpr_erasure.md (EN+TR) + dispatcher docstring updated. * F-W3-06 / F-W3-PS-06 / F-W3S-02 — audit-dir default `output_dir → /audit/`. The Article 15 forensic record no longer co-locates with the corpus the subject is asking about. * F-W3-PS-03 / F-W3-07 / F-W3S-03 — ReDoS guard for `--type custom` via per-file SIGALRM budget (POSIX-only; 30s). A pathological pattern surfaces as OSError → EXIT_TRAINING_ERROR + failure audit event instead of pegging a CPU core indefinitely. * F-W3-08 — regex validation runs before salt resolution. An unparseable `--type custom` regex no longer creates the salt file as a side effect of the config-error exit. * F-W3-09 — `error_message` audit field bounded to 200 chars, closing the future-leak risk where a refactor could surface raw query data through `str(exc)`. * F-W3-PS-04 — `--salt-source per_dir` with `FORGELM_AUDIT_SECRET` set now refuses with a direction-aware diagnostic naming the unset path. * F-W3-PS-07 — `salt_source` field recorded in every audit event (plaintext / per_dir / env_var); compliance reviewer sees how two digests were salted. * F-W3-12 — directory positional argument produces a targeted diagnostic naming the glob form, not a generic empty-glob error. * F-W3-XPR-PS-02 — text-mode envelope prints a privacy banner warning operators not to redirect snippet output to a persistent log. Test-rigor HIGH absorptions: * F-W3T-01 — failure-path audit event no-leak assertion (the same json.dumps membership check as the success path). * F-W3T-02 — `unacceptable + safety.disabled` raise pinned via parametrize over both `_STRICT_RISK_TIERS` members. * F-W3T-03 — setext heading exclusion regression in the parity tool's heading extractor. * F-W3T-05 — multi-byte UTF-8 (CJK + emoji) snippet truncation test. * F-W3T-07 — orphan-TR-mirror meta-test pinning the `_PAIRS` registry as source of truth for guides + reference dirs (with documented allowlist for safety_compliance-tr.md whose EN→TR completion is a Wave 4 task). * F-W3T-08 — docstring on the live-repo smoke test explaining the deliberate redundancy with the CI lint job. * F-W3T-09 — `WebhookConfig.test_defaults` pins `timeout == 10` next to every other field default. * F-W3T-10 — `--audit-dir` explicit + audit-init OSError fail-closed regression. * F-W3T-12 — overlapping-glob dedupe regression. Standards low-cost: * F-W3S-04 — webhook.py timeout literal-5 fallback now reads from `WebhookConfig.model_fields["timeout"].default`; clamp message reports the real default. * F-W3S-06 — tools/check_bilingual_parity.py module docstring now documents its 0/1 exit-code contract for CI gate authors. * F-W3S-07 — `.claude/skills/add-test/SKILL.md` and `add-trainer-feature/SKILL.md` updated to reference the post-rename `tests/test_integration.py` (was `test_integration_smoke.py`). * F-W3S-09 — `_validate_identifier_type` docstring notes argparse enforces the choice set; the dispatcher guard is for direct library callers. Inline PR review comments: * `closure-plan-202604300906.md:130` — "Kalan iş" listesi Wave 3 kapanış özeti ile senkronize: Faz 24 + 28 + 38 kaldırıldı, pending sayısı 7 → 4 + 1 release. * `audit_event_catalog{,-tr}.md` — `data.access_request_query` Article column "15" → "GDPR Art. 15" / "GDPR Md. 15" (disambiguates from EU AI Act Article 15); payload column refreshed for new fields (salted query_hash, salt_source, literal type). * `data-formats.md` (EN + TR L145) — "✓ no PII detected (5 medium severity, see report)" mantıksal çelişkisi → "⚠ PII detected: 5 medium severity (see report)". Both locales fixed; example now self-consistent. * `simpo.md` (TR L48) — uzun Markdown satırı iki satıra bölündü. * `_reverse_pii.py` `--audit-dir` flow — explicit-flag awareness + fail-closed handler, per the inline ask. Skipped with reasons: * `wave3-review-prompt.md:18-19` SHA refresh — review prompts are frozen artefacts pointing at the snapshot the reviewers reviewed (61d74fa). Refreshing it now would invalidate the connection between the 4 review reports we are absorbing AND the snapshot they reviewed. The next review wave will write its own prompt pointing at a fresh SHA. * `architecture-tr.md` MD022 blank-lines-after-H3 — the EN counterpart uses the same no-blank-line pattern; no markdownlint runs in CI; blank-line-only changes do not affect parity. Adding blank lines to TR alone would diverge from EN unnecessarily. If MD022 ever becomes a CI gate, fix both mirrors in the same commit. * F-W3-PS-05 (symlink follow), F-W3-PS-08 (DoS via crafted corpus size cap), F-W3-PS-10 (grapheme cluster split), F-W3-PS-13/14 (NITs), F-W3-XPR-PS-01 (per-event field allowlist refactor), F-W3T-04 (true purge → reverse-pii integration test), F-W3T-13/14 (NITs), F-W3S-05 (INFO startup line), F-W3S-08 (audit-catalog row format polish) — defer to Wave 4. * F-W3-PS-09 / F-W3-PS-12 / F-W3T-06 — subsumed by the F-W3-PS-01 + F-W3-04 + F-W3-PS-03 absorptions. Suite 1333 → 1344 tests (+11 net). Lint clean. Dry-run green. Bilingual parity passes against 9 doc pairs (was 8; added audit_event_catalog). Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/skills/add-test/SKILL.md | 2 +- .claude/skills/add-trainer-feature/SKILL.md | 2 +- CHANGELOG.md | 49 +- .../code_reviews/closure-plan-202604300906.md | 2 +- docs/guides/gdpr_erasure-tr.md | 37 +- docs/guides/gdpr_erasure.md | 34 +- docs/reference/audit_event_catalog-tr.md | 2 +- docs/reference/audit_event_catalog.md | 2 +- docs/usermanuals/en/concepts/data-formats.md | 2 +- docs/usermanuals/en/reference/json-output.md | 12 +- docs/usermanuals/tr/concepts/data-formats.md | 2 +- docs/usermanuals/tr/reference/json-output.md | 12 +- docs/usermanuals/tr/training/simpo.md | 3 +- forgelm/cli/__init__.py | 2 + forgelm/cli/_parser.py | 8 +- forgelm/cli/subcommands/_reverse_pii.py | 478 ++++++++++++++---- forgelm/webhook.py | 14 +- tests/test_check_bilingual_parity.py | 67 ++- tests/test_config.py | 5 + tests/test_eu_ai_act.py | 17 +- tests/test_reverse_pii.py | 269 +++++++++- tools/check_bilingual_parity.py | 11 + 22 files changed, 826 insertions(+), 206 deletions(-) diff --git a/.claude/skills/add-test/SKILL.md b/.claude/skills/add-test/SKILL.md index 3160d2a5..ce66a5e8 100644 --- a/.claude/skills/add-test/SKILL.md +++ b/.claude/skills/add-test/SKILL.md @@ -29,7 +29,7 @@ Do **not** use for: |---|---| | Pydantic config field or validator | `test_config.py` | | A new alignment trainer (DPO-style) | `test_alignment.py` | -| A full pipeline scenario | `test_integration_smoke.py` | +| A full pipeline scenario | `test_integration.py` | | CLI argument parsing, exit codes | `test_cli.py` or `test_cli_subcommands.py` | | Compliance artifact generation | `test_compliance.py` or `test_eu_ai_act.py` | | Safety evaluation | `test_safety_advanced.py` | diff --git a/.claude/skills/add-trainer-feature/SKILL.md b/.claude/skills/add-trainer-feature/SKILL.md index 3589487c..6bfe48de 100644 --- a/.claude/skills/add-trainer-feature/SKILL.md +++ b/.claude/skills/add-trainer-feature/SKILL.md @@ -50,7 +50,7 @@ Every trainer-level feature touches roughly the same list of files. Tick them as - [ ] **`tests/test_.py`** — unit tests for the new module - [ ] **`tests/test_config.py`** — validation tests for new config -- [ ] **`tests/test_integration_smoke.py`** — dry-run that exercises the feature +- [ ] **`tests/test_integration.py`** — dry-run that exercises the feature - [ ] **`tests/conftest.py`** — new fixture if reusable across tests ### Config diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ba361e5..5a167a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,18 +11,45 @@ Single integration branch covering three closure-plan phases: **Faz 38 — `forgelm reverse-pii` (GDPR Article 15 right-of-access)** - New CLI subcommand: `forgelm reverse-pii --query VALUE [--type - email|phone|tr_id|us_ssn|iban|credit_card|custom] [--salt-source - per_dir|env_var] JSONL_GLOB...`. Walks JSONL corpora, reports - every line where the supplied identifier appears. Two scan modes: - *plaintext residual* (mask-leak detection) and *hash-mask* - (reuses `forgelm purge`'s per-output-dir salt to re-derive the - digest). Centre-truncated snippets cap log spam. + literal|email|phone|tr_id|us_ssn|iban|credit_card|custom] + [--salt-source per_dir|env_var] JSONL_GLOB...`. Walks JSONL + corpora, reports every line where the supplied identifier appears. + Two scan modes: *plaintext residual* (mask-leak detection) and + *hash-mask* (reuses `forgelm purge`'s per-output-dir salt to + re-derive the digest, so a purge → reverse-pii cycle for the same + subject yields matching digests). Snippets are centred on the + matched span and capped at 160 chars so the operator can always + eyeball the hit. - New audit event `data.access_request_query` (catalogued bilingually). - The identifier is **SHA-256-hashed before audit emission** — - Article 15 access requests must not themselves leak the subject's - data into the audit log. -- 18 regression tests; library re-exports through `forgelm.cli` - facade; help epilog + dispatcher row added. + The identifier is **salted-and-hashed before audit emission**, + reusing the same per-output-dir salt that purge uses for + `target_id` — Article 15 access requests must not themselves leak + the subject's data into the audit log, AND a wordlist attack + against the audit chain requires the operator's salt file. The + `salt_source` field is recorded in every event so a compliance + reviewer can correlate Article 17 + Article 15 events for the same + subject. +- Default `--type` is `literal` (not `custom`) — a stray + `--query alice@example.com` matches the literal e-mail substring, + not the regex shape `alice@exampleXcom`. Operators wanting raw + regex pass `--type custom` explicitly; on POSIX a 30s SIGALRM + budget guards against ReDoS hangs. +- Audit fail-closed: AuditLogger init `OSError`/`ValueError` + refuses the run with `EXIT_TRAINING_ERROR`, naming the audit-dir. + `ConfigError` (operator-identity unavailable) still skips with a + WARNING — the only "best-effort" branch. +- Audit-dir default: `/audit/` (was `output_dir`); the + Article 15 forensic record no longer co-locates with the corpus + the subject is asking about. +- Mid-scan UTF-8 / I/O failures and ReDoS timeouts emit a failure- + flavoured `data.access_request_query` event before exit — same + no-leak invariant as the success path. +- 28 regression tests (was 18) covering: literal-default no + false-positives, salted audit hash, purge ↔ reverse-pii digest + correlation, failure-path no-leak, multi-byte UTF-8 truncation, + malformed UTF-8 corpus, explicit-audit-dir fail-closed, + per_dir-with-env-var symmetric refusal, overlapping-glob dedupe, + directory-arg diagnostic, no-salt-side-effect on regex parse error. **Faz 24 — Bilingual TR mirror sweep + parity CI guard** diff --git a/docs/analysis/code_reviews/closure-plan-202604300906.md b/docs/analysis/code_reviews/closure-plan-202604300906.md index e873ef81..fec9812b 100644 --- a/docs/analysis/code_reviews/closure-plan-202604300906.md +++ b/docs/analysis/code_reviews/closure-plan-202604300906.md @@ -127,7 +127,7 @@ Her faz bir veya birden çok PR'la kapanır. Sıralama dependency-driven; parale **Wave 3 kapanış özeti (2026-05-05, PR review aşamasında):** 3 faz tamamlandı (Faz 24 bilingual TR mirror sweep + `tools/check_bilingual_parity.py` CI guard, Faz 28 curated tier 1+2 cleanup, Faz 38 `forgelm reverse-pii` GDPR Article 15 subcommand). Branch: `closure/wave3-integration`. Yeni: 1 CLI subcommand (`reverse-pii`), 1 audit event (`data.access_request_query`), 1 yeni CI guard tool. Behaviour change: high-risk + safety-disabled artık `ConfigError` raise eder (F-compliance-110). Suite 1298 → 1333 tests (+35 net). -**Kalan iş (Wave 3+):** 7 faz + 1 release. Pending: Faz 22 (ISO 27001/SOC 2 design), Faz 23 (ISO 27001/SOC 2 impl), Faz 24 (bilingual TR mirror sweep), Faz 26 (QMS bilingual mirror), Faz 28 (Major+Minor cleanup), Faz 30 (final docs + site finalization), Faz 38 (`forgelm reverse-pii` GDPR Art. 15). Release: Faz 33 (v0.5.5 PyPI publish). +**Kalan iş (Wave 3+):** 4 faz + 1 release. Pending: Faz 22 (ISO 27001/SOC 2 design), Faz 23 (ISO 27001/SOC 2 impl), Faz 26 (QMS bilingual mirror), Faz 30 (final docs + site finalization). Release: Faz 33 (v0.5.5 PyPI publish). (Faz 24 + Faz 28 + Faz 38 Wave 3 ile tamamlandı; bkz. yukarıdaki kapanış özeti.) --- diff --git a/docs/guides/gdpr_erasure-tr.md b/docs/guides/gdpr_erasure-tr.md index 1d066a06..7cb89f47 100644 --- a/docs/guides/gdpr_erasure-tr.md +++ b/docs/guides/gdpr_erasure-tr.md @@ -136,25 +136,32 @@ satırlarda görünüyor" sorusuna yanıt verir. $ forgelm reverse-pii --query "ali@example.com" --type email \ --output-dir ./outputs data/*.jsonl -# Hash-mask scan: corpus ForgeLM'in hash stratejisi ile maskelenmiş. -# `purge`'ün `target_id` hash'lemesinde kullandığı per-output-dir -# salt'ı yeniden kullanır — burada hesaplanan digest, corpus'a -# yazılmış digest ile eşleşir. +# Hash-mask scan: corpus DIŞSAL bir pipeline ile maskelenmiş — +# `purge`'ün `target_id` audit-event hash'lemesinde kullandığı +# per-output-dir salt'ını paylaşan SHA256(salt + identifier) +# digest'leri corpus'a gömülmüş. ForgeLM'in kendisi bir +# hash-replacement ingest stratejisi YOLLAMAZ; bu mod, purge'ün +# salt'ını ortak gizli olarak kullanan harici bir pipeline kuran +# operatörler içindir. $ forgelm reverse-pii --query "ali@example.com" --type email \ --salt-source per_dir --output-dir ./outputs data/*.jsonl ``` -Audit chain `data.access_request_query`'yi identifier *hash'lenmiş* -olarak kaydeder — Madde 15 erişim talepleri kendisi de subject'in -verisini audit log'a sızdırmamalıdır. Hash'lenmiş form, compliance -reviewer'ın iki sorguyu cleartext görmeden aynı subject'e -ilişkilendirmesini sağlayan stabil per-identifier fingerprint'tir. - -**Identifier tipleri** (`--type`): `email`, `phone`, `tr_id`, -`us_ssn`, `iban`, `credit_card`, `custom`. `custom` dışındaki tüm -tipler query'i literal string olarak ele alır (regex şekil-match -yapmaz — o iş audit-time detector'ın görevi, erişim talebinin değil). -`custom` query'i Python regex olarak yorumlar. +Audit chain `data.access_request_query`'yi identifier *salt'lı +hash'lenmiş* olarak kaydeder — `forgelm purge`'ün `target_id` field'ı +için kullandığı aynı per-output-dir salt'ı yeniden kullanılır. +Madde 15 erişim talepleri kendisi de subject'in verisini audit log'a +sızdırmamalıdır. Salt'lı form, compliance reviewer'ın aynı subject +için Madde 17 (purge) ve Madde 15 (reverse-pii) olaylarını cleartext +görmeden ilişkilendirmesini sağlar (digest'ler eşleşir). + +**Identifier tipleri** (`--type`): `literal` (varsayılan), `email`, +`phone`, `tr_id`, `us_ssn`, `iban`, `credit_card`, `custom`. +`custom` dışındaki tüm tipler query'i literal substring olarak ele +alır (regex şekil-match yapmaz — o iş audit-time detector'ın görevi, +erişim talebinin değil). `custom` query'i Python regex olarak +yorumlar; POSIX sistemlerde 30s'lik per-file SIGALRM bütçesi ReDoS +hang'lerine karşı koruma sağlar. **Exit kodları:** `0` = scan tamamlandı (matches listesi boş olabilir); `1` = config hatası (boş query, malformed regex, boş diff --git a/docs/guides/gdpr_erasure.md b/docs/guides/gdpr_erasure.md index 13d5b30e..7e91beb9 100644 --- a/docs/guides/gdpr_erasure.md +++ b/docs/guides/gdpr_erasure.md @@ -135,24 +135,32 @@ line where my identifier appears." $ forgelm reverse-pii --query "alice@example.com" --type email \ --output-dir ./outputs data/*.jsonl -# Hash-mask scan: corpus was masked through ForgeLM's hash strategy. -# Reuses the per-output-dir salt purge uses for `target_id` hashing, -# so the digest computed here matches the digest written into the corpus. +# Hash-mask scan: corpus was masked through an EXTERNAL pipeline +# that embedded SHA256(salt + identifier) digests using the same +# per-output-dir salt purge uses for `target_id` audit-event +# hashing. ForgeLM does not ship a hash-replacement ingest +# strategy of its own; this mode is for operators who built one +# outside the toolkit using purge's salt as the shared secret. $ forgelm reverse-pii --query "alice@example.com" --type email \ --salt-source per_dir --output-dir ./outputs data/*.jsonl ``` The audit chain records `data.access_request_query` with the -identifier *hashed* — Article 15 access requests must not themselves -leak the subject's data into the audit log. The hashed form is a -stable per-identifier fingerprint a compliance reviewer can correlate -across queries without seeing the cleartext. - -**Identifier types** (`--type`): `email`, `phone`, `tr_id`, `us_ssn`, -`iban`, `credit_card`, `custom`. All non-`custom` types treat the -query as a literal string (no regex shape match — that's the -*audit-time* detector's job, not the access-request answer). -`custom` interprets the query as a Python regex. +identifier *salted-and-hashed* using the same per-output-dir salt +that `forgelm purge` uses for its `target_id` field — Article 15 +access requests must not themselves leak the subject's data into the +audit log. The salted form is a stable per-identifier fingerprint +that lets a compliance reviewer correlate Article 17 (purge) and +Article 15 (reverse-pii) events for the same subject (the digests +match) without seeing the cleartext. + +**Identifier types** (`--type`): `literal` (default), `email`, +`phone`, `tr_id`, `us_ssn`, `iban`, `credit_card`, `custom`. All +non-`custom` types treat the query as a literal substring (no regex +shape match — that's the *audit-time* detector's job, not the +access-request answer). `custom` interprets the query as a Python +regex; on POSIX systems a 30s per-file SIGALRM budget guards against +ReDoS hangs. **Exit codes:** `0` = scan completed (matches list may be empty); `1` = config error (empty query, malformed regex, empty glob); `2` = diff --git a/docs/reference/audit_event_catalog-tr.md b/docs/reference/audit_event_catalog-tr.md index 9399da4a..c650ecfe 100644 --- a/docs/reference/audit_event_catalog-tr.md +++ b/docs/reference/audit_event_catalog-tr.md @@ -69,7 +69,7 @@ Hash zinciri, satır diske düştükten (`flush` + `fsync`) sonra ilerler; kirli | Event | Ne zaman yayılır | Payload | Madde | |--------------------------------|---------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|-------| -| `data.access_request_query` | Her `forgelm reverse-pii` çağrısında, scan tamamlandıktan sonra (veya mid-scan I/O hatası sonrası — `error_*` field'larıyla). | `query_hash` (raw identifier'ın SHA-256'sı — asla raw), `identifier_type` ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`, `scan_mode` ∈ `{plaintext, hash}`, `files_scanned` (path'ler), `match_count`, opsiyonel `error_class`/`error_message` | 15 | +| `data.access_request_query` | Her `forgelm reverse-pii` çağrısında, scan tamamlandıktan sonra (veya mid-scan I/O hatası sonrası — `error_*` field'larıyla). | `query_hash` (raw identifier'ın salt'lı SHA-256'sı — asla raw; purge'ün per-output-dir salt'ını yeniden kullanır), `identifier_type` ∈ `{literal, email, phone, tr_id, us_ssn, iban, credit_card, custom}`, `scan_mode` ∈ `{plaintext, hash}`, `salt_source` ∈ `{plaintext, per_dir, env_var}`, `files_scanned` (path'ler), `match_count`, opsiyonel `error_class`/`error_message` | GDPR Md. 15 | ### Air-gap pre-cache (Phase 35 — `forgelm cache-models` / `cache-tasks`) diff --git a/docs/reference/audit_event_catalog.md b/docs/reference/audit_event_catalog.md index 2193debd..25e98298 100644 --- a/docs/reference/audit_event_catalog.md +++ b/docs/reference/audit_event_catalog.md @@ -69,7 +69,7 @@ The hash chain advances after the line lands on disk (`flush` + `fsync`), so an | Event | When emitted | Payload | Article | |--------------------------------|----------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|---------| -| `data.access_request_query` | Every `forgelm reverse-pii` invocation, after the scan completes (or after a mid-scan I/O failure — with `error_*`). | `query_hash` (SHA-256 of raw identifier — never raw), `identifier_type` ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`, `scan_mode` ∈ `{plaintext, hash}`, `files_scanned` (paths), `match_count`, optional `error_class`/`error_message` | 15 | +| `data.access_request_query` | Every `forgelm reverse-pii` invocation, after the scan completes (or after a mid-scan I/O failure — with `error_*`). | `query_hash` (salted SHA-256 of raw identifier — never raw; reuses purge's per-output-dir salt), `identifier_type` ∈ `{literal, email, phone, tr_id, us_ssn, iban, credit_card, custom}`, `scan_mode` ∈ `{plaintext, hash}`, `salt_source` ∈ `{plaintext, per_dir, env_var}`, `files_scanned` (paths), `match_count`, optional `error_class`/`error_message` | GDPR Art. 15 | ### Air-gap pre-cache (Phase 35 — `forgelm cache-models` / `cache-tasks`) diff --git a/docs/usermanuals/en/concepts/data-formats.md b/docs/usermanuals/en/concepts/data-formats.md index d55b2cd9..8b4abe32 100644 --- a/docs/usermanuals/en/concepts/data-formats.md +++ b/docs/usermanuals/en/concepts/data-formats.md @@ -162,7 +162,7 @@ Always run `forgelm audit` before training: ```shell $ forgelm audit data/preferences.jsonl ✓ format: preference (12,400 rows, 3 splits) -✓ no PII detected (5 medium severity, see report) +⚠ PII detected: 5 medium severity (see report) ⚠ 12 chosen-rejected identical rows — likely collection bug ✓ no cross-split leakage ``` diff --git a/docs/usermanuals/en/reference/json-output.md b/docs/usermanuals/en/reference/json-output.md index c741069e..1e68d6a8 100644 --- a/docs/usermanuals/en/reference/json-output.md +++ b/docs/usermanuals/en/reference/json-output.md @@ -268,11 +268,12 @@ Wave 3 Phase 38 — GDPR Article 15 right-of-access companion to `forgelm purge` "query_hash": "abc...64-hex", "identifier_type": "email", "scan_mode": "plaintext", + "salt_source": "per_dir", "matches": [ { "file": "/work/data/train.jsonl", "line": 42, - "snippet": "Contact: alice@example.com regarding the appointment" + "snippet": "…Contact: alice@example.com regarding the appointment…" } ], "files_scanned": [ @@ -284,14 +285,15 @@ Wave 3 Phase 38 — GDPR Article 15 right-of-access companion to `forgelm purge` | Key | Type | Notes | |---|---|---| -| `query_hash` | str | 64-character lowercase hex SHA-256 digest of the operator's raw `--query`. The raw value never appears in the envelope. | -| `identifier_type` | str | Echo of `--type` ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`. | +| `query_hash` | str | 64-character lowercase hex SHA-256 digest of `salt + raw --query`, where `salt` is the per-output-dir audit salt that `forgelm purge` also uses for `target_id`. The raw value never appears in the envelope; cross-tool correlation with purge's `target_id` works out of the box. | +| `identifier_type` | str | Echo of `--type` ∈ `{literal, email, phone, tr_id, us_ssn, iban, credit_card, custom}`. Default is `literal` — the query is matched as a literal substring. | | `scan_mode` | str | `"plaintext"` (default — verbatim search; mask-leak detection) or `"hash"` (`--salt-source` set; SHA-256 of `salt + query` searched in the corpus). | -| `matches` | list[object] | One entry per line that matched. `snippet` is centre-truncated to 160 chars (`…` separator) so unbounded log spam is impossible. | +| `salt_source` | str | Which salt-resolution path was used: `"plaintext"` (no `--salt-source`), `"per_dir"` (per-output-dir salt file only), or `"env_var"` (XOR with `FORGELM_AUDIT_SECRET`). Mirrored in the audit event so a compliance reviewer can tell whether two digests were salted the same way. | +| `matches` | list[object] | One entry per line that matched. `snippet` is centred on the matched span and capped at 160 chars (`…` separator); the matched identifier is always preserved within the window. | | `files_scanned` | list[object] | Per-file match counts, in glob-resolution order. The list is the source of truth for which paths the scan walked. | | `match_count` | int | Equals `len(matches)`; convenience for CI gates that branch on "any match." | -**Exit code mapping:** `0` = scan completed (matches list may be empty); `1` = config error (empty `--query`, malformed regex on `--type custom`, empty glob expansion, `--salt-source env_var` without `FORGELM_AUDIT_SECRET`); `2` = runtime error (mid-scan I/O failure). +**Exit code mapping:** `0` = scan completed (matches list may be empty); `1` = config error (empty `--query`, malformed regex on `--type custom`, empty glob expansion, `--salt-source env_var` without `FORGELM_AUDIT_SECRET` — or `--salt-source per_dir` with `FORGELM_AUDIT_SECRET` set); `2` = runtime error (mid-scan I/O failure, malformed UTF-8 corpus, custom-regex ReDoS timeout, audit-init failure when `--audit-dir` was explicit). ## `forgelm cache-models` diff --git a/docs/usermanuals/tr/concepts/data-formats.md b/docs/usermanuals/tr/concepts/data-formats.md index 9ed2f048..cd9dfb43 100644 --- a/docs/usermanuals/tr/concepts/data-formats.md +++ b/docs/usermanuals/tr/concepts/data-formats.md @@ -142,7 +142,7 @@ Eğitimden önce her zaman `forgelm audit` çalıştırın: ```shell $ forgelm audit data/preferences.jsonl ✓ format: preference (12,400 satır, 3 split) -✓ PII tespit edilmedi (5 orta ciddiyet, rapora bakın) +⚠ PII bulundu: 5 orta ciddiyet (rapora bakın) ⚠ 12 chosen-rejected aynı satır — muhtemelen toplama hatası ✓ split'ler arası sızıntı yok ``` diff --git a/docs/usermanuals/tr/reference/json-output.md b/docs/usermanuals/tr/reference/json-output.md index 0d81d097..c60d361f 100644 --- a/docs/usermanuals/tr/reference/json-output.md +++ b/docs/usermanuals/tr/reference/json-output.md @@ -268,11 +268,12 @@ Wave 3 Phase 38 — GDPR Madde 15 erişim hakkı, `forgelm purge`'ün companion' "query_hash": "abc...64-hex", "identifier_type": "email", "scan_mode": "plaintext", + "salt_source": "per_dir", "matches": [ { "file": "/work/data/train.jsonl", "line": 42, - "snippet": "Contact: alice@example.com regarding the appointment" + "snippet": "…Contact: alice@example.com regarding the appointment…" } ], "files_scanned": [ @@ -284,14 +285,15 @@ Wave 3 Phase 38 — GDPR Madde 15 erişim hakkı, `forgelm purge`'ün companion' | Anahtar | Tip | Notlar | |---|---|---| -| `query_hash` | str | Operatörün raw `--query`'sinin 64-karakter küçük-harfli hex SHA-256 digest'i. Raw değer hiçbir zaman zarfta yer almaz. | -| `identifier_type` | str | `--type` echo'su ∈ `{email, phone, tr_id, us_ssn, iban, credit_card, custom}`. | +| `query_hash` | str | `salt + raw --query`'nin 64-karakter küçük-harfli hex SHA-256 digest'i — `salt`, `forgelm purge`'ün `target_id` için de kullandığı per-output-dir audit salt'ı. Raw değer hiçbir zaman zarfta yer almaz; purge'ün `target_id`'si ile cross-tool korelasyon hazır gelir. | +| `identifier_type` | str | `--type` echo'su ∈ `{literal, email, phone, tr_id, us_ssn, iban, credit_card, custom}`. Varsayılan `literal` — query literal substring olarak eşleştirilir. | | `scan_mode` | str | `"plaintext"` (varsayılan — verbatim arama; mask-leak detection) veya `"hash"` (`--salt-source` set; corpus'ta `salt + query`'nin SHA-256'sı aranır). | -| `matches` | list[object] | Eşleşen her satır için bir entry. `snippet` 160 karaktere centre-truncated (`…` ayraç) — sınırsız log spam imkansız. | +| `salt_source` | str | Hangi salt-resolution path kullanıldı: `"plaintext"` (`--salt-source` yok), `"per_dir"` (yalnız per-output-dir salt dosyası) veya `"env_var"` (`FORGELM_AUDIT_SECRET` ile XOR). Audit event'inde de yansıtılır; bir compliance reviewer iki digest'in aynı şekilde salt'lanıp salt'lanmadığını anlayabilir. | +| `matches` | list[object] | Eşleşen her satır için bir entry. `snippet` eşleşen span'in etrafında merkezlenip 160 karaktere kapanır (`…` ayraç); eşleşen identifier her zaman pencere içinde kalır. | | `files_scanned` | list[object] | Glob-resolution sırasında per-file match sayıları. Hangi path'lerin tarandığının kaynağı budur. | | `match_count` | int | `len(matches)`'a eşit; "match var mı yok mu" branch'leyen CI gate'leri için convenience. | -**Exit kodu:** `0` = scan tamamlandı (matches listesi boş olabilir); `1` = config hatası (boş `--query`, `--type custom` malformed regex, empty glob expansion, `FORGELM_AUDIT_SECRET` olmadan `--salt-source env_var`); `2` = runtime hatası (mid-scan I/O failure). +**Exit kodu:** `0` = scan tamamlandı (matches listesi boş olabilir); `1` = config hatası (boş `--query`, `--type custom` malformed regex, empty glob expansion, `FORGELM_AUDIT_SECRET` olmadan `--salt-source env_var` — veya `FORGELM_AUDIT_SECRET` set iken `--salt-source per_dir`); `2` = runtime hatası (mid-scan I/O failure, malformed UTF-8 corpus, custom-regex ReDoS timeout, `--audit-dir` explicit verildiğinde audit-init başarısızlığı). ## `forgelm cache-models` diff --git a/docs/usermanuals/tr/training/simpo.md b/docs/usermanuals/tr/training/simpo.md index 710c4e69..1bd6cf3d 100644 --- a/docs/usermanuals/tr/training/simpo.md +++ b/docs/usermanuals/tr/training/simpo.md @@ -45,7 +45,8 @@ output: ## Veri formatı -DPO ile aynı — `prompt`, `chosen`, `rejected` taşıyan `preference` formatı. Bkz. [Veri Formatları](#/concepts/data-formats). +DPO ile aynı — `prompt`, `chosen`, `rejected` taşıyan `preference` formatı. +Bkz. [Veri Formatları](#/concepts/data-formats). ## Parametreler diff --git a/forgelm/cli/__init__.py b/forgelm/cli/__init__.py index 9e132ba8..497bb1af 100644 --- a/forgelm/cli/__init__.py +++ b/forgelm/cli/__init__.py @@ -203,8 +203,10 @@ # Reverse-PII subcommand (Phase 38 — GDPR Article 15 right-of-access). from .subcommands._reverse_pii import ( _build_search_pattern, # noqa: F401 — re-export for tests + _emit_audit_event, # noqa: F401 — re-export for tests _emit_reverse_pii_result, # noqa: F401 — re-export for tests _hash_for_audit, # noqa: F401 — re-export for tests + _resolve_audit_dir, # noqa: F401 — re-export for tests _resolve_files, # noqa: F401 — re-export for tests _resolve_query_form, # noqa: F401 — re-export for tests _run_reverse_pii_cmd, # noqa: F401 — re-export for tests diff --git a/forgelm/cli/_parser.py b/forgelm/cli/_parser.py index c2588893..98600ed5 100644 --- a/forgelm/cli/_parser.py +++ b/forgelm/cli/_parser.py @@ -920,9 +920,9 @@ def _add_reverse_pii_subcommand(subparsers) -> None: p.add_argument( "--type", type=str, - default="custom", - choices=["email", "phone", "tr_id", "us_ssn", "iban", "credit_card", "custom"], - help="Identifier category. `custom` (default) treats --query as an arbitrary regular expression; every other value treats --query as a literal string and matches it verbatim.", + default="literal", + choices=["literal", "email", "phone", "tr_id", "us_ssn", "iban", "credit_card", "custom"], + help="Identifier category. `literal` (default) treats --query as a literal substring (re.escape applied) — the safe choice for Article 15 access requests where dots in e-mails must NOT match arbitrary characters. Type-specific values (email/phone/...) behave the same way; `custom` treats --query as an arbitrary Python regex (use with care; ReDoS-guarded by a per-file timeout on POSIX).", ) p.add_argument( "--salt-source", @@ -949,7 +949,7 @@ def _add_reverse_pii_subcommand(subparsers) -> None: type=str, default=None, metavar="DIR", - help="Where to write the audit chain entries (default: same as --output-dir). Use this when scans run config-free but the audit log lives elsewhere.", + help="Where to write the audit chain entries (default: /audit/, isolated from the corpus directory). Pass an explicit directory to override; explicit values fail loudly when unwritable rather than silently dropping the Article 15 forensic record.", ) _add_common_subparser_flags(p, include_output_format=True) diff --git a/forgelm/cli/subcommands/_reverse_pii.py b/forgelm/cli/subcommands/_reverse_pii.py index 33fd26c7..528ed12f 100644 --- a/forgelm/cli/subcommands/_reverse_pii.py +++ b/forgelm/cli/subcommands/_reverse_pii.py @@ -10,30 +10,45 @@ regex, or hash-masked digest) appears. Two complementary scan modes: 1. **Plaintext residual scan** (default). Searches for the identifier - verbatim — the path that surfaces *mask leaks* (operator believed - the corpus was masked but a residual span was missed by the - masking pass). This is the audit-time honest-PII check. + as a literal substring (``re.escape`` applied; the dot in + ``alice@example.com`` is a literal dot, not "any character") — the + path that surfaces *mask leaks* (operator believed the corpus was + masked but a residual span was missed by the masking pass). This + is the audit-time honest-PII check. 2. **Hash-mask scan** (``--salt-source per_dir`` or ``--salt-source env_var``). Computes ``SHA256(salt + identifier)`` using the same per-output-dir salt that ``forgelm purge`` uses for its ``target_id`` hashing, then searches every JSONL line for that - digest. This is the path for corpora that were masked through - ForgeLM's own ``hash`` replacement strategy. + digest. This mode targets corpora that an external pipeline masked + by embedding ``SHA256(salt + identifier)`` digests. ForgeLM itself + does not ship a hash-replacement ingest strategy; this mode is for + operators who built one outside the toolkit using purge's salt. Audit chain: every invocation writes a ``data.access_request_query`` -event with the identifier *hashed* (never raw — Article 15 access -requests must not themselves leak the subject's identifier into the -audit log) and the per-file match count. An operator who later runs -``forgelm verify-audit`` sees a forensic record that the request was -processed without the audit log itself becoming a residual leak. +event with the identifier *salted-and-hashed* (never raw — Article 15 +access requests must not themselves leak the subject's identifier +into the audit log; the salt comes from the same per-output-dir +file purge uses, so a wordlist attack against the audit log would +require the operator's salt file too) and the per-file match count. +An operator who later runs ``forgelm verify-audit`` sees a forensic +record that the request was processed without the audit log itself +becoming a residual leak. Cross-tool correlation: the digest written +here matches the ``target_id`` digest ``forgelm purge`` writes for +the same identifier in the same output_dir, so a compliance reviewer +auditing "every event about subject X" sees a single connected +timeline. Exit codes (per ``docs/standards/error-handling.md``): - 0 — command ran (matches reported, may be empty). - 1 — config error (empty/whitespace ``--query``, unparseable regex - for ``--type custom``, glob pattern resolved to zero files). + for ``--type custom``, glob pattern resolved to zero files, + ``--salt-source`` requested without the matching salt source + available). - 2 — runtime error (I/O failure walking the corpus, audit-log write - failure when an audit dir was supplied). + failure when an audit dir was supplied, custom-regex ReDoS timeout, + malformed UTF-8 mid-corpus, audit-init failure when --audit-dir was + explicitly supplied). """ from __future__ import annotations @@ -43,6 +58,7 @@ import json import os import re +import signal as _signal import sys from typing import Any, Dict, List, NoReturn, Optional, Tuple @@ -56,11 +72,32 @@ # itself leak unbounded surrounding context. _SNIPPET_MAX_CHARS = 160 -# Identifier-type keys. Mirror the categories that -# ``forgelm/data_audit/_pii_regex.py::_PII_PATTERNS`` knows so the -# operator can ask "is there a phone matching this exact number" and -# get a result that the audit's own detector would have flagged. -_IDENTIFIER_TYPES: Tuple[str, ...] = ("email", "phone", "tr_id", "us_ssn", "iban", "credit_card", "custom") +# Bound the audit event's ``error_message`` field so a future refactor +# cannot accidentally leak operator-controlled raw data through +# ``str(exc)`` (e.g. ``raise OSError(f"row {line!r} malformed")``). +_AUDIT_ERROR_MESSAGE_MAX = 200 + +# Custom-regex ReDoS guard: per-file scan budget in seconds. POSIX +# only (SIGALRM); on Windows the guard is a no-op and operators are +# expected to vet their regex themselves. +_CUSTOM_REGEX_TIMEOUT_S = 30 + +# Identifier-type keys. ``literal`` (default) treats ``--query`` as +# an exact substring; every category-specific type does the same plus +# a mnemonic in audit + envelope; ``custom`` interprets ``--query`` as +# an arbitrary Python regex. The literal-default fix closes the +# F-W3-02 false-positive trap where a default-``custom`` regex +# silently treated dots in e-mails as wildcards. +_IDENTIFIER_TYPES: Tuple[str, ...] = ( + "literal", + "email", + "phone", + "tr_id", + "us_ssn", + "iban", + "credit_card", + "custom", +) def _output_error_and_exit(output_format: str, msg: str, exit_code: int) -> NoReturn: @@ -90,11 +127,16 @@ def _validate_query(query: Optional[str], output_format: str) -> str: def _validate_identifier_type(identifier_type: str, output_format: str) -> str: - """Refuse identifier types we don't know how to scan for.""" + """Refuse identifier types we don't know how to scan for. + + Argparse already enforces the choice set via ``choices=``; this + dispatcher-side guard is a safety net for direct library callers + who construct the ``args`` namespace themselves. + """ if identifier_type not in _IDENTIFIER_TYPES: _output_error_and_exit( output_format, - f"--type {identifier_type!r} is not recognised. Choose one of: {', '.join(sorted(_IDENTIFIER_TYPES))}.", + f"--type {identifier_type!r} is not recognised. Choose one of: {', '.join(_IDENTIFIER_TYPES)}.", EXIT_CONFIG_ERROR, ) return identifier_type @@ -104,11 +146,11 @@ def _build_search_pattern(query: str, identifier_type: str, output_format: str) """Compile the regex used to scan each JSONL line. For ``--type custom`` the operator's ``--query`` is interpreted as - a regex (escaped only to fix anchoring / boundaries). For every - other type the query is treated as a literal string and surrounded - with ``re.escape`` — the goal is "find this exact identifier", - not "find anything matching its shape" (the audit-time detector - already handles the latter). + a regex. For every other type (``literal``, ``email``, …) the + query is treated as a literal string and wrapped in ``re.escape`` + — the goal is "find this exact identifier", not "find anything + matching its shape" (the audit-time detector already handles the + latter). """ if identifier_type == "custom": try: @@ -129,11 +171,19 @@ def _resolve_files(globs: List[str], output_format: str) -> List[str]: set as a config error — the operator either typed a wrong path or pointed at an empty directory; either way "no files scanned" is not a successful answer to "is this identifier in the corpus." + + Operators who pass a literal directory get a targeted diagnostic + pointing them at the glob-form they probably meant — silent skip + of the whole input set is too easy to miss. """ resolved: List[str] = [] seen: set[str] = set() + saw_directory: Optional[str] = None for pattern in globs: for match in sorted(_glob.glob(pattern, recursive=True)): + if os.path.isdir(match): + saw_directory = match + continue if not os.path.isfile(match): continue absolute = os.path.abspath(match) @@ -142,6 +192,13 @@ def _resolve_files(globs: List[str], output_format: str) -> List[str]: seen.add(absolute) resolved.append(absolute) if not resolved: + if saw_directory is not None: + _output_error_and_exit( + output_format, + f"{saw_directory!r} is a directory, not a JSONL file. " + f"Pass a glob (e.g. {saw_directory!r}/*.jsonl) or a concrete file path.", + EXIT_CONFIG_ERROR, + ) _output_error_and_exit( output_format, f"No files matched the supplied glob pattern(s): {globs!r}. " @@ -151,22 +208,30 @@ def _resolve_files(globs: List[str], output_format: str) -> List[str]: return resolved -def _resolve_query_form(query: str, salt_source: Optional[str], output_dir: str, output_format: str) -> Tuple[str, str]: +def _resolve_query_form( + query: str, + salt_source: Optional[str], + output_dir: str, + output_format: str, +) -> Tuple[str, str, Optional[bytes], str]: """Map the operator-supplied identifier to the form actually searched in the corpus. - Returns ``(scan_query, scan_mode)`` where ``scan_mode`` is one of: + Returns ``(scan_query, scan_mode, salt, salt_source_label)``: - - ``"plaintext"`` — search for ``query`` verbatim (mask-leak path). - - ``"hash"`` — compute ``SHA256(salt + query)`` and search for the - digest (the corpus was masked via ForgeLM's hash strategy). + - ``scan_query`` — the literal/regex string to compile. + - ``scan_mode`` — ``"plaintext"`` or ``"hash"``. + - ``salt`` — the per-output-dir salt bytes, resolved if the audit + hash needs it (always, post F-W3-PS-01) or ``None`` for the + degraded path where salt resolution is unavailable. + - ``salt_source_label`` — ``"per_dir" | "env_var" | "plaintext"``; + recorded in the audit event so a reviewer can tell whether two + digests were salted the same way. ``--salt-source`` triggers hash mode and reuses the per-output-dir salt resolution from ``forgelm purge`` so the two subcommands cannot drift on hashing semantics. """ - if salt_source is None: - return query, "plaintext" # Late import: ``_resolve_salt`` lives in the purge subcommand and # pulling it eagerly would create a load-time cycle. The two # subcommands intentionally share the salt-resolution path so a @@ -181,17 +246,52 @@ def _resolve_query_form(query: str, salt_source: Optional[str], output_dir: str, f"Could not resolve audit salt for {output_dir!r}: {exc}", EXIT_TRAINING_ERROR, ) + + if salt_source is None: + return query, "plaintext", salt, resolved_source + if salt_source != resolved_source: - # The operator asked for ``env_var`` but the env wasn't set - # (or vice versa). Refuse rather than silently scan with the - # wrong salt source — the resulting digest would never match. + # Direction-aware diagnostic: telling the operator which + # specific knob to adjust beats a generic "or vice versa" + # message that requires them to re-derive the failure mode. + if salt_source == "env_var": + hint = "Set FORGELM_AUDIT_SECRET to the env-mode secret and retry." + else: # salt_source == "per_dir" + hint = ( + "Unset FORGELM_AUDIT_SECRET (e.g. `env -u FORGELM_AUDIT_SECRET forgelm reverse-pii ...`) " + "to force per-dir mode and retry." + ) _output_error_and_exit( output_format, - f"--salt-source={salt_source!r} requested but ``_resolve_salt`` returned {resolved_source!r}. " - "Set FORGELM_AUDIT_SECRET (env_var mode) or unset it (per_dir mode) and retry.", + f"--salt-source={salt_source!r} requested but ``_resolve_salt`` returned {resolved_source!r}. {hint}", EXIT_CONFIG_ERROR, ) - return _hash_target_id(query, salt), "hash" + return _hash_target_id(query, salt), "hash", salt, resolved_source + + +def _truncate_snippet(line: str, match_span: Tuple[int, int]) -> str: + """Centre the snippet on the match span, capped at ``_SNIPPET_MAX_CHARS``. + + The whole point of the snippet (operator-eyeball verification of + the hit) is defeated if a long-line head+tail truncation drops the + matched span itself — F-W3-03. We anchor a window of + ``_SNIPPET_MAX_CHARS`` characters around the match and ellide the + ends only. Short lines pass through unchanged. + + Operates on Python ``str`` (code-points), so multi-byte UTF-8 + runes are not split mid-byte. + """ + if len(line) <= _SNIPPET_MAX_CHARS: + return line + start, end = match_span + budget = _SNIPPET_MAX_CHARS - 2 # reserve room for two literal "…" + match_len = max(end - start, 0) + ctx = max((budget - match_len) // 2, 0) + win_start = max(start - ctx, 0) + win_end = min(end + ctx, len(line)) + head = "…" if win_start > 0 else "" + tail = "…" if win_end < len(line) else "" + return f"{head}{line[win_start:win_end]}{tail}" def _scan_file(path: str, pattern: re.Pattern[str]) -> List[Dict[str, Any]]: @@ -202,61 +302,216 @@ def _scan_file(path: str, pattern: re.Pattern[str]) -> List[Dict[str, Any]]: - ``file`` — absolute path of the file. - ``line`` — 1-based line number. - ``snippet`` — at most ``_SNIPPET_MAX_CHARS`` characters around - the match, with the matched span left intact so the operator can + the match, with the matched span centred so the operator can eyeball the hit. We deliberately do NOT mask the snippet: the whole point of the access request is to surface the verbatim data the subject asked about. - Raises :class:`OSError` on read failure; the caller handles it. + Raises :class:`OSError` on read failure (decoded as a runtime + error by the dispatcher); raises :class:`UnicodeDecodeError` on + malformed UTF-8 (also decoded as runtime error; F-W3-04). """ matches: List[Dict[str, Any]] = [] with open(path, "r", encoding="utf-8") as fh: for line_no, line in enumerate(fh, start=1): stripped = line.rstrip("\n") - if not pattern.search(stripped): + match = pattern.search(stripped) + if match is None: continue matches.append( { "file": path, "line": line_no, - "snippet": _truncate_snippet(stripped), + "snippet": _truncate_snippet(stripped, match.span()), } ) return matches -def _truncate_snippet(line: str) -> str: - """Centre-truncate ``line`` to at most ``_SNIPPET_MAX_CHARS``. +def _scan_files_with_redos_guard( + files: List[str], + pattern: re.Pattern[str], + identifier_type: str, + output_format: str, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Walk every file once; centralise the per-file ReDoS budget. + + On POSIX a ``SIGALRM`` per-file budget of ``_CUSTOM_REGEX_TIMEOUT_S`` + seconds bounds the worst-case backtracking cost of a + ``--type custom`` pattern (F-W3-PS-03 / F-W3-07 / F-W3S-03). On + Windows ``SIGALRM`` is unavailable; the scan runs without a guard + and the operator is expected to vet their regex. The exit-code + contract is unchanged either way — a timeout converts to + ``OSError`` which the dispatcher converts to ``EXIT_TRAINING_ERROR`` + plus a failure-flavoured audit event. + """ + matches: List[Dict[str, Any]] = [] + files_scanned: List[Dict[str, Any]] = [] + use_alarm = identifier_type == "custom" and os.name == "posix" and hasattr(_signal, "SIGALRM") + for path in files: + if use_alarm: + file_matches = _scan_file_with_alarm(path, pattern) + else: + file_matches = _scan_file(path, pattern) + matches.extend(file_matches) + files_scanned.append({"path": path, "match_count": len(file_matches)}) + return matches, files_scanned + + +def _scan_file_with_alarm(path: str, pattern: re.Pattern[str]) -> List[Dict[str, Any]]: + """POSIX-only wrapper enforcing ``_CUSTOM_REGEX_TIMEOUT_S``. - For lines longer than the budget we keep both ends with a literal - ``…`` separator so the operator can see the start and tail context - around the match. Short lines pass through unchanged. + Translates a ReDoS hang into a clean ``OSError`` so the dispatcher + handles it via the same audit-event path as a read failure. """ - if len(line) <= _SNIPPET_MAX_CHARS: - return line - half = (_SNIPPET_MAX_CHARS - 1) // 2 - return f"{line[:half]}…{line[-half:]}" + def _alarm(_sig, _frame): # pragma: no cover — signal handler + raise OSError(f"reverse-pii scan of {path!r} exceeded {_CUSTOM_REGEX_TIMEOUT_S}s (custom regex ReDoS guard)") -def _hash_for_audit(query: str) -> str: - """SHA-256 the operator's raw query for the audit event. + previous = _signal.signal(_signal.SIGALRM, _alarm) + _signal.alarm(_CUSTOM_REGEX_TIMEOUT_S) + try: + return _scan_file(path, pattern) + finally: + _signal.alarm(0) + _signal.signal(_signal.SIGALRM, previous) + + +def _hash_for_audit(query: str, salt: Optional[bytes]) -> str: + """Salted SHA-256 of the operator's raw query for the audit event. Article 15 access requests must not write the subject's identifier into the audit log: the very thing we're letting the subject query - is precisely what would be re-introduced if we logged it raw. We - therefore log a salt-free SHA-256 (the salt's job is collision - resistance for ``target_id`` matching, not for audit obfuscation; - here we only need a stable per-identifier fingerprint so a - compliance reviewer can correlate two queries about the same - subject without seeing the subject's data). + is precisely what would be re-introduced if we logged it raw. + + We reuse the per-output-dir salt that ``forgelm purge`` uses for + its ``target_id`` hashing (F-W3-PS-01). Two consequences: + + 1. A wordlist attack against the audit log requires the operator's + ``.forgelm_audit_salt`` file (and, if ``FORGELM_AUDIT_SECRET`` + is set, the env secret too). Salt-free SHA-256 of low-entropy + identifiers (e-mails, phone numbers) is brute-forcible from a + commodity wordlist; the salt closes that gap. + 2. A purge → reverse-pii cycle for the same subject in the same + ``output_dir`` produces matching digests, so a compliance + reviewer correlating Article 17 + Article 15 events for one + subject sees a connected timeline. + + The salt is ``None`` only on the legacy / pre-resolution code + path; production callers always pass the resolved salt. """ - return hashlib.sha256(query.encode("utf-8")).hexdigest() + raw = query.encode("utf-8") + if salt is None: + return hashlib.sha256(raw).hexdigest() + return hashlib.sha256(salt + raw).hexdigest() + + +def _bound_audit_error_message(message: str) -> str: + """Cap ``error_message`` so a future refactor cannot leak operator + data through ``str(exc)`` (F-W3-09).""" + if len(message) <= _AUDIT_ERROR_MESSAGE_MAX: + return message + return message[:_AUDIT_ERROR_MESSAGE_MAX] + "…[truncated]" + + +def _resolve_audit_dir(output_dir: str, audit_dir_override: Optional[str]) -> str: + """Return the audit-log root for this invocation. + + F-W3-06 / F-W3S-02 fix: the previous default (``audit_dir = + output_dir``) wrote ``audit_log.jsonl`` next to the corpus the + subject was asking about — bad colocation for an Article 12 + durable record. We now default to ``/audit/``, + matching the ``forgelm audit`` subcommand's convention. An + explicit ``--audit-dir`` always wins. + """ + if audit_dir_override: + return audit_dir_override + return os.path.join(output_dir, "audit") + + +def _maybe_audit_logger( + audit_dir: str, + *, + explicit: bool, + output_format: str, +): + """Construct the AuditLogger; fail closed when the operator opted in. + + F-W3-01 / F-W3-PS-02 / F-W3S-01 fix. Two distinct policies: + + - ``ConfigError`` (operator-identity not resolvable) is the only + "best-effort" branch. We log at WARNING (not DEBUG, so the + operator sees the audit was skipped) and return ``None``. + - Every other exception class — ``OSError`` from a read-only audit + dir, ``ValueError`` from a corrupt chain head, etc. — is a hard + failure. When the operator passed ``--audit-dir`` explicitly + (``explicit=True``) we refuse with ``EXIT_TRAINING_ERROR`` so a + regulator never sees a "scan completed" envelope without a + matching forensic-chain entry. When the dir is the implicit + default we still fail closed because the audit-chain contract is + a non-negotiable Article 15 requirement. + """ + try: + from forgelm.compliance import AuditLogger + from forgelm.config import ConfigError + + return AuditLogger(audit_dir) + except ConfigError as exc: + logger.warning( + "reverse-pii: AuditLogger init refused (%s); access request " + "will run but the regulatory chain will NOT record it. Set " + "FORGELM_OPERATOR= or FORGELM_ALLOW_ANONYMOUS_OPERATOR=1 " + "to restore audit emission.", + exc, + ) + return None + except (OSError, ValueError) as exc: + _output_error_and_exit( + output_format, + ( + f"reverse-pii: AuditLogger init failed for {audit_dir!r} ({exc.__class__.__name__}: {exc}). " + "Refusing to process an Article 15 access request without a forensic record. " + f"{'Pass --audit-dir ' if not explicit else 'Fix the audit-dir permission / chain integrity'} " + "and retry." + ), + EXIT_TRAINING_ERROR, + ) + + +def _emit_audit_event( + audit, + *, + query_hash: str, + identifier_type: str, + scan_mode: str, + salt_source: str, + files_scanned: List[Dict[str, Any]], + match_count: int, + error_class: Optional[str] = None, + error_message: Optional[str] = None, +) -> None: + """Centralise the ``data.access_request_query`` emit so success + and failure paths cannot drift on payload shape. ``salt_source`` + is recorded in every event (F-W3-PS-07).""" + payload: Dict[str, Any] = { + "query_hash": query_hash, + "identifier_type": identifier_type, + "scan_mode": scan_mode, + "salt_source": salt_source, + "files_scanned": [fs["path"] for fs in files_scanned], + "match_count": match_count, + } + if error_class is not None: + payload["error_class"] = error_class + if error_message is not None: + payload["error_message"] = _bound_audit_error_message(error_message) + audit.log_event(_EVT_ACCESS_REQUEST_QUERY, **payload) def _run_reverse_pii_cmd(args, output_format: str) -> None: """Top-level dispatcher for ``forgelm reverse-pii``.""" query = _validate_query(getattr(args, "query", None), output_format) - identifier_type = _validate_identifier_type(getattr(args, "type", "custom") or "custom", output_format) + identifier_type = _validate_identifier_type(getattr(args, "type", "literal") or "literal", output_format) globs = list(getattr(args, "files", None) or []) if not globs: _output_error_and_exit( @@ -269,31 +524,49 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: files = _resolve_files(globs, output_format) output_dir = getattr(args, "output_dir", None) or os.path.dirname(os.path.abspath(files[0])) or "." salt_source = getattr(args, "salt_source", None) - scan_query, scan_mode = _resolve_query_form(query, salt_source, output_dir, output_format) + + # F-W3-08 fix: validate the regex before any filesystem side effects + # (salt-file creation). We build the search pattern over the raw + # query first; if the operator typed an invalid custom regex they + # get exit 1 without leaving a salt file behind. + _build_search_pattern(query, identifier_type, output_format) + + scan_query, scan_mode, salt, salt_source_label = _resolve_query_form(query, salt_source, output_dir, output_format) pattern = _build_search_pattern(scan_query, identifier_type, output_format) - audit = _maybe_audit_logger(getattr(args, "audit_dir", None) or output_dir) - query_hash = _hash_for_audit(query) + audit_dir_override = getattr(args, "audit_dir", None) + audit_dir = _resolve_audit_dir(output_dir, audit_dir_override) + audit = _maybe_audit_logger( + audit_dir, + explicit=audit_dir_override is not None, + output_format=output_format, + ) + query_hash = _hash_for_audit(query, salt) matches: List[Dict[str, Any]] = [] files_scanned: List[Dict[str, Any]] = [] try: - for path in files: - file_matches = _scan_file(path, pattern) - matches.extend(file_matches) - files_scanned.append({"path": path, "match_count": len(file_matches)}) - except OSError as exc: + matches, files_scanned = _scan_files_with_redos_guard(files, pattern, identifier_type, output_format) + except (OSError, UnicodeDecodeError) as exc: if audit is not None: - audit.log_event( - _EVT_ACCESS_REQUEST_QUERY, - query_hash=query_hash, - identifier_type=identifier_type, - scan_mode=scan_mode, - files_scanned=[fs["path"] for fs in files_scanned], - match_count=len(matches), - error_class=exc.__class__.__name__, - error_message=str(exc), - ) + try: + _emit_audit_event( + audit, + query_hash=query_hash, + identifier_type=identifier_type, + scan_mode=scan_mode, + salt_source=salt_source_label, + files_scanned=files_scanned, + match_count=len(matches), + error_class=exc.__class__.__name__, + error_message=str(exc), + ) + except Exception as audit_exc: # noqa: BLE001 — fail-closed on audit error. + _output_error_and_exit( + output_format, + f"reverse-pii: failed to write the Article 15 failure audit event ({audit_exc}).", + EXIT_TRAINING_ERROR, + ) _output_error_and_exit( output_format, f"Failed reading corpus file mid-scan: {exc}. Partial matches surfaced before the failure are not emitted.", @@ -301,20 +574,29 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: ) if audit is not None: - audit.log_event( - _EVT_ACCESS_REQUEST_QUERY, - query_hash=query_hash, - identifier_type=identifier_type, - scan_mode=scan_mode, - files_scanned=[fs["path"] for fs in files_scanned], - match_count=len(matches), - ) + try: + _emit_audit_event( + audit, + query_hash=query_hash, + identifier_type=identifier_type, + scan_mode=scan_mode, + salt_source=salt_source_label, + files_scanned=files_scanned, + match_count=len(matches), + ) + except Exception as audit_exc: # noqa: BLE001 — fail-closed on audit error. + _output_error_and_exit( + output_format, + f"reverse-pii: failed to write the Article 15 success audit event ({audit_exc}).", + EXIT_TRAINING_ERROR, + ) payload: Dict[str, Any] = { "success": True, "query_hash": query_hash, "identifier_type": identifier_type, "scan_mode": scan_mode, + "salt_source": salt_source_label, "matches": matches, "files_scanned": files_scanned, "match_count": len(matches), @@ -337,6 +619,7 @@ def _emit_reverse_pii_result(payload: Dict[str, Any], output_format: str) -> Non f"{len(files_scanned)} file(s)." ) return + print("WARNING: snippets below contain raw corpus content (PII). Do not redirect this output to a persistent log.") print( f"Found {match_count} match(es) for identifier hash {payload['query_hash'][:16]}… " f"({payload['identifier_type']}, {payload['scan_mode']} mode):" @@ -349,27 +632,6 @@ def _emit_reverse_pii_result(payload: Dict[str, Any], output_format: str) -> Non print(f" {fs['path']}: {fs['match_count']} match(es)") -def _maybe_audit_logger(audit_dir: str): - """Best-effort construct the AuditLogger. - - Mirrors :func:`forgelm.cli.subcommands._cache._maybe_audit_logger` — - a missing operator identity should not abort an Article 15 access - request (the subject still gets their answer; the chain just - skips the breadcrumb). - """ - try: - from forgelm.compliance import AuditLogger - from forgelm.config import ConfigError - - return AuditLogger(audit_dir) - except ConfigError as exc: - logger.debug("reverse-pii: AuditLogger init failed (%s); continuing without audit log.", exc) - return None - except Exception as exc: # noqa: BLE001 — best-effort: audit is optional context here. # NOSONAR - logger.debug("reverse-pii: AuditLogger init crashed (%s); continuing without audit log.", exc) - return None - - __all__ = [ "_run_reverse_pii_cmd", "_validate_query", @@ -381,4 +643,6 @@ def _maybe_audit_logger(audit_dir: str): "_truncate_snippet", "_hash_for_audit", "_emit_reverse_pii_result", + "_resolve_audit_dir", + "_emit_audit_event", ] diff --git a/forgelm/webhook.py b/forgelm/webhook.py index b80dbba5..1c4e51b7 100644 --- a/forgelm/webhook.py +++ b/forgelm/webhook.py @@ -87,14 +87,20 @@ def _post_payload(self, url: str, payload: dict, event: str) -> None: ca_bundle = getattr(self.config, "tls_ca_bundle", None) # Timeout floor — webhook keeps the historical 1s floor (``safe_post`` - # rejects 0/None unconditionally). - timeout = getattr(self.config, "timeout", 5) + # rejects 0/None unconditionally). Wave 3 F-compliance-106 + # raised the WebhookConfig default 5 → 10; the clamp message + # and fallback value follow that default. + from .config import WebhookConfig as _WebhookConfig + + default_timeout = _WebhookConfig.model_fields["timeout"].default + timeout = getattr(self.config, "timeout", default_timeout) if not isinstance(timeout, (int, float)) or timeout < 1: logger.warning( - "Webhook timeout=%r is below the 1s floor; clamping to 5s.", + "Webhook timeout=%r is below the 1s floor; clamping to %ds.", timeout, + default_timeout, ) - timeout = 5 + timeout = default_timeout allow_private = bool(getattr(self.config, "allow_private_destinations", False)) diff --git a/tests/test_check_bilingual_parity.py b/tests/test_check_bilingual_parity.py index 0d3fcb87..5d7e8b09 100644 --- a/tests/test_check_bilingual_parity.py +++ b/tests/test_check_bilingual_parity.py @@ -84,6 +84,22 @@ def test_tilde_fence_also_recognised(self, tmp_path: Path) -> None: def test_missing_file_returns_empty(self, tmp_path: Path) -> None: assert extract_headings(tmp_path / "missing.md") == [] + def test_setext_headings_are_not_matched(self, tmp_path: Path) -> None: + """F-W3T-03 regression: setext (``===``, ``---``) underlines + must NOT register as headings. The project's docs use ATX + exclusively per ``docs/standards/documentation.md`` and a setext + heading in a mirror is itself a lint finding (see ``_HEADING_RE`` + comment in the production code). A regex tweak that accidentally + widened the matcher to setext would silently inflate the parity + tool's counts on every doc that mixes setext.""" + src = _write( + tmp_path / "doc.md", + "Setext H1\n=========\n## Real H2\nSetext H2\n---------\n", + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [2] + assert [h.text for h in headings] == ["Real H2"] + # --------------------------------------------------------------------------- # Pair diff: pass / fail / missing @@ -198,7 +214,16 @@ class TestCanonicalRepoPasses: def test_repository_pairs_pass_strict(self) -> None: """The repo's actual EN/TR pairs must pass ``--strict`` so the CI gate stays green. This is the test that fails first if a - future PR drifts a mirror without fixing it.""" + future PR drifts a mirror without fixing it. + + DELIBERATELY REDUNDANT with the CI lint job at + ``.github/workflows/ci.yml`` (F-W3T-08 docstring) — a drift + here surfaces in the pytest run too so contributors see it + locally before push. Do not delete one in favour of the + other; they catch different categories of regression (this + one catches a drift introduced in the same PR as the test + edit; the lint job catches a drift introduced after the test + ran).""" repo_root = Path(__file__).parent.parent rc = main(["--strict", "--repo-root", str(repo_root)]) assert rc == 0, ( @@ -206,3 +231,43 @@ def test_repository_pairs_pass_strict(self) -> None: "`python3 tools/check_bilingual_parity.py` for the per-pair " "drift report." ) + + def test_every_tr_mirror_appears_in_pair_registry(self) -> None: + """F-W3T-07 regression: a ``*-tr.md`` file added under the + parity tool's curated domain (``docs/guides/`` + + ``docs/reference/``) without a ``_PAIRS`` registry entry would + silently bypass the strict gate. This meta-test asserts the + registry is the source of truth for those two directories. + + Out-of-scope: ``docs/usermanuals/`` has its own structural + validator (``tools/build_usermanuals.py``); top-level docs + like ``docs/roadmap-tr.md`` and ``docs/product_strategy-tr.md`` + diverge intentionally; ``docs/analysis/`` is internal / + gitignored research. + + Allowlist: ``safety_compliance-tr.md`` carries a known + in-progress structural drift (34 H2/H3/H4 deltas as of Wave + 3); the EN → TR completion is tracked as a separate Wave 4 + translation task and the file would block this gate today. + Remove from the allowlist once parity is achieved. + """ + repo_root = Path(__file__).parent.parent + scoped_dirs = [repo_root / "docs" / "guides", repo_root / "docs" / "reference"] + # Files whose TR mirror is acknowledged as structurally + # incomplete; do NOT add new entries here without an explicit + # tracking ticket. + in_progress_allowlist = {"docs/guides/safety_compliance-tr.md"} + tr_files: list[str] = [] + for scoped_dir in scoped_dirs: + if not scoped_dir.is_dir(): + continue + for path in scoped_dir.rglob("*-tr.md"): + tr_files.append(path.relative_to(repo_root).as_posix()) + tr_files.sort() + registered = {tr for _, tr in check_bilingual_parity._PAIRS} + orphans = [path for path in tr_files if path not in registered and path not in in_progress_allowlist] + assert orphans == [], ( + f"{orphans} carry a TR mirror but are not registered in " + "_PAIRS — register them in tools/check_bilingual_parity.py " + "or the strict gate will silently skip drift detection." + ) diff --git a/tests/test_config.py b/tests/test_config.py index a8025bd6..bced1c2a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -129,6 +129,11 @@ def test_defaults(self): assert w.notify_on_start is True assert w.notify_on_success is True assert w.notify_on_failure is True + # F-W3T-09 regression: pin the F-compliance-106 default + # (5 → 10) here next to every other field default so a + # contributor editing WebhookConfig sees the pin in the + # obvious reading position. + assert w.timeout == 10 def test_url_env(self): w = WebhookConfig(url_env="MY_WEBHOOK_URL") diff --git a/tests/test_eu_ai_act.py b/tests/test_eu_ai_act.py index 982f62aa..a9fc77ec 100644 --- a/tests/test_eu_ai_act.py +++ b/tests/test_eu_ai_act.py @@ -141,18 +141,21 @@ def test_high_risk_warnings(self, caplog, minimal_config): assert "high-risk" in caplog.text assert "auto_revert" in caplog.text - def test_high_risk_safety_disabled_raises_config_error(self, minimal_config): - """F-compliance-110 regression: high-risk + safety.enabled=false - must surface as ``ConfigError`` (was a warning prior to v0.5.5). - EU AI Act Article 9 risk-management evidence cannot be derived - from a disabled safety eval; operators who genuinely want a - sandboxed run must lower the risk_classification.""" + @pytest.mark.parametrize("tier", ["high-risk", "unacceptable"]) + def test_strict_tier_safety_disabled_raises_config_error(self, minimal_config, tier): + """F-compliance-110 + F-W3T-02 regression: BOTH tiers in + ``_STRICT_RISK_TIERS`` (high-risk + unacceptable) must surface as + ``ConfigError`` when safety is disabled — pinning the failure leg + for both Article 9 (high-risk) and Article 5 (unacceptable / + prohibited) tiers. EU AI Act risk-management evidence cannot be + derived from a disabled safety eval; operators who genuinely + want a sandboxed run must lower the risk_classification.""" from forgelm.config import ConfigError with pytest.raises(ConfigError, match="evaluation.safety.enabled"): ForgeConfig( **minimal_config( - risk_assessment={"risk_category": "high-risk"}, + risk_assessment={"risk_category": tier}, # No safety block → safety disabled by default. ) ) diff --git a/tests/test_reverse_pii.py b/tests/test_reverse_pii.py index 54d37491..090452da 100644 --- a/tests/test_reverse_pii.py +++ b/tests/test_reverse_pii.py @@ -11,7 +11,10 @@ same per-output-dir salt as `forgelm purge`; matches the masked corpus. 3. Audit-event emission — `data.access_request_query` written; - identifier hashed (NEVER raw). + identifier hashed (NEVER raw) and salted (Wave 3 absorption + F-W3-PS-01: salt-free SHA-256 of low-entropy identifiers is + brute-forcible from a wordlist; the audit hash now reuses the + per-output-dir salt purge already uses). 4. Glob expansion — multiple files scanned in deterministic order; per-file match count surfaced. 5. Failure paths — empty query / unparseable custom regex / empty @@ -52,7 +55,7 @@ def _read_audit_events(audit_log_path: Path) -> list[dict]: def _build_args( *, query: str | None = None, - type: str | None = "custom", + type: str | None = "literal", salt_source: str | None = None, files: list[str] | None = None, output_dir: str | None = None, @@ -140,6 +143,29 @@ def test_no_residual_found_returns_empty_match_list(self, tmp_path: Path, capsys assert len(payload["files_scanned"]) == 1 assert payload["files_scanned"][0]["match_count"] == 0 + def test_default_type_is_literal_not_regex(self, tmp_path: Path, capsys) -> None: + """F-W3-02 regression: the default --type must NOT interpret the + query as a regex. ``alice@example.com`` (with literal dot) must + not match ``alice@exampleXcom`` (where ``.`` would be wildcarded + under the previous default of ``custom``).""" + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "row-A", "text": "contact alice@exampleXcom"}]) + # No --type → default = "literal". + args = _build_args( + query="alice@example.com", + type=None, + files=[str(corpus)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["match_count"] == 0, "literal default must not regex-match wildcards" + assert payload["identifier_type"] == "literal" + # --------------------------------------------------------------------------- # §2 Test — Hash-mask scan: SHA256(salt + identifier) found in masked corpus @@ -204,6 +230,29 @@ def test_env_var_salt_source_requires_secret_env(self, tmp_path: Path, capsys, m _run_reverse_pii_cmd(args, output_format="json") assert ei.value.code == 1, "env_var without FORGELM_AUDIT_SECRET must surface as EXIT_CONFIG_ERROR" + def test_per_dir_salt_source_refuses_when_env_var_set(self, tmp_path: Path, capsys, monkeypatch) -> None: + """F-W3-11 / F-W3-PS-04 regression: the symmetric direction — + operator asked for ``per_dir`` but ``FORGELM_AUDIT_SECRET`` is + set in the shell environment — must also refuse, with a + direction-aware diagnostic that names the env-var unset path.""" + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + monkeypatch.setenv("FORGELM_AUDIT_SECRET", "leftover-from-shell") + corpus = tmp_path / "masked.jsonl" + _seed_corpus(corpus, [{"id": "row-A", "text": "x"}]) + args = _build_args( + query="alice@example.com", + type="email", + salt_source="per_dir", + files=[str(corpus)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1 + payload = json.loads(capsys.readouterr().out) + assert "FORGELM_AUDIT_SECRET" in payload["error"] + # --------------------------------------------------------------------------- # §3 Test — Audit event emitted; identifier hashed (NEVER raw) @@ -212,6 +261,7 @@ def test_env_var_salt_source_requires_secret_env(self, tmp_path: Path, capsys, m class TestAuditEventDoesNotLeakIdentifier: def test_access_request_event_carries_hashed_query_only(self, tmp_path: Path) -> None: + from forgelm.cli.subcommands._purge import _hash_target_id, _resolve_salt from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd audit_dir = tmp_path / "audit" @@ -234,15 +284,42 @@ def test_access_request_event_carries_hashed_query_only(self, tmp_path: Path) -> access_events = [e for e in events if e["event"] == "data.access_request_query"] assert len(access_events) == 1 evt = access_events[0] - # Hashed identifier present; raw email NOT. - expected_hash = hashlib.sha256(b"alice@example.com").hexdigest() + # F-W3-PS-01 absorption: the hash is salted with the per-output-dir + # salt purge uses for target_id; an unsalted SHA-256 must NOT be + # present (otherwise a wordlist attack against the audit log + # would recover the subject's identifier). + salt, _ = _resolve_salt(str(tmp_path)) + expected_hash = _hash_target_id("alice@example.com", salt) + unsalted = hashlib.sha256(b"alice@example.com").hexdigest() assert evt["query_hash"] == expected_hash + assert evt["query_hash"] != unsalted, ( + "audit query_hash must be salted; otherwise a wordlist attack " + "recovers the subject's identifier from the chain" + ) # Critical: raw identifier appears NOWHERE in the event payload. as_str = json.dumps(evt) assert "alice@example.com" not in as_str, f"audit event leaked raw identifier:\n{as_str}" assert evt["identifier_type"] == "email" assert evt["scan_mode"] == "plaintext" assert evt["match_count"] == 1 + # F-W3-PS-07 absorption: salt_source recorded in every event. + assert evt["salt_source"] in {"plaintext", "per_dir", "env_var"} + + def test_purge_target_id_matches_reverse_pii_query_hash_on_same_output_dir(self, tmp_path: Path) -> None: + """F-W3-PS-09 absorption: the audit chain must let a compliance + reviewer correlate Article 17 (purge) and Article 15 (reverse-pii) + events for the same subject in the same ``output_dir``. The + digests are equal iff both subcommands use the same per-output-dir + salt — which they now do.""" + from forgelm.cli.subcommands._purge import _hash_target_id, _resolve_salt + from forgelm.cli.subcommands._reverse_pii import _hash_for_audit + + salt, _ = _resolve_salt(str(tmp_path)) + purge_target_id = _hash_target_id("alice@example.com", salt) + reverse_pii_query_hash = _hash_for_audit("alice@example.com", salt) + assert purge_target_id == reverse_pii_query_hash, ( + "cross-tool audit correlation requires both subcommands to salt with the same per-output-dir salt" + ) # --------------------------------------------------------------------------- @@ -299,6 +376,25 @@ def test_recursive_glob_descends_into_subdirs(self, tmp_path: Path, capsys) -> N payload = json.loads(capsys.readouterr().out) assert payload["match_count"] == 1 + def test_overlapping_globs_deduped_to_one_scan(self, tmp_path: Path, capsys) -> None: + """F-W3T-12 regression: two globs that match the same file must + not double-count matches.""" + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + _seed_corpus(tmp_path / "train.jsonl", [{"id": "1", "text": "alice@example.com"}]) + args = _build_args( + query="alice@example.com", + type="email", + files=[str(tmp_path / "*.jsonl"), str(tmp_path / "train*.jsonl")], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + payload = json.loads(capsys.readouterr().out) + assert len(payload["files_scanned"]) == 1 + assert payload["match_count"] == 1 + # --------------------------------------------------------------------------- # §5 Test — Failure paths @@ -316,16 +412,26 @@ def test_empty_query_exits_config_error(self, tmp_path: Path) -> None: _run_reverse_pii_cmd(args, output_format="json") assert ei.value.code == 1 - def test_unparseable_custom_regex_exits_config_error(self, tmp_path: Path) -> None: + def test_unparseable_custom_regex_exits_config_error_without_salt_side_effect(self, tmp_path: Path) -> None: + """F-W3-08 regression: a config error (unparseable regex) must + not leave a salt file behind on disk. Validation runs before + any filesystem side effect.""" from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd corpus = tmp_path / "train.jsonl" _seed_corpus(corpus, [{"id": "1", "text": "x"}]) # Unbalanced bracket; re.compile will raise. - args = _build_args(query="[invalid", type="custom", files=[str(corpus)], output_dir=str(tmp_path)) + args = _build_args( + query="[invalid", + type="custom", + salt_source="per_dir", # would trigger salt creation if validation order was wrong + files=[str(corpus)], + output_dir=str(tmp_path), + ) with pytest.raises(SystemExit) as ei: _run_reverse_pii_cmd(args, output_format="json") assert ei.value.code == 1 + assert not (tmp_path / ".forgelm_audit_salt").exists(), "config error must not create a salt file on disk" def test_unknown_identifier_type_rejected(self, tmp_path: Path) -> None: # The argparse layer also catches this, but the dispatcher @@ -352,6 +458,30 @@ def test_empty_glob_exits_config_error(self, tmp_path: Path, capsys) -> None: _run_reverse_pii_cmd(args, output_format="json") assert ei.value.code == 1 + def test_directory_argument_diagnoses_glob_form(self, tmp_path: Path, capsys) -> None: + """F-W3-12 regression: passing a directory must produce a + targeted diagnostic naming the glob form, not a generic empty-glob + error. Closes the UX paper-cut where the operator typed `data/` + intending `data/*.jsonl`.""" + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + # Seed a file inside the directory so glob expansion finds the + # directory but not its contents (positional is the directory + # itself, not a glob). + _seed_corpus(tmp_path / "child.jsonl", [{"id": "1", "text": "x"}]) + args = _build_args( + query="x", + type="email", + files=[str(tmp_path)], + output_dir=str(tmp_path), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 1 + payload = json.loads(capsys.readouterr().out) + assert "directory" in payload["error"].lower() + assert ".jsonl" in payload["error"] + def test_no_files_argument_exits_config_error(self, tmp_path: Path) -> None: from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd @@ -360,7 +490,11 @@ def test_no_files_argument_exits_config_error(self, tmp_path: Path) -> None: _run_reverse_pii_cmd(args, output_format="json") assert ei.value.code == 1 - def test_mid_scan_io_failure_writes_failed_audit_event(self, tmp_path: Path, monkeypatch) -> None: + def test_mid_scan_io_failure_writes_failed_audit_event_without_leaking_identifier( + self, tmp_path: Path, monkeypatch + ) -> None: + """F-W3T-01 regression: the failure-path audit event must carry + the same no-leak invariant as the success path.""" from forgelm.cli.subcommands import _reverse_pii corpus = tmp_path / "train.jsonl" @@ -402,19 +536,83 @@ def _flaky_scan(path: str, pattern): evt = access_events[0] assert evt["error_class"] == "OSError" assert "simulated mid-scan I/O failure" in evt["error_message"] + # Critical: failure-path event must carry the same no-leak invariant + # as the success path (F-W3T-01). + as_str = json.dumps(evt) + assert "alice@example.com" not in as_str, f"failure-path audit event leaked raw identifier:\n{as_str}" + + def test_malformed_utf8_corpus_exits_runtime_error_with_audit_event(self, tmp_path: Path) -> None: + """F-W3-04 regression: a UnicodeDecodeError mid-scan must surface + as ``EXIT_TRAINING_ERROR=2`` with a failure-flavoured audit + event — not as Python's default exit 1 with no audit record.""" + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "bad.jsonl" + # Valid first line, then a multi-byte rune fragment. + corpus.write_bytes(b'{"id":1,"text":"valid"}\n\xff\xfe garbage\n') + audit_dir = tmp_path / "audit" + audit_dir.mkdir() + + args = _build_args( + query="x", + type="email", + files=[str(corpus)], + output_dir=str(tmp_path), + audit_dir=str(audit_dir), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 2 + + events = _read_audit_events(audit_dir / "audit_log.jsonl") + access_events = [e for e in events if e["event"] == "data.access_request_query"] + assert len(access_events) == 1 + evt = access_events[0] + assert evt["error_class"] in {"UnicodeDecodeError", "OSError"} + + def test_explicit_audit_dir_unwritable_fails_closed(self, tmp_path: Path, monkeypatch) -> None: + """F-W3-01 / F-W3-PS-02 regression: when --audit-dir is explicit + and AuditLogger init crashes with a non-ConfigError, the run + must refuse with EXIT_TRAINING_ERROR rather than silently + proceeding without an Article 15 forensic record.""" + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + from forgelm.compliance import AuditLogger + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "1", "text": "x"}]) + + def _crashing_init(self, *_args, **_kw): + raise OSError("simulated audit-init failure (read-only volume)") + + monkeypatch.setattr(AuditLogger, "__init__", _crashing_init) + args = _build_args( + query="x", + type="email", + files=[str(corpus)], + output_dir=str(tmp_path), + audit_dir=str(tmp_path / "audit"), + ) + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 2, ( + "explicit --audit-dir + audit init OSError must fail closed; " + "silently dropping the chain entry breaches the Article 15 contract" + ) # --------------------------------------------------------------------------- -# §6 Test — Snippet truncation defends unbounded log spam +# §6 Test — Snippet truncation defends unbounded log spam AND preserves match # --------------------------------------------------------------------------- class TestSnippetTruncation: - def test_long_line_centre_truncated(self, tmp_path: Path, capsys) -> None: + def test_long_line_centred_on_match_preserves_identifier(self, tmp_path: Path, capsys) -> None: + """F-W3-03 regression: snippet truncation must centre on the + match span so the operator can verify the hit. The previous + head+tail strategy dropped the matched span on long lines.""" from forgelm.cli.subcommands._reverse_pii import _SNIPPET_MAX_CHARS, _run_reverse_pii_cmd - # Construct a line longer than the snippet budget with the - # match in the middle. + # Long line with the match buried in the middle. prefix = "x" * 200 suffix = "y" * 200 corpus = tmp_path / "train.jsonl" @@ -433,8 +631,30 @@ def test_long_line_centre_truncated(self, tmp_path: Path, capsys) -> None: payload = json.loads(capsys.readouterr().out) snippet = payload["matches"][0]["snippet"] assert len(snippet) <= _SNIPPET_MAX_CHARS - # Ellipsis marks where the middle was elided. + # Ellipsis marks where context was elided. assert "…" in snippet + # Critical: the identifier itself must survive truncation + # (otherwise the operator cannot verify the hit). + assert "alice@example.com" in snippet, ( + "centred truncation must preserve the matched span — the whole point is operator verification" + ) + + def test_truncation_respects_multibyte_utf8(self, tmp_path: Path, capsys) -> None: + """F-W3T-05 regression: code-point slicing must not produce + broken UTF-8 sequences when the line mixes CJK + emoji.""" + from forgelm.cli.subcommands._reverse_pii import _SNIPPET_MAX_CHARS, _truncate_snippet + + # Mix CJK + emoji; bytes per char vary 1/3/4. + line = ("前置テキスト" * 30) + " alice@example.com " + ("🚀後続" * 30) + # Locate the match span by hand for the unit test. + start = line.index("alice@example.com") + end = start + len("alice@example.com") + snippet = _truncate_snippet(line, (start, end)) + assert len(snippet) <= _SNIPPET_MAX_CHARS + 2 # head + tail "…" + # Round-trip through utf-8 must succeed (no broken surrogates). + snippet.encode("utf-8").decode("utf-8") + # The matched span survives the truncation. + assert "alice@example.com" in snippet # --------------------------------------------------------------------------- @@ -444,18 +664,12 @@ def test_long_line_centre_truncated(self, tmp_path: Path, capsys) -> None: class TestReversePiiFacade: def test_facade_re_exports_dispatcher_and_helpers(self) -> None: + """F-W3-10 regression: facade must re-export every name in + ``_reverse_pii.__all__``, not a hand-maintained subset.""" from forgelm import cli as _cli_facade + from forgelm.cli.subcommands import _reverse_pii as _mod - for name in ( - "_run_reverse_pii_cmd", - "_validate_query", - "_validate_identifier_type", - "_resolve_files", - "_scan_file", - "_truncate_snippet", - "_hash_for_audit", - "_emit_reverse_pii_result", - ): + for name in _mod.__all__: assert hasattr(_cli_facade, name), f"forgelm.cli must re-export {name!r}" def test_parser_registers_reverse_pii_subcommand(self) -> None: @@ -483,13 +697,16 @@ def test_parser_accepts_minimum_arguments(self, tmp_path: Path) -> None: assert args.command == "reverse-pii" assert args.query == "alice@example.com" assert args.files == [str(corpus)] - # Defaults round-trip. - assert args.type == "custom" + # Defaults round-trip — Wave 3 absorption: default --type is now + # ``literal`` (was ``custom``); F-W3-02 fix. + assert args.type == "literal" assert args.salt_source is None def test_dispatch_table_registers_reverse_pii(self) -> None: - """The dispatcher table must include the reverse-pii row so the - CLI binary actually routes the subcommand to its handler.""" + """Dispatch routing only — patching the function under test is + intentional; the contract is "command name routes to the + registered handler", not "the handler does the right thing" + (which is covered by the rest of this file).""" from forgelm.cli._dispatch import _dispatch_subcommand # Exercise the dispatch via a monkeypatch on the facade so we diff --git a/tools/check_bilingual_parity.py b/tools/check_bilingual_parity.py index 0f92248f..5331e966 100644 --- a/tools/check_bilingual_parity.py +++ b/tools/check_bilingual_parity.py @@ -27,6 +27,16 @@ # Restrict to one pair (useful while editing). python3 tools/check_bilingual_parity.py --only docs/guides/ingestion.md +Exit codes (per the parity tool's own contract — ``forgelm/`` exit +codes do NOT apply since this script lives under ``tools/``): + + 0 — parity (or advisory mode reporting drift but not gating). + 1 — strict drift detected, OR invalid argument (unknown ``--only``, + unparseable ``--levels``). + +CI gate authors must NOT wrap the strict invocation in ``|| true`` — +exit 1 IS the gate. + The tool is AST-free and Pydantic-free — it only needs ``re`` and the filesystem, so it runs in the same lint job as ``ruff`` without any optional extra. @@ -58,6 +68,7 @@ ("docs/guides/ingestion.md", "docs/guides/ingestion-tr.md"), # docs/reference/ ("docs/reference/architecture.md", "docs/reference/architecture-tr.md"), + ("docs/reference/audit_event_catalog.md", "docs/reference/audit_event_catalog-tr.md"), ("docs/reference/configuration.md", "docs/reference/configuration-tr.md"), ("docs/reference/data_preparation.md", "docs/reference/data_preparation-tr.md"), ("docs/reference/distributed_training.md", "docs/reference/distributed_training-tr.md"), From 6ba93be858487e8547ef68e0aa5f25f75c2f693a Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 18:15:18 +0300 Subject: [PATCH 04/11] fix(wave-3-followup): absorb 3 SonarCloud + 1 hotspot findings on PR #31 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * python:S1172 (MAJOR) — `_scan_files_with_redos_guard` `output_format` parameter is unused; dropped (and from the call site). No public-API break; the helper is private to the dispatcher module. * python:S2201 (MAJOR) — multi-byte UTF-8 truncation regression test was discarding the `bytes.decode` return value. Now asserts the round-trip equals the original snippet, which is the actual contract ("no broken surrogates from a slice landing mid-rune"). * python:S3776 (CRITICAL) — `ForgeConfig._warn_high_risk_compliance` cognitive complexity 20 vs 15 allowed. Refactored into three focused helpers (`_resolve_risk_label`, `_warn_unacceptable_practice`, `_enforce_safety_gate_for_strict_tier`) plus a thin orchestrator. No behaviour change: the F-compliance-110 ConfigError raise, the Article 5 prohibited-practices banner, and the auto_revert / track_categories warnings all fire identically; the existing parametrised tests in test_eu_ai_act.py + test_integration.py continue to pin the contract. * DOS hotspot (MEDIUM) — `tools/check_bilingual_parity.py:_HEADING_RE` used `^(#{1,6})\s+(.+?)\s*#*\s*$`, which chains a lazy `(.+?)` against a trailing `\s*#*\s*$` — polynomial backtracking on a pathological all-spaces-and-hashes line. Split heading recognition into a linear-time prefix match (`^(#{1,6}) +`) plus a deterministic trailing-hash trim (`\s+#+$`); no nested quantifiers, no chained lazy/greedy interplay. Live-tree `--strict` invocation still passes against all 9 doc pairs. Skipped: * publicly-writable-directories hotspot at `test_integration.py:285` flags `model_path="/tmp/model"` — a string literal passed as a structured-log payload, never written to disk. False positive on test fixture data; would suppress in Sonar UI rather than alter the test. Codacy 2.0 / v3 issue API requires auth; the 21 Codacy issues (2 high · 19 minor) overlap heavily with Sonar's smaller authoritative set. Re-running CI on this push will surface any remaining Codacy finding that Sonar did not catch. Suite: 1344 passed, 14 skipped (no regressions). Lint clean. Bilingual parity: 9/9 OK against the live tree. Co-Authored-By: Claude Opus 4.7 (1M context) --- forgelm/cli/subcommands/_reverse_pii.py | 3 +- forgelm/config.py | 104 ++++++++++++++---------- tests/test_reverse_pii.py | 6 +- tools/check_bilingual_parity.py | 41 +++++++--- 4 files changed, 95 insertions(+), 59 deletions(-) diff --git a/forgelm/cli/subcommands/_reverse_pii.py b/forgelm/cli/subcommands/_reverse_pii.py index 528ed12f..91da129b 100644 --- a/forgelm/cli/subcommands/_reverse_pii.py +++ b/forgelm/cli/subcommands/_reverse_pii.py @@ -332,7 +332,6 @@ def _scan_files_with_redos_guard( files: List[str], pattern: re.Pattern[str], identifier_type: str, - output_format: str, ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: """Walk every file once; centralise the per-file ReDoS budget. @@ -546,7 +545,7 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: matches: List[Dict[str, Any]] = [] files_scanned: List[Dict[str, Any]] = [] try: - matches, files_scanned = _scan_files_with_redos_guard(files, pattern, identifier_type, output_format) + matches, files_scanned = _scan_files_with_redos_guard(files, pattern, identifier_type) except (OSError, UnicodeDecodeError) as exc: if audit is not None: try: diff --git a/forgelm/config.py b/forgelm/config.py index 7b3fafaa..11b5b5bc 100644 --- a/forgelm/config.py +++ b/forgelm/config.py @@ -840,6 +840,59 @@ def _warn_general_consistency(self) -> None: self.training.save_steps, ) + def _resolve_risk_label(self) -> Optional[str]: + """Return the active risk label across both sibling fields. + + ``risk_assessment.risk_category`` and ``compliance.risk_classification`` + share the same RiskTier set; either is accepted as authoritative + for the strict-tier warnings. + """ + if self.risk_assessment and self.risk_assessment.risk_category: + return self.risk_assessment.risk_category + if self.compliance and self.compliance.risk_classification: + return self.compliance.risk_classification + return None + + def _warn_unacceptable_practice(self) -> None: + """Article 5 — prohibited-practices banner. + + Louder operator notice on top of the auto_revert nudge — the + deployment itself is unlawful in the EU regardless of how well + the safety gates are wired up. + """ + logger.warning( + "Risk classification 'unacceptable' corresponds to EU AI Act Article 5 prohibited " + "practices. ForgeLM will not refuse the run, but deploying such a system inside the " + "EU is unlawful — confirm operator intent before continuing." + ) + + def _enforce_safety_gate_for_strict_tier(self, label: Optional[str]) -> None: + """Article 9 — risk management evidence requires safety eval enabled. + + Wave 3 / Faz 28 (F-compliance-110): a high-risk / unacceptable + classification REQUIRES an enabled safety evaluation gate to + back the EU AI Act Article 9 risk-management claim. Earlier + versions only emitted a warning, which let regulated runs + ship Annex IV bundles whose risk-management section was not + actually evidenced. v0.5.5 escalates the warning to a hard + ``ConfigError``: operators who genuinely want a sandboxed run + without safety eval must lower the risk_classification (e.g. + to ``limited-risk``) or enable ``evaluation.safety``. + """ + safety = self.evaluation.safety if self.evaluation else None + if not safety or not safety.enabled: + raise ConfigError( + f"Risk classification {label!r} requires evaluation.safety.enabled: true " + "(EU AI Act Article 9 risk-management evidence cannot be derived " + "from a disabled safety eval). Either enable safety evaluation " + "or lower the risk_classification to a non-strict tier." + ) + if not safety.track_categories: + logger.warning( + "High-risk AI: harm category tracking (track_categories: true) is recommended " + "for detailed EU AI Act compliance documentation." + ) + def _warn_high_risk_compliance(self) -> None: """EU AI Act compliance recommendations for strict risk tiers. @@ -849,57 +902,18 @@ def _warn_high_risk_compliance(self) -> None: and ``unacceptable`` should never get *less* gating than ``high-risk`` because the underlying use case is not allowed at all under the Act. """ - is_strict = (self.risk_assessment and self.risk_assessment.risk_category in _STRICT_RISK_TIERS) or ( - self.compliance and self.compliance.risk_classification in _STRICT_RISK_TIERS - ) - if not is_strict: + label = self._resolve_risk_label() + if label not in _STRICT_RISK_TIERS: return if not self.evaluation or not self.evaluation.auto_revert: logger.warning( "Risk classification %r requires evaluation.auto_revert: true " "for EU AI Act compliance. Safety gates should be enabled.", - # Use whichever field is set; both share the same RiskTier. - (self.compliance and self.compliance.risk_classification) - or (self.risk_assessment and self.risk_assessment.risk_category), - ) - # Article 5 prohibited practices warrant a louder operator notice on - # top of the auto_revert nudge — the deployment itself is unlawful in - # the EU regardless of how well the safety gates are wired up. - unacceptable = (self.risk_assessment and self.risk_assessment.risk_category == "unacceptable") or ( - self.compliance and self.compliance.risk_classification == "unacceptable" - ) - if unacceptable: - logger.warning( - "Risk classification 'unacceptable' corresponds to EU AI Act Article 5 prohibited " - "practices. ForgeLM will not refuse the run, but deploying such a system inside the " - "EU is unlawful — confirm operator intent before continuing." - ) - safety = self.evaluation.safety if self.evaluation else None - if not safety or not safety.enabled: - # Wave 3 / Faz 28 (F-compliance-110): a high-risk / unacceptable - # classification REQUIRES an enabled safety evaluation gate to - # back the EU AI Act Article 9 risk-management claim. Earlier - # versions only emitted a warning, which let regulated runs - # ship Annex IV bundles whose risk-management section was not - # actually evidenced. v0.5.5 escalates the warning to a hard - # ``ConfigError``: operators who genuinely want a sandboxed run - # without safety eval must lower the risk_classification (e.g. - # to ``limited-risk``) or enable ``evaluation.safety``. - raise ConfigError( - "Risk classification %r requires evaluation.safety.enabled: true " - "(EU AI Act Article 9 risk-management evidence cannot be derived " - "from a disabled safety eval). Either enable safety evaluation " - "or lower the risk_classification to a non-strict tier." - % ( - (self.compliance and self.compliance.risk_classification) - or (self.risk_assessment and self.risk_assessment.risk_category), - ) - ) - elif not safety.track_categories: - logger.warning( - "High-risk AI: harm category tracking (track_categories: true) is recommended " - "for detailed EU AI Act compliance documentation." + label, ) + if label == "unacceptable": + self._warn_unacceptable_practice() + self._enforce_safety_gate_for_strict_tier(label) def _validate_galore(self) -> None: if not self.training.galore_enabled: diff --git a/tests/test_reverse_pii.py b/tests/test_reverse_pii.py index 090452da..b712c856 100644 --- a/tests/test_reverse_pii.py +++ b/tests/test_reverse_pii.py @@ -651,8 +651,10 @@ def test_truncation_respects_multibyte_utf8(self, tmp_path: Path, capsys) -> Non end = start + len("alice@example.com") snippet = _truncate_snippet(line, (start, end)) assert len(snippet) <= _SNIPPET_MAX_CHARS + 2 # head + tail "…" - # Round-trip through utf-8 must succeed (no broken surrogates). - snippet.encode("utf-8").decode("utf-8") + # Round-trip through utf-8 must produce the same string (no + # broken surrogates from a slice landing mid-rune). + round_tripped = snippet.encode("utf-8").decode("utf-8") + assert round_tripped == snippet # The matched span survives the truncation. assert "alice@example.com" in snippet diff --git a/tools/check_bilingual_parity.py b/tools/check_bilingual_parity.py index 5331e966..c19108b6 100644 --- a/tools/check_bilingual_parity.py +++ b/tools/check_bilingual_parity.py @@ -75,12 +75,23 @@ ("docs/reference/usage.md", "docs/reference/usage-tr.md"), ) -# Match a Markdown ATX heading: 1-6 leading hashes, then a single -# space, then the heading text. We deliberately exclude setext -# headings (``===``/``---`` underlines) — the project's docs use ATX -# exclusively per ``docs/standards/documentation.md``, and a setext -# heading in a mirror would itself be a lint finding. -_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$") +# Match a Markdown ATX heading prefix: 1-6 leading hashes followed by +# at least one space. We deliberately exclude setext headings +# (``===``/``---`` underlines) — the project's docs use ATX exclusively +# per ``docs/standards/documentation.md``, and a setext heading in a +# mirror would itself be a lint finding. +# +# Sonar python:S5852 hotspot avoidance: the previous single-shot regex +# ``^(#{1,6})\s+(.+?)\s*#*\s*$`` chained a lazy ``(.+?)`` against +# ``\s*#*\s*$``, exposing polynomial backtracking on a pathological line +# of all spaces and hashes. We split heading recognition into a +# linear-time prefix match plus deterministic Python string trimming — +# no backtracking surface. +_HEADING_PREFIX_RE = re.compile(r"^(#{1,6}) +") +# Trailing ``\s+#+`` is the ATX-closing form (``## Foo ##``). Bound it +# at one or more whitespace + one or more hashes; no nested +# quantifiers. +_HEADING_TRAILING_HASHES_RE = re.compile(r"\s+#+$") # Code fence opening / closing — heading-like lines inside a code # block are content, not document structure. We track ``open / close`` @@ -134,11 +145,21 @@ def extract_headings(path: Path) -> List[Heading]: continue if in_fence: continue - heading_match = _HEADING_RE.match(line) - if heading_match is None: + prefix_match = _HEADING_PREFIX_RE.match(line) + if prefix_match is None: + continue + level = len(prefix_match.group(1)) + # Strip the prefix, then the optional ATX-closing run of + # hashes, then any leftover whitespace. Two regex passes + # over independent suffixes — neither has a backtracking + # surface against the body text. + body = line[prefix_match.end() :] + body = _HEADING_TRAILING_HASHES_RE.sub("", body) + text = body.strip() + if not text: + # ``# `` or ``## ##`` with no body is not a heading we + # want to compare structurally — skip. continue - level = len(heading_match.group(1)) - text = heading_match.group(2).strip() headings.append(Heading(level=level, text=text, line=line_no)) return headings From 730a015193c12705c9feb534f1c37dc957d16095 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 18:22:03 +0300 Subject: [PATCH 05/11] fix(wave-3-followup): close 2 remaining SonarCloud hotspots on PR #31 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * python:S5852 (MEDIUM, dos) `tools/check_bilingual_parity.py:94` — the previous absorption split heading recognition into a prefix regex + a trailing-hash regex (`\s+#+$`). Sonar conservative analysis still flagged the trailing regex's adjacent quantifiers even though the two character classes (`\s` vs `#`) cannot overlap — no actual backtracking surface exists. To clear the hotspot rather than annotate it, replaced the trailing regex entirely with a deterministic O(n) Python walk (`_strip_atx_close_and_whitespace`); zero regex on the ATX-close-trim path. Behaviour identical: ATX-closing hashes are stripped only when preceded by whitespace, matching CommonMark §4.2. All 16 parity-tool regression tests still pass; live-tree `--strict` still 9/9 OK. * python:S5443 (LOW) `tests/test_integration.py:285` — the `model_path="/tmp/model"` string literal was a structured-log payload, never written to disk, so this was a Sonar false positive on test fixture data. Routed the path-shaped literal through pytest's per-test ``tmp_path`` fixture instead; cosmetic clean-up. Test still asserts the same audit-event chain shape. Suite: 1344 passed, 14 skipped (no regressions). Lint clean. Bilingual parity: 9/9 OK. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_integration.py | 6 +++++- tools/check_bilingual_parity.py | 37 +++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 117c179e..d5f31bef 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -282,7 +282,11 @@ def test_full_event_chain(self, tmp_path): audit.log_event("training.started") audit.log_event("evaluation.loss_check", eval_loss=0.5, passed=True) audit.log_event("evaluation.safety", safe_ratio=0.95, passed=True) - audit.log_event("human_approval.required", model_path="/tmp/model") + # Sonar python:S5443 hotspot avoidance: route the path-shaped + # literal through the per-test ``tmp_path`` fixture so it cannot + # be misread as a publicly-writable directory reference. The + # value is a structured-log payload only — no file is created. + audit.log_event("human_approval.required", model_path=str(tmp_path / "model")) audit.log_event("pipeline.completed", success=True) log_path = os.path.join(str(tmp_path), "audit_log.jsonl") diff --git a/tools/check_bilingual_parity.py b/tools/check_bilingual_parity.py index c19108b6..3a4e4813 100644 --- a/tools/check_bilingual_parity.py +++ b/tools/check_bilingual_parity.py @@ -84,14 +84,11 @@ # Sonar python:S5852 hotspot avoidance: the previous single-shot regex # ``^(#{1,6})\s+(.+?)\s*#*\s*$`` chained a lazy ``(.+?)`` against # ``\s*#*\s*$``, exposing polynomial backtracking on a pathological line -# of all spaces and hashes. We split heading recognition into a -# linear-time prefix match plus deterministic Python string trimming — -# no backtracking surface. +# of all spaces and hashes. We use one minimal regex for the prefix +# (single quantifier on a fixed character class) and a pure-Python +# deterministic trimmer for the optional ATX-closing run of hashes — +# no nested quantifiers anywhere on the heading-recognition path. _HEADING_PREFIX_RE = re.compile(r"^(#{1,6}) +") -# Trailing ``\s+#+`` is the ATX-closing form (``## Foo ##``). Bound it -# at one or more whitespace + one or more hashes; no nested -# quantifiers. -_HEADING_TRAILING_HASHES_RE = re.compile(r"\s+#+$") # Code fence opening / closing — heading-like lines inside a code # block are content, not document structure. We track ``open / close`` @@ -149,13 +146,8 @@ def extract_headings(path: Path) -> List[Heading]: if prefix_match is None: continue level = len(prefix_match.group(1)) - # Strip the prefix, then the optional ATX-closing run of - # hashes, then any leftover whitespace. Two regex passes - # over independent suffixes — neither has a backtracking - # surface against the body text. body = line[prefix_match.end() :] - body = _HEADING_TRAILING_HASHES_RE.sub("", body) - text = body.strip() + text = _strip_atx_close_and_whitespace(body) if not text: # ``# `` or ``## ##`` with no body is not a heading we # want to compare structurally — skip. @@ -164,6 +156,25 @@ def extract_headings(path: Path) -> List[Heading]: return headings +def _strip_atx_close_and_whitespace(body: str) -> str: + """Strip ATX-closing hashes (``Foo ##`` → ``Foo``) deterministically. + + Pure-Python O(n) walk — no regex, no backtracking surface. Only + treats a trailing run of hashes as an ATX close when at least one + whitespace character precedes it, matching CommonMark §4.2. + """ + stripped = body.rstrip() + end = len(stripped) + hash_start = end + while hash_start > 0 and stripped[hash_start - 1] == "#": + hash_start -= 1 + if hash_start < end and hash_start > 0 and stripped[hash_start - 1] in " \t": + # ``stripped[:hash_start - 1]`` drops the separator + the run of + # hashes; rstrip the result to clear any inner trailing space. + return stripped[: hash_start - 1].rstrip() + return stripped + + @dataclass(frozen=True) class PairDrift: """A single drift finding for an EN/TR doc pair.""" From cd66c54765542fc4de554a0438fb387d013fede9 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 18:43:24 +0300 Subject: [PATCH 06/11] fix(wave-3-followup): close gemini-code-assist fence-length finding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tools/check_bilingual_parity.py` previously truncated fence markers to 3 characters (`fence_match.group(1)[:3]`), so a 4-backtick opener could be closed by a 3-backtick line — violating CommonMark §4.5 (closing fence must be at least as long as the opener) AND letting heading-shaped content inside a long fence sneak into the parity tool's heading counts. Now stores the full opener and requires the closer to: 1. Use the same marker character (backtick vs tilde — already partly enforced; now explicit), AND 2. Be at least as long as the opener (the new bit). Two regression tests pin both invariants: * `test_longer_opening_fence_not_closed_by_shorter_run` — 4-backtick fence with a 3-backtick "close" inside; headings inside stay as code, not as document structure. * `test_backtick_fence_not_closed_by_tilde_fence` — tilde line cannot close a backtick fence. Suite 1344 → 1346. Lint clean. Live-tree `--strict` 9/9 OK. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_check_bilingual_parity.py | 26 ++++++++++++++++++++++++++ tools/check_bilingual_parity.py | 9 +++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/test_check_bilingual_parity.py b/tests/test_check_bilingual_parity.py index 5d7e8b09..76c6404f 100644 --- a/tests/test_check_bilingual_parity.py +++ b/tests/test_check_bilingual_parity.py @@ -81,6 +81,32 @@ def test_tilde_fence_also_recognised(self, tmp_path: Path) -> None: headings = extract_headings(src) assert [h.level for h in headings] == [2, 2] + def test_longer_opening_fence_not_closed_by_shorter_run(self, tmp_path: Path) -> None: + """CommonMark §4.5: a closing fence must use the same marker + character AND be at least as long as the opening fence. A + 4-backtick opener is NOT closed by a 3-backtick line; the + ``## inside`` line in between must stay treated as code, not + as a heading.""" + src = _write( + tmp_path / "doc.md", + "## Real H2\n````\n## inside long fence\n```\n## still inside\n````\n## After close\n", + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [2, 2] + assert [h.text for h in headings] == ["Real H2", "After close"] + + def test_backtick_fence_not_closed_by_tilde_fence(self, tmp_path: Path) -> None: + """CommonMark §4.5: closing fence must use the same marker + character as the opener. A ``~~~`` line cannot close a + ```` ``` ```` fence.""" + src = _write( + tmp_path / "doc.md", + "## Real H2\n```\n## inside backtick fence\n~~~\n## still inside\n```\n## After close\n", + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [2, 2] + assert [h.text for h in headings] == ["Real H2", "After close"] + def test_missing_file_returns_empty(self, tmp_path: Path) -> None: assert extract_headings(tmp_path / "missing.md") == [] diff --git a/tools/check_bilingual_parity.py b/tools/check_bilingual_parity.py index 3a4e4813..0217db5f 100644 --- a/tools/check_bilingual_parity.py +++ b/tools/check_bilingual_parity.py @@ -132,11 +132,16 @@ def extract_headings(path: Path) -> List[Heading]: line = raw.rstrip("\n") fence_match = _FENCE_RE.match(line) if fence_match: - marker = fence_match.group(1)[:3] # collapse runs to first three chars + marker = fence_match.group(1) if not in_fence: + # Remember the full opening run; CommonMark §4.5 + # requires the closing fence to use the same + # marker character (backtick vs tilde) AND at + # least as many of them. Storing the literal + # opener lets us enforce both invariants. in_fence = True fence_marker = marker - elif fence_marker is not None and marker == fence_marker: + elif fence_marker is not None and marker[0] == fence_marker[0] and len(marker) >= len(fence_marker): in_fence = False fence_marker = None continue From 7adecb1db474ce5739aa9527e7dbdddf7a0e1849 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 19:40:37 +0300 Subject: [PATCH 07/11] fix(wave-3-followup): warn on implicit --output-dir + --salt-source; drop unreachable fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_run_reverse_pii_cmd` previously had two issues at the ``output_dir`` resolution step: * The trailing ``or "."`` fallback in ``getattr(args, "output_dir", None) or os.path.dirname(os.path.abspath(files[0])) or "."`` was unreachable: ``os.path.abspath()`` always returns an absolute path, and ``os.path.dirname()`` of an absolute path is at minimum ``"/"`` — never falsy. Removed. * When ``--salt-source`` was set but ``--output-dir`` was not, the dispatcher silently fell back to the corpus parent directory for salt-file resolution. ``_resolve_salt(corpus_parent)`` then created ``.forgelm_audit_salt`` next to the corpus, breaking the cross-tool correlation invariant: a ``forgelm purge --output-dir=./outputs`` invocation for the same subject computes a different salt and therefore a different digest. Now emits a WARNING naming the resolved fallback dir + the cross-tool correlation consequence so the operator can either supply ``--output-dir`` or accept the scope. No hard refusal — plaintext-mode invocations are common and need no warning, and even hash-mask operators may legitimately scan corpus dirs directly. Two regression tests pin both halves of the fix: * ``test_implicit_output_dir_with_salt_source_warns_about_correlation_risk`` — implicit + ``--salt-source per_dir`` surfaces a WARNING with "Cross-tool" in the message and the salt file lands in the corpus dir as expected. * ``test_implicit_output_dir_without_salt_source_does_not_warn`` — plaintext-mode runs (no ``--salt-source``) do NOT trigger the cross-tool nag; the warning is scoped to hash-mask invocations. Suite 1346 → 1348. Lint clean. Live-tree parity 9/9 OK. Co-Authored-By: Claude Opus 4.7 (1M context) --- forgelm/cli/subcommands/_reverse_pii.py | 24 +++++++++- tests/test_reverse_pii.py | 62 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/forgelm/cli/subcommands/_reverse_pii.py b/forgelm/cli/subcommands/_reverse_pii.py index 91da129b..e2028250 100644 --- a/forgelm/cli/subcommands/_reverse_pii.py +++ b/forgelm/cli/subcommands/_reverse_pii.py @@ -521,8 +521,12 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: ) files = _resolve_files(globs, output_format) - output_dir = getattr(args, "output_dir", None) or os.path.dirname(os.path.abspath(files[0])) or "." salt_source = getattr(args, "salt_source", None) + explicit_output_dir = getattr(args, "output_dir", None) + # ``os.path.abspath(files[0])`` always returns an absolute path, so + # ``dirname`` returns at minimum ``"/"`` — the previous trailing + # ``or "."`` fallback was unreachable and is dropped. + output_dir = explicit_output_dir or os.path.dirname(os.path.abspath(files[0])) # F-W3-08 fix: validate the regex before any filesystem side effects # (salt-file creation). We build the search pattern over the raw @@ -530,6 +534,24 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: # get exit 1 without leaving a salt file behind. _build_search_pattern(query, identifier_type, output_format) + if explicit_output_dir is None and salt_source is not None: + # F-W3-PS-06 / F-W3T-10 follow-up: an implicit ``output_dir`` + # creates ``.forgelm_audit_salt`` next to the operator's corpus. + # In hash-mask mode, this silently breaks cross-tool correlation + # with any ``forgelm purge`` invocation that supplied an + # explicit ``--output-dir`` (the two would compute different + # salts). Surface the risk loudly so the operator can pin the + # output dir or accept the consequence. + logger.warning( + "reverse-pii: --salt-source=%r set but --output-dir is not. " + "Falling back to the corpus parent (%r); the per-dir salt file " + ".forgelm_audit_salt will be created/read there. Cross-tool " + "correlation with `forgelm purge` requires both subcommands " + "to use the SAME --output-dir.", + salt_source, + output_dir, + ) + scan_query, scan_mode, salt, salt_source_label = _resolve_query_form(query, salt_source, output_dir, output_format) pattern = _build_search_pattern(scan_query, identifier_type, output_format) diff --git a/tests/test_reverse_pii.py b/tests/test_reverse_pii.py index b712c856..dfe3f8e2 100644 --- a/tests/test_reverse_pii.py +++ b/tests/test_reverse_pii.py @@ -230,6 +230,68 @@ def test_env_var_salt_source_requires_secret_env(self, tmp_path: Path, capsys, m _run_reverse_pii_cmd(args, output_format="json") assert ei.value.code == 1, "env_var without FORGELM_AUDIT_SECRET must surface as EXIT_CONFIG_ERROR" + def test_implicit_output_dir_with_salt_source_warns_about_correlation_risk(self, tmp_path: Path, caplog) -> None: + """Wave 3 follow-up: when ``--salt-source`` is set but + ``--output-dir`` is not, the dispatcher falls back to the corpus + parent for salt-file resolution. That silently breaks + cross-tool correlation with any ``forgelm purge`` invocation + that supplied an explicit ``--output-dir``. The fallback + must surface a WARNING naming the resolved dir so the + operator can pin it or accept the consequence.""" + import logging + + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + # Place the corpus in a subdirectory so the fallback + # ``output_dir`` is a real, distinct path the warning can name. + corpus_dir = tmp_path / "data" + corpus_dir.mkdir() + corpus = corpus_dir / "masked.jsonl" + _seed_corpus(corpus, [{"id": "row-A", "text": "x"}]) + args = _build_args( + query="alice@example.com", + type="email", + salt_source="per_dir", + files=[str(corpus)], + output_dir=None, # implicit — falls back to corpus_dir + ) + with caplog.at_level(logging.WARNING, logger="forgelm.cli"): + with pytest.raises(SystemExit) as ei: + _run_reverse_pii_cmd(args, output_format="json") + assert ei.value.code == 0 + warnings = [ + r + for r in caplog.records + if r.levelname == "WARNING" and "reverse-pii" in r.message and "Cross-tool" in r.message + ] + assert warnings, "implicit --output-dir + --salt-source must surface a cross-tool correlation warning" + # The salt file landed in the corpus dir per the warning text. + assert (corpus_dir / ".forgelm_audit_salt").exists() + + def test_implicit_output_dir_without_salt_source_does_not_warn(self, tmp_path: Path, caplog) -> None: + """The cross-tool correlation warning is scoped to hash-mask + invocations; plaintext-mode runs without ``--salt-source`` use + the salt only for the audit hash and must not nag the operator + about cross-tool correlation.""" + import logging + + from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + + corpus = tmp_path / "train.jsonl" + _seed_corpus(corpus, [{"id": "row-A", "text": "x"}]) + args = _build_args( + query="alice@example.com", + type="email", + salt_source=None, + files=[str(corpus)], + output_dir=None, + ) + with caplog.at_level(logging.WARNING, logger="forgelm.cli"): + with pytest.raises(SystemExit): + _run_reverse_pii_cmd(args, output_format="json") + cross_tool_warns = [r for r in caplog.records if "Cross-tool" in r.message] + assert cross_tool_warns == [], "plaintext mode must not warn about cross-tool correlation" + def test_per_dir_salt_source_refuses_when_env_var_set(self, tmp_path: Path, capsys, monkeypatch) -> None: """F-W3-11 / F-W3-PS-04 regression: the symmetric direction — operator asked for ``per_dir`` but ``FORGELM_AUDIT_SECRET`` is From c8de7180a8ce2a2763cca97087833e7b5ba22eb0 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 19:47:56 +0300 Subject: [PATCH 08/11] docs(wave-3): multi-agent follow-up review prompt for absorption commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Frozen at 7adecb1. Scopes the second-pass review to the 5 absorption commits since 95e2bf8 (the original 4-agent review snapshot): 5491572 — 4-agent review + 6 inline PR comment findings 6ba93be — 3 SonarCloud + 1 hotspot 730a015 — 2 remaining hotspots cd66c54 — gemini-code-assist fence-length 7adecb1 — implicit --output-dir cross-tool correlation warning Two-question framing: 1. Did each absorption correctly fix the original finding it claims to absorb? ("F- [verified]" / "[partial]" / "[regressed]" bookkeeping in the deliverable.) 2. Did the absorption itself introduce defects? (5491572 rewrote most of `_reverse_pii.py` — same bar as new code.) Same 4-agent split as the original review: code-correctness / privacy-security / standards / test-rigor. Each report saved as `wave3fu-.md` so the original `wave3-.md` reports stay readable as the pre-absorption baseline. Re-allowed in .gitignore alongside the original review prompt (public artefact, not internal analysis). Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 + .../wave3-followup-review-prompt.md | 573 ++++++++++++++++++ 2 files changed, 574 insertions(+) create mode 100644 docs/analysis/code_reviews/wave3-followup-review-prompt.md diff --git a/.gitignore b/.gitignore index f216509d..351536e2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ docs/analysis/code_reviews/* !docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md !docs/analysis/code_reviews/wave2b-final-merge-review-prompt.md !docs/analysis/code_reviews/wave3-review-prompt.md +!docs/analysis/code_reviews/wave3-followup-review-prompt.md # Generated by tools/build_usermanuals.py — produced from # docs/usermanuals//**/*.md during the CI deploy. Not committed; diff --git a/docs/analysis/code_reviews/wave3-followup-review-prompt.md b/docs/analysis/code_reviews/wave3-followup-review-prompt.md new file mode 100644 index 00000000..13a20787 --- /dev/null +++ b/docs/analysis/code_reviews/wave3-followup-review-prompt.md @@ -0,0 +1,573 @@ +# Wave 3 Follow-up — Multi-Agent Code Review Prompt (post-absorption) + +> **Use this prompt verbatim** to launch the multi-agent review on the +> 5 absorption commits that landed on PR #31 after the original +> 4-agent review completed. The prompt is self-contained: each +> reviewing agent can pick it up cold without context from the prior +> review round. +> +> The original Wave 3 review (per +> [`wave3-review-prompt.md`](wave3-review-prompt.md), HEAD `61d74fa`) +> produced 4 reports under `docs/analysis/code_reviews/wave3-*.md`. +> The maintainer absorbed those reports plus inline PR-comment +> findings (CodeRabbit / gemini-code-assist) plus SonarCloud quality +> gate failures across **5 commits**. This second-pass review asks: +> *did the absorption correctly address the original findings, and +> did the absorption itself introduce any new defects?* + +--- + +## Repo & branch under review + +- **Repo:** `cemililik/ForgeLM` +- **PR:** [#31](https://github.com/cemililik/ForgeLM/pull/31) + (`closure/wave3-integration` → `development`) +- **HEAD SHA at prompt freeze:** `7adecb1` +- **Pre-absorption SHA (the snapshot the original 4-agent review + read):** `95e2bf8` (before that, `61d74fa`) +- **Absorption-only diff range:** `git diff 95e2bf8..7adecb1` + — 24 files changed, ~1051 insertions, ~265 deletions. +- **5 absorption commits in scope** (oldest → newest): + + | SHA | Title | Findings absorbed | + |-----------|---------------------------------------------------------------------------|------| + | `5491572` | absorb 4-agent review + 6 inline PR comment findings | F-W3-01..14, F-W3-PS-01..09, F-W3S-01..09, F-W3T-01..12 + 6 inline | + | `6ba93be` | absorb 3 SonarCloud + 1 hotspot findings on PR #31 | python:S1172, S2201, S3776 + dos hotspot | + | `730a015` | close 2 remaining SonarCloud hotspots on PR #31 | python:S5852 (final), S5443 | + | `cd66c54` | close gemini-code-assist fence-length finding | CommonMark §4.5 fence length | + | `7adecb1` | warn on implicit --output-dir + --salt-source; drop unreachable fallback | implicit-output-dir cross-tool correlation risk | + +## What ForgeLM is (60 seconds) + +A **config-driven, enterprise-grade LLM fine-tuning toolkit** — YAML +in, fine-tuned model + compliance artifacts out. Built for CI/CD +pipelines, not notebooks. Six alignment paradigms +(SFT/DPO/SimPO/KTO/ORPO/GRPO), integrated safety evaluation +(Llama Guard), EU AI Act compliance artefacts (Articles 9–17 + Annex +IV), append-only audit log, opt-in human approval gate, auto-revert +on quality regression, GDPR Article 15 + 17 subject rights tooling. +Read [docs/product_strategy.md](../../product_strategy.md) for the +fuller background. + +Project rulebook lives under [docs/standards/](../../standards/) — read +[coding.md](../../standards/coding.md), +[error-handling.md](../../standards/error-handling.md), +[testing.md](../../standards/testing.md), +[logging-observability.md](../../standards/logging-observability.md), +[regex.md](../../standards/regex.md), +[localization.md](../../standards/localization.md), and +[code-review.md](../../standards/code-review.md) before commenting. +Project-wide guidance for AI agents is in the root `CLAUDE.md`. + +The original 4 review reports under +[`docs/analysis/code_reviews/`](.) document the prior bar: + +- [`wave3-code-correctness.md`](wave3-code-correctness.md) — 15 findings. +- [`wave3-privacy-security.md`](wave3-privacy-security.md) — 14 findings + 2 cross-phase. +- [`wave3-standards.md`](wave3-standards.md) — 9 findings + 7 cross-phase passes. +- [`wave3-test-rigor.md`](wave3-test-rigor.md) — 14 findings + 5 cross-phase notes. + +The maintainer's absorption summary (per-finding disposition) is at +[PR #31 issuecomment-4380828720](https://github.com/cemililik/ForgeLM/pull/31#issuecomment-4380828720). + +## What this round is for + +**Goal:** decide whether `closure/wave3-integration` at `7adecb1` is +mergeable to `development`, given the absorptions claim to have +addressed the original 4-agent review. + +**Question shape:** the previous review framed defects in code shipped +*before* absorption. This review asks two complementary questions: + +1. **Did the absorptions correctly fix the original findings?** + Walk every CRITICAL / HIGH from the 4 prior reports. For each, + verify the claimed fix lands at the cited file:line and actually + addresses the original concern (not "a fix that compiles" but + "the contract the original finding asked about now holds"). +2. **Did the absorptions themselves introduce defects?** The diff + touches 24 files; the largest absorption (`5491572`) rewrote + most of `forgelm/cli/subcommands/_reverse_pii.py`. Treat the + rewrite as new code — same severity bar as the original review. + +Be concrete. Every finding must include: + +1. **Severity tag** — `CRITICAL` / `HIGH` / `MEDIUM` / `LOW` / `NIT`. +2. **Finding ID** — `F-W3FU-NN` (e.g. `F-W3FU-01`, `F-W3FU-12`). +3. **File:line citation** — `forgelm/cli/subcommands/_reverse_pii.py:524` + or a short range `:520-545`. +4. **One-paragraph reasoning** — what is wrong, what is the user-visible + consequence, and why it qualifies for that severity tier. +5. **Suggested fix** — a code or text snippet, not a vague direction. +6. **Test that would have caught it** — name the test file + a stub + of the assertion. +7. **Trace-back to original finding (if any)** — when re-evaluating + a claimed absorption, cite the original finding ID + (e.g. F-W3-03, F-W3-PS-01, F-W3S-01, F-W3T-01) so the maintainer + can compare against the pre-absorption code. + +Severity bar (unchanged from the original prompt — same calibration): + +- `CRITICAL` — would corrupt data, silently lose audit events, leak + PII / secrets, break the public exit-code contract (`0/1/2/3/4`), + break the `--offline` / air-gap promise, or invalidate the + audit-log hash chain. Blocks merge. +- `HIGH` — runtime crash on a documented happy path, missing test + for a documented contract, schema mismatch with `pyproject.toml` + / `config.py`, documentation claim with no code backing, or a + behaviour change that breaks 0.5.x users without a clear migration + path. Blocks merge. +- `MEDIUM` — logic bug in an off-the-happy-path branch, observability + gap, or design-doc statement that contradicts shipped code. + Should be fixed before merge but the call is the maintainer's. +- `LOW` — defensive-coding improvement, minor inconsistency, missing + edge-case test that would not have changed PR direction. +- `NIT` — naming, line wrap, comment phrasing. Note in passing + only; do not stack these. + +## Areas to scrutinise + +These are the surfaces most likely to harbour real issues *post-absorption*. + +### A. `_reverse_pii.py` — heaviest rewrite (`5491572` + `6ba93be` + `7adecb1`) + +The dispatcher grew from 384 lines to ~635 lines. Walk every new +helper for soundness, and every removed branch for missed callers. + +#### A.1 — Audit fail-closed semantics (claims to absorb F-W3-01 + F-W3-PS-02 + F-W3S-01) + +- **File:** `forgelm/cli/subcommands/_reverse_pii.py:_maybe_audit_logger` +- The new policy: `ConfigError` → WARNING + `None` (best-effort); + every other exception (`OSError`, `ValueError`) → fail-closed + with `EXIT_TRAINING_ERROR`. Walk: + - Is the `(OSError, ValueError)` catch tuple complete? Could + `AuditLogger.__init__` raise something not in the tuple + (`PermissionError` is `OSError` subclass; `RuntimeError`, + `KeyError` from a corrupt manifest)? + - The `explicit=` kwarg distinguishes operator-supplied vs + inferred audit dirs but the failure path treats both the same + (`fail closed`). Is that consistent with the docstring's + "explicit values fail loudly when unwritable rather than + silently dropping" framing? Or should the implicit default + fall back to a different writable location instead of failing? + - The two `try / except Exception as audit_exc: _output_error_and_exit` + wrappers around `_emit_audit_event` (success path + failure + path) — does the failure path's nested try/except correctly + preserve the original exception's audit semantics? + +#### A.2 — Salted audit hash + cross-tool correlation (claims to absorb F-W3-PS-01 + F-W3-PS-09) + +- **File:** `_reverse_pii.py:_hash_for_audit`, + `_resolve_query_form`, `_run_reverse_pii_cmd` +- The hash is now `SHA256(salt + raw_query)` using the same + per-output-dir salt purge uses for `target_id`. Walk: + - The `salt: Optional[bytes]` signature has a `None` legacy path + (`hashlib.sha256(raw).hexdigest()` unsalted). Can any caller + in production reach the `None` branch, or is it dead code? + If dead, drop it; if reachable, document why an unsalted hash + is acceptable on that path. + - `_resolve_query_form` ALWAYS calls `_resolve_salt(output_dir)` + even in plaintext mode (because the audit hash needs a salt). + Side effect: every reverse-pii invocation creates + `.forgelm_audit_salt` in the resolved `output_dir`. Is that + documented? Is it the same semantic as `forgelm purge` (does + purge also unconditionally create the salt file on first run)? + - The new `salt_source_label` field is recorded in the audit + event. In plaintext mode it gets the `_resolve_salt` return + value (`per_dir` or `env_var`) — i.e. the audit chain says + `"scan_mode": "plaintext", "salt_source": "per_dir"`. Is + that label semantically right (the salt is irrelevant to the + plaintext scan; it's only used for the audit hash) or + misleading? + +#### A.3 — `--type literal` default (claims to absorb F-W3-02) + +- **File:** `_parser.py:_add_reverse_pii_subcommand` + + `_reverse_pii.py:_build_search_pattern` +- The default changed `custom` → `literal`. The previous + `custom`-default would `re.compile(query)` directly; the new + `literal`-default `re.escape`s the query. Walk: + - Is this a *user-visible* breaking change that 0.5.4 operators + would notice? E.g. an operator who previously passed a + regex without `--type custom` and got regex semantics by + default now gets literal semantics — the documented behaviour + is correct but the migration is silent. + - Is the new `literal` choice surfaced in every doc location + (`json-output.md`, `cli.md`, `gdpr_erasure.md`, audit catalog)? + Cross-check. + - The choice list at `_parser.py:921-925` adds `literal` first; + confirm `argparse choices=` set is identical between + `_parser.py` and `_IDENTIFIER_TYPES` in `_reverse_pii.py`. + +#### A.4 — Snippet match-span centring (claims to absorb F-W3-03) + +- **File:** `_reverse_pii.py:_truncate_snippet`, + `_scan_file` +- The signature changed: now takes `match_span: Tuple[int, int]` + and centres the window. Walk: + - Does `_scan_file` always pass a valid span? Could + `pattern.search(stripped)` return `None` on a line that the + iterator believes matched? (Defensive: should the helper + handle `match_span=None`?) + - The budget calculation + `ctx = max((budget - match_len) // 2, 0)` has a degenerate + case when `match_len > budget`. What happens if the operator + matches a regex that captures most of a 200-char line? + Verify the snippet is still bounded at `_SNIPPET_MAX_CHARS`. + - Multi-byte / grapheme cluster handling — F-W3T-05 absorption + pinned a regression test, but does it cover edge cases like + a match span that lands on a single emoji ZWJ sequence? + - The trailing-ellipsis logic uses `< len(line)` to decide + whether to append `…`. If `win_end == len(line)` exactly, + no ellipsis; if `win_end == len(line) - 1`, ellipsis. Off- + by-one risk? + +#### A.5 — `UnicodeDecodeError` handling (claims to absorb F-W3-04) + +- **File:** `_run_reverse_pii_cmd` exception handler +- The `except (OSError, UnicodeDecodeError) as exc:` block now + catches both classes and emits a failure audit event. Walk: + - Does the audit event correctly record `error_class = + "UnicodeDecodeError"` (not `OSError`)? The + `exc.__class__.__name__` should preserve the real class. + - Other `ValueError` subclasses from `open()` / `read()` (e.g. + `LookupError` from a custom encoding) — same treatment, or + do they bubble? Compare to the comment claim at + `_reverse_pii.py:48-51` about "malformed UTF-8 mid-corpus". + +#### A.6 — Audit-dir default move (claims to absorb F-W3-06 + F-W3-PS-06 + F-W3S-02) + +- **File:** `_reverse_pii.py:_resolve_audit_dir` +- The default moved from `output_dir` to + `/audit/`. Walk: + - Does this break any operator workflow that pinned + `audit_log.jsonl` to the corpus dir? CHANGELOG note + sufficient? + - The `forgelm purge` and `forgelm audit` subcommands — what + are their audit-dir defaults? Are the three subcommands + consistent? + - When the audit dir doesn't exist yet, `AuditLogger.__init__` + creates it via `os.makedirs(... exist_ok=True)`. Any + permission edge case for the new `/audit/` + sub-creation? + +#### A.7 — ReDoS guard (claims to absorb F-W3-PS-03 + F-W3-07 + F-W3S-03) + +- **File:** `_reverse_pii.py:_scan_file_with_alarm`, + `_scan_files_with_redos_guard` +- POSIX SIGALRM-based per-file 30s budget. Walk: + - On Windows the guard is a no-op. Is that documented? Does + the audit event say anything about the guard being skipped? + - SIGALRM is a process-wide signal. If `_scan_file_with_alarm` + is called from a thread, the alarm will fire on the main + thread. Are there any test fixtures or ForgeLM invocation + paths that run reverse-pii in a thread? + - The alarm restoration logic (`finally` clause that resets + the alarm + restores the previous handler) — does it + correctly handle re-entrant calls? Could a previous + handler's signal be lost? + - Is 30s the right budget? Compare to the audit chain's + expected per-event latency. + +#### A.8 — Implicit `output_dir` warning (claims to absorb the new finding from `7adecb1`) + +- **File:** `_run_reverse_pii_cmd:524-541` +- When `--salt-source` is set + `--output-dir` is implicit, a + WARNING fires about cross-tool correlation risk. Walk: + - Is the warning's message text accurate? It names "Cross-tool" + — is that operator-meaningful? + - Should this be an ERROR instead of WARNING? The original + finding offered both (a) error, (b) warning; the maintainer + chose (b). Is that the right call given the privacy + sensitivity of Article 15 access requests? + - The warning suppression — does `--quiet` mute it? Per + `logging-observability.md`, quiet mode should still surface + warnings *to the run log*, not stdout. + +### B. `forgelm/config.py` — F-compliance-110 refactor (`6ba93be`) + +#### B.1 — Cognitive-complexity refactor (claims to absorb python:S3776) + +- **File:** `config.py:_warn_high_risk_compliance` + + `_resolve_risk_label`, `_warn_unacceptable_practice`, + `_enforce_safety_gate_for_strict_tier` +- The function was refactored into 3 helpers + a thin orchestrator. + Walk: + - **Behaviour invariance:** does the new code produce identical + output for every input the old code handled? Specifically: + - `risk_assessment` AND `compliance` BOTH set to high-risk + vs only one set — does `_resolve_risk_label` pick + consistently with the previous `(self.compliance and …) or + (self.risk_assessment and …)` ordering? The old code + preferred `self.compliance.risk_classification` last; the + new code prefers `risk_assessment.risk_category` first + (line 858). Is that an intentional swap? + - The `auto_revert` warning previously fired BEFORE the + `unacceptable` extra notice; the new code preserves that + order via `_warn_unacceptable_practice` after the + auto_revert log. Confirm via test_eu_ai_act caplog. + - `track_categories` warning — previously inside `elif` + against safety not enabled; new code is in the + `_enforce_safety_gate_for_strict_tier` body after the + raise. Equivalent? + - The error message format — old code used `% (label,)` lazy + formatting; new code uses `f"... {label!r} ..."` eager. Is + `label!r` rendering identical to the old `%r`? Quoting style + matters for the regression test + `pytest.raises(ConfigError, match="evaluation.safety.enabled")`. + +### C. `tools/check_bilingual_parity.py` — heading recogniser rewrite (`5491572` + `6ba93be` + `730a015` + `cd66c54`) + +#### C.1 — Heading prefix split (claims to absorb python:S5852) + +- The single `_HEADING_RE` was split into `_HEADING_PREFIX_RE` + (compiled regex) + `_strip_atx_close_and_whitespace` (pure + Python). Walk: + - **CommonMark §4.2 conformance for ATX-close strip:** the + helper requires whitespace BEFORE the trailing run of hashes + for them to count as ATX-close. Spec says "An optional + closing sequence of `#` characters preceded by a space". + Verify the helper matches: `## Foo ##` → `Foo` (yes), `## Foo## ` + (no space before closing `##`) → `Foo##` (kept as content)? + The new helper checks `stripped[hash_start - 1] in " \t"`. + Tab is acceptable per CommonMark; space is acceptable; a + multi-byte whitespace character (NBSP) is NOT. Is this a + real risk for any project doc? + +#### C.2 — Fence length / type enforcement (claims to absorb gemini-code-assist medium) + +- The fence detection now stores the full opener and requires the + closer to have the same first character AND length ≥ opener. + Walk: + - The check `marker[0] == fence_marker[0]` only inspects the + first character. CommonMark §4.5: "The closing code fence + must use the same character as the opening fence (backticks + or tildes)" — first-char check is sufficient since the + `_FENCE_RE` already restricts to `(```+|~~~+)` (homogeneous + runs). Confirm. + - Indented fences — CommonMark allows up to 3 spaces of indent + on the fence line, but the parity tool's regex `^(```+|~~~+)` + is anchored at column 0. Is that acceptable for ForgeLM's + docs (which use no leading indent)? + - The new `test_longer_opening_fence_not_closed_by_shorter_run` + + `test_backtick_fence_not_closed_by_tilde_fence` regressions + cover the headline fix. Is there a third edge case (closer + longer than opener) that needs a test? + +#### C.3 — Pair registry growth (8 → 9 pairs) + +- `audit_event_catalog{,-tr}.md` was added to `_PAIRS`. + `safety_compliance-tr.md` was deliberately NOT added (in-progress + allowlist via `test_every_tr_mirror_appears_in_pair_registry`). + Walk: + - Is the allowlist mechanism the right shape? Or should the + test just track an issue link for safety_compliance-tr.md's + Wave 4 completion? + - Are there any other `*-tr.md` files outside + `docs/{guides,reference}/` that DO have parity contracts the + test should cover (but currently don't because the test scope + is limited to those two directories)? + +### D. `forgelm/webhook.py` — timeout fallback (claims to absorb F-W3S-04) + +- **File:** `webhook.py:80-100` +- Now reads + `WebhookConfig.model_fields["timeout"].default`. Walk: + - Late-import inside the method body — defensible (avoids + circular imports with `config.py`)? Or could it be a top-of- + file import? + - The clamp message format (`%ds`) — does it correctly render + `10` for the new default? + - The `not isinstance(timeout, (int, float))` guard remains. + Could a Pydantic-validated `WebhookConfig` ever produce a + non-numeric `timeout`? If not, is the guard dead code? + +### E. Test rigour (claims to absorb F-W3T-01 + F-W3T-02 + F-W3T-03 + …) + +#### E.1 — Failure-path no-leak invariant (F-W3T-01) + +- **File:** `test_reverse_pii.py:test_mid_scan_io_failure_writes_failed_audit_event_without_leaking_identifier` +- The test asserts + `"alice@example.com" not in json.dumps(evt)` on the failure + path. Walk: + - Is the assertion strong enough? An attacker could embed a + transformed form (base64, hex, ROT13) that the substring + check misses. Should the test also assert + `evt["query_hash"] == _hash_for_audit("alice@example.com", salt)` + (positive shape)? + - The test fixture passes a custom `_flaky_scan` that raises + `OSError` on the second call. Does the test exercise the + `UnicodeDecodeError` path too, or is that + `test_malformed_utf8_corpus_exits_runtime_error_with_audit_event` + only? If the no-leak assertion is in the OSError test only, + the UnicodeDecodeError audit event isn't pinned for no-leak. + +#### E.2 — Strict-tier parametrize (F-W3T-02) + +- **File:** `test_eu_ai_act.py:test_strict_tier_safety_disabled_raises_config_error` +- Now `@pytest.mark.parametrize("tier", ["high-risk", "unacceptable"])`. + Walk: + - Is the `match=` regex on the `pytest.raises` permissive + enough for both tiers? The error message includes the tier + label, so `match="evaluation.safety.enabled"` should match + both — confirm. + - Are both tiers exercised end-to-end (not just at config + validation)? An integration test in `test_integration.py` + pins one of them; does it pin both? + +#### E.3 — Setext exclusion (F-W3T-03) + +- **File:** `test_check_bilingual_parity.py:test_setext_headings_are_not_matched` +- Walk: the test plants `Setext H1\n=====` and asserts only + `## Real H2` registers. Does it also cover the case where a + setext underline appears AFTER the structural heading set is + built (i.e., line N is `Some text\n========\n## H2`)? The + `=====` is on its own line and would not match `_HEADING_PREFIX_RE` + — confirmed safe by inspection, but is there a test? + +### F. Cross-cutting (XPR) checks + +#### F.1 — Documentation drift (post-absorption) + +- **F-W3FU-XPR-01** — Did every absorption that touched code also + update the matching doc? Specifically: + - `--type literal` default change → `cli.md`, `json-output.md`, + `gdpr_erasure.md`, audit catalog — all four updated? + - `salt_source` field in audit event → audit catalog (EN+TR) + + `json-output.md` (EN+TR) updated? + - Audit-dir default `/audit/` → operator guide, + `cli.md`, `_parser.py` help text — all consistent? + +#### F.2 — Wave 3 closure-plan honesty + +- **F-W3FU-XPR-02** — `closure-plan-202604300906.md:130` "Kalan + iş" list now says 4 phases pending. Cross-check against the + table at L75-110: are exactly 4 phases marked PENDING and + exactly 3 (24 + 28 + 38) marked Wave 3 done? + +#### F.3 — Coverage / behaviour parity + +- **F-W3FU-XPR-03** — The original review's "Verified absorptions" + section in each report (e.g. + `wave3-code-correctness.md:619-663`) lists contracts that were + PASSING before absorption. Did any absorption inadvertently + break one of those passing contracts? Spot-check 3 random + passing items per original report. + +#### F.4 — Backwards compatibility (post-absorption) + +- **F-W3FU-XPR-04** — Walk every public-API surface touched in + the diff: + - `forgelm.cli` facade re-exports — added `_emit_audit_event`, + `_resolve_audit_dir`. Is that acceptable additive change? + - `_scan_files_with_redos_guard` signature lost `output_format`. + Is it a public symbol? If yes, who imports it? (The + `__all__` list in `_reverse_pii.py` doesn't include it; the + facade doesn't either — should be safe.) + +#### F.5 — Lint / format equivalence + +- **F-W3FU-XPR-05** — The absorptions ran `ruff format` between + commits. Spot-check whether any auto-format change altered + *behaviour* vs purely cosmetic. E.g. changes to docstring + indentation, line splitting that affected an f-string's + embedded expression. + +## Ground rules + +- **Read the relevant standard before commenting.** The project + [docs/standards/](../../standards/) directory is the rulebook. +- **Re-evaluate, don't just trust.** An "absorbed" finding in + the maintainer's PR comment summary is a *claim*. Verify the + fix against the cited file:line. If the fix is correct, say + so explicitly in your "Verified absorptions" section. If the + fix is incomplete or wrong, that's a `F-W3FU-NN` finding. +- **Cite line numbers.** If you cannot point at a line, the + finding is not actionable. +- **Distinguish original vs newly introduced.** Tag every + finding with whether it traces back to an original + (`F-W3-NN` / `F-W3-PS-NN` / `F-W3S-NN` / `F-W3T-NN`) or is + *new* (introduced by the absorption itself). Use the + `Trace-back to original finding` field in the deliverable. +- **No phantom severity inflation.** A doc typo is `NIT`. A + user-visible CLI bug on the happy path is `HIGH` minimum. +- **No re-litigating closed scope.** "Should ForgeLM also do X" + is not a finding. Stay in the diff. +- **Reverse-pii is a privacy-sensitive surface.** Same rule as + the original prompt: any path where cleartext can leak into + the audit log is `CRITICAL`. + +## Required deliverable structure + +Each reviewing agent returns a single Markdown report with this +skeleton, saved as +`docs/analysis/code_reviews/wave3fu-.md`: + +```markdown +# Wave 3 Follow-up — Review of `closure/wave3-integration` + +> Branch: `closure/wave3-integration` · HEAD `7adecb1` · Reviewer focus: +> . Re-evaluating absorptions since `95e2bf8`. + +## Summary +- Verdict: [Block / Conditional / Approve] +- CRITICAL: N · HIGH: N · MEDIUM: N · LOW: N · NIT: N +- One-sentence headline of the highest-severity finding. + +## Findings + +### F-W3FU-NN · · +- **File:** path:line[-line] +- **Trace-back:** F-W3-NN / F-W3-PS-NN / F-W3S-NN / F-W3T-NN / NEW +- **What's wrong:** ... +- **User-visible consequence:** ... +- **Suggested fix:** ```code``` +- **Regression test:** ```python ...``` + +(repeat per finding) + +## Verified absorptions + +For each CRITICAL / HIGH from the four original reports + the +4 inline PR comments + 5 SonarCloud items, state: + +- **F- [verified]** — fix lands at ; the + contract now holds. Walked: . +- **F- [partial]** — fix addresses but misses + ; new finding F-W3FU-NN tracks it. +- **F- [regressed]** — fix introduces ; new + finding F-W3FU-NN tracks it. + +## What this report deliberately did not cover + +(scope notes, deferred items, out-of-PR areas) + +## Merge recommendation + +(Block / Conditional / Approve, with the headline reasons) +``` + +## How to launch + +Spawn 4 parallel sub-agents (you may use any harness that supports +parallel workers; the original review used Claude `Agent` calls +in a single message). Pass each agent the SAME prompt, with one +extra line at the top stating its agent name + focus: + +1. **Code-correctness agent.** Focus areas: §A.1, A.4, A.5, A.7, + B.1, C.2, F.4, F.5. +2. **Privacy / security agent.** Focus areas: §A.1, A.2, A.6, A.8, + E.1. Privacy bar reigns. +3. **Standards / compliance agent.** Focus areas: §A.3, A.6, B.1, + D, F.1, F.2, F.3. Cite docs/standards/* extensively. +4. **Test-rigour agent.** Focus areas: §A.4, A.7, C.2, C.3, E (all), + F.3. Vacuous-test bar reigns. + +Agents run independently. Maintainer absorbs in a single round +(per the established Wave 3 cadence). + +--- + +*Frozen at `7adecb1` on 2026-05-05.* *If any reviewer finds the +prompt itself ambiguous, surface it as a `F-W3FU-PROMPT-NN` +finding before merging — the prompt itself is part of the audit +trail.* From 549d438919c0c1b5657c6347b174ff3fef79d606 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 20:27:58 +0300 Subject: [PATCH 09/11] fix(wave-3-followup): absorb 4-agent follow-up review (HIGH+MEDIUM convergent findings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 4 wave-3 follow-up review reports identified 5 HIGH-severity regressions in the prior absorption commits plus a set of MEDIUM/LOW sharpening fixes. All convergent items absorbed in this single commit; deferred items documented at the end. HIGH absorptions ================ * F-W3FU-01 (cc) + F-W3FU-S-01 (std) — F-compliance-110 strict-gate OR-across-fields semantics regressed. The 6ba93be cognitive- complexity refactor that absorbed Sonar python:S3776 silently inverted the gate's OR semantics: when ``risk_assessment.risk_category`` AND ``compliance.risk_classification`` disagree (one strict, one not), the resolver picked ``risk_assessment`` first, the strict membership test failed, and the F-compliance-110 ConfigError raise was silently bypassed. Asymmetric-tier YAML reachable from hand- written configs. Fix: split label resolution (display-only) from strict-tier detection (OR across both sibling fields) via ``_risk_tiers`` + ``_is_strict_tier`` + ``_is_unacceptable`` helpers. 4-tier parametrize regression test (``test_asymmetric_strict_tier_still_raises_when_safety_disabled``) + Article-5 banner test pin both the gate AND the unacceptable warning across asymmetric configs. * F-W3FU-01 (priv) + F-W3FU-S-03 (std) — audit-dir default move (``/audit/``) broke cross-tool correlation with ``forgelm purge`` (which writes to ``/audit_log.jsonl`` directly). ``verify-audit`` correlating Article 17 + Article 15 events for the same subject saw two disjoint chains. Fix: revert ``_resolve_audit_dir`` default to ``output_dir`` so both subcommands write into one chain. The cross-tenant write concern that motivated the move is now addressed by the implicit-output-dir warning at the dispatcher entry (which names the resolved fallback dir loudly). Parser help text + CHANGELOG + gdpr_erasure operator guide (EN+TR) updated. * F-W3FU-T-01 — POSIX SIGALRM ReDoS guard had ZERO regression coverage. The 5491572 absorption shipped 251 lines of new code (`_scan_files_with_redos_guard` + `_scan_file_with_alarm`) without pinning any of the three load-bearing contracts (pathological pattern terminates within budget, alarm-reset on success, thread- safety on non-main thread). Fix: 3 focused tests under new ``TestReDoSGuard`` class — one POSIX-only catastrophic-backtracking test that monkeypatches ``_CUSTOM_REGEX_TIMEOUT_S`` to 1s and verifies exit 2 + failure audit event; one outer-alarm-budget preservation test (also pins the F-W3FU-03 fix below); one worker-thread no-op test that exercises ``_scan_files_with_redos_guard`` from a ``threading.Thread`` and asserts no crash. * F-W3FU-02 (cc) + F-W3FU-T-02 — ``_truncate_snippet`` exceeded ``_SNIPPET_MAX_CHARS`` when the match span itself was wider than the budget. The centring math (``ctx = max((budget - match_len) // 2, 0)``) floored at 0 in the degenerate case, returning the full match span unbounded. A ``--type custom --query ".+"`` against a 500-char line emerged as a 502-char snippet. Fix: when ``match_len >= budget``, slice a centred window of the match itself with leading/trailing ellipses. Regression test (``test_match_span_wider_than_budget_still_bounded``) plants a 250-char match in a 264-char line and pins the cap. * F-W3FU-05 (priv) + F-W3FU-T-03 — UnicodeDecodeError failure-path no-leak invariant unpinned. The OSError leg's no-leak assertion did not transfer to the symmetric UnicodeDecodeError test, and the test used a low-entropy ``query="x"`` that would have made any leak invisible. Fix: switch to ``query="alice@example.com"``, add ``"alice@example.com" not in json.dumps(evt)`` assertion, add positive-shape assertion ``evt["query_hash"] == _hash_for_audit(...)`` to pin the salted-hash contract. Test renamed ``test_malformed_utf8_corpus_exits_runtime_error_with_audit_event_and_no_leak``. MEDIUM absorptions ================== * F-W3FU-02 (priv) — ``salt_source_label`` never returned ``"plaintext"`` despite the docstring promise; plaintext-mode scans recorded the audit-hash salt source (``per_dir`` / ``env_var``) and a forensic reviewer could not distinguish hash-mask scans from plaintext ones. Fix: ``_resolve_query_form`` returns ``"plaintext"`` for the scan label in plaintext mode. * F-W3FU-03 (priv) — audit fail-closed ``(OSError, ValueError)`` catch tuple was narrower than the original F-W3-PS-02 ask. A future hardening hook in ``AuditLogger.__init__`` raising RuntimeError / KeyError / etc. would silently re-introduce the silent-drop failure mode. Fix: widen to bare ``Exception`` (with BLE001 noqa + comment) — the contract is "every non-ConfigError fails closed", not "every class we currently know about". * F-W3FU-03 (cc) — ``signal.alarm(N)`` returns the previously- scheduled alarm's remaining seconds. ``_scan_file_with_alarm`` was discarding the return value and calling ``signal.alarm(0)`` in ``finally``, permanently cancelling any outer SIGALRM budget. Fix: capture ``previous_alarm_remaining`` and re-arm in the ``finally`` clause (rounded up to 1s — sub-second remainders cannot survive alarm-syscall granularity). * F-W3FU-04 (cc) — ``signal.signal(SIGALRM, ...)`` raises ``ValueError`` from non-main threads on POSIX. The guard at ``_scan_files_with_redos_guard`` only checked ``os.name == "posix"`` + ``hasattr(SIGALRM)``; a thread-context call would crash. Fix: also require ``threading.current_thread() is threading.main_thread()`` before use. Worker thread test pins the fall-through. * F-W3FU-T-04 — failure-path no-leak test gained the positive-shape hash assertion + base64/hex transformed-form negative-shape assertions. Closes the substring-only-check weakness called out in F-W3-PS-12. * F-W3FU-T-05 — strict-tier ``unacceptable`` integration coverage: ``test_high_risk_without_safety_raises_config_error`` parametrised over both ``high-risk`` and ``unacceptable`` tiers. Pins the YAML-round-trip + Pydantic-validator interaction at the integration layer. * F-W3FU-T-06 — ``test_shorter_opening_fence_closed_by_longer_run`` pins the CommonMark §4.5 ``len(closer) >= len(opener)`` semantics so a hand-edit dropping ``>=`` to ``==`` would surface in CI. LOW / NIT absorptions ===================== * F-W3FU-06 (priv) — implicit-output-dir warning broadened to fire in plaintext mode too (the salt-file side effect is present regardless of scan mode). Tag the message with whether hash-mask is also in play. Test ``test_plaintext_implicit_output_dir_also_warns_about_salt_file`` replaces the previous ``..._does_not_warn`` test (the contract inverted with the broadening). * F-W3FU-06 (cc) — ``_emit_audit_event`` failure-branch chains the original mid-scan exception's diagnostic context into the audit-failure message so a double-fault (corpus read fails AND audit chain write fails) preserves the underlying cause. * F-W3FU-08 (priv) — dropped the unreachable ``salt: Optional[bytes]`` ``None`` branch in ``_hash_for_audit``. Type-tightened to ``salt: bytes``; a future refactor passing ``None`` is now a static error rather than a silent unsalted-hash downgrade. * F-W3FU-S-04 — webhook ``isinstance(timeout, (int, float))`` guard dropped (dead under Pydantic int-validation). Numeric-floor clamp retained. * F-W3FU-S-02 — ``gdpr_erasure.md`` (EN+TR) operator guide gained an "Audit-dir default" paragraph naming the ``/audit_log.jsonl`` fallback and the explicit-override fail-closed behaviour. * F-W3FU-T-09 — ``test_setext_underline_after_atx_heading_does_not_register`` pins setext-after-ATX coverage as a sibling to the existing setext-at-doc-start test. * F-W3FU-T-11 — facade re-exports test strengthened: each name must be ``callable``, not just ``hasattr``. * F-W3FU-T-12 — cross-tool warning substring match made case-insensitive (``"cross-tool" in r.message.lower()``) so a future log-style sweep that lower-cased warning prefixes does not break the test brittlely. Skipped with reason =================== * F-W3FU-T-07 — true purge → reverse-pii driver-level integration test. Helper-shape correlation already pinned at ``test_purge_target_id_matches_reverse_pii_query_hash_on_same_output_dir``. Driver-level test is a Wave 4 task per the original report's classification. * F-W3FU-T-08 — indented fence (CommonMark §4.5 ≤3 leading spaces). ForgeLM docs do not use indented fences; strict no-indent policy is acceptable for the parity tool's curated domain. Defer xfail pin to Wave 4. * F-W3FU-T-10 — ``safety_compliance-tr.md`` allowlist tracking ticket. Comment in test docstring is sufficient for the current cycle; xfail-shape refactor is a Wave 4 nicety. * F-W3FU-S-05 — audit-event catalog row formatting (NIT, F-W3S-08 carry-over). Cosmetic; defer. * F-W3FU-S-06 — parity tool summary message hard-codes ``H2/H3/H4`` even when ``--levels`` is overridden. NIT; defer. * F-W3FU-S-08 — deprecated ``--data-audit`` alias help text references stale ``./audit/`` default. NIT; flagged for future-author awareness only. * F-W3FU-07 (priv) — implicit-output-dir warning severity (WARNING vs ERROR). Judgement call; WARNING is defensible — operator is informed, scan still runs. Promotion to ERROR would break the documented quickstart workflow where operators legitimately scan corpus dirs directly. * F-W3FU-05 (cc) — ``_maybe_audit_logger`` ConfigError fall-through on explicit ``--audit-dir``. Privacy-policy call: the ConfigError-best-effort branch is documented in the docstring as the only "ConfigError is the operator-identity branch" exception to fail-closed; switching to fail-closed on explicit ``--audit-dir`` with ConfigError would force CI runners without ``FORGELM_OPERATOR`` set to refuse the run. Acceptable trade-off; can revisit if a privacy regulator surfaces the gap. Suite 1348 → 1360 tests (+12 net). Lint clean. Dry-run green. Bilingual parity 9/9 OK. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 12 +- docs/guides/gdpr_erasure-tr.md | 9 + docs/guides/gdpr_erasure.md | 9 + forgelm/cli/_parser.py | 2 +- forgelm/cli/subcommands/_reverse_pii.py | 160 ++++++++++++----- forgelm/config.py | 79 +++++++-- forgelm/webhook.py | 9 +- tests/test_check_bilingual_parity.py | 28 +++ tests/test_eu_ai_act.py | 60 +++++++ tests/test_integration.py | 17 +- tests/test_reverse_pii.py | 226 +++++++++++++++++++++--- 11 files changed, 522 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a167a9d..b994349a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,13 +34,17 @@ Single integration branch covering three closure-plan phases: not the regex shape `alice@exampleXcom`. Operators wanting raw regex pass `--type custom` explicitly; on POSIX a 30s SIGALRM budget guards against ReDoS hangs. -- Audit fail-closed: AuditLogger init `OSError`/`ValueError` +- Audit fail-closed: AuditLogger init failure on any non-`ConfigError` + exception class (per Wave-3-followup F-W3FU-03 absorption — was + narrowed to `(OSError, ValueError)` only, now bare `Exception`) refuses the run with `EXIT_TRAINING_ERROR`, naming the audit-dir. `ConfigError` (operator-identity unavailable) still skips with a WARNING — the only "best-effort" branch. -- Audit-dir default: `/audit/` (was `output_dir`); the - Article 15 forensic record no longer co-locates with the corpus - the subject is asking about. +- Audit-dir default: same as `--output-dir` (matching `forgelm purge` + so `verify-audit` correlates Article 17 + Article 15 events for the + same subject in one chain — Wave-3-followup F-W3FU-01 reverted the + intermediate `/audit/` move that broke cross-tool + correlation). - Mid-scan UTF-8 / I/O failures and ReDoS timeouts emit a failure- flavoured `data.access_request_query` event before exit — same no-leak invariant as the success path. diff --git a/docs/guides/gdpr_erasure-tr.md b/docs/guides/gdpr_erasure-tr.md index 7cb89f47..401fcfcc 100644 --- a/docs/guides/gdpr_erasure-tr.md +++ b/docs/guides/gdpr_erasure-tr.md @@ -163,6 +163,15 @@ erişim talebinin değil). `custom` query'i Python regex olarak yorumlar; POSIX sistemlerde 30s'lik per-file SIGALRM bütçesi ReDoS hang'lerine karşı koruma sağlar. +**Audit-dir varsayılanı**: audit chain varsayılan olarak +`/audit_log.jsonl`'a yazılır — `forgelm purge` ile aynı +yola, böylece bir `verify-audit` koşumu aynı subject için Madde 17 +(silme) ve Madde 15 (erişim) olaylarını tek bir chain üzerinde +ilişkilendirir. Override için `--audit-dir ` geçin; +explicit verilen `--audit-dir`'a yazılamıyorsa dispatcher Madde 15 +forensic kaydını sessizce düşürmek yerine `EXIT_TRAINING_ERROR` ile +reddeder. + **Exit kodları:** `0` = scan tamamlandı (matches listesi boş olabilir); `1` = config hatası (boş query, malformed regex, boş glob); `2` = runtime hatası (mid-scan I/O failure). JSON envelope diff --git a/docs/guides/gdpr_erasure.md b/docs/guides/gdpr_erasure.md index 7e91beb9..eecf5a5c 100644 --- a/docs/guides/gdpr_erasure.md +++ b/docs/guides/gdpr_erasure.md @@ -162,6 +162,15 @@ access-request answer). `custom` interprets the query as a Python regex; on POSIX systems a 30s per-file SIGALRM budget guards against ReDoS hangs. +**Audit-dir default**: the audit chain is written to +`/audit_log.jsonl` by default — the same path +`forgelm purge` uses, so a `verify-audit` run correlates Article 17 +(erasure) and Article 15 (access) events for the same subject in +one chain. Pass `--audit-dir ` to override; an +explicit `--audit-dir` that the dispatcher cannot write to refuses +the run with `EXIT_TRAINING_ERROR` rather than silently dropping +the Article 15 forensic record. + **Exit codes:** `0` = scan completed (matches list may be empty); `1` = config error (empty query, malformed regex, empty glob); `2` = runtime error (mid-scan I/O failure). See [`reference/json-output.md`](#/reference/json-output) diff --git a/forgelm/cli/_parser.py b/forgelm/cli/_parser.py index 98600ed5..e7890858 100644 --- a/forgelm/cli/_parser.py +++ b/forgelm/cli/_parser.py @@ -949,7 +949,7 @@ def _add_reverse_pii_subcommand(subparsers) -> None: type=str, default=None, metavar="DIR", - help="Where to write the audit chain entries (default: /audit/, isolated from the corpus directory). Pass an explicit directory to override; explicit values fail loudly when unwritable rather than silently dropping the Article 15 forensic record.", + help="Where to write the audit chain entries (default: same as --output-dir, matching `forgelm purge` so verify-audit correlates Article 17 + Article 15 events for the same subject in one chain). Pass an explicit directory to override; explicit values fail loudly when unwritable rather than silently dropping the Article 15 forensic record.", ) _add_common_subparser_flags(p, include_output_format=True) diff --git a/forgelm/cli/subcommands/_reverse_pii.py b/forgelm/cli/subcommands/_reverse_pii.py index e2028250..9590a904 100644 --- a/forgelm/cli/subcommands/_reverse_pii.py +++ b/forgelm/cli/subcommands/_reverse_pii.py @@ -248,7 +248,18 @@ def _resolve_query_form( ) if salt_source is None: - return query, "plaintext", salt, resolved_source + # F-W3FU-02 (priv) fix: in plaintext mode the salt is used only + # to compute the audit event's ``query_hash`` (so the digest is + # wordlist-resistant); it is NOT applied to the corpus scan. + # Recording ``salt_source_label = resolved_source`` here would + # mislead a forensic reviewer into thinking the env-var/per-dir + # salt was applied to the scan. Record ``"plaintext"`` so the + # audit event distinguishes plaintext scans from hash-mask scans + # cleanly; the audit-hash salt source remains ``resolved_source`` + # internally but is not separately surfaced (a future absorption + # may add an ``audit_hash_salt_source`` field if forensic depth + # demands it). + return query, "plaintext", salt, "plaintext" if salt_source != resolved_source: # Direction-aware diagnostic: telling the operator which @@ -280,13 +291,31 @@ def _truncate_snippet(line: str, match_span: Tuple[int, int]) -> str: Operates on Python ``str`` (code-points), so multi-byte UTF-8 runes are not split mid-byte. + + F-W3FU-02 / F-W3FU-T-02 fix: when ``match_len > budget`` (e.g. + a ``--type custom`` greedy regex matches most of a long line), the + centred window cannot fit the whole match without breaching the + cap. We truncate the match itself in that case — the snippet's + job is "operator can verify the hit", not "operator sees the + entire match"; for matches that overflow the budget the snippet + shows the centre slice with leading/trailing ellipses. """ if len(line) <= _SNIPPET_MAX_CHARS: return line start, end = match_span budget = _SNIPPET_MAX_CHARS - 2 # reserve room for two literal "…" match_len = max(end - start, 0) - ctx = max((budget - match_len) // 2, 0) + if match_len >= budget: + # Degenerate case: the match itself exceeds the budget. Slice + # a centred window of the match, eliding both ends. + mid = (start + end) // 2 + half = budget // 2 + win_start = max(mid - half, 0) + win_end = min(win_start + budget, len(line)) + head = "…" if win_start > 0 else "" + tail = "…" if win_end < len(line) else "" + return f"{head}{line[win_start:win_end]}{tail}" + ctx = (budget - match_len) // 2 win_start = max(start - ctx, 0) win_end = min(end + ctx, len(line)) head = "…" if win_start > 0 else "" @@ -335,18 +364,27 @@ def _scan_files_with_redos_guard( ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: """Walk every file once; centralise the per-file ReDoS budget. - On POSIX a ``SIGALRM`` per-file budget of ``_CUSTOM_REGEX_TIMEOUT_S`` - seconds bounds the worst-case backtracking cost of a - ``--type custom`` pattern (F-W3-PS-03 / F-W3-07 / F-W3S-03). On - Windows ``SIGALRM`` is unavailable; the scan runs without a guard + On POSIX **main-thread** runs a ``SIGALRM`` per-file budget of + ``_CUSTOM_REGEX_TIMEOUT_S`` seconds bounds the worst-case + backtracking cost of a ``--type custom`` pattern (F-W3-PS-03 / + F-W3-07 / F-W3S-03). On Windows AND on POSIX worker threads + (`signal.signal` raises ``ValueError`` outside the main thread — + F-W3FU-04) the guard is a no-op; the scan runs without a wrapper and the operator is expected to vet their regex. The exit-code contract is unchanged either way — a timeout converts to ``OSError`` which the dispatcher converts to ``EXIT_TRAINING_ERROR`` plus a failure-flavoured audit event. """ + import threading as _threading + matches: List[Dict[str, Any]] = [] files_scanned: List[Dict[str, Any]] = [] - use_alarm = identifier_type == "custom" and os.name == "posix" and hasattr(_signal, "SIGALRM") + use_alarm = ( + identifier_type == "custom" + and os.name == "posix" + and hasattr(_signal, "SIGALRM") + and _threading.current_thread() is _threading.main_thread() + ) for path in files: if use_alarm: file_matches = _scan_file_with_alarm(path, pattern) @@ -358,25 +396,39 @@ def _scan_files_with_redos_guard( def _scan_file_with_alarm(path: str, pattern: re.Pattern[str]) -> List[Dict[str, Any]]: - """POSIX-only wrapper enforcing ``_CUSTOM_REGEX_TIMEOUT_S``. + """POSIX main-thread wrapper enforcing ``_CUSTOM_REGEX_TIMEOUT_S``. Translates a ReDoS hang into a clean ``OSError`` so the dispatcher handles it via the same audit-event path as a read failure. + + F-W3FU-03 fix: ``signal.alarm(N)`` returns the seconds remaining + on any previously-scheduled alarm. The previous implementation + discarded that value and called ``signal.alarm(0)`` in ``finally``, + permanently cancelling any outer alarm budget. We now capture the + previous-alarm remainder and re-arm it after our wrapper returns + so a caller that nests this helper inside its own SIGALRM budget + keeps that budget. """ def _alarm(_sig, _frame): # pragma: no cover — signal handler raise OSError(f"reverse-pii scan of {path!r} exceeded {_CUSTOM_REGEX_TIMEOUT_S}s (custom regex ReDoS guard)") - previous = _signal.signal(_signal.SIGALRM, _alarm) - _signal.alarm(_CUSTOM_REGEX_TIMEOUT_S) + previous_handler = _signal.signal(_signal.SIGALRM, _alarm) + previous_alarm_remaining = _signal.alarm(_CUSTOM_REGEX_TIMEOUT_S) try: return _scan_file(path, pattern) finally: _signal.alarm(0) - _signal.signal(_signal.SIGALRM, previous) + _signal.signal(_signal.SIGALRM, previous_handler) + if previous_alarm_remaining > 0: + # Re-arm the outer alarm with whatever budget it had left + # (rounded up to 1s — sub-second remainders cannot survive + # the alarm-syscall granularity, but losing the alarm + # entirely is worse than rounding up). + _signal.alarm(max(previous_alarm_remaining, 1)) -def _hash_for_audit(query: str, salt: Optional[bytes]) -> str: +def _hash_for_audit(query: str, salt: bytes) -> str: """Salted SHA-256 of the operator's raw query for the audit event. Article 15 access requests must not write the subject's identifier @@ -396,13 +448,15 @@ def _hash_for_audit(query: str, salt: Optional[bytes]) -> str: reviewer correlating Article 17 + Article 15 events for one subject sees a connected timeline. - The salt is ``None`` only on the legacy / pre-resolution code - path; production callers always pass the resolved salt. + F-W3FU-08 (priv) fix: ``salt`` is required (was ``Optional[bytes]`` + with an unsalted-fallback branch). The legacy ``None`` branch was + unreachable from any production caller, but a future refactor that + accidentally passed ``None`` would have silently downgraded to + unsalted SHA-256 — exactly the failure mode F-W3-PS-01 closed. + Type-tightening the signature makes the regression statically + impossible. """ - raw = query.encode("utf-8") - if salt is None: - return hashlib.sha256(raw).hexdigest() - return hashlib.sha256(salt + raw).hexdigest() + return hashlib.sha256(salt + query.encode("utf-8")).hexdigest() def _bound_audit_error_message(message: str) -> str: @@ -416,16 +470,21 @@ def _bound_audit_error_message(message: str) -> str: def _resolve_audit_dir(output_dir: str, audit_dir_override: Optional[str]) -> str: """Return the audit-log root for this invocation. - F-W3-06 / F-W3S-02 fix: the previous default (``audit_dir = - output_dir``) wrote ``audit_log.jsonl`` next to the corpus the - subject was asking about — bad colocation for an Article 12 - durable record. We now default to ``/audit/``, - matching the ``forgelm audit`` subcommand's convention. An - explicit ``--audit-dir`` always wins. + F-W3FU-01 (priv) regression fix: the previous Wave 3 absorption + moved the default to ``/audit/`` to address F-W3-06's + cross-tenant write concern, but that broke the cross-tool + correlation contract (`F-W3-PS-09`) — `forgelm purge` writes to + ``/audit_log.jsonl`` directly, so `verify-audit` + correlating Article 17 + Article 15 events for the same subject + saw two disjoint chains. We revert to ``output_dir`` so both + subcommands write into the same chain. An explicit + ``--audit-dir`` always wins; the implicit-output-dir warning at + the dispatcher entry handles the cross-tenant concern by naming + the resolved fallback dir loudly. """ if audit_dir_override: return audit_dir_override - return os.path.join(output_dir, "audit") + return output_dir def _maybe_audit_logger( @@ -464,7 +523,14 @@ def _maybe_audit_logger( exc, ) return None - except (OSError, ValueError) as exc: + except Exception as exc: # noqa: BLE001 — F-W3FU-03 (priv): fail-closed contract is "every non-ConfigError class". + # The original (OSError, ValueError) tuple was narrower than the + # original F-W3-PS-02 ask ("everything except ConfigError fails + # closed"); a future hardening hook in AuditLogger.__init__ that + # raises e.g. RuntimeError would silently re-introduce the + # silent-drop failure mode. Catching bare Exception here is the + # right shape: the contract should not depend on enumerating + # every exception class the audit chain might raise. _output_error_and_exit( output_format, ( @@ -534,22 +600,25 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: # get exit 1 without leaving a salt file behind. _build_search_pattern(query, identifier_type, output_format) - if explicit_output_dir is None and salt_source is not None: - # F-W3-PS-06 / F-W3T-10 follow-up: an implicit ``output_dir`` - # creates ``.forgelm_audit_salt`` next to the operator's corpus. - # In hash-mask mode, this silently breaks cross-tool correlation - # with any ``forgelm purge`` invocation that supplied an - # explicit ``--output-dir`` (the two would compute different - # salts). Surface the risk loudly so the operator can pin the - # output dir or accept the consequence. + if explicit_output_dir is None: + # F-W3FU-06 (priv) follow-up: the salt-file side effect is + # present in BOTH plaintext and hash-mask modes, because the + # audit hash always uses the per-output-dir salt (F-W3-PS-01). + # The previous warning was scoped to ``salt_source is not + # None``, leaving plaintext-mode implicit-output-dir runs + # silently creating ``.forgelm_audit_salt`` in the corpus + # parent. Always warn when the resolution is implicit; tag + # the message with whether hash-mask scanning is also in play + # so the operator sees the full set of consequences. + scope = "audit-hash AND hash-mask scanning" if salt_source is not None else "audit-hash" logger.warning( - "reverse-pii: --salt-source=%r set but --output-dir is not. " - "Falling back to the corpus parent (%r); the per-dir salt file " - ".forgelm_audit_salt will be created/read there. Cross-tool " - "correlation with `forgelm purge` requires both subcommands " - "to use the SAME --output-dir.", - salt_source, + "reverse-pii: --output-dir not set; falling back to the corpus " + "parent (%r). The per-dir salt file .forgelm_audit_salt will " + "be created/read there for the %s. Cross-tool correlation " + "with `forgelm purge` requires both subcommands to use the " + "SAME --output-dir.", output_dir, + scope, ) scan_query, scan_mode, salt, salt_source_label = _resolve_query_form(query, salt_source, output_dir, output_format) @@ -583,9 +652,18 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: error_message=str(exc), ) except Exception as audit_exc: # noqa: BLE001 — fail-closed on audit error. + # F-W3FU-06 (cc) fix: chain the original mid-scan + # exception's diagnostic context into the audit-failure + # message so a double-fault doesn't lose the underlying + # cause (otherwise the operator sees only "audit chain + # write failed" and has to re-run to diagnose). _output_error_and_exit( output_format, - f"reverse-pii: failed to write the Article 15 failure audit event ({audit_exc}).", + ( + f"reverse-pii: failed to write the Article 15 failure audit event " + f"({audit_exc.__class__.__name__}: {audit_exc}); the original " + f"mid-scan failure was {exc.__class__.__name__}: {exc}." + ), EXIT_TRAINING_ERROR, ) _output_error_and_exit( diff --git a/forgelm/config.py b/forgelm/config.py index 11b5b5bc..3701cd3a 100644 --- a/forgelm/config.py +++ b/forgelm/config.py @@ -1,7 +1,7 @@ import logging import os import warnings -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional, Tuple import yaml from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator @@ -840,18 +840,64 @@ def _warn_general_consistency(self) -> None: self.training.save_steps, ) + def _risk_tiers(self) -> Tuple[Optional[str], Optional[str]]: + """Return the (risk_assessment.risk_category, compliance.risk_classification) pair. + + Both sibling fields share the same RiskTier Literal but are + independent — Pydantic does not enforce equality between them, + so a hand-written YAML can reach an asymmetric state where the + technical and compliance views disagree. All strict-gate + decisions OR across both fields (F-W3FU-S-01 / F-W3FU-01 + regression fix): if EITHER is in the strict tier, the gate + fires; if EITHER is ``unacceptable``, the Article 5 banner + fires. ``_resolve_risk_label`` produces a *display* label for + log messages; the gate boolean is computed independently to + avoid the asymmetric-tier silent bypass. + """ + ra = self.risk_assessment.risk_category if self.risk_assessment else None + cm = self.compliance.risk_classification if self.compliance else None + return ra, cm + def _resolve_risk_label(self) -> Optional[str]: - """Return the active risk label across both sibling fields. + """Return the active risk label for log messages. - ``risk_assessment.risk_category`` and ``compliance.risk_classification`` - share the same RiskTier set; either is accepted as authoritative - for the strict-tier warnings. + Display-only — used to fill the ``%r`` slot in the auto_revert + warning and the ``ConfigError`` raise. See ``_risk_tiers`` / + ``_is_strict_tier`` / ``_is_unacceptable`` for the actual gate + logic, which OR's across both sibling fields rather than + picking one. + """ + ra, cm = self._risk_tiers() + # When the two siblings disagree, prefer whichever side carries + # a strict tier so the warning message names the strict label + # the operator needs to address. + if ra in _STRICT_RISK_TIERS: + return ra + if cm in _STRICT_RISK_TIERS: + return cm + return ra or cm + + def _is_strict_tier(self) -> bool: + """True iff EITHER sibling field is in ``_STRICT_RISK_TIERS``. + + The OR-across-fields semantics matches the pre-Wave-3 behaviour + and is required so that an asymmetric config (e.g. + ``risk_assessment.risk_category="limited-risk"`` AND + ``compliance.risk_classification="high-risk"``) cannot silently + bypass the F-compliance-110 strict gate. """ - if self.risk_assessment and self.risk_assessment.risk_category: - return self.risk_assessment.risk_category - if self.compliance and self.compliance.risk_classification: - return self.compliance.risk_classification - return None + ra, cm = self._risk_tiers() + return ra in _STRICT_RISK_TIERS or cm in _STRICT_RISK_TIERS + + def _is_unacceptable(self) -> bool: + """True iff EITHER sibling field is ``"unacceptable"``. + + Article 5 prohibited-practice banner fires whenever either view + marks the deployment unacceptable — disagreement between the + technical and compliance views must NOT silence the notice. + """ + ra, cm = self._risk_tiers() + return ra == "unacceptable" or cm == "unacceptable" def _warn_unacceptable_practice(self) -> None: """Article 5 — prohibited-practices banner. @@ -901,17 +947,24 @@ def _warn_high_risk_compliance(self) -> None: to nudge the operator into running with auto-revert + safety eval, and ``unacceptable`` should never get *less* gating than ``high-risk`` because the underlying use case is not allowed at all under the Act. + + Strict-tier detection ORs across both sibling fields + (``risk_assessment.risk_category`` and + ``compliance.risk_classification``) — see ``_is_strict_tier`` + for rationale. The display label is whichever sibling carries + a strict tier; this is NOT the gate decision (the gate is the + OR), only the message text. """ - label = self._resolve_risk_label() - if label not in _STRICT_RISK_TIERS: + if not self._is_strict_tier(): return + label = self._resolve_risk_label() if not self.evaluation or not self.evaluation.auto_revert: logger.warning( "Risk classification %r requires evaluation.auto_revert: true " "for EU AI Act compliance. Safety gates should be enabled.", label, ) - if label == "unacceptable": + if self._is_unacceptable(): self._warn_unacceptable_practice() self._enforce_safety_gate_for_strict_tier(label) diff --git a/forgelm/webhook.py b/forgelm/webhook.py index 1c4e51b7..ef83f66b 100644 --- a/forgelm/webhook.py +++ b/forgelm/webhook.py @@ -89,12 +89,17 @@ def _post_payload(self, url: str, payload: dict, event: str) -> None: # Timeout floor — webhook keeps the historical 1s floor (``safe_post`` # rejects 0/None unconditionally). Wave 3 F-compliance-106 # raised the WebhookConfig default 5 → 10; the clamp message - # and fallback value follow that default. + # and fallback value follow that default. F-W3FU-S-04 fix: + # the previous ``isinstance(timeout, (int, float))`` half of + # the guard was dead under Pydantic (the field is ``int``-typed + # and validated at config load), so a non-numeric ``timeout`` + # cannot reach this site through any documented call path. + # Drop the isinstance check; keep the numeric-floor clamp. from .config import WebhookConfig as _WebhookConfig default_timeout = _WebhookConfig.model_fields["timeout"].default timeout = getattr(self.config, "timeout", default_timeout) - if not isinstance(timeout, (int, float)) or timeout < 1: + if timeout < 1: logger.warning( "Webhook timeout=%r is below the 1s floor; clamping to %ds.", timeout, diff --git a/tests/test_check_bilingual_parity.py b/tests/test_check_bilingual_parity.py index 76c6404f..e0fe10cb 100644 --- a/tests/test_check_bilingual_parity.py +++ b/tests/test_check_bilingual_parity.py @@ -107,6 +107,34 @@ def test_backtick_fence_not_closed_by_tilde_fence(self, tmp_path: Path) -> None: assert [h.level for h in headings] == [2, 2] assert [h.text for h in headings] == ["Real H2", "After close"] + def test_shorter_opening_fence_closed_by_longer_run(self, tmp_path: Path) -> None: + """F-W3FU-T-06 regression: CommonMark §4.5 allows a closer + LONGER than the opener (the spec says "at least as long as", + not "equal to"). Pin the production ``len(marker) >= + len(fence_marker)`` semantics so a hand-edit dropping ``>=`` + to ``==`` would surface in CI.""" + src = _write( + tmp_path / "doc.md", + "## Real H2\n```\n## inside short opener\n````\n## After\n", + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [2, 2] + assert [h.text for h in headings] == ["Real H2", "After"] + + def test_setext_underline_after_atx_heading_does_not_register(self, tmp_path: Path) -> None: + """F-W3FU-T-09 regression: ``test_setext_headings_are_not_matched`` + pins the case where setext appears at the start of a doc. + This sibling test pins the case where a setext underline + appears AFTER an ATX heading — production should still skip + the setext (only ATX is recognised).""" + src = _write( + tmp_path / "doc.md", + "## ATX H2 first\nSome paragraph\n=========\n## After\n", + ) + headings = extract_headings(src) + assert [h.level for h in headings] == [2, 2] + assert [h.text for h in headings] == ["ATX H2 first", "After"] + def test_missing_file_returns_empty(self, tmp_path: Path) -> None: assert extract_headings(tmp_path / "missing.md") == [] diff --git a/tests/test_eu_ai_act.py b/tests/test_eu_ai_act.py index a9fc77ec..c0f3abcd 100644 --- a/tests/test_eu_ai_act.py +++ b/tests/test_eu_ai_act.py @@ -160,6 +160,66 @@ def test_strict_tier_safety_disabled_raises_config_error(self, minimal_config, t ) ) + @pytest.mark.parametrize( + "ra,cm", + [ + ("limited-risk", "high-risk"), + ("limited-risk", "unacceptable"), + ("high-risk", "limited-risk"), + ("unacceptable", "limited-risk"), + ], + ) + def test_asymmetric_strict_tier_still_raises_when_safety_disabled(self, minimal_config, ra, cm): + """F-W3FU-S-01 / F-W3FU-01 regression: ``risk_assessment.risk_category`` + and ``compliance.risk_classification`` are independent + ``RiskTier`` Literals; Pydantic does not enforce equality. An + asymmetric YAML where ONE sibling is strict and the other is + non-strict must still trip the F-compliance-110 gate — the + cognitive-complexity refactor that absorbed Sonar python:S3776 + accidentally inverted this OR-across-fields semantics by + single-label-resolution-then-strict-check, silently bypassing + the gate when the ``risk_assessment``-first preference picked + the non-strict sibling.""" + from forgelm.config import ConfigError + + with pytest.raises(ConfigError, match="evaluation.safety.enabled"): + ForgeConfig( + **minimal_config( + risk_assessment={"risk_category": ra}, + compliance={ + "provider_name": "Acme", + "system_name": "Bot", + "risk_classification": cm, + }, + ) + ) + + def test_compliance_unacceptable_fires_article_5_banner_even_when_risk_assessment_says_high_risk( + self, caplog, minimal_config + ): + """F-W3FU-01 regression: the Article 5 banner must fire whenever + EITHER sibling marks the deployment ``unacceptable`` — the + post-refactor single-label-resolution would have suppressed the + banner when ``risk_assessment.risk_category="high-risk"`` won + the preference ordering. Both fields must contribute to the + unacceptable check.""" + import logging + + with caplog.at_level(logging.WARNING, logger="forgelm.config"): + ForgeConfig( + **minimal_config( + risk_assessment={"risk_category": "high-risk"}, + compliance={ + "provider_name": "Acme", + "system_name": "Bot", + "risk_classification": "unacceptable", + }, + evaluation={"safety": {"enabled": True}}, + ) + ) + assert "Article 5" in caplog.text + assert "prohibited" in caplog.text + def test_unacceptable_risk_warnings(self, caplog, minimal_config): """``unacceptable`` (Article 5) must trip the strict gate AND emit the dedicated prohibited-practices warning on top of the auto_revert diff --git a/tests/test_integration.py b/tests/test_integration.py index d5f31bef..9fe23f17 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -159,15 +159,20 @@ def test_full_config_yaml_round_trip(self, tmp_path): assert cfg.risk_assessment.foreseeable_misuse[0] == "Users may ask for medical advice" assert cfg.data.governance.collection_method == "Manual curation by domain experts" - def test_high_risk_without_safety_raises_config_error(self): - """Wave 3 / Faz 28 F-compliance-110: high-risk classification with - safety eval disabled is now a hard ``ConfigError`` (was a warning - prior to v0.5.5). EU AI Act Article 9 risk-management evidence - cannot be derived from a disabled safety eval; operators who - genuinely want a sandboxed run must lower the risk_classification.""" + @pytest.mark.parametrize("tier", ["high-risk", "unacceptable"]) + def test_strict_tier_without_safety_raises_config_error(self, tier): + """Wave 3 / Faz 28 F-compliance-110 + F-W3FU-T-05: BOTH strict + tiers (Article 9 ``high-risk`` AND Article 5 ``unacceptable``) + are hard ``ConfigError`` when safety eval is disabled at the + full-config integration layer. EU AI Act risk-management + evidence cannot be derived from a disabled safety eval; the + unit test in test_eu_ai_act.py pins the validator-layer + contract, and this test pins the YAML round-trip + Pydantic + validator interaction at the integration layer.""" from forgelm.config import ConfigError data = _full_config() + data["risk_assessment"]["risk_category"] = tier del data["evaluation"]["safety"] with pytest.raises(ConfigError, match="evaluation.safety.enabled"): ForgeConfig(**data) diff --git a/tests/test_reverse_pii.py b/tests/test_reverse_pii.py index dfe3f8e2..f6deabd5 100644 --- a/tests/test_reverse_pii.py +++ b/tests/test_reverse_pii.py @@ -262,35 +262,49 @@ def test_implicit_output_dir_with_salt_source_warns_about_correlation_risk(self, warnings = [ r for r in caplog.records - if r.levelname == "WARNING" and "reverse-pii" in r.message and "Cross-tool" in r.message + if r.levelname == "WARNING" and "reverse-pii" in r.message and "cross-tool" in r.message.lower() ] assert warnings, "implicit --output-dir + --salt-source must surface a cross-tool correlation warning" # The salt file landed in the corpus dir per the warning text. assert (corpus_dir / ".forgelm_audit_salt").exists() - def test_implicit_output_dir_without_salt_source_does_not_warn(self, tmp_path: Path, caplog) -> None: - """The cross-tool correlation warning is scoped to hash-mask - invocations; plaintext-mode runs without ``--salt-source`` use - the salt only for the audit hash and must not nag the operator - about cross-tool correlation.""" + def test_plaintext_implicit_output_dir_also_warns_about_salt_file(self, tmp_path: Path, caplog) -> None: + """F-W3FU-06 (priv) regression: the salt-file side effect is + present in BOTH plaintext and hash-mask modes (the audit hash + always uses the per-output-dir salt per F-W3-PS-01). An + operator running plaintext-mode reverse-pii with an implicit + ``--output-dir`` must therefore ALSO get the cross-tool + correlation warning — the previous absorption scoped the + warning to hash-mask mode only, leaving plaintext-mode runs + silently creating ``.forgelm_audit_salt`` in the corpus + parent dir without warning.""" import logging from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd - corpus = tmp_path / "train.jsonl" + corpus_dir = tmp_path / "data" + corpus_dir.mkdir() + corpus = corpus_dir / "train.jsonl" _seed_corpus(corpus, [{"id": "row-A", "text": "x"}]) args = _build_args( query="alice@example.com", type="email", - salt_source=None, + salt_source=None, # plaintext mode files=[str(corpus)], output_dir=None, ) with caplog.at_level(logging.WARNING, logger="forgelm.cli"): with pytest.raises(SystemExit): _run_reverse_pii_cmd(args, output_format="json") - cross_tool_warns = [r for r in caplog.records if "Cross-tool" in r.message] - assert cross_tool_warns == [], "plaintext mode must not warn about cross-tool correlation" + warnings = [ + r + for r in caplog.records + if r.levelname == "WARNING" and "reverse-pii" in r.message and "cross-tool" in r.message.lower() + ] + assert warnings, ( + "plaintext mode with implicit --output-dir must still warn — the " + "salt file side effect is present regardless of scan mode" + ) def test_per_dir_salt_source_refuses_when_env_var_set(self, tmp_path: Path, capsys, monkeypatch) -> None: """F-W3-11 / F-W3-PS-04 regression: the symmetric direction — @@ -598,16 +612,37 @@ def _flaky_scan(path: str, pattern): evt = access_events[0] assert evt["error_class"] == "OSError" assert "simulated mid-scan I/O failure" in evt["error_message"] - # Critical: failure-path event must carry the same no-leak invariant - # as the success path (F-W3T-01). + # F-W3FU-T-04 / F-W3FU-04: failure-path event must carry the same + # no-leak invariant AND the same positive-shape (salted hash) + # contract as the success path. Substring-only check is too weak + # — a future refactor that base64-encoded the query into the event + # under a different field name would slip past it. + from forgelm.cli.subcommands._purge import _resolve_salt + from forgelm.cli.subcommands._reverse_pii import _hash_for_audit + + salt, _ = _resolve_salt(str(tmp_path)) + expected_hash = _hash_for_audit("alice@example.com", salt) + assert evt["query_hash"] == expected_hash, ( + "failure-path event must record the salted hash of the query (positive shape)" + ) as_str = json.dumps(evt) assert "alice@example.com" not in as_str, f"failure-path audit event leaked raw identifier:\n{as_str}" - - def test_malformed_utf8_corpus_exits_runtime_error_with_audit_event(self, tmp_path: Path) -> None: - """F-W3-04 regression: a UnicodeDecodeError mid-scan must surface - as ``EXIT_TRAINING_ERROR=2`` with a failure-flavoured audit - event — not as Python's default exit 1 with no audit record.""" - from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd + # Defence-in-depth: no encoded form of the identifier in the event. + import base64 + + raw = b"alice@example.com" + for encoded in (raw.hex(), raw.hex().upper(), base64.b64encode(raw).decode()): + assert encoded not in as_str, f"event leaked encoded identifier ({encoded!r})" + + def test_malformed_utf8_corpus_exits_runtime_error_with_audit_event_and_no_leak(self, tmp_path: Path) -> None: + """F-W3-04 + F-W3FU-T-03 regression: a UnicodeDecodeError mid-scan + must surface as ``EXIT_TRAINING_ERROR=2`` with a failure-flavoured + audit event AND that event must carry the same no-leak invariant + as the OSError sibling test (a future change that wrapped + ``str(exc)`` with corpus byte context could otherwise leak a + high-entropy identifier into the chain).""" + from forgelm.cli.subcommands._purge import _resolve_salt + from forgelm.cli.subcommands._reverse_pii import _hash_for_audit, _run_reverse_pii_cmd corpus = tmp_path / "bad.jsonl" # Valid first line, then a multi-byte rune fragment. @@ -616,7 +651,7 @@ def test_malformed_utf8_corpus_exits_runtime_error_with_audit_event(self, tmp_pa audit_dir.mkdir() args = _build_args( - query="x", + query="alice@example.com", # high-entropy identifier; pinned by no-leak assertion below type="email", files=[str(corpus)], output_dir=str(tmp_path), @@ -631,6 +666,13 @@ def test_malformed_utf8_corpus_exits_runtime_error_with_audit_event(self, tmp_pa assert len(access_events) == 1 evt = access_events[0] assert evt["error_class"] in {"UnicodeDecodeError", "OSError"} + # F-W3FU-T-03: same no-leak invariant as the OSError leg. + as_str = json.dumps(evt) + assert "alice@example.com" not in as_str, f"UnicodeDecodeError audit event leaked raw identifier:\n{as_str}" + # Positive-shape: digest is the salted hash purge would compute. + salt, _ = _resolve_salt(str(tmp_path)) + expected_hash = _hash_for_audit("alice@example.com", salt) + assert evt["query_hash"] == expected_hash def test_explicit_audit_dir_unwritable_fails_closed(self, tmp_path: Path, monkeypatch) -> None: """F-W3-01 / F-W3-PS-02 regression: when --audit-dir is explicit @@ -701,6 +743,27 @@ def test_long_line_centred_on_match_preserves_identifier(self, tmp_path: Path, c "centred truncation must preserve the matched span — the whole point is operator verification" ) + def test_match_span_wider_than_budget_still_bounded(self) -> None: + """F-W3FU-02 / F-W3FU-T-02 regression: when a ``--type custom`` + greedy regex matches a span longer than ``_SNIPPET_MAX_CHARS``, + the previous centring math floored ``ctx`` at 0 and returned + the whole match span, breaching the documented cap. The + degenerate case must still respect the cap.""" + from forgelm.cli.subcommands._reverse_pii import _SNIPPET_MAX_CHARS, _truncate_snippet + + line = "before " + ("A" * 250) + " after" + # The whole "AAAA..." run is the match — wider than the budget. + match_start = line.index("A") + match_end = match_start + 250 + out = _truncate_snippet(line, (match_start, match_end)) + # Allow up to 2 extra chars for head + tail "…" markers. + assert len(out) <= _SNIPPET_MAX_CHARS + 2, ( + f"degenerate match-len > budget case must still respect the cap; got {len(out)}" + ) + # Both ends must show ellipses since context was elided on both sides. + assert out.startswith("…") + assert out.endswith("…") + def test_truncation_respects_multibyte_utf8(self, tmp_path: Path, capsys) -> None: """F-W3T-05 regression: code-point slicing must not produce broken UTF-8 sequences when the line mixes CJK + emoji.""" @@ -721,6 +784,121 @@ def test_truncation_respects_multibyte_utf8(self, tmp_path: Path, capsys) -> Non assert "alice@example.com" in snippet +# --------------------------------------------------------------------------- +# §6b Test — POSIX SIGALRM ReDoS guard (F-W3FU-T-01) +# --------------------------------------------------------------------------- + + +class TestReDoSGuard: + """Wave-3-followup: pin the SIGALRM-based ReDoS guard contracts. + + Three invariants: + 1. A pathological ``--type custom`` regex terminates within the + budget and surfaces as ``EXIT_TRAINING_ERROR=2`` with a + failure-flavoured audit event (no hang). + 2. The wrapper restores any outer alarm budget after returning. + 3. On non-main threads (where ``signal.signal`` raises) the guard + is a no-op rather than a crash; the scan still runs. + """ + + @pytest.mark.skipif(os.name != "posix", reason="SIGALRM is POSIX-only") + def test_pathological_custom_regex_terminates_within_budget(self, tmp_path: Path, monkeypatch) -> None: + from forgelm.cli.subcommands import _reverse_pii + + # Cut the budget hard so the test is fast; the contract is + # "timeout fires", not "exact 30s". + monkeypatch.setattr(_reverse_pii, "_CUSTOM_REGEX_TIMEOUT_S", 1) + # Plant a corpus line that triggers catastrophic backtracking + # for ``(a+)+$`` against a long mismatching tail. + corpus = tmp_path / "evil.jsonl" + corpus.write_text("a" * 40 + "X" + "\n", encoding="utf-8") + audit_dir = tmp_path / "audit" + audit_dir.mkdir() + + args = _build_args( + query=r"(a+)+$", + type="custom", + files=[str(corpus)], + output_dir=str(tmp_path), + audit_dir=str(audit_dir), + ) + import time + + started = time.monotonic() + with pytest.raises(SystemExit) as ei: + _reverse_pii._run_reverse_pii_cmd(args, output_format="json") + elapsed = time.monotonic() - started + # The 1s budget plus dispatcher overhead must not stretch past 10s. + assert elapsed < 10, f"ReDoS guard did not fire fast enough; elapsed={elapsed:.2f}s" + assert ei.value.code == 2 + + events = _read_audit_events(audit_dir / "audit_log.jsonl") + access_events = [e for e in events if e["event"] == "data.access_request_query"] + assert len(access_events) == 1 + evt = access_events[0] + assert evt["error_class"] == "OSError" + assert "ReDoS" in evt["error_message"] or "exceeded" in evt["error_message"] + + @pytest.mark.skipif(os.name != "posix", reason="SIGALRM is POSIX-only") + def test_alarm_wrapper_restores_outer_alarm_budget(self, tmp_path: Path) -> None: + """F-W3FU-03: a previously-scheduled outer SIGALRM budget must + survive the per-file wrapper. The wrapper's old shape called + ``signal.alarm(0)`` in ``finally``, permanently cancelling the + outer alarm; the fix captures and re-arms it.""" + import re as _re + import signal as _signal + + from forgelm.cli.subcommands._reverse_pii import _scan_file_with_alarm + + corpus = tmp_path / "x.jsonl" + corpus.write_text('{"id":1,"text":"x"}\n', encoding="utf-8") + pattern = _re.compile("x") + + previous_handler = _signal.signal(_signal.SIGALRM, lambda *_: None) + try: + _signal.alarm(60) # outer budget + _scan_file_with_alarm(str(corpus), pattern) + remaining = _signal.alarm(0) + assert remaining > 0, ( + "outer alarm budget must survive the wrapper — F-W3FU-03 " + "regression: signal.alarm(0) in finally was cancelling the outer alarm" + ) + finally: + _signal.alarm(0) + _signal.signal(_signal.SIGALRM, previous_handler) + + @pytest.mark.skipif(os.name != "posix", reason="SIGALRM is POSIX-only") + def test_redos_guard_no_op_on_worker_thread(self, tmp_path: Path) -> None: + """F-W3FU-04: ``signal.signal`` raises ``ValueError`` from a + non-main thread; the guard must skip gracefully rather than + crash the whole scan.""" + import re as _re + import threading as _threading + + from forgelm.cli.subcommands._reverse_pii import _scan_files_with_redos_guard + + corpus = tmp_path / "x.jsonl" + corpus.write_text('{"id":1,"text":"abc"}\n', encoding="utf-8") + pattern = _re.compile("abc") + + results: list = [] + errors: list = [] + + def _worker() -> None: + try: + results.append(_scan_files_with_redos_guard([str(corpus)], pattern, identifier_type="custom")) + except Exception as exc: # pragma: no cover — fails the test below + errors.append(exc) + + thread = _threading.Thread(target=_worker) + thread.start() + thread.join() + assert not errors, f"worker thread crashed instead of skipping the alarm guard: {errors}" + assert len(results) == 1 + matches, files_scanned = results[0] + assert len(matches) == 1 + + # --------------------------------------------------------------------------- # §7 Test — Facade re-exports + parser wiring + dispatch # --------------------------------------------------------------------------- @@ -728,13 +906,17 @@ def test_truncation_respects_multibyte_utf8(self, tmp_path: Path, capsys) -> Non class TestReversePiiFacade: def test_facade_re_exports_dispatcher_and_helpers(self) -> None: - """F-W3-10 regression: facade must re-export every name in - ``_reverse_pii.__all__``, not a hand-maintained subset.""" + """F-W3-10 + F-W3FU-T-11 regression: facade must re-export every + name in ``_reverse_pii.__all__`` AND each re-export must be a + live callable. The previous ``hasattr`` check would have passed + even if a re-export was set to ``None`` — strengthen to pin the + functional contract, not just the name presence.""" from forgelm import cli as _cli_facade from forgelm.cli.subcommands import _reverse_pii as _mod for name in _mod.__all__: - assert hasattr(_cli_facade, name), f"forgelm.cli must re-export {name!r}" + attr = getattr(_cli_facade, name, None) + assert callable(attr), f"forgelm.cli.{name} must be a callable re-export, got {attr!r}" def test_parser_registers_reverse_pii_subcommand(self) -> None: """`forgelm reverse-pii --help` must succeed without --config.""" From e3c44be5de110fb01aa7d3b095dd6861e82acde8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 22:40:23 +0300 Subject: [PATCH 10/11] fix(wave-3-followup): absorb 6 inline review comments on the absorption commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `gdpr_erasure{,-tr}.md` JSON envelope link — the previous link mixed a ``.md``-shaped link text with a SPA hash-router href (``[reference/json-output.md](#/reference/json-output)``). Replaced with bilingual repo-relative paths (``../usermanuals/{en,tr}/reference/json-output.md``) so the link navigates correctly from both GitHub and the SPA viewer. * `_parser.py` ``--output-dir`` help text — was misleading post- F-W3-PS-01 ("only consulted when --salt-source is set"); the salt is now read/created on every invocation for the audit-event hash. Help text rewritten to name the side effect for both plaintext and hash-mask modes. * `_scan_file_with_alarm` outer-alarm preservation — the previous ``F-W3FU-03`` absorption captured ``previous_alarm_remaining`` and re-armed it in ``finally``, but did NOT subtract the elapsed wall-clock seconds spent in ``_scan_file``. A nested call would silently EXTEND the outer caller's deadline by however long the scan ran. Fixed: capture ``time.monotonic()`` at entry, compute ``remaining = max(int(math.ceil(prev - elapsed)), 1)`` in the ``finally`` clause. Clamp to ≥1 so we never lose the outer alarm (alarm-syscall granularity is whole seconds; rounding down would silently shave 0-1s off on every nested call). * `webhook.py` timeout clamp — the F-W3S-04 absorption replaced the old literal ``5`` fallback with ``WebhookConfig.model_fields["timeout"].default`` (10s), but the documented contract is "below 1s → clamp to 1s floor", not "jump to the model default". An operator who set ``timeout: 0`` was getting their budget silently 10x'd. Fixed: ``timeout = 1`` and clamp message updated to "clamping to 1s". * `tests/test_reverse_pii.py` UnicodeDecodeError assertion — the failure-path test allowed either ``"UnicodeDecodeError"`` or ``"OSError"`` in ``evt["error_class"]``. The fixture plants ``\xff\xfe`` after a newline which produces ``UnicodeDecodeError`` deterministically when Python's text-mode iterator decodes the second line; pinning the exact class makes a refactor that converted it to a generic OSError before logging surface here. * `_resolve_query_form` per-dir bypass — the previous absorption refused ``--salt-source=per_dir`` when ``FORGELM_AUDIT_SECRET`` was exported (because ``_resolve_salt`` always XOR'd the env var in and the post-resolution mismatch check fired). Refusing was conservative-but-wrong: an explicit ``per_dir`` choice should HONOR the operator's intent, ignoring the unrelated env var for this invocation. Added ``_read_persistent_salt`` helper to ``_purge.py`` (exposes just the on-disk persistent half of the salt-resolution path) and routed the ``per_dir`` branch through it. ``env_var`` mode now refuses cleanly when ``FORGELM_AUDIT_SECRET`` is unset (its prerequisite). ``test_per_dir_salt_source_refuses_when_env_var_set`` flipped to ``test_per_dir_salt_source_honored_when_env_var_set`` — pins the new contract: explicit per_dir wins, scan_mode=hash, salt_source records "per_dir" in the audit event. Suite stays at 1360 passed, 14 skipped (the per-dir test was flipped, not added; new behaviour is covered by the same test in the inverted shape). Lint clean. Bilingual parity 9/9 OK. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/guides/gdpr_erasure-tr.md | 3 +- docs/guides/gdpr_erasure.md | 3 +- forgelm/cli/_parser.py | 2 +- forgelm/cli/subcommands/_purge.py | 44 +++++--- forgelm/cli/subcommands/_reverse_pii.py | 131 +++++++++++++++--------- forgelm/webhook.py | 21 ++-- tests/test_reverse_pii.py | 44 ++++++-- 7 files changed, 161 insertions(+), 87 deletions(-) diff --git a/docs/guides/gdpr_erasure-tr.md b/docs/guides/gdpr_erasure-tr.md index 401fcfcc..b0a69d0b 100644 --- a/docs/guides/gdpr_erasure-tr.md +++ b/docs/guides/gdpr_erasure-tr.md @@ -175,7 +175,8 @@ reddeder. **Exit kodları:** `0` = scan tamamlandı (matches listesi boş olabilir); `1` = config hatası (boş query, malformed regex, boş glob); `2` = runtime hatası (mid-scan I/O failure). JSON envelope -şeması için bkz. [`reference/json-output.md`](#/reference/json-output). +şeması için bkz. +[`../usermanuals/tr/reference/json-output.md`](../usermanuals/tr/reference/json-output.md). ## Bkz. diff --git a/docs/guides/gdpr_erasure.md b/docs/guides/gdpr_erasure.md index eecf5a5c..8169a951 100644 --- a/docs/guides/gdpr_erasure.md +++ b/docs/guides/gdpr_erasure.md @@ -173,7 +173,8 @@ the Article 15 forensic record. **Exit codes:** `0` = scan completed (matches list may be empty); `1` = config error (empty query, malformed regex, empty glob); `2` = -runtime error (mid-scan I/O failure). See [`reference/json-output.md`](#/reference/json-output) +runtime error (mid-scan I/O failure). See +[`../usermanuals/en/reference/json-output.md`](../usermanuals/en/reference/json-output.md) for the JSON envelope schema. ## See also diff --git a/forgelm/cli/_parser.py b/forgelm/cli/_parser.py index e7890858..8e087f68 100644 --- a/forgelm/cli/_parser.py +++ b/forgelm/cli/_parser.py @@ -942,7 +942,7 @@ def _add_reverse_pii_subcommand(subparsers) -> None: type=str, default=None, metavar="DIR", - help="Output directory containing the per-output-dir salt file (only consulted when --salt-source is set). Defaults to the parent of the first resolved corpus file.", + help="Output directory containing the per-output-dir salt file (`.forgelm_audit_salt`). The salt is read/created here for BOTH the audit-event hash (every invocation, plaintext or hash-mask mode) AND for hash-mask scanning when --salt-source is set. Defaults to the parent of the first resolved corpus file; an implicit fallback emits a WARNING naming the resolved dir so the operator can pin --output-dir for cross-tool correlation with `forgelm purge`.", ) p.add_argument( "--audit-dir", diff --git a/forgelm/cli/subcommands/_purge.py b/forgelm/cli/subcommands/_purge.py index e9aafb37..913d9034 100644 --- a/forgelm/cli/subcommands/_purge.py +++ b/forgelm/cli/subcommands/_purge.py @@ -109,21 +109,14 @@ def _output_error_and_exit(output_format: str, msg: str, exit_code: int) -> NoRe # --------------------------------------------------------------------------- -def _resolve_salt(output_dir: str) -> Tuple[bytes, str]: - """Return ``(salt_bytes, salt_source)`` for ``target_id`` hashing. +def _read_persistent_salt(output_dir: str) -> bytes: + """Read (or create) the per-output-dir persistent salt — no env XOR. - Salt resolution per design §5.4: - - - The persistent per-output-dir salt at - ``/.forgelm_audit_salt`` is **always** consulted - (created on first call with mode 0600). - - When ``FORGELM_AUDIT_SECRET`` is set, its first 16 bytes are - XOR'd with the persistent salt to derive the actual hashing - salt; ``salt_source`` is recorded as ``"env_var"`` so a salt- - source toggle between invocations is detectable in the chain - via the ``salt_source`` audit-event field. - - When the env var is absent, the persistent salt is used - verbatim; ``salt_source = "per_dir"``. + Companion to :func:`_resolve_salt`; this helper exposes JUST the + on-disk persistent half of the salt resolution so callers that + explicitly want per-dir-only semantics (e.g. ``forgelm reverse-pii + --salt-source per_dir``) can bypass the env-var XOR step that + ``_resolve_salt`` performs unconditionally. Raises :class:`OSError` when the salt file cannot be created or read; the caller surfaces the I/O error as ``EXIT_TRAINING_ERROR``. @@ -156,6 +149,29 @@ def _resolve_salt(output_dir: str) -> Tuple[bytes, str]: f"Salt file {salt_path!r} is shorter than {_SALT_BYTES} bytes " "(corrupted or truncated). Delete the file to regenerate." ) + return persistent + + +def _resolve_salt(output_dir: str) -> Tuple[bytes, str]: + """Return ``(salt_bytes, salt_source)`` for ``target_id`` hashing. + + Salt resolution per design §5.4: + + - The persistent per-output-dir salt at + ``/.forgelm_audit_salt`` is **always** consulted + (created on first call with mode 0600). + - When ``FORGELM_AUDIT_SECRET`` is set, its first 16 bytes are + XOR'd with the persistent salt to derive the actual hashing + salt; ``salt_source`` is recorded as ``"env_var"`` so a salt- + source toggle between invocations is detectable in the chain + via the ``salt_source`` audit-event field. + - When the env var is absent, the persistent salt is used + verbatim; ``salt_source = "per_dir"``. + + Raises :class:`OSError` when the salt file cannot be created or + read; the caller surfaces the I/O error as ``EXIT_TRAINING_ERROR``. + """ + persistent = _read_persistent_salt(output_dir) env_secret = os.environ.get("FORGELM_AUDIT_SECRET", "").encode("utf-8") if env_secret: env_prefix = env_secret[:_SALT_BYTES].ljust(_SALT_BYTES, b"\x00") diff --git a/forgelm/cli/subcommands/_reverse_pii.py b/forgelm/cli/subcommands/_reverse_pii.py index 9590a904..ff5e29fd 100644 --- a/forgelm/cli/subcommands/_reverse_pii.py +++ b/forgelm/cli/subcommands/_reverse_pii.py @@ -232,52 +232,72 @@ def _resolve_query_form( salt resolution from ``forgelm purge`` so the two subcommands cannot drift on hashing semantics. """ - # Late import: ``_resolve_salt`` lives in the purge subcommand and - # pulling it eagerly would create a load-time cycle. The two + # Late import: salt-resolution helpers live in the purge subcommand + # and pulling them eagerly would create a load-time cycle. The two # subcommands intentionally share the salt-resolution path so a - # purge-then-reverse-pii cycle works on the same digest. - from ._purge import _hash_target_id, _resolve_salt - + # purge → reverse-pii cycle works on the same digest. + from ._purge import _hash_target_id, _read_persistent_salt, _resolve_salt + + if salt_source == "per_dir": + # F-W3FU follow-up duplicate finding: honor an explicit + # ``--salt-source=per_dir`` even when ``FORGELM_AUDIT_SECRET`` + # is exported. The previous absorption refused this case + # (because ``_resolve_salt`` would XOR the env var in and + # disagree with the requested source); refusing was conservative + # but contradicted the operator's explicit choice. We now read + # ONLY the persistent per-dir salt and ignore the env var for + # this invocation. Cross-tool correlation with ``forgelm purge`` + # holds when the operator runs both subcommands with the same + # ``--salt-source=per_dir`` (purge does not surface a salt-source + # flag yet, but the audit-event ``salt_source`` field records + # which path each invocation used so a reviewer sees the choice). + try: + salt = _read_persistent_salt(output_dir) + except OSError as exc: + _output_error_and_exit( + output_format, + f"Could not resolve audit salt for {output_dir!r}: {exc}", + EXIT_TRAINING_ERROR, + ) + return _hash_target_id(query, salt), "hash", salt, "per_dir" + + if salt_source == "env_var": + # Env-var mode requires FORGELM_AUDIT_SECRET; if unset there is + # no env half to XOR, and silently falling through to per-dir + # would produce a digest that does not match the operator's + # intent. Refuse with a direction-aware diagnostic. + if not os.environ.get("FORGELM_AUDIT_SECRET"): + _output_error_and_exit( + output_format, + "--salt-source=env_var requested but FORGELM_AUDIT_SECRET is not set. " + "Export the env-mode secret (e.g. `export FORGELM_AUDIT_SECRET=`) and retry.", + EXIT_CONFIG_ERROR, + ) + try: + salt, _ = _resolve_salt(output_dir) + except OSError as exc: + _output_error_and_exit( + output_format, + f"Could not resolve audit salt for {output_dir!r}: {exc}", + EXIT_TRAINING_ERROR, + ) + return _hash_target_id(query, salt), "hash", salt, "env_var" + + # Plaintext mode (salt_source is None): resolve the salt for the + # audit-hash and tag the scan as plaintext. F-W3FU-02 (priv) fix: + # the salt is used only to compute ``query_hash``, NOT for the + # corpus scan; recording ``resolved_source`` would mislead a + # forensic reviewer into thinking it was applied to the scan, so + # we tag the salt-source label as ``"plaintext"``. try: - salt, resolved_source = _resolve_salt(output_dir) + salt, _ = _resolve_salt(output_dir) except OSError as exc: _output_error_and_exit( output_format, f"Could not resolve audit salt for {output_dir!r}: {exc}", EXIT_TRAINING_ERROR, ) - - if salt_source is None: - # F-W3FU-02 (priv) fix: in plaintext mode the salt is used only - # to compute the audit event's ``query_hash`` (so the digest is - # wordlist-resistant); it is NOT applied to the corpus scan. - # Recording ``salt_source_label = resolved_source`` here would - # mislead a forensic reviewer into thinking the env-var/per-dir - # salt was applied to the scan. Record ``"plaintext"`` so the - # audit event distinguishes plaintext scans from hash-mask scans - # cleanly; the audit-hash salt source remains ``resolved_source`` - # internally but is not separately surfaced (a future absorption - # may add an ``audit_hash_salt_source`` field if forensic depth - # demands it). - return query, "plaintext", salt, "plaintext" - - if salt_source != resolved_source: - # Direction-aware diagnostic: telling the operator which - # specific knob to adjust beats a generic "or vice versa" - # message that requires them to re-derive the failure mode. - if salt_source == "env_var": - hint = "Set FORGELM_AUDIT_SECRET to the env-mode secret and retry." - else: # salt_source == "per_dir" - hint = ( - "Unset FORGELM_AUDIT_SECRET (e.g. `env -u FORGELM_AUDIT_SECRET forgelm reverse-pii ...`) " - "to force per-dir mode and retry." - ) - _output_error_and_exit( - output_format, - f"--salt-source={salt_source!r} requested but ``_resolve_salt`` returned {resolved_source!r}. {hint}", - EXIT_CONFIG_ERROR, - ) - return _hash_target_id(query, salt), "hash", salt, resolved_source + return query, "plaintext", salt, "plaintext" def _truncate_snippet(line: str, match_span: Tuple[int, int]) -> str: @@ -401,31 +421,42 @@ def _scan_file_with_alarm(path: str, pattern: re.Pattern[str]) -> List[Dict[str, Translates a ReDoS hang into a clean ``OSError`` so the dispatcher handles it via the same audit-event path as a read failure. - F-W3FU-03 fix: ``signal.alarm(N)`` returns the seconds remaining - on any previously-scheduled alarm. The previous implementation - discarded that value and called ``signal.alarm(0)`` in ``finally``, - permanently cancelling any outer alarm budget. We now capture the - previous-alarm remainder and re-arm it after our wrapper returns - so a caller that nests this helper inside its own SIGALRM budget - keeps that budget. + Outer-alarm preservation (F-W3FU-03 + post-followup fix): + ``signal.alarm(N)`` returns the seconds remaining on any + previously-scheduled alarm. We capture that remainder and re-arm + it in ``finally`` so a caller that nests this helper inside its + own SIGALRM budget keeps that budget. The naive shape + ``signal.alarm(previous_alarm_remaining)`` would silently EXTEND + the outer caller's deadline by however long ``_scan_file`` ran; + we therefore subtract the elapsed wall-clock seconds before + re-arming, clamping to ≥1 so we never lose the outer alarm + entirely (alarm-syscall granularity is whole seconds). """ + import math + import time def _alarm(_sig, _frame): # pragma: no cover — signal handler raise OSError(f"reverse-pii scan of {path!r} exceeded {_CUSTOM_REGEX_TIMEOUT_S}s (custom regex ReDoS guard)") previous_handler = _signal.signal(_signal.SIGALRM, _alarm) previous_alarm_remaining = _signal.alarm(_CUSTOM_REGEX_TIMEOUT_S) + started = time.monotonic() try: return _scan_file(path, pattern) finally: _signal.alarm(0) _signal.signal(_signal.SIGALRM, previous_handler) if previous_alarm_remaining > 0: - # Re-arm the outer alarm with whatever budget it had left - # (rounded up to 1s — sub-second remainders cannot survive - # the alarm-syscall granularity, but losing the alarm - # entirely is worse than rounding up). - _signal.alarm(max(previous_alarm_remaining, 1)) + # Subtract the wall-clock seconds spent in _scan_file from + # the outer caller's remaining budget. ceil() so we round + # up partial seconds (alarm-syscall granularity is whole + # seconds; rounding down would silently shave 0-1s off the + # outer budget on every nested call). Clamp to ≥1 so we + # never accidentally cancel the outer alarm — losing it + # entirely is strictly worse than re-arming with 1s. + elapsed = time.monotonic() - started + remaining = max(int(math.ceil(previous_alarm_remaining - elapsed)), 1) + _signal.alarm(remaining) def _hash_for_audit(query: str, salt: bytes) -> str: diff --git a/forgelm/webhook.py b/forgelm/webhook.py index ef83f66b..8c464924 100644 --- a/forgelm/webhook.py +++ b/forgelm/webhook.py @@ -87,25 +87,24 @@ def _post_payload(self, url: str, payload: dict, event: str) -> None: ca_bundle = getattr(self.config, "tls_ca_bundle", None) # Timeout floor — webhook keeps the historical 1s floor (``safe_post`` - # rejects 0/None unconditionally). Wave 3 F-compliance-106 - # raised the WebhookConfig default 5 → 10; the clamp message - # and fallback value follow that default. F-W3FU-S-04 fix: - # the previous ``isinstance(timeout, (int, float))`` half of - # the guard was dead under Pydantic (the field is ``int``-typed - # and validated at config load), so a non-numeric ``timeout`` - # cannot reach this site through any documented call path. - # Drop the isinstance check; keep the numeric-floor clamp. + # rejects 0/None unconditionally). Sub-1 values are clamped to + # the floor (NOT to the model default). Pre-Wave-3-followup the + # branch jumped to ``default_timeout`` (10s) on a sub-1 value, + # which silently 10x'd the operator's chosen budget; the + # documented contract is "below 1s → clamp to 1s", so we now + # clamp to the floor literally. F-W3FU-S-04 also dropped the + # dead ``isinstance(timeout, (int, float))`` check (Pydantic + # already enforces the int type at config load). from .config import WebhookConfig as _WebhookConfig default_timeout = _WebhookConfig.model_fields["timeout"].default timeout = getattr(self.config, "timeout", default_timeout) if timeout < 1: logger.warning( - "Webhook timeout=%r is below the 1s floor; clamping to %ds.", + "Webhook timeout=%r is below the 1s floor; clamping to 1s.", timeout, - default_timeout, ) - timeout = default_timeout + timeout = 1 allow_private = bool(getattr(self.config, "allow_private_destinations", False)) diff --git a/tests/test_reverse_pii.py b/tests/test_reverse_pii.py index f6deabd5..aba24bd1 100644 --- a/tests/test_reverse_pii.py +++ b/tests/test_reverse_pii.py @@ -306,16 +306,35 @@ def test_plaintext_implicit_output_dir_also_warns_about_salt_file(self, tmp_path "salt file side effect is present regardless of scan mode" ) - def test_per_dir_salt_source_refuses_when_env_var_set(self, tmp_path: Path, capsys, monkeypatch) -> None: - """F-W3-11 / F-W3-PS-04 regression: the symmetric direction — - operator asked for ``per_dir`` but ``FORGELM_AUDIT_SECRET`` is - set in the shell environment — must also refuse, with a - direction-aware diagnostic that names the env-var unset path.""" + def test_per_dir_salt_source_honored_when_env_var_set(self, tmp_path: Path, capsys, monkeypatch) -> None: + """Wave-3-followup duplicate-finding regression: explicit + ``--salt-source=per_dir`` must be HONORED even when + ``FORGELM_AUDIT_SECRET`` is exported in the shell environment. + + The previous absorption refused this case (because + ``_resolve_salt`` XOR'd the env var into the salt and the + post-resolution mismatch check fired), but the operator's + explicit ``per_dir`` choice should override an unrelated env + var. The audit event records ``salt_source = "per_dir"`` so a + forensic reviewer sees the choice was respected. + """ + from forgelm.cli.subcommands._purge import _hash_target_id, _read_persistent_salt from forgelm.cli.subcommands._reverse_pii import _run_reverse_pii_cmd monkeypatch.setenv("FORGELM_AUDIT_SECRET", "leftover-from-shell") + + # Compute the per-dir-only digest the operator would have + # written into the masked corpus (no env XOR). + salt = _read_persistent_salt(str(tmp_path)) + masked_digest = _hash_target_id("alice@example.com", salt) + corpus = tmp_path / "masked.jsonl" - _seed_corpus(corpus, [{"id": "row-A", "text": "x"}]) + _seed_corpus( + corpus, + [ + {"id": "row-A", "text": f"hashed identifier embedded: {masked_digest}"}, + ], + ) args = _build_args( query="alice@example.com", type="email", @@ -325,9 +344,11 @@ def test_per_dir_salt_source_refuses_when_env_var_set(self, tmp_path: Path, caps ) with pytest.raises(SystemExit) as ei: _run_reverse_pii_cmd(args, output_format="json") - assert ei.value.code == 1 + assert ei.value.code == 0, "explicit --salt-source=per_dir must be honored, not refused" payload = json.loads(capsys.readouterr().out) - assert "FORGELM_AUDIT_SECRET" in payload["error"] + assert payload["scan_mode"] == "hash" + assert payload["salt_source"] == "per_dir" + assert payload["match_count"] == 1 # --------------------------------------------------------------------------- @@ -665,7 +686,12 @@ def test_malformed_utf8_corpus_exits_runtime_error_with_audit_event_and_no_leak( access_events = [e for e in events if e["event"] == "data.access_request_query"] assert len(access_events) == 1 evt = access_events[0] - assert evt["error_class"] in {"UnicodeDecodeError", "OSError"} + # The fixture plants ``\xff\xfe`` after a newline, which produces + # ``UnicodeDecodeError`` deterministically when Python's text-mode + # iterator decodes the second line. Pin the exact class so a + # future refactor that converted it to a generic OSError before + # logging would surface here. + assert evt["error_class"] == "UnicodeDecodeError" # F-W3FU-T-03: same no-leak invariant as the OSError leg. as_str = json.dumps(evt) assert "alice@example.com" not in as_str, f"UnicodeDecodeError audit event leaked raw identifier:\n{as_str}" From 4137395ccd01aa81a53e67b8e9451c6ed99cd29f Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 5 May 2026 23:01:52 +0300 Subject: [PATCH 11/11] fix(wave-3-followup): absorb 5 follow-up review nitpicks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `gdpr_erasure{,-tr}.md` (EN+TR) — ReDoS guard text amended to name the main-thread requirement. Previous text said "POSIX 30s SIGALRM budget" without mentioning that ``_scan_files_with_redos_guard`` also gates on ``threading.current_thread() is threading.main_thread()``, so worker-thread invocations on POSIX silently fall through to the unguarded path. Operators running ``forgelm reverse-pii --type custom`` from a worker thread or on Windows must vet their regex. * `_run_reverse_pii_cmd` cognitive complexity — extracted ``_scan_with_audit`` helper that owns the full scan + audit-emit lifecycle (try/except around ``_scan_files_with_redos_guard``, failure-path audit emission, success-path audit emission). The dispatcher now delegates and receives ``(matches, files_scanned)`` back. Helper added to ``__all__`` + facade re-exports for symmetry with the rest of the module surface. No behaviour change; pure structural refactor to bring the function under Sonar's cognitive-complexity ceiling. * `webhook._post_payload` docstring — replaced the historical "5s default" text with the current contract: ``timeout`` resolves from ``self.config.timeout`` and falls back to ``WebhookConfig.model_fields["timeout"].default`` (10s now), with sub-1 values clamped to 1s (NOT to the model default — explicit per the F-W3FU-followup framing). * `tests/test_reverse_pii.py::test_redos_guard_no_op_on_worker_thread` — replaced unused ``files_scanned`` with ``_`` to silence Sonar S1481. The contract pinned by the test is "the worker thread did not crash on signal.signal", not the per-file accounting; the unused unpack was incidental. * `tests/test_reverse_pii.py::test_alarm_wrapper_restores_outer_alarm_budget` — strengthened beyond ``remaining > 0``. Now measures ``time.monotonic()`` either side of the wrapper, computes ``expected_max = max(int(math.ceil(previous_budget - elapsed)), 1)`` (mirroring the production code's clamp), and asserts both ``remaining > 0`` AND ``remaining <= expected_max``. A regression that drops the elapsed-time subtraction in the production ``finally`` clause would push ``remaining`` above the bound and fail the test. Suite stays at 1360 passed, 14 skipped (the two test edits do not add cases — one renames a variable, the other tightens an existing assertion). Lint clean. Bilingual parity 9/9 OK. Dry-run green. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/guides/gdpr_erasure-tr.md | 8 +- docs/guides/gdpr_erasure.md | 8 +- forgelm/cli/__init__.py | 1 + forgelm/cli/subcommands/_reverse_pii.py | 157 +++++++++++++++--------- forgelm/webhook.py | 13 +- tests/test_reverse_pii.py | 38 +++++- 6 files changed, 157 insertions(+), 68 deletions(-) diff --git a/docs/guides/gdpr_erasure-tr.md b/docs/guides/gdpr_erasure-tr.md index b0a69d0b..56d5e9fa 100644 --- a/docs/guides/gdpr_erasure-tr.md +++ b/docs/guides/gdpr_erasure-tr.md @@ -160,8 +160,12 @@ görmeden ilişkilendirmesini sağlar (digest'ler eşleşir). `custom` dışındaki tüm tipler query'i literal substring olarak ele alır (regex şekil-match yapmaz — o iş audit-time detector'ın görevi, erişim talebinin değil). `custom` query'i Python regex olarak -yorumlar; POSIX sistemlerde 30s'lik per-file SIGALRM bütçesi ReDoS -hang'lerine karşı koruma sağlar. +yorumlar; POSIX **ana iş parçacığında** çalışırken 30s'lik per-file +SIGALRM bütçesi ReDoS hang'lerine karşı koruma sağlar. Windows VE +POSIX worker thread'lerinde SIGALRM guard no-op olur (sinyal +handler'ları yalnızca ana iş parçacığından kurulabilir); worker +thread'den veya Windows üzerinde `forgelm reverse-pii --type custom` +çalıştıran operatörler regex'lerini kendileri doğrulamalıdır. **Audit-dir varsayılanı**: audit chain varsayılan olarak `/audit_log.jsonl`'a yazılır — `forgelm purge` ile aynı diff --git a/docs/guides/gdpr_erasure.md b/docs/guides/gdpr_erasure.md index 8169a951..0c23aa37 100644 --- a/docs/guides/gdpr_erasure.md +++ b/docs/guides/gdpr_erasure.md @@ -159,8 +159,12 @@ match) without seeing the cleartext. non-`custom` types treat the query as a literal substring (no regex shape match — that's the *audit-time* detector's job, not the access-request answer). `custom` interprets the query as a Python -regex; on POSIX systems a 30s per-file SIGALRM budget guards against -ReDoS hangs. +regex; on POSIX **main-thread** invocations a 30s per-file SIGALRM +budget guards against ReDoS hangs. On Windows AND on POSIX worker +threads the SIGALRM guard is a no-op (signal handlers must be +installed from the main thread); operators running +`forgelm reverse-pii --type custom` from a worker thread or on +Windows must vet their regex themselves. **Audit-dir default**: the audit chain is written to `/audit_log.jsonl` by default — the same path diff --git a/forgelm/cli/__init__.py b/forgelm/cli/__init__.py index 497bb1af..f964df61 100644 --- a/forgelm/cli/__init__.py +++ b/forgelm/cli/__init__.py @@ -211,6 +211,7 @@ _resolve_query_form, # noqa: F401 — re-export for tests _run_reverse_pii_cmd, # noqa: F401 — re-export for tests _scan_file, # noqa: F401 — re-export for tests + _scan_with_audit, # noqa: F401 — re-export for tests _truncate_snippet, # noqa: F401 — re-export for tests _validate_identifier_type, # noqa: F401 — re-export for tests _validate_query, # noqa: F401 — re-export for tests diff --git a/forgelm/cli/subcommands/_reverse_pii.py b/forgelm/cli/subcommands/_reverse_pii.py index ff5e29fd..bb779085 100644 --- a/forgelm/cli/subcommands/_reverse_pii.py +++ b/forgelm/cli/subcommands/_reverse_pii.py @@ -604,6 +604,96 @@ def _emit_audit_event( audit.log_event(_EVT_ACCESS_REQUEST_QUERY, **payload) +def _scan_with_audit( + *, + files: List[str], + pattern: re.Pattern[str], + identifier_type: str, + audit, + query_hash: str, + scan_mode: str, + salt_source_label: str, + output_format: str, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Run the corpus scan and emit the matching audit event. + + Extracted from ``_run_reverse_pii_cmd`` to keep the dispatcher + under Sonar's cognitive-complexity ceiling (Wave-3-followup + nitpick). Owns the full scan + audit-emit lifecycle: + + 1. Walk every resolved file via ``_scan_files_with_redos_guard``. + 2. On corpus-read failure (``OSError`` / ``UnicodeDecodeError``) + OR a custom-regex ReDoS timeout — which surfaces as ``OSError`` + via ``_scan_file_with_alarm`` — emit a failure-flavoured + ``data.access_request_query`` event (best-effort; double-fault + on audit chain write surfaces with chained context per the + F-W3FU-06 fix) and exit ``EXIT_TRAINING_ERROR``. + 3. On success, emit the matching success event and return + ``(matches, files_scanned)`` to the dispatcher. + + Audit-chain failure is fail-closed in both branches: the chain + is the Article 15 contract and silently dropping it would breach + the very framing this subcommand is built to deliver. + """ + matches: List[Dict[str, Any]] = [] + files_scanned: List[Dict[str, Any]] = [] + try: + matches, files_scanned = _scan_files_with_redos_guard(files, pattern, identifier_type) + except (OSError, UnicodeDecodeError) as exc: + if audit is not None: + try: + _emit_audit_event( + audit, + query_hash=query_hash, + identifier_type=identifier_type, + scan_mode=scan_mode, + salt_source=salt_source_label, + files_scanned=files_scanned, + match_count=len(matches), + error_class=exc.__class__.__name__, + error_message=str(exc), + ) + except Exception as audit_exc: # noqa: BLE001 — fail-closed on audit error. + # Chain the original mid-scan exception's diagnostic + # context into the audit-failure message so a + # double-fault doesn't lose the underlying cause + # (otherwise the operator sees only "audit chain + # write failed" and has to re-run to diagnose). + _output_error_and_exit( + output_format, + ( + f"reverse-pii: failed to write the Article 15 failure audit event " + f"({audit_exc.__class__.__name__}: {audit_exc}); the original " + f"mid-scan failure was {exc.__class__.__name__}: {exc}." + ), + EXIT_TRAINING_ERROR, + ) + _output_error_and_exit( + output_format, + f"Failed reading corpus file mid-scan: {exc}. Partial matches surfaced before the failure are not emitted.", + EXIT_TRAINING_ERROR, + ) + + if audit is not None: + try: + _emit_audit_event( + audit, + query_hash=query_hash, + identifier_type=identifier_type, + scan_mode=scan_mode, + salt_source=salt_source_label, + files_scanned=files_scanned, + match_count=len(matches), + ) + except Exception as audit_exc: # noqa: BLE001 — fail-closed on audit error. + _output_error_and_exit( + output_format, + f"reverse-pii: failed to write the Article 15 success audit event ({audit_exc}).", + EXIT_TRAINING_ERROR, + ) + return matches, files_scanned + + def _run_reverse_pii_cmd(args, output_format: str) -> None: """Top-level dispatcher for ``forgelm reverse-pii``.""" query = _validate_query(getattr(args, "query", None), output_format) @@ -664,62 +754,16 @@ def _run_reverse_pii_cmd(args, output_format: str) -> None: ) query_hash = _hash_for_audit(query, salt) - matches: List[Dict[str, Any]] = [] - files_scanned: List[Dict[str, Any]] = [] - try: - matches, files_scanned = _scan_files_with_redos_guard(files, pattern, identifier_type) - except (OSError, UnicodeDecodeError) as exc: - if audit is not None: - try: - _emit_audit_event( - audit, - query_hash=query_hash, - identifier_type=identifier_type, - scan_mode=scan_mode, - salt_source=salt_source_label, - files_scanned=files_scanned, - match_count=len(matches), - error_class=exc.__class__.__name__, - error_message=str(exc), - ) - except Exception as audit_exc: # noqa: BLE001 — fail-closed on audit error. - # F-W3FU-06 (cc) fix: chain the original mid-scan - # exception's diagnostic context into the audit-failure - # message so a double-fault doesn't lose the underlying - # cause (otherwise the operator sees only "audit chain - # write failed" and has to re-run to diagnose). - _output_error_and_exit( - output_format, - ( - f"reverse-pii: failed to write the Article 15 failure audit event " - f"({audit_exc.__class__.__name__}: {audit_exc}); the original " - f"mid-scan failure was {exc.__class__.__name__}: {exc}." - ), - EXIT_TRAINING_ERROR, - ) - _output_error_and_exit( - output_format, - f"Failed reading corpus file mid-scan: {exc}. Partial matches surfaced before the failure are not emitted.", - EXIT_TRAINING_ERROR, - ) - - if audit is not None: - try: - _emit_audit_event( - audit, - query_hash=query_hash, - identifier_type=identifier_type, - scan_mode=scan_mode, - salt_source=salt_source_label, - files_scanned=files_scanned, - match_count=len(matches), - ) - except Exception as audit_exc: # noqa: BLE001 — fail-closed on audit error. - _output_error_and_exit( - output_format, - f"reverse-pii: failed to write the Article 15 success audit event ({audit_exc}).", - EXIT_TRAINING_ERROR, - ) + matches, files_scanned = _scan_with_audit( + files=files, + pattern=pattern, + identifier_type=identifier_type, + audit=audit, + query_hash=query_hash, + scan_mode=scan_mode, + salt_source_label=salt_source_label, + output_format=output_format, + ) payload: Dict[str, Any] = { "success": True, @@ -770,6 +814,7 @@ def _emit_reverse_pii_result(payload: Dict[str, Any], output_format: str) -> Non "_resolve_files", "_resolve_query_form", "_scan_file", + "_scan_with_audit", "_truncate_snippet", "_hash_for_audit", "_emit_reverse_pii_result", diff --git a/forgelm/webhook.py b/forgelm/webhook.py index 8c464924..cfe7c513 100644 --- a/forgelm/webhook.py +++ b/forgelm/webhook.py @@ -65,11 +65,14 @@ def _post_payload(self, url: str, payload: dict, event: str) -> None: the codebase shares the same policy. Webhook-specific behaviour kept here: - * The webhook config defaults ``timeout`` to 5s for historical - compatibility — ``safe_post`` is invoked with ``min_timeout=1.0`` - so an existing operator config that uses ``timeout=5`` keeps - working (every ``http*`` call site outside webhook gets the - stricter 10s floor). + * The local ``timeout`` variable resolves from + ``self.config.timeout`` and falls back to + ``WebhookConfig.model_fields["timeout"].default`` (currently 10s + per Wave 3 / F-compliance-106 — was 5s historically) when the + attribute is absent on a hand-rolled config namespace. Sub-1 + values are clamped to the 1s floor (NOT to the model default — + see the inline comment around ``timeout < 1`` for the + F-W3FU-followup framing); 0 / negative budgets are not honoured. * On policy rejection or transport error we log a warning and *swallow* — ``notify_*`` is never allowed to fail the training run. * Response body suppression on non-2xx — receivers (Slack, Teams) diff --git a/tests/test_reverse_pii.py b/tests/test_reverse_pii.py index aba24bd1..fa1b2019 100644 --- a/tests/test_reverse_pii.py +++ b/tests/test_reverse_pii.py @@ -870,9 +870,23 @@ def test_alarm_wrapper_restores_outer_alarm_budget(self, tmp_path: Path) -> None """F-W3FU-03: a previously-scheduled outer SIGALRM budget must survive the per-file wrapper. The wrapper's old shape called ``signal.alarm(0)`` in ``finally``, permanently cancelling the - outer alarm; the fix captures and re-arms it.""" + outer alarm; the fix captures and re-arms it. + + Wave-3-followup tightening: also pin the budget-decrement + contract. A naive re-arm with ``previous_alarm_remaining`` + (without subtracting the elapsed wall-clock) silently EXTENDS + the outer caller's deadline by however long ``_scan_file`` + ran. We measure ``time.monotonic()`` either side of the + wrapper, derive an upper bound from + ``previous_budget - ceil(elapsed)`` (clamped to ≥1 — the + production code's clamp), and assert the re-armed alarm is + within that bound. A regression that drops the elapsed + subtraction would push ``remaining`` above the bound. + """ + import math import re as _re import signal as _signal + import time from forgelm.cli.subcommands._reverse_pii import _scan_file_with_alarm @@ -880,15 +894,30 @@ def test_alarm_wrapper_restores_outer_alarm_budget(self, tmp_path: Path) -> None corpus.write_text('{"id":1,"text":"x"}\n', encoding="utf-8") pattern = _re.compile("x") + previous_budget = 60 previous_handler = _signal.signal(_signal.SIGALRM, lambda *_: None) try: - _signal.alarm(60) # outer budget + _signal.alarm(previous_budget) # outer budget + started = time.monotonic() _scan_file_with_alarm(str(corpus), pattern) + elapsed = time.monotonic() - started remaining = _signal.alarm(0) + + # The wrapper's contract: re-arm with + # ``max(int(ceil(previous - elapsed)), 1)``. ``remaining`` + # must be > 0 (outer budget preserved) AND not exceed the + # production clamp (otherwise the outer caller's deadline + # was extended by however long the inner scan ran). + expected_max = max(int(math.ceil(previous_budget - elapsed)), 1) assert remaining > 0, ( "outer alarm budget must survive the wrapper — F-W3FU-03 " "regression: signal.alarm(0) in finally was cancelling the outer alarm" ) + assert remaining <= expected_max, ( + f"alarm wrapper extended the outer budget — remaining={remaining}s " + f"but expected_max={expected_max}s after {elapsed:.3f}s of inner scan; " + "Wave-3-followup tightening: the re-arm must subtract the wall-clock spent inside the wrapper" + ) finally: _signal.alarm(0) _signal.signal(_signal.SIGALRM, previous_handler) @@ -921,7 +950,10 @@ def _worker() -> None: thread.join() assert not errors, f"worker thread crashed instead of skipping the alarm guard: {errors}" assert len(results) == 1 - matches, files_scanned = results[0] + # files_scanned is the second tuple element; not checked in this test + # because the contract being pinned is "the worker thread did not + # crash on signal.signal", not the per-file accounting shape. + matches, _ = results[0] assert len(matches) == 1