From ac5ba2ca1d68f34f29a6f47db38d7626c9e56155 Mon Sep 17 00:00:00 2001 From: Stefano Maffeis Date: Thu, 22 Oct 2026 10:40:11 +0000 Subject: [PATCH 1/3] fix(docs): make the ingestion quick-start example runnable (#395) --- CHANGELOG.md | 2 ++ docs/usermanuals/en/data/ingestion.md | 16 ++++++--- docs/usermanuals/tr/data/ingestion.md | 16 ++++++--- .../test_usermanual_reference_consistency.py | 34 +++++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b9461b5..ebe39d6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,6 +125,8 @@ All notable changes to ForgeLM are documented here. ### Fixed +- Make the Document Ingestion quick-start example runnable in the English and Turkish user manuals: pair `--chunk-tokens` with `--tokenizer` and show the real `summarize_result` output instead of a layout the tool never prints (#395). + - Make the GDPR, erasure, retention, PII-masking, GPU-cost and chat documentation say what ForgeLM does, so a deployer can copy a documented command or YAML block, and quote a documented limit in a DPIA or an access reply, without it being untrue. The GDPR / KVKK manual page was rewritten earlier in this cycle; this completes it and the pages around it (#101): diff --git a/docs/usermanuals/en/data/ingestion.md b/docs/usermanuals/en/data/ingestion.md index 1fe04e3a..7e546baf 100644 --- a/docs/usermanuals/en/data/ingestion.md +++ b/docs/usermanuals/en/data/ingestion.md @@ -27,12 +27,20 @@ $ forgelm ingest ./policies/ \ --recursive \ --strategy markdown \ --chunk-tokens 1024 \ + --tokenizer "Qwen/Qwen2.5-7B-Instruct" \ --all-mask \ --output data/policies.jsonl -✓ scanned 47 files (12 PDF, 8 DOCX, 27 MD) -✓ extracted 12,240 chunks (avg 743 tokens) -✓ masked 18 PII matches, 0 secret matches -✓ wrote data/policies.jsonl (8.2 MB) +Ingestion summary + Output JSONL : data/policies.jsonl + Files (in/out): processed=47 skipped=0 + Chunks : 12240 + Total chars : 28410553 + By format : docx=8, md=27, pdf=12 + Note : PII masking redacted 18 span(s): email=11, phone=7 + Note : Secrets masking enabled — no credentials detected in this corpus + +Next: forgelm audit data/policies.jsonl --output ./audit/ +Or train: forgelm --config (set data.dataset_name_or_path: data/policies.jsonl) ``` `--all-mask` is the documented shorthand for `--secrets-mask --pii-mask` diff --git a/docs/usermanuals/tr/data/ingestion.md b/docs/usermanuals/tr/data/ingestion.md index 63421c4e..c05eef67 100644 --- a/docs/usermanuals/tr/data/ingestion.md +++ b/docs/usermanuals/tr/data/ingestion.md @@ -27,12 +27,20 @@ $ forgelm ingest ./policies/ \ --recursive \ --strategy markdown \ --chunk-tokens 1024 \ + --tokenizer "Qwen/Qwen2.5-7B-Instruct" \ --all-mask \ --output data/policies.jsonl -✓ 47 dosya tarandı (12 PDF, 8 DOCX, 27 MD) -✓ 12,240 chunk çıkarıldı (ortalama 743 token) -✓ 18 PII eşleşmesi maskelendi, 0 sır -✓ data/policies.jsonl yazıldı (8.2 MB) +Ingestion summary + Output JSONL : data/policies.jsonl + Files (in/out): processed=47 skipped=0 + Chunks : 12240 + Total chars : 28410553 + By format : docx=8, md=27, pdf=12 + Note : PII masking redacted 18 span(s): email=11, phone=7 + Note : Secrets masking enabled — no credentials detected in this corpus + +Next: forgelm audit data/policies.jsonl --output ./audit/ +Or train: forgelm --config (set data.dataset_name_or_path: data/policies.jsonl) ``` `--all-mask`, `--secrets-mask --pii-mask`'in doğru sıradaki belgelenen diff --git a/tests/test_usermanual_reference_consistency.py b/tests/test_usermanual_reference_consistency.py index ee15e8cf..8ae609a8 100644 --- a/tests/test_usermanual_reference_consistency.py +++ b/tests/test_usermanual_reference_consistency.py @@ -21,6 +21,7 @@ import bisect import re +import shlex from pathlib import Path import pytest @@ -621,6 +622,39 @@ def test_gpu_cost_page_lists_every_priced_gpu_and_cost_source(lang: str) -> None assert documented == {"user_config", "auto_detected", "fuzzy_match"}, f"{lang}/operations/gpu-cost.md: {documented}" +@pytest.mark.parametrize("lang", ["en", "tr"]) +class TestIngestionPageQuickExampleStaysRunnable: + """CR-A12-004: the Document Ingestion quick example must be a command that runs and its real output.""" + + @staticmethod + def _page(lang: str) -> str: + return (_REPO_ROOT / "docs" / "usermanuals" / lang / "data" / "ingestion.md").read_text(encoding="utf-8") + + def _quick_example_tokens(self, lang: str) -> list[str]: + text = self._page(lang) + block = next( + (block for block in re.findall(r"```shell[ \t]*\n(.*?)```", text, re.DOTALL) if "forgelm ingest" in block), + None, + ) + assert block is not None, f"ingestion.md ({lang}) must show a shell block with `forgelm ingest`" + command = re.sub(r"\\\n\s*", " ", block) + line = next(line for line in command.splitlines() if "forgelm ingest" in line) + return shlex.split(line.lstrip("$ ").strip()) + + def test_chunk_tokens_is_paired_with_a_tokenizer(self, lang: str) -> None: + # `forgelm ingest --chunk-tokens` refuses to run without `--tokenizer` (exit 1), so the first command a reader + # copies must carry both. + tokens = self._quick_example_tokens(lang) + assert "--chunk-tokens" in tokens, f"ingestion.md ({lang}) must keep the documented --chunk-tokens example" + assert "--tokenizer" in tokens, f"ingestion.md ({lang}) must pass --tokenizer with --chunk-tokens" + + def test_page_shows_the_real_summarize_result_labels(self, lang: str) -> None: + # The old block showed tick-mark progress lines that `summarize_result` never prints. + text = self._page(lang) + assert "Ingestion summary" in text, f"ingestion.md ({lang}) must show the real summarize_result heading" + assert "Files (in/out)" in text, f"ingestion.md ({lang}) must show the real Files (in/out) line" + + @pytest.mark.parametrize("lang", ["en", "tr"]) def test_pii_masking_page_counts_and_validates_the_real_detectors(lang: str) -> None: from forgelm.data_audit import PII_TYPES From d4962d61f87dd35f949bd6fdcea0a36186b8d38d Mon Sep 17 00:00:00 2001 From: Stefano Maffeis Date: Thu, 22 Oct 2026 11:11:38 +0000 Subject: [PATCH 2/3] test(docs): require a model name after --tokenizer in the quick-start guard Addresses the Sourcery review on #1060: the previous assertion only checked that the flag was present, so a bare `--tokenizer` (which argparse rejects) would still pass. Assert the flag is followed by a non-flag value. --- tests/test_usermanual_reference_consistency.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_usermanual_reference_consistency.py b/tests/test_usermanual_reference_consistency.py index 8ae609a8..09d016b6 100644 --- a/tests/test_usermanual_reference_consistency.py +++ b/tests/test_usermanual_reference_consistency.py @@ -647,6 +647,10 @@ def test_chunk_tokens_is_paired_with_a_tokenizer(self, lang: str) -> None: tokens = self._quick_example_tokens(lang) assert "--chunk-tokens" in tokens, f"ingestion.md ({lang}) must keep the documented --chunk-tokens example" assert "--tokenizer" in tokens, f"ingestion.md ({lang}) must pass --tokenizer with --chunk-tokens" + tokenizer_at = tokens.index("--tokenizer") + assert tokenizer_at + 1 < len(tokens) and not tokens[tokenizer_at + 1].startswith("-"), ( + f"ingestion.md ({lang}) must give --tokenizer a model name, not just the bare flag" + ) def test_page_shows_the_real_summarize_result_labels(self, lang: str) -> None: # The old block showed tick-mark progress lines that `summarize_result` never prints. From 313ddbceba462bc8e09f670cc02c4bee388a7324 Mon Sep 17 00:00:00 2001 From: Stefano Maffeis Date: Thu, 22 Oct 2026 12:22:28 +0000 Subject: [PATCH 3/3] docs: show the resolved absolute output path in the ingestion quick-start Apply the maintainer's review suggestions on #1060: `ingest_path` resolves `--output` with `Path(...).resolve()`, so `summarize_result` prints an absolute path in the `Output JSONL`, `Next:` and `Or train:` lines. Both the English and Turkish pages now show `/srv/project/data/policies.jsonl`. Also harden the guard so a bare `--chunk-tokens` (no token count) fails, and wrap the CHANGELOG entry at the neighbours' width. --- CHANGELOG.md | 4 +++- docs/usermanuals/en/data/ingestion.md | 6 +++--- docs/usermanuals/tr/data/ingestion.md | 6 +++--- tests/test_usermanual_reference_consistency.py | 5 +++++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebe39d6b..6bab99b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,7 +125,9 @@ All notable changes to ForgeLM are documented here. ### Fixed -- Make the Document Ingestion quick-start example runnable in the English and Turkish user manuals: pair `--chunk-tokens` with `--tokenizer` and show the real `summarize_result` output instead of a layout the tool never prints (#395). +- Make the Document Ingestion quick-start example runnable in the English and Turkish user manuals: pair `--chunk-tokens` + with `--tokenizer` and show the real `summarize_result` output, with `--output` resolved to the absolute path the tool + prints, instead of a layout the tool never prints (#395). - Make the GDPR, erasure, retention, PII-masking, GPU-cost and chat documentation say what ForgeLM does, so a deployer can copy a documented command or YAML block, and quote a documented limit in a DPIA or an access reply, without it being untrue. The GDPR diff --git a/docs/usermanuals/en/data/ingestion.md b/docs/usermanuals/en/data/ingestion.md index 7e546baf..76b5bbfa 100644 --- a/docs/usermanuals/en/data/ingestion.md +++ b/docs/usermanuals/en/data/ingestion.md @@ -31,7 +31,7 @@ $ forgelm ingest ./policies/ \ --all-mask \ --output data/policies.jsonl Ingestion summary - Output JSONL : data/policies.jsonl + Output JSONL : /srv/project/data/policies.jsonl Files (in/out): processed=47 skipped=0 Chunks : 12240 Total chars : 28410553 @@ -39,8 +39,8 @@ Ingestion summary Note : PII masking redacted 18 span(s): email=11, phone=7 Note : Secrets masking enabled — no credentials detected in this corpus -Next: forgelm audit data/policies.jsonl --output ./audit/ -Or train: forgelm --config (set data.dataset_name_or_path: data/policies.jsonl) +Next: forgelm audit /srv/project/data/policies.jsonl --output ./audit/ +Or train: forgelm --config (set data.dataset_name_or_path: /srv/project/data/policies.jsonl) ``` `--all-mask` is the documented shorthand for `--secrets-mask --pii-mask` diff --git a/docs/usermanuals/tr/data/ingestion.md b/docs/usermanuals/tr/data/ingestion.md index c05eef67..7d12712c 100644 --- a/docs/usermanuals/tr/data/ingestion.md +++ b/docs/usermanuals/tr/data/ingestion.md @@ -31,7 +31,7 @@ $ forgelm ingest ./policies/ \ --all-mask \ --output data/policies.jsonl Ingestion summary - Output JSONL : data/policies.jsonl + Output JSONL : /srv/project/data/policies.jsonl Files (in/out): processed=47 skipped=0 Chunks : 12240 Total chars : 28410553 @@ -39,8 +39,8 @@ Ingestion summary Note : PII masking redacted 18 span(s): email=11, phone=7 Note : Secrets masking enabled — no credentials detected in this corpus -Next: forgelm audit data/policies.jsonl --output ./audit/ -Or train: forgelm --config (set data.dataset_name_or_path: data/policies.jsonl) +Next: forgelm audit /srv/project/data/policies.jsonl --output ./audit/ +Or train: forgelm --config (set data.dataset_name_or_path: /srv/project/data/policies.jsonl) ``` `--all-mask`, `--secrets-mask --pii-mask`'in doğru sıradaki belgelenen diff --git a/tests/test_usermanual_reference_consistency.py b/tests/test_usermanual_reference_consistency.py index 09d016b6..696a2b56 100644 --- a/tests/test_usermanual_reference_consistency.py +++ b/tests/test_usermanual_reference_consistency.py @@ -647,6 +647,11 @@ def test_chunk_tokens_is_paired_with_a_tokenizer(self, lang: str) -> None: tokens = self._quick_example_tokens(lang) assert "--chunk-tokens" in tokens, f"ingestion.md ({lang}) must keep the documented --chunk-tokens example" assert "--tokenizer" in tokens, f"ingestion.md ({lang}) must pass --tokenizer with --chunk-tokens" + chunk_tokens_at = tokens.index("--chunk-tokens") + chunk_value = tokens[chunk_tokens_at + 1] if chunk_tokens_at + 1 < len(tokens) else "" + assert chunk_value.isdigit() and int(chunk_value) > 0, ( + f"ingestion.md ({lang}) must give --chunk-tokens a positive token count, not just the bare flag" + ) tokenizer_at = tokens.index("--tokenizer") assert tokenizer_at + 1 < len(tokens) and not tokens[tokenizer_at + 1].startswith("-"), ( f"ingestion.md ({lang}) must give --tokenizer a model name, not just the bare flag"