diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ded461b9..af214684 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,6 +158,13 @@ jobs: print('synthetic.py import: OK') " + - name: Site-as-tested-surface guard (strict) + # Diffs site/*.html claims against the Python sources of truth + # (compliance artefacts, quickstart templates, GPU profile count, + # pyproject version). Fails the build on any drift; see + # tools/check_site_claims.py for what's checked and why. + run: python3 tools/check_site_claims.py --strict + - name: Bilingual doc H2 parity check run: | python -c " diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6fa60dd9..2fe177fe 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,16 +1,22 @@ -name: Publish to PyPI +name: Release - Build, cross-OS validate, publish to PyPI + +# Tag-driven release pipeline. A pushed v* tag fans out into: +# build -> cross-os-tests (3 OS x 4 Python = 12) -> publish +# Any matrix combo failure breaks the needs: chain so PyPI publish does +# not run unless every supported (OS, Python) pair installs the packaged +# wheel cleanly and runs the test suite green. on: - release: - types: [published] + push: + tags: ['v*'] -# No workflow-level permissions — each job declares the minimal set it needs. +# No workflow-level permissions; each job declares the minimum it needs. jobs: build: runs-on: ubuntu-latest permissions: - contents: read # checkout needs read access to the release ref + contents: read # checkout reads the tagged ref steps: - uses: actions/checkout@v5 @@ -18,15 +24,15 @@ jobs: with: python-version: "3.11" - - name: Install build tools - run: pip install build + - name: Install build tooling + run: python -m pip install --upgrade pip build - - name: Build sdist and wheel - run: python -m build + - name: Build wheel + sdist + run: python -m build # outputs dist/*.whl and dist/*.tar.gz - - name: Verify package + - name: Verify package metadata run: | - pip install twine + python -m pip install twine twine check dist/* - uses: actions/upload-artifact@v5 @@ -34,40 +40,75 @@ jobs: name: dist path: dist/ - # TestPyPI: triggered by pre-release tags (e.g., v0.3.0-rc1, v0.3.0-beta) - # or releases created from test/development branches - publish-testpypi: + cross-os-tests: needs: build - runs-on: ubuntu-latest - if: github.event.release.prerelease == true - environment: - name: testpypi - url: https://test.pypi.org/p/forgelm + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ['3.10', '3.11', '3.12', '3.13'] + runs-on: ${{ matrix.os }} permissions: - contents: read # download-artifact reads workflow run state - id-token: write # required for trusted publishing to TestPyPI + contents: read # checkout reads the tagged ref for tests/ + tools/ steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@v5 with: name: dist - path: dist/ + path: dist - - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Install packaged wheel (NOT editable) + shell: bash # POSIX shell on Windows runners too + run: | + # Install exactly what PyPI users will pip install. Transitive + # dependencies resolve from PyPI; the local dist/ only contains + # the forgelm wheel, not its dependency closure (so no + # --no-index --find-links here). + python -m pip install --upgrade pip + python -m pip install dist/*.whl + + - name: Install Linux-only extras (qlora, ingestion, eval) + if: runner.os == 'Linux' + shell: bash + # qlora pulls bitsandbytes which only ships Linux wheels. + # ingestion + eval are cross-platform but bundled here so a single + # conditional install covers the breadth-test surface on Linux. + run: python -m pip install 'forgelm[qlora,ingestion,eval]' + + - name: Install pytest + shell: bash + run: python -m pip install pytest pytest-cov + + - name: Test (CPU-only, no-network) + shell: bash + # test_cost_estimation is excluded per Faz 31 closure plan: it + # depends on a snapshot pricing fixture that drifts on a different + # cadence than the release matrix. + run: pytest tests/ -q --ignore=tests/test_cost_estimation.py + + - name: Generate SBOM (CycloneDX 1.5) + shell: bash + run: python tools/generate_sbom.py > sbom-${{ matrix.os }}-py${{ matrix.python }}.json + + - uses: actions/upload-artifact@v5 with: - repository-url: https://test.pypi.org/legacy/ + name: sbom-${{ matrix.os }}-py${{ matrix.python }} + path: sbom-${{ matrix.os }}-py${{ matrix.python }}.json - # PyPI: triggered by stable releases (non-prerelease) from main - publish-pypi: - needs: build + publish: + needs: cross-os-tests # blocked until every matrix combo is green runs-on: ubuntu-latest - if: github.event.release.prerelease == false environment: name: pypi url: https://pypi.org/p/forgelm permissions: - contents: read # download-artifact reads workflow run state - id-token: write # required for trusted publishing to PyPI + contents: read # download-artifact reads workflow run state + id-token: write # OIDC trusted publishing to PyPI steps: - uses: actions/download-artifact@v5 with: diff --git a/.github/workflows/site-deploy.yml b/.github/workflows/site-deploy.yml index fc2b65cb..3c49abd5 100644 --- a/.github/workflows/site-deploy.yml +++ b/.github/workflows/site-deploy.yml @@ -59,10 +59,13 @@ jobs: run: python3 tools/build_usermanuals.py - name: Verify the generated tree + # Only the languages in SUPPORTED_LANGUAGES (build_usermanuals.py) are + # emitted today — see docs/standards/localization.md → Supported + # languages. Deferred bags (de/fr/es/zh) are intentionally absent. run: | test -f site/js/usermanuals/_index.js test -f site/js/usermanuals/en.js - for lang in en tr de fr es zh; do + for lang in en tr; do test -f "site/js/usermanuals/${lang}.js" || { echo "missing ${lang}.js"; exit 1; } done diff --git a/.github/workflows/usermanuals-validate.yml b/.github/workflows/usermanuals-validate.yml index 5b89e29f..10e04e18 100644 --- a/.github/workflows/usermanuals-validate.yml +++ b/.github/workflows/usermanuals-validate.yml @@ -56,3 +56,37 @@ jobs: raise SystemExit(1) print("OK — every declared page has an English source.") PY + + - name: Orphan-markdown detection (strict) + # The reverse of the smoke test above: a markdown file under + # docs/usermanuals//
/.md that is NOT declared + # in _meta.yaml will silently never appear on the site. Strict + # mode fails the build so abandoned drafts are caught before + # they merge to main. + run: | + python3 - <<'PY' + import pathlib, sys, yaml + meta = yaml.safe_load(pathlib.Path("docs/usermanuals/_meta.yaml").read_text(encoding="utf-8")) + declared = set() + for sec in meta.get("sections", []): + for page in sec.get("pages", []): + declared.add(f"{sec['id']}/{page['id']}.md") + root = pathlib.Path("docs/usermanuals") + orphans = [] + for md in root.rglob("*.md"): + rel = md.relative_to(root) + parts = rel.parts + if len(parts) < 3: + continue + # parts = (lang, section, page.md, ...). Only the
/.md + # portion is matched against _meta.yaml. + key = "/".join(parts[1:]) + if key not in declared: + orphans.append(str(rel)) + if orphans: + print("ERROR: markdown files not declared in _meta.yaml (strict orphan check):") + for o in sorted(orphans): + print(f" - {o}") + sys.exit(1) + print("OK — no orphan markdown under docs/usermanuals/.") + PY diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..9b2d32f7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,37 @@ +# Pre-commit hooks for ForgeLM (optional, developer-installed). +# +# CI is the enforcement boundary: ruff and gitleaks already run as separate +# steps in .github/workflows/ci.yml. These hooks exist purely as a local +# ergonomic optimization for contributors who want fast feedback before push. +# +# Install: +# pip install pre-commit && pre-commit install +# +# Run all hooks against the whole tree: +# pre-commit run --all-files +# +# Skip a hook (e.g. when committing a known-test fixture): +# SKIP=gitleaks git commit -m "..." +# +# Versions are pinned per closure-plan-202604300906.md Faz 32. + +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.0 + hooks: + - id: ruff + - id: ruff-format + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.18.0 + hooks: + - id: gitleaks + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-merge-conflict diff --git a/CHANGELOG.md b/CHANGELOG.md index b82451cb..61781716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,68 @@ All notable changes to ForgeLM are documented here. > Per-PR CHANGELOG entries below collapse into the v0.5.5 release > notes at tag time. +### Added — Wave 1 closure (Faz 9, 11, 12, 13, 25, 31, 32 — see PR description) + +- **Article 14 staging directory + `forgelm approve` / `forgelm reject` (Faz 9)** — + When `evaluation.require_human_approval=true`, the trainer now saves the + final adapters to `final_model.staging/` instead of writing to + `final_model/` before review. Two new CLI subcommands manage the gate: + `forgelm approve --output-dir ` atomically renames + `final_model.staging/` → `final_model/` (with a `shutil.move` fallback on + cross-device output mounts) and emits a `human_approval.granted` audit + event plus a `notify_success` webhook; `forgelm reject ` records a + `human_approval.rejected` event and leaves the staging directory in place + for forensic review. Both commands resolve the approver identity via + `FORGELM_OPERATOR` → `getpass.getuser()` → `"anonymous"`, mirroring + `AuditLogger.operator`. The `human_approval.required` audit event payload + now also carries `staging_path` and `run_id` so downstream tooling can + cross-check the approval against the originating run. +- **`evaluation.staging_ttl_days` config field (Faz 9)** — documents the + retention horizon for `final_model.staging/` after a `forgelm reject`; + default 7 days. Auto-deletion enforcement is deferred to Phase 21 + (GDPR right-to-erasure); v0.5.5 surfaces the policy in the compliance + manifest only. +- **`forgelm.wizard._print` indirection (Faz 11)** — 85 `print()` calls + replaced with a testable `_print()` helper (mirrors the chat.py pattern). + Coverage omit list emptied; wizard is now visible to coverage measurement. + Closes F-code-105, F-test-003, F-code-019. +- **`tests/_helpers/factories.py` (Faz 12)** — single canonical + `minimal_config(**overrides)` factory replaces 4 scattered local + `_minimal_config` definitions and 7 `from conftest import` indirections. + Closes F-test-004, F-test-005, F-code-015. +- **`forgelm --data-audit` deprecation (Faz 13)** — legacy flag emits + `DeprecationWarning` + a `cli.legacy_flag_invoked` audit event; + scheduled for removal in v0.7.0. Closes F-code-107, F-business-024. +- **6 enum-shaped config fields tightened to `Literal[...]` (Faz 10)** — + `LoraConfig.bias`, `DistributedConfig.fsdp_backward_prefetch` / + `fsdp_state_dict_type`, `SafetyConfig.scoring`, + `ComplianceMetadataConfig.risk_classification`, `TrainingConfig.galore_optim` / + `galore_proj_type`. Pydantic now validates whitelist at parse time; + bespoke runtime validators removed. Closes F-code-101, F-compliance-105. +- **`tools/check_site_claims.py` (Faz 25)** — site-as-tested-surface CI + guard; AST-parses `forgelm/compliance.py`, `forgelm/quickstart.py`, + `forgelm/trainer.py`, `pyproject.toml` and diffs against site HTML to + catch claim/code drift. Wired into `ci.yml` (`--strict` mode). +- **`docs/standards/localization.md` "Supported languages" section (Faz 25)** + — codifies that EN+TR are authored at site AND user-manual levels, while + DE/FR/ES/ZH are site-translated only and the user-manual side falls back + to English via the i18n chain. Closes F-loc-001, F-loc-003, Theme α. +- **`.github/workflows/publish.yml` cross-OS release matrix (Faz 31)** — + tag-driven `build → cross-os-tests → publish` chain over 3 OS × 4 Python + = 12 combinations; packaged-wheel install (not editable); SBOM artifact + upload per combo; OIDC trusted publishing. Closes F-test-007. +- **`tools/generate_sbom.py` (Faz 31)** — stdlib-only CycloneDX 1.5 + emitter; called from each `cross-os-tests` matrix combo to produce a + per-OS-and-Python SBOM artifact. +- **`.pre-commit-config.yaml` (Faz 32; optional)** — opt-in local hooks + (`ruff`, `ruff-format`, `gitleaks`, trailing-whitespace, + end-of-file-fixer, check-yaml/-toml, check-merge-conflict). CI keeps + enforcing the same checks; pre-commit is ergonomic optimization, not a + duplicate enforcement boundary. Closes F-test-008. +- `tests/test_human_approval_gate.py` — 15 new tests covering the staging + → approve / reject flow, stale-staging detection, atomic-rename race, + cross-device move, audit chain integrity. Total suite: 951+ tests. + ### Added — Foundation bundle (PR #19, Faz 1-8) - `forgelm verify-audit` subcommand + library function @@ -88,6 +150,72 @@ All notable changes to ForgeLM are documented here. [split-design-data_audit-cli-202604300906.md](docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md) (Faz 14-15 forward-looking). +### Deprecated + +- **`forgelm --data-audit PATH`** — the legacy flag now emits a + `DeprecationWarning` and an `cli.legacy_flag_invoked` audit-log event + on every invocation. Behaviour is unchanged; the flag is scheduled for + removal in **v0.7.0**. Migrate to the `forgelm audit PATH` subcommand + (same output, same exit codes). See + [docs/standards/release.md](docs/standards/release.md#deprecation-cadence) for + the removal timeline. + +### Changed (Wave 1) + +- `WebhookNotifier.notify_awaiting_approval(run_name, model_path)` is now + wired into the human-approval gate so an operator-facing webhook fires the + moment the model is staged. Receivers can opt out via + `webhook.notify_on_awaiting_approval=false`. +- **`forgelm/data_audit.py` -> `forgelm/data_audit/` package (Faz 14)** — + the 3098-line monolith was split into a 14-module package + (`_optional`, `_types`, `_pii_regex`, `_pii_ml`, `_secrets`, `_simhash`, + `_minhash`, `_quality`, `_streaming`, `_aggregator`, `_splits`, + `_summary`, `_croissant`, `_orchestrator`) per the + cohesion ceiling in `docs/standards/architecture.md`. The public + `forgelm.data_audit.X` import surface — including the + test-touched private helpers (`_HAS_PRESIDIO`, `_get_presidio_analyzer`, + `_token_digest`, `_strip_code_fences`, `_row_quality_flags`, + `_read_jsonl_split`, `_count_leaked_rows`, `_find_near_duplicates_brute`, + `_is_credit_card`, `_is_tr_id`, `_require_presidio`, + `_PRESIDIO_ENTITY_MAP`, etc.) — is preserved by `__init__.py` re-exports + so external callers (`forgelm.ingestion`, `forgelm.wizard`) and the + test suite keep working without code changes. Closes F-code-103 (Major). + See + [split-design-data_audit-cli-202604300906.md](docs/analysis/code_reviews/split-design-data_audit-cli-202604300906.md) + §1 for the design. +- **`forgelm/cli.py` → `forgelm/cli/` package (Faz 15)** — the ~2300-line + monolith was split into a 24-module package (subcommands/, `_dispatch`, + `_training`, `_dry_run`, `_result`, `_resume`, `_logging`, `_exit_codes`, + etc.). The `forgelm.cli:main` entry point and `python -m forgelm.cli` are + preserved; dispatcher uses late-binding facade re-resolution so test + monkeypatches (`forgelm.cli._run_chat_cmd` etc.) keep resolving correctly. + Closes F-code-104. See split-design §2. +- **16 broad `except Exception` sites narrowed (Faz 27)** — `_streaming.py`, + `trainer.py`, `safety.py`, `judge.py`, `compliance.py`, `ingestion.py` + narrow to specific exception classes; 7 sites retained with `# noqa: BLE001` + and rationale comments per `docs/standards/error-handling.md` carve-out. + C-55 resolved: MoE expert-name resolver migrated from hardcoded substring + match to regex-registry (`_EXPERT_NAME_PATTERNS`) covering Mixtral, Qwen 3 + MoE, DeepSeek-V3, Phi-MoE. Closes F-code-106. +- **Audit event catalog and CLI sample drift fixed (Faz 29)** — placeholder + `` entries in `audit_event_catalog.md` filled; trailing-whitespace + cleaned; CLI help sample in `docs/reference/usage.md` brought in sync with + current subcommand surface. + +### Removed + +- **`[ingestion-secrets]` extra (`detect-secrets>=1.5.0,<2.0.0`)** — + reserved during Phase 12 for a follow-up integration that never landed. + The `detect-secrets` scanner expects file paths while ForgeLM audits + row-level JSONL streams, so the wire-up was rejected as architecturally + incompatible. The prefix-anchored regex set in + `forgelm/data_audit/_secrets.py` (9-family coverage: AWS, GitHub, + Slack, OpenAI, Google API, JWT, OpenSSH, PGP, Azure storage) is the + sole detection backend and stays the sole detection backend. Removed + the dead extra from `pyproject.toml`, the install snippet from + `README.md`, and the "fallback regex set" framing from the secrets + module docstring. Closes C-53 (deferred from v0.5.0). + --- ## [0.5.0] — 2026-04-30 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca99966a..2d1e93ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,6 +95,37 @@ ruff format . Configuration is in `pyproject.toml` under `[tool.ruff]`. +### Pre-commit hooks (optional) + +ForgeLM ships a [`.pre-commit-config.yaml`](.pre-commit-config.yaml) that mirrors +the CI checks (`ruff`, `ruff-format`, `gitleaks`, plus a few hygiene hooks for +trailing whitespace, EOF newlines, and YAML/TOML syntax). The hooks are an +**optional ergonomic optimization** — CI enforces the same checks on every PR, +so installing them locally is purely about getting feedback before you push. + +To opt in: + +```bash +pip install pre-commit +pre-commit install +``` + +Run all hooks against the whole tree at any time: + +```bash +pre-commit run --all-files +``` + +If a hook ever flags a known-good fixture (e.g. a credential-shaped test +string the gitleaks hook can't tell apart from a real secret), skip just that +hook for the commit: + +```bash +SKIP=gitleaks git commit -m "test: add credential-shape fixture" +``` + +CI remains the enforcement boundary; skipping a hook locally never bypasses CI. + ## Guidelines ### Code diff --git a/README.md b/README.md index e2cb5f0b..4c2bbde5 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![CI](https://github.com/cemililik/ForgeLM/actions/workflows/ci.yml/badge.svg)](https://github.com/cemililik/ForgeLM/actions/workflows/ci.yml) +**Runs on Linux, macOS, and Windows.** The PyPI metadata's `Operating System :: OS Independent` classifier is backed by a release-tag CI matrix that builds the wheel on Linux and re-installs + tests it across **3 operating systems × 4 Python versions = 12 combinations** (Ubuntu, macOS, Windows × Python 3.10, 3.11, 3.12, 3.13) — every combo must pass before PyPI publish runs. See [`.github/workflows/publish.yml`](.github/workflows/publish.yml). Linux-only extras (`qlora`, `unsloth`) are flagged on each `pip install "forgelm[...]"` line below. + **ForgeLM** is a config-driven, enterprise-ready LLM fine-tuning toolkit. It supports the full modern post-training stack — from supervised fine-tuning to preference alignment to reasoning RL — with integrated safety evaluation, EU AI Act compliance, and CI/CD-native design. ## Features @@ -174,7 +176,6 @@ pip install "forgelm[distributed]" # DeepSpeed multi-GPU pip install "forgelm[merging]" # mergekit model merging pip install "forgelm[ingestion]" # PDF/DOCX/EPUB/Markdown → JSONL + langdetect + xxhash pip install "forgelm[ingestion-scale]" # MinHash LSH dedup (datasketch) for >50K-row corpora -pip install "forgelm[ingestion-secrets]" # detect-secrets scanner for SFT corpora (falls back to regex if absent) pip install "forgelm[ingestion-pii-ml]" # Presidio ML-NER for person/organization/location PII (Phase 12.5; ALSO needs `python -m spacy download en_core_web_lg`) pip install "forgelm[export]" # GGUF export via llama-cpp-python pip install "forgelm[chat]" # Rich terminal rendering for `forgelm chat` diff --git a/docs/analysis/code_reviews/closure-plan-202604300906.md b/docs/analysis/code_reviews/closure-plan-202604300906.md index 2a2b77f1..4495b1d4 100644 --- a/docs/analysis/code_reviews/closure-plan-202604300906.md +++ b/docs/analysis/code_reviews/closure-plan-202604300906.md @@ -2,7 +2,7 @@ **Plan ID:** `closure-plan-202604300906` **Created:** 2026-04-30 -**Last revised:** 2026-04-30 (v2 — kullanıcı kararları sonrası tek-sürüm modeline geçiş) +**Last revised:** 2026-05-02 (v2.2 — ghost feature analysis absorption: Faz 34-38 yeni; Wave 1 IN PROGRESS işaretleri) **Branch:** `main` **Commit (baseline):** `6b515ed912f8f22304194c1b3f55ed07a26f519c` **Source review:** [`master-review-opus-202604300906.md`](./master-review-opus-202604300906.md) @@ -78,33 +78,43 @@ Her faz bir veya birden çok PR'la kapanır. Sıralama dependency-driven; parale | **6** | `forgelm verify-audit` subcommand + library function | Tek PR | 1 | — | ✅ PR #19 | | **7** | `safe_post` HTTP discipline extension (Theme ε) | Tek PR | 1 | — | ✅ PR #19 | | **8** | Webhook lifecycle vocabulary | Tek PR | 1 | — | ✅ PR #19 | -| **9** | Article 14: honesty fix + real staging + `forgelm approve` | Tek PR | 1 | Faz 6 (verify-audit benzeri pattern) | ⏳ | -| **10** | Pydantic Literal sweep (6 kalan field) | Tek PR | 1 | — | ⏳ | -| **11** | Wizard `_print` indirection + drop coverage omit | Tek PR | 1 | — | ⏳ | -| **12** | Fixture consolidation (`tests/_helpers/`) | Tek PR | 1 | — | ⏳ | -| **13** | `--data-audit` deprecation discipline | Tek PR | 1 | — | ⏳ | -| **14** | `data_audit/` package split (multi-PR series) | 5 PR (D-1..D-5) | 5 | — | ⏳ | -| **15** | `cli/` package split (multi-PR series) | 6 PR (C-1..C-6) | 6 | — | ⏳ | -| **16** | Pydantic `description=` migration with CI guard (multi-PR) | 4 PR | 4 | Faz 10 (Literal'lar tamamen önce) | ⏳ | -| **17** | `--workers N` audit determinism | Tek PR | 1 | Faz 14 (data_audit/) + Faz 15 (cli/) | ⏳ | -| **18** | **Library API support — Analysis & design** | Tek PR (design dosyası) | 1 | Faz 14 + 15 | ⏳ | -| **19** | **Library API support — Implementation** | Tek PR | 1 | Faz 18 | ⏳ | -| **20** | **GDPR right-to-erasure — Analysis & design** | Tek PR | 1 | Faz 6 + 9 (audit + staging) | ⏳ | -| **21** | **GDPR right-to-erasure — Implementation** | Tek PR | 1 | Faz 20 | ⏳ | -| **22** | **ISO 27001 / SOC 2 Type II — Analysis & gap assessment** | Tek PR | 1 | Faz 3, 6, 7, 8, 9, 21 (security + compliance baseline) | ⏳ | -| **23** | **ISO 27001 / SOC 2 Type II — Implementation** | Tek PR (büyük) | 1 | Faz 22 | ⏳ | -| **24** | Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN/TR user-manual H2 drift) | Tek PR | 1 | Faz 16 (description= migration ile config docs auto-gen) | ⏳ | -| **25** | Site picker honesty + site-as-tested-surface CI guard | Tek PR | 1 | Faz 1 (site honesty fixes önce) | ⏳ | -| **26** | QMS bilingual EN+TR mirror + compliance docs cleanup | Tek PR | 1 | Faz 23 (ISO/SOC 2 dokümanları QMS ile lockstep) | ⏳ | -| **27** | Silent `except Exception:` sweep (Theme D residual) | Tek PR | 1 | — | ⏳ | -| **28** | Remaining Major + Minor cleanup | Tek PR | 1 | Hepsi öncesindekilere bağlı | ⏳ | -| **29** | Nit sweep (40 madde mass-edit) | Tek PR | 1 | — | ⏳ | -| **30** | **Final documentation + site finalization (EN+TR only)** — tüm `docs/` + `docs/usermanuals/{en,tr}/` + `site/*` v0.5.5 nihai durumuna senkron | Tek PR | 1 | **Faz 1-29** (tüm fonksiyonel iş bitmiş olmalı) | ⏳ | -| **31** | **Cross-OS release-tag matrix workflow** | Tek PR | 1 | — (release öncesi her şeyden) | ⏳ | -| **32** | `.pre-commit-config.yaml` (opsiyonel — kullanıcı tercihi) | Tek PR | 1 | — | ⏳ | -| **33** | **v0.5.5 RELEASE** (CHANGELOG, version bump, tag, PyPI) | Tek PR | 1 | **Tüm önceki fazlar** | ⏳ | - -**Toplam:** 33 faz, ~47 PR (multi-PR seri'ler dahil). +| **9** | Article 14: honesty fix + real staging + `forgelm approve` | Tek PR | 1 | Faz 6 (verify-audit benzeri pattern) | ✅ Wave 1 PR | +| **10** | Pydantic Literal sweep (6 kalan field) | Tek PR | 1 | — | ✅ Wave 1 PR | +| **11** | Wizard `_print` indirection + drop coverage omit | Tek PR | 1 | — | ✅ Wave 1 PR | +| **12** | Fixture consolidation (`tests/_helpers/`) | Tek PR | 1 | — | ✅ Wave 1 PR | +| **13** | `--data-audit` deprecation discipline | Tek PR | 1 | — | ✅ Wave 1 PR | +| **14** | `data_audit/` package split (multi-PR series) | 5 PR (D-1..D-5) | 5 | — | ✅ Wave 1 PR | +| **15** | `cli/` package split (multi-PR series) | 6 PR (C-1..C-6) | 6 | — | ✅ Wave 1 PR | +| **16** | Pydantic `description=` migration with CI guard (multi-PR) | 4 PR | 4 | Faz 10 (Literal'lar tamamen önce) | 🔄 Wave 1 IP | +| **17** | `--workers N` audit determinism | Tek PR | 1 | Faz 14 (data_audit/) + Faz 15 (cli/) | 🔄 Wave 1 IP | +| **18** | **Library API support — Analysis & design** | Tek PR (design dosyası) | 1 | Faz 14 + 15 | 🔄 Wave 1 IP | +| **19** | **Library API support — Implementation** | Tek PR | 1 | Faz 18 | 🔄 Wave 1 IP | +| **20** | **GDPR right-to-erasure — Analysis & design** | Tek PR | 1 | Faz 6 + 9 (audit + staging) | 🔄 Wave 1 IP | +| **21** | **GDPR right-to-erasure — Implementation** | Tek PR | 1 | Faz 20 | 🔄 Wave 1 IP | +| **22** | **ISO 27001 / SOC 2 Type II — Analysis & gap assessment** | Tek PR | 1 | Faz 3, 6, 7, 8, 9, 21 (security + compliance baseline) | 🔄 Wave 1 IP | +| **23** | **ISO 27001 / SOC 2 Type II — Implementation** | Tek PR (büyük) | 1 | Faz 22 | 🔄 Wave 1 IP | +| **24** | Bilingual TR mirror sweep (configuration-tr, distributed-tr, ingestion-tr, EN/TR user-manual H2 drift) | Tek PR | 1 | Faz 16 (description= migration ile config docs auto-gen) | 🔄 Wave 1 IP | +| **25** | Site picker honesty + site-as-tested-surface CI guard | Tek PR | 1 | Faz 1 (site honesty fixes önce) | ✅ Wave 1 PR | +| **26** | QMS bilingual EN+TR mirror + compliance docs cleanup | Tek PR | 1 | Faz 23 (ISO/SOC 2 dokümanları QMS ile lockstep) | 🔄 Wave 1 IP | +| **27** | Silent `except Exception:` sweep (Theme D residual) | Tek PR | 1 | — | ✅ Wave 1 PR | +| **28** | Remaining Major + Minor cleanup | Tek PR | 1 | Hepsi öncesindekilere bağlı | 🔄 Wave 1 IP | +| **29** | Nit sweep (40 madde mass-edit) | Tek PR | 1 | — | ✅ Wave 1 PR | +| **30** | **Final documentation + site finalization (EN+TR only)** — tüm `docs/` + `docs/usermanuals/{en,tr}/` + `site/*` v0.5.5 nihai durumuna senkron | Tek PR | 1 | **Faz 1-29 + 34-38** (tüm fonksiyonel iş bitmiş olmalı) | 🔄 Wave 1 IP | +| **31** | **Cross-OS release-tag matrix workflow** | Tek PR | 1 | — (release öncesi her şeyden) | ✅ Wave 1 PR | +| **32** | `.pre-commit-config.yaml` (opsiyonel — kullanıcı tercihi) | Tek PR | 1 | — | ✅ Wave 1 PR | +| **34** | **`forgelm doctor` env-check subcommand** (GH-001 — onboarding bloker) | Tek PR | 1 | Faz 15 (cli/ split) | 🔄 Wave 1 IP | +| **35** | **Air-gap pre-cache subcommands** (`cache-models` + `cache-tasks`) (GH-002, GH-003) | Tek PR | 1 | Faz 15 + Faz 34 (doctor `--offline` paydaş) | 🔄 Wave 1 IP | +| **36** | **Compliance verification toolbelt** (`safety-eval` standalone, `verify-annex-iv`, `verify-gguf`) (GH-004, GH-005, GH-009) | Tek PR | 1 | Faz 6 (verify-audit pattern), Faz 9 (compliance baseline) | 🔄 Wave 1 IP | +| **37** | **`forgelm approvals` listing subcommand** — Faz 9 follow-up (GH-007) | Tek PR | 1 | Faz 9 (`approve` + `reject` + staging directory) | 🔄 Wave 1 IP | +| **38** | **`forgelm reverse-pii` GDPR Article 15 subcommand** (GH-014) | Tek PR | 1 | Faz 21 (GDPR erasure baseline) | 🔄 Wave 1 IP | +| **33** | **v0.5.5 RELEASE** (CHANGELOG, version bump, tag, PyPI) | Tek PR | 1 | **Tüm önceki fazlar (1-32, 34-38)** | ⏳ POST-WAVE | + +**Toplam:** 38 faz, ~52 PR (multi-PR seri'ler dahil). + +**Status legend:** +- ✅ Wave 1 PR — kod yazıldı, Wave 1 PR'ında live (review/merge bekliyor) +- 🔄 Wave 1 IP — Wave 1 kapsamında, in progress veya başlanmamış +- ⏳ POST-WAVE — Wave 1 sonrasına kalır (release) --- @@ -322,7 +332,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. --- -### Faz 9 — Article 14: honesty fix + real staging directory + `forgelm approve` (Tek PR) +### Faz 9 — Article 14: honesty fix + real staging directory + `forgelm approve` (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-compliance-101 (Critical) — **PATH (a) + PATH (b)** birlikte @@ -384,7 +394,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. ## 5. Code hygiene fazları (Faz 10-13) -### Faz 10 — Pydantic Literal sweep (Tek PR) +### Faz 10 — Pydantic Literal sweep (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-code-101 (Major), F-compliance-105 (Major) @@ -406,7 +416,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. --- -### Faz 11 — Wizard `_print` indirection + drop coverage omit (Tek PR) +### Faz 11 — Wizard `_print` indirection + drop coverage omit (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-code-105 (Major), F-test-003 (Major), F-code-019 (Major carry-over) @@ -426,7 +436,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. --- -### Faz 12 — Fixture consolidation (`tests/_helpers/`) (Tek PR) +### Faz 12 — Fixture consolidation (`tests/_helpers/`) (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-test-004 (Major), F-test-005 (Major), F-code-015 (Major carry-over) @@ -449,7 +459,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. --- -### Faz 13 — `--data-audit` deprecation discipline (Tek PR) +### Faz 13 — `--data-audit` deprecation discipline (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-code-107 (Major), F-business-024 (Minor) @@ -475,7 +485,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. ## 6. Architectural splits (Faz 14-16) -### Faz 14 — `data_audit/` package split (5 PR series) +### Faz 14 — `data_audit/` package split (5 PR series) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-code-103 (Major) @@ -493,7 +503,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. --- -### Faz 15 — `cli/` package split (6 PR series) +### Faz 15 — `cli/` package split (6 PR series) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-code-104 (Major) @@ -582,7 +592,7 @@ Diğer her şeyden bağımsız. Paralel açılabilir. - Experimental: `Notifier`, advanced trainer hooks - Internal: `_*` her şey - **Type stub strategy** — `forgelm/py.typed` marker; type hints PEP 561 compliant; `mypy --strict` pass. - - **Lazy import discipline** — `forgelm.ForgeTrainer` access torch'u tetikler ama `import forgelm` etmemeli (mevcut `__getattr__` paterni doğru). + - **Lazy import discipline** — `forgelm.ForgeTrainer` access torch'u tetikler ama `import forgelm` etmemeli (mevcut `__getattr__` paterni doğru). **Wave 1 round-5 carry-over:** `forgelm/cli/__init__.py` şu an 153 satır eager `from .submodule import ...` çağrıları içeriyor (153 satır için `python -c "import forgelm.cli"` cold-import maliyetli). Faz 18 design'da bu facade için module-level `__getattr__` + `__dir__` mapping'i karara bağlanır; symbol→submodule lookup tablosu, monkeypatch path resolution ile uyumlu kalır. Şu anki davranış known limitation; reactive olarak değil tasarımlı olarak yapılır. - **Versioned API contract** — semver explicit document; breaking change detection (CI guard?). - **Integration test strategy** — `tests/test_library_api.py` (yeni); CLI'siz pure Python integration paths. - **Documentation surface** — `docs/reference/library_api.md` + `-tr.md` mirror; example notebook (`notebooks/library_api_example.ipynb`). @@ -816,7 +826,7 @@ Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.m --- -### Faz 25 — Site picker honesty + site-as-tested-surface CI guard (Tek PR) +### Faz 25 — Site picker honesty + site-as-tested-surface CI guard (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-loc-001 (Major), F-loc-003 (Major), Theme α (site claim/code drift) @@ -877,7 +887,7 @@ Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.m --- -### Faz 27 — Silent `except Exception:` sweep (Theme D residual) (Tek PR) +### Faz 27 — Silent `except Exception:` sweep (Theme D residual) (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-code-106 (Major) @@ -931,7 +941,7 @@ Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.m --- -### Faz 29 — Nit sweep (Tek PR) +### Faz 29 — Nit sweep (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** Kalan 40 Nit (Faz 28'da bitmemiş olanlar). @@ -939,6 +949,239 @@ Yeni dosya: `docs/analysis/code_reviews/iso27001-soc2-alignment-202604YYMMHHMM.m --- +## 9.5 Ghost feature closure (Faz 34-38) + +**Kaynak:** [`ghost-features-analysis-20260502.md`](./ghost-features-analysis-20260502.md) — 28 ghost entry (15 subcommand + 10 flag/option + 1 config + 2 env var) tespit edildi. Tier 1 (no-code naming/alias fix) doğrudan **Faz 30 Task O**'ya absorb edildi. Tier 2 (implement or remove) bu §9.5'teki yeni fazlar (34-38) ile kapatılır. Tier 3 (Phase 13 Pro CLI scope, Phase 14 chains scope) plan kapsamı dışı; Faz 30'da "planned for future release" callout'u ile docs'tan ya kaldırılır ya işaretlenir. + +**Bağımlılık:** Faz 15 (cli/ split bitmeli — yeni subcommand'lar `forgelm/cli/subcommands/_*.py` pattern'ine yerleşir). Faz 9 (Article 14 staging) + Faz 21 (GDPR erasure) bazı fazlar için baseline. + +**Kapsam dışı (deferred — Faz 30'da docs callout):** + +| Ghost | Karar | Hedef sürüm | +|---|---|---| +| GH-006 `forgelm trend` | Phase 13 Pro CLI scope (eval-history.jsonl + dashboard) | v0.6.0+ | +| GH-010 `forgelm compare-runs` | Phase 13 Pro CLI scope (experiment browser) | v0.6.0+ | +| GH-012 `forgelm batch-chat` | Phase 13 ya da `forgelm chat --batch` flag | v0.6.0+ | +| GH-015 `forgelm merge-sweep` | Phase 14 pipeline chains scope | v0.6.0+ | +| GH-017 `--estimate-cost` | Phase 13 cost estimation feature | v0.6.0+ | + +--- + +### Faz 34 — `forgelm doctor` env-check subcommand (Tek PR) 🔄 Wave 1 IN PROGRESS + +**Closes:** GH-001 (Critical — onboarding bloker; 30 referans across 8 doc dosyası, hem EN hem TR mirror). + +**Bağımlılık:** Faz 15 (cli/ split — yeni subcommand pattern). + +**Tasks:** + +1. **`forgelm/cli/subcommands/_doctor.py`** (yeni): + - `_run_doctor_cmd(args, output_format)` dispatcher. + - **Checks (sırayla, tabular output):** + - Python version (≥3.10 zorunlu, ≥3.11 önerilen) + - `torch.cuda.is_available()` + CUDA version + - GPU count + per-GPU VRAM (`torch.cuda.get_device_properties`) + - Optional extras varlığı: `[qlora]` (bitsandbytes), `[unsloth]`, `[deepspeed]`, `[evaluation]` (lm-eval), `[wandb]`, `[mergekit]`, `[ingestion]` (pdf/docx parsers), `[ingestion-pii-ml]` (Presidio), `[chunking-semantic]` (sentence-transformers; C-52 sonrası). + - HF Hub erişimi (HEAD `https://huggingface.co/api/models` — `--offline` ile skip). + - Disk free space (≥10 GB warn, ≥50 GB pass). + - `FORGELM_OPERATOR` ortam değişkeni set mi (Article 12 compliance hatırlatma). + - JSON çıktı: her check için `{"name": ..., "status": "pass|warn|fail", "detail": "..."}`. +2. **`--offline` variant:** + - HF Hub probe atlanır. + - `HF_HUB_OFFLINE=1` set mi kontrol et. + - Local HF cache (`~/.cache/huggingface/hub` veya `HF_HOME`) doluluk + boyut. +3. **`forgelm/cli/_parser.py`** — `_add_doctor_subcommand` kayıt et; `--offline`, `--output-format json|text` flag'leri. +4. **`forgelm/cli/_dispatch.py`** — `doctor` dispatch satırı. +5. **Exit codes:** + - 0: tüm check'ler pass. + - 1: en az bir fail (zorunlu): Python version mismatch, no GPU when explicitly required, vb. — config-error sınıfı. + - 2: subprocess/IO hatası (ör. CUDA query crash). +6. **Tests** (`tests/test_doctor.py`, yeni): + - GPU yokken `cuda_available=False` warn (CPU-only legitimate). + - `--offline` Hub probe yapmıyor (mock'la doğrula). + - JSON output structure stable. + - Optional extra eksikse `status: "warn"` (fail değil; opsiyonel). +7. **Dokümantasyon güncellemeleri** (Faz 30 Task A'da finalize): + - Mevcut `installation.md`, `first-run.md`, `troubleshooting.md`, `air-gap.md` referansları artık çalışıyor — değiştirme gerekmez. + +**Faz 34 Acceptance:** +- `forgelm doctor` çalışıyor; tüm check'ler tabular text + structured JSON döner. +- `forgelm doctor --offline` HF Hub'a istek atmıyor (Wireshark/mock doğrulamalı). +- 8 doc dosyasındaki 30 referans artık geçerli komut. +- 8 test yeşil. + +--- + +### Faz 35 — Air-gap pre-cache subcommands (Tek PR) 🔄 Wave 1 IN PROGRESS + +**Closes:** GH-002 (`cache-models`, High — air-gap workflow bloker), GH-003 (`cache-tasks`, High — air-gap workflow bloker). + +**Bağımlılık:** Faz 15 (cli/ split) + Faz 34 (`doctor --offline` ile semantik paydaş). + +**Tasks:** + +1. **`forgelm/cli/subcommands/_cache.py`** (yeni; iki subcommand'ı tek modülde host'la): + - `_run_cache_models_cmd(args, output_format)`: + - Flag: `--model ` (zorunlu; tekrarlanabilir), `--safety ` (opsiyonel — Llama Guard pre-cache). + - `huggingface_hub.snapshot_download` (`local_files_only=False`, `cache_dir=HF_HOME`). + - Her model için download size + time + final cache path raporla. + - JSON output: `{"success": bool, "models": [{"name": ..., "cached_path": ..., "size_mb": ..., "duration_s": ...}], "total_size_mb": ...}`. + - `_run_cache_tasks_cmd(args, output_format)`: + - Flag: `--tasks ` (zorunlu; örn. `hellaswag,arc_easy,truthfulqa`). + - lm-evaluation-harness task dataset cache (datasets library wrapper). + - `lm_eval.tasks.get_task_dict(task_names)` ile task'ları enumerate, sonra `dataset.download_and_prepare()` her biri için. + - JSON output: `{"success": bool, "tasks": [{"name": ..., "cached": bool, "size_mb": ...}]}`. +2. **`forgelm/cli/_parser.py`** — iki subcommand kayıt: `cache-models`, `cache-tasks`. Her birinin kendi argparse subparser'ı. +3. **`forgelm/cli/_dispatch.py`** — dispatch satırları. +4. **Optional extras dependency:** + - `cache-models` → `huggingface_hub` (zaten core dep). + - `cache-tasks` → `lm-eval` (`[evaluation]` extra). Eksikse hata mesajı: "lm-eval not installed; pip install 'forgelm[evaluation]'". +5. **Exit codes:** + - 0: tüm modeller/task'lar başarılı cache. + - 1: invalid model name veya task name (config error). + - 2: network / HF Hub / disk-space hatası (runtime error). +6. **Tests** (`tests/test_cache_subcommands.py`, yeni): + - `cache-models --model gpt2` (tiny model) test fixture'ı; cache path verify. + - `cache-tasks --tasks hellaswag` mock'lu lm-eval call (ağırsız test). + - Eksik extra → ImportError + install hint. + - Network unreachable → graceful failure + exit 2. +7. **Documentation:** + - `air-gap.md` ve `docker.md` referansları artık çalışıyor — Faz 30'da revalidate. + +**Faz 35 Acceptance:** +- `forgelm cache-models --model meta-llama/Llama-3.2-3B` çalışır; HF cache populate eder; sonraki training run `local_files_only=True` ile çalışabilir. +- `forgelm cache-tasks --tasks hellaswag,arc_easy` lm-eval task dataset'lerini önceden indirir. +- 4 doc dosyasındaki 16 referans artık çalışıyor. +- 5 test yeşil. + +--- + +### Faz 36 — Compliance verification toolbelt (Tek PR) 🔄 Wave 1 IN PROGRESS + +**Closes:** GH-004 (`verify-annex-iv`, High — compliance bloker), GH-005 (`safety-eval` standalone, High — eval gap), GH-009 (`verify-gguf`, Medium — deployment integrity). + +**Bağımlılık:** Faz 6 (verify-audit pattern reuse) + Faz 9 (Article 14 baseline). + +**Tasks:** + +1. **`forgelm/cli/subcommands/_verify_annex_iv.py`** (yeni): + - `forgelm verify-annex-iv ` — Annex IV artifact field-completeness check + manifest hash re-compute. + - **Required Annex IV fields** (EU AI Act Annex IV §1-9): system identification, intended purpose, system components, computational resources, data and data governance, technical documentation, monitoring and logging, performance metrics, risk management. + - `forgelm/compliance.py`'da `verify_annex_iv_artifact(path) -> VerifyResult` library function (verify-audit pattern'i ile aynı). + - Manifest hash re-compute: `SHA-256(canonical JSON)` ↔ `metadata.manifest_hash` field karşılaştırması. + - Tampering detection: hash mismatch → `VerifyResult(valid=False, reason="manifest hash mismatch")`. +2. **`forgelm/cli/subcommands/_safety_eval.py`** (yeni): + - `forgelm safety-eval --model [--probes ] [--default-probes] [--output ]`. + - `forgelm/safety.py`'da mevcut `run_safety_evaluation` library function'ını standalone subcommand olarak expose et (training pipeline'dan bağımsız). + - GGUF model desteği: `model_path.endswith(".gguf")` ise `llama-cpp-python` backend'le yükle (zaten `forgelm/inference.py`'de pattern var). + - Default probe set: `forgelm/safety_prompts/default_probes.jsonl` (eğer yoksa, Faz 36'da minimal 50-prompt fixture oluştur). + - JSON output: harm category breakdown + per-prompt verdicts. +3. **`forgelm/cli/subcommands/_verify_gguf.py`** (yeni): + - `forgelm verify-gguf ` — GGUF dosyasının integrity check'i. + - Magic bytes (`GGUF` 4-byte header) doğrula. + - Metadata block parse (existing GGUF parser via `gguf` Python package). + - Tensor count + tensor shape consistency check. + - SHA-256 manifest karşılaştırması (export sırasında yazılan `model.gguf.sha256` dosyası varsa). +4. **`forgelm/cli/_parser.py`** — üç subcommand registrar. +5. **`forgelm/cli/_dispatch.py`** — üç dispatch satırı. +6. **Tests** (`tests/test_verification_toolbelt.py`, yeni): + - `verify-annex-iv`: valid artifact pass; missing field fail; tampered manifest hash fail. + - `safety-eval --default-probes`: minimal 50-prompt run; structured output. + - `safety-eval --model model.gguf`: GGUF backend smoke test (mock'lu). + - `verify-gguf`: valid GGUF pass; corrupted magic bytes fail; SHA mismatch fail. +7. **Library API exposure** (Faz 19 ile koordineli): + - `from forgelm import verify_annex_iv, run_safety_evaluation, verify_gguf` — public surface. + +**Faz 36 Acceptance:** +- Üç subcommand çalışır + JSON output stable. +- 8+ test yeşil. +- `annex-iv.md`, `evaluation/safety.md`, `deployment/gguf-export.md` referansları artık geçerli. + +--- + +### Faz 37 — `forgelm approvals` listing subcommand (Tek PR) 🔄 Wave 1 IN PROGRESS + +**Closes:** GH-007 (Medium — Faz 9'un eksik kalan listing parçası; operator pending approval'ları manuel filesystem inspection'la bulamamalı). + +**Bağımlılık:** Faz 9 (`approve` + `reject` + staging directory + audit event vocabulary). + +**Tasks:** + +1. **`forgelm/cli/subcommands/_approvals.py`** (yeni): + - `forgelm approvals --pending [--output-dir ]` — pending approval'ları listele. + - `forgelm approvals --show ` — tek run'ın detaylı staging info + audit event chain. +2. **Pending detection logic:** + - `` altında `*.staging.` directory pattern'ı tara. + - Audit log'tan `human_approval.required` event'lerini topla (her run için son event). + - `human_approval.granted` veya `human_approval.rejected` event'i olan run_id'leri filtrele. + - Geriye kalan "pending" set. +3. **Output format:** + - **Text:** Tabular — run_id, staging_path, requested_at (audit event timestamp), age (now - timestamp), config hash. + - **JSON:** `[{"run_id": ..., "staging_path": ..., "requested_at": ..., "age_seconds": ..., "metrics": {...}, "config_hash": ...}, ...]`. +4. **`--show `:** + - Staging dir varlığı + içerik listesi (model files, audit summary, metrics.json). + - Tüm `human_approval.*` chain (required → granted/rejected eventleri). + - Webhook config bilgisi (ne zaman bildirim gitti). +5. **`forgelm/cli/_parser.py`** — `_add_approvals_subcommand` kayıt et. +6. **`forgelm/cli/_dispatch.py`** — `approvals` dispatch. +7. **Exit codes:** + - 0: command çalıştı (pending listesi boş veya dolu, fark etmez). + - 1: `--show` için run_id bulunamadı veya invalid. + - 2: audit log read failure / staging dir scan failure. +8. **Tests** (`tests/test_approvals_listing.py`, yeni): + - Empty staging dir → empty list. + - 3 staging + 1 already-granted → 2 pending. + - `--show ` granted run → "no longer pending" notice. + - `--show ` → exit 1 + clear error message. + - JSON output schema stable. + +**Faz 37 Acceptance:** +- `forgelm approvals --pending` operator için pending review queue gösterir. +- `forgelm approvals --show ` audit chain + metrics summary. +- `human-oversight.md` referansları artık çalışıyor. +- 5 test yeşil. + +--- + +### Faz 38 — `forgelm reverse-pii` GDPR Article 15 subcommand (Tek PR) 🔄 Wave 1 IN PROGRESS + +**Closes:** GH-014 (Medium — GDPR Article 15 right of access; data subject "benim verim eğitim setinde mi?" sorgusuna yanıt veren tooling). + +**Bağımlılık:** Faz 21 (GDPR erasure baseline + retention config + audit erasure events). + +**Tasks:** + +1. **`forgelm/cli/subcommands/_reverse_pii.py`** (yeni): + - `forgelm reverse-pii --query "" ` — masked JSONL'lerde verilen identifier'ı ara. + - Identifier tipleri: e-posta, telefon, isim, custom regex. +2. **Forward-PII baseline kullanımı:** + - `forgelm/data_audit/_pii_regex.py`'da `_PII_PATTERNS` registry zaten var. + - Reverse path: identifier'ı hash'le (eğer corpus hash-masked ise: `SHA256(salt + identifier)` karşılaştır), veya plaintext residual aramaları (mask-leak detection). +3. **Glob desteği:** `data/*.jsonl`, `corpora/**/train.jsonl` gibi pattern'ler — `glob.glob(recursive=True)`. +4. **Output:** + - **Text:** "Identifier 'ali@example.com' found in: data/train.jsonl:42 (line snippet); not found in: data/eval.jsonl, data/test.jsonl". + - **JSON:** `{"query": "...", "matches": [{"file": ..., "line": ..., "snippet": "..."}], "files_scanned": N, "match_count": M}`. +5. **Audit event:** `data.access_request_query` event yazılır — query identifier hash'lenir (raw identifier audit'e sızmasın). +6. **`forgelm/cli/_parser.py`** + `_dispatch.py` registrar/dispatch. +7. **Tests** (`tests/test_reverse_pii.py`, yeni): + - Plaintext match (mask-leak): identifier residual found → flag. + - Hash-mask match: identifier salt+hash hesaplanır, JSONL'de aranır. + - Non-existent identifier → empty match list. + - Glob pattern: 3 dosya tarar, doğru sayım. + - Audit event written; raw identifier hash'lenmiş (plaintext değil). +8. **Exit codes:** + - 0: command ran (match var/yok). + - 1: invalid query / unreadable JSONL. + - 2: glob expansion empty + audit log error. + +**Faz 38 Acceptance:** +- `forgelm reverse-pii --query "ali@example.com" data/*.jsonl` çalışır; structured match list. +- Audit event `data.access_request_query` yazıldığında identifier raw değil hash'li. +- `gdpr.md` Article 15 section artık geçerli komut referans veriyor. +- 5 test yeşil. + +--- + ## 10. Final documentation + site finalization (Faz 30) ### Faz 30 — Final documentation + site finalization (EN+TR only) (Tek PR) @@ -974,6 +1217,15 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa | **Semantic chunking** (C-52, Faz 28 ships code) | `ingestion.md` chunking-strategies tablosuna `semantic` satırı (+tr) | `configuration.md` `[chunking-semantic]` extra docs | `data/ingestion.md` chunking section (+ tr) | | **`detect-secrets` extra removal** (C-53, Faz 14 PR D-1 ships code) | `data_audit.md:269-274` + `data_audit-tr.md:269-273` paragrafı sil; `ingestion.md:138` + `ingestion-tr.md:137-138` cümlesi sil | (yok) | `data/secrets.md` regex set tablosunu mevcut `_SECRET_PATTERNS`'a hizala (+tr) — Anthropic/Stripe/SendGrid/Twilio drift kararı (kısalt veya code'a ekle) | | **GRPO reward token honesty** (C-57, Faz 28 ships docstring) | `alignment.md:177-188` GRPO section'a "built-in reward GSM8K/MATH-tuned" disclaimer + custom reward callable örneği (TR mirror Faz 24 scope'u) | (yok) | `training/grpo.md` aynı disclaimer (+ tr) | +| **`forgelm doctor`** (Faz 34 ships code, GH-001) | `getting-started.md` (yeni; +tr) — "Run `forgelm doctor` first" canonical onboarding step | yeni `doctor_subcommand.md` (+ -tr) — flag listing + check categories | `getting-started/installation.md`, `first-run.md`, `troubleshooting.md`, `air-gap.md` (+ tr) — mevcut referansları code-aligned yap (rename/typo değişiklikleri yok; sadece beklenen output update) | +| **`forgelm cache-models` + `cache-tasks`** (Faz 35 ships code, GH-002, GH-003) | `air_gap_deployment.md` (yeni; +tr) — pre-cache workflow end-to-end | yeni `cache_subcommands.md` (+ -tr) — flag listing | `operations/air-gap.md`, `operations/docker.md`, `reference/cli.md`, `reference/yaml-templates.md` (+ tr) — referansları çalışıyor olarak revalidate | +| **`forgelm verify-annex-iv`** (Faz 36 ships code, GH-004) | `safety_compliance.md` Annex IV verification section ekle (+tr) | yeni `verify_annex_iv_subcommand.md` (+ -tr) | `compliance/annex-iv.md`, `operations/troubleshooting.md`, `reference/cli.md` (+ tr) — kullanım örnekleri | +| **`forgelm safety-eval` standalone** (Faz 36 ships code, GH-005) | `safety_compliance.md` standalone eval section (+tr) | yeni `safety_eval_subcommand.md` (+ -tr) — probe set + GGUF support docs | `evaluation/safety.md`, `deployment/gguf-export.md`, `reference/cli.md` (+ tr) | +| **`forgelm verify-gguf`** (Faz 36 ships code, GH-009) | `safety_compliance.md` GGUF integrity section (+tr) | yeni `verify_gguf_subcommand.md` (+ -tr) | `deployment/gguf-export.md`, `reference/cli.md` (+ tr) | +| **`forgelm approvals` listing** (Faz 37 ships code, GH-007) | `human_approval_gate.md` (Faz 9'da yazılmıştı) — listing section ekle (+tr) | yeni `approvals_subcommand.md` (+ -tr) — `--pending` + `--show` semantics | `compliance/human-oversight.md`, `reference/cli.md` (+ tr) — listing flow eklenir | +| **`forgelm reverse-pii`** (Faz 38 ships code, GH-014) | `gdpr_erasure.md` (Faz 21'de yazılmıştı) — Article 15 right of access section ekle (+tr) | yeni `reverse_pii_subcommand.md` (+ -tr) | `compliance/gdpr.md` (+ tr) — Article 15 sub-section çalışır komutla | +| **Phase 13 deferred ghost features** (GH-006 trend, GH-010 compare-runs, GH-012 batch-chat, GH-017 estimate-cost) | `phase-13-pro-cli.md` roadmap dosyasına explicit ekle (+tr) | (yok) | `evaluation/trend-tracking.md`, `operations/experiment-tracking.md`, `deployment/chat.md` (+ tr) — `> Note: This command is planned for v0.6.0+ Pro CLI tier` callout ile değiştir; `--estimate-cost` flag ref kaldırılır veya callout | +| **Phase 14 deferred ghost feature** (GH-015 merge-sweep) | `phase-14-pipeline-chains.md` roadmap dosyasına explicit ekle (+tr) | (yok) | `deployment/model-merging.md` (+ tr) — `> Note: planned for v0.6.0+ pipeline chains scope` callout | **Yapılacak:** Yukarıdaki tablodaki her satır için, eksik olan dokümanı yaz veya mevcut olanı v0.5.5'e güncelle. Bilingual (EN+TR) zorunlu. @@ -1074,6 +1326,27 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa 1. `tools/check_anchor_resolution.py` (Faz 26'da eklenmişti) tüm markdown anchor'ları resolve 2. Opsiyonel: `markdownlint-cli2` rule'ları temiz +##### O. Ghost feature naming + flag drift sweep (Tier 1 — no-code fix) + +**Kaynak:** [`ghost-features-analysis-20260502.md`](./ghost-features-analysis-20260502.md) Tier 1 listesi. Tüm değişiklikler doc-only; kod değişmez. EN + TR mirror çiftleri zorunlu. + +| ID | Düzeltme | Dosyalar | +|---|---|---| +| GH-008 | `verify-log` → `verify-audit` rename (Faz 6'da `verify-audit` shipped; doc'lar eski adı kullanıyor) | `compliance/audit-log.md`, `reference/cli.md` (+ tr) | +| GH-011 | `forgelm benchmark --model ...` (subcommand form) → `forgelm --config ... --benchmark-only PATH` (flag form) | `evaluation/benchmarks.md`, `reference/cli.md` (+ tr) | +| GH-016 | `--export-bundle` → `--compliance-export` rename — QMS SOPs'da gözden kaçmıştı | `docs/qms/sop_model_training.md`, `docs/qms/roles_responsibilities.md`, `docs/roadmap/completed-phases.md` | +| GH-018 | Deploy targets: `kserve`, `triton` listesinden çıkar (parser choices yalnızca `ollama\|vllm\|tgi\|hf-endpoints`) | `reference/cli.md` (+ tr) | +| GH-019 | Chat flags: `--base`, `--top-p`, `--safety`, `--load` → kaldır; `--max-tokens` → `--max-new-tokens` | `reference/cli.md` (+ tr) | +| GH-020 | Ingest flags: `--max-tokens` → `--chunk-tokens`; `--pii-locale` → `--pii-ml-language`; `--language`/`--include`/`--exclude`/`--format` → kaldır; `--strategy` choices `tokens\|markdown\|paragraph\|sentence` → `sliding\|paragraph\|markdown` | `reference/cli.md` (+ tr) | +| GH-021 | Chat slash commands: `/load`, `/top_p`, `/max_tokens`, `/safety` → kaldır; `/quit` → `/exit` veya ikisi bağlantılı | `reference/cli.md` (+ tr) | +| GH-022 | Export quant: `q6_k` → kaldır (parser'da yok); `fp16` → `f16` (parser actual string) | `reference/cli.md` (+ tr) | +| GH-024 | `FORGELM_RESUME_TOKEN` env var → kaldır (kodda kullanılmıyor) | `reference/cli.md` (+ tr) | +| GH-025 | `FORGELM_CACHE_DIR` env var → kaldır; `HF_HOME`'u canonical olarak işaretle | `reference/cli.md` (+ tr) | + +**Doğrulama:** +- Her düzeltme sonrası `grep -rn "" docs/` boş olmalı (false-positive olmadığı kontrol edilir). +- `tools/check_cli_help_consistency.py` (yukarıda Task J'de yeni eklendi) Tier 1 fix'lerin tamamını mekanik olarak yakalar — Tier 1'leri doğrularken bu check'in regression test'i olarak iş gördüğünü teyit et. + #### Faz 30 Acceptance - [ ] Tablo A'daki her feature için EN+TR `docs/guides/`, `docs/reference/`, `docs/usermanuals/{en,tr}/` üçlüsü tam. @@ -1089,12 +1362,15 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa - [ ] Site/* HTML pages v0.5.5 features highlighted (özellikle features.html, compliance.html, index.html). - [ ] DE/FR/ES/ZH user-manual'lara dokunulmamış (kullanıcı kararı); deferred policy formalized. - [ ] CHANGELOG `[Unreleased]` `[0.5.5]` draft hazır (Faz 33'te tamamlanacak). +- [ ] **Task O ghost feature naming/flag drift sweep tamamlandı** — `grep -rn "verify-log\|--export-bundle\|FORGELM_RESUME_TOKEN\|FORGELM_CACHE_DIR\|q6_k\|fp16" docs/` boş; tüm 10 Tier 1 fix uygulandı. +- [ ] **Phase 13 + Phase 14 deferred ghost callout'ları** — `evaluation/trend-tracking.md`, `operations/experiment-tracking.md`, `deployment/chat.md`, `deployment/model-merging.md` (+ tr) içine "planned for v0.6.0+" callout eklendi; ghost subcommand'lar artık doc'ta "şu an çalışan komut" olarak sunulmuyor. +- [ ] **Faz 34-38 yeni subcommand'ların doc üçlüsü** (guides + reference + usermanual EN+TR) tablo A'da listelenen tüm satırlar için tam. --- ## 11. Release prep + release fazları (Faz 31-33) -### Faz 31 — Cross-OS release-tag matrix workflow (Tek PR) +### Faz 31 — Cross-OS release-tag matrix workflow (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-test-007 (Major) — kullanıcı kararı: release-tag-only matrix. @@ -1191,7 +1467,7 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa --- -### Faz 32 — `.pre-commit-config.yaml` (opsiyonel) (Tek PR) +### Faz 32 — `.pre-commit-config.yaml` (opsiyonel) (Tek PR) ✅ TAMAMLANDI — Wave 1 PR **Closes:** F-test-008 (Major) — kullanıcı kararı için: **A (opsiyonel) öneriliyor.** @@ -1239,7 +1515,7 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa **Closes:** Plan kapanış — v0.5.5 PyPI'da. -**Bağımlılık:** **Tüm önceki fazlar (1-32)**. +**Bağımlılık:** **Tüm önceki fazlar (1-32, 34-38)**. **Tasks:** 1. `pyproject.toml` version `0.5.1rc1` → `0.5.5`. @@ -1281,6 +1557,7 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa - [ ] Notebook'lar PyPI'dan install. - [ ] "What ForgeLM is NOT" disiplini scope creep'siz. - [ ] `risks-and-decisions.md` güncel. +- [ ] **Ghost feature closure:** ghost-features-analysis-20260502.md'deki 28 entry → ya implement (Faz 34-38), ya doc-rename (Faz 30 Task O), ya Phase 13/14 callout (Faz 30 Task A deferred row). Açık ghost yok; `tools/check_cli_help_consistency.py` --strict yeşil. ### Code 5/5 - [ ] `data_audit/` package; en büyük dosya <400 satır. @@ -1433,7 +1710,12 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa - **Faz 23 → Faz 26** (QMS yeni dosyalar TR mirror'la birlikte) - **Faz 1 → Faz 25** (site honesty fixes önce, sonra site CI guard) - **Faz 16 → Faz 24** (description= migration → configuration-tr.md auto-gen) -- **Faz 1-29 → Faz 30** (final doc sweep tüm fonksiyonel iş bittikten sonra) +- **Faz 15 → Faz 34** (cli/ split sonrası `_doctor.py` subcommand pattern) +- **Faz 15 + Faz 34 → Faz 35** (cli/ split + doctor `--offline` semantik paydaşlık) +- **Faz 6 + Faz 9 → Faz 36** (verify-audit pattern + Article 14 baseline) +- **Faz 9 → Faz 37** (`approve`/`reject` + staging dir layout + audit event vocabulary) +- **Faz 21 → Faz 38** (GDPR erasure baseline + retention config + access_request audit event family) +- **Faz 1-29 + Faz 34-38 → Faz 30** (final doc sweep tüm fonksiyonel iş bittikten sonra; yeni subcommand'lar dahil) - **Faz 30 → Faz 31, 32 → Faz 33** (final docs → release prep workflows → release) ### Paralel-safe pair'lar @@ -1442,6 +1724,9 @@ Her v0.5.5 yeni feature için aşağıdaki üçlü zorunlu — eksik olan varsa - Faz 10, 11, 12, 13 → bağımsız - Faz 14 ↔ Faz 15 (split-design §3'teki D-X || C-X pair'lar) - Faz 24, 27, 28, 29 → büyük ölçüde bağımsız +- **Faz 34 ↔ Faz 36 ↔ Faz 37** (üçü de cli/ split sonrası bağımsız subcommand additions; tek-PR-per-faz disiplini ile paralel açılabilir) +- **Faz 35** Faz 34'e bağımlı (`doctor --offline` ile koordinasyon); kalan ikisinden bağımsız +- **Faz 38** Faz 21'e bağımlı; Faz 34-37'den bağımsız - Faz 31, 32 → bağımsız (her ikisi de Faz 30 sonrası release öncesi) - Faz 30 → bağımsız değil; tüm fonksiyonel iş bitmiş olmalı (final sweep) @@ -1596,9 +1881,50 @@ Plan §8 Faz 22 "ISO 27001 / SOC 2 Type II — Analysis & gap assessment" design - **Foundation PR carry-over (C-6):** "Audit event coverage matrix" başlığı altında — `Notifier` → `AuditLogger` dependency tartışması. Mevcut durum: `_revert_model` `model.reverted` audit event'i emit ediyor; webhook ayrıca `training.reverted` event'i emit ediyor. Çift kayıt mu, single source of truth mu? ISO 27001 A.12.4 Logging perspektifinden karar. +- **Wave 1 review carry-over (Round-5):** Concurrent `forgelm approve` / `forgelm reject` race protection. Mevcut durum: `_atomic_rename_or_move`'un `os.rename` adımı POSIX'te atomic olduğu için iki concurrent approve'dan sadece biri promote edebiliyor; ancak iki concurrent **reject** çağrısı staging dir'i değiştirmiyor, sadece audit log'a yazıyor — decision-event guard'ı her ikisi de geçer ve duplicate `human_approval.rejected` event'i yazılır. ISO 27001 A.12.4 + A.18.1.4 (record integrity) perspektifinden per-run interprocess lock (`fcntl.flock` POSIX + `msvcrt.locking` Windows, ya da pure-Python `filelock` paketi) Faz 22 kapsamında karara bağlanır. Şu anki davranış: gerçek tek-operatör senaryosunda race görünmez; CI/CD parallel-job senaryosunda known limitation. + +### Ghost feature analysis (2026-05-02) carry-over (C-58 toplu blok) + +**Kaynak:** [`ghost-features-analysis-20260502.md`](./ghost-features-analysis-20260502.md). Analiz Wave 1 PR'ı review aşamasındayken yapıldı; 28 ghost entry tespit edildi. Her birinin closure plan içinde nereye absorb edildiği aşağıda. Hiçbiri "open kuyrukta" bırakılmadı. + +| Ghost ID | Kategori | Çözüm yolu | Hedef faz | +|---|---|---|---| +| GH-001 (`forgelm doctor`) | Critical — onboarding bloker | **Yeni subcommand implement** | **Faz 34** (yeni) | +| GH-002 (`cache-models`) | High — air-gap bloker | **Yeni subcommand implement** | **Faz 35** (yeni) | +| GH-003 (`cache-tasks`) | High — air-gap bloker | **Yeni subcommand implement** | **Faz 35** (yeni) | +| GH-004 (`verify-annex-iv`) | High — compliance bloker | **Yeni subcommand implement** | **Faz 36** (yeni) | +| GH-005 (`safety-eval` standalone) | High — eval gap | **Yeni subcommand implement** | **Faz 36** (yeni) | +| GH-006 (`forgelm trend`) | Medium | **Phase 13 Pro CLI'a defer** + Faz 30'da "planned for v0.6.0+" callout | Faz 30 Task A deferred row | +| GH-007 (`forgelm approvals`) | Medium — Faz 9 follow-up (listing eksik) | **Yeni subcommand implement** | **Faz 37** (yeni) | +| GH-008 (`verify-log` → `verify-audit`) | Medium — naming alias | **Doc-only rename** | **Faz 30 Task O** | +| GH-009 (`verify-gguf`) | Medium | **Yeni subcommand implement** | **Faz 36** (yeni) | +| GH-010 (`compare-runs`) | Low-Medium | **Phase 13 Pro CLI'a defer** | Faz 30 Task A deferred row | +| GH-011 (`forgelm benchmark` wrong form) | Low-Medium | **Doc-only — `--benchmark-only` flag form'una düzelt** | **Faz 30 Task O** | +| GH-012 (`batch-chat`) | Low | **Phase 13'e defer** veya `chat --batch` flag (Faz 28 değerlendirme) | Faz 30 Task A deferred row | +| GH-013 (`forgelm purge`) | Informational — planned | **Zaten Faz 21'de planlı** (no-op) | Faz 21 | +| GH-014 (`reverse-pii`) | Medium — GDPR Art. 15 | **Yeni subcommand implement** (Faz 21 follow-up) | **Faz 38** (yeni) | +| GH-015 (`merge-sweep`) | Low-Medium | **Phase 14 pipeline chains'e defer** | Faz 30 Task A deferred row | +| GH-016 (`--export-bundle` rename) | High — QMS audit risk | **Doc-only rename → `--compliance-export`** | **Faz 30 Task O** | +| GH-017 (`--estimate-cost`) | Low-Medium | **Phase 13 Pro CLI'a defer** | Faz 30 Task A deferred row | +| GH-018 (deploy `kserve`/`triton`) | Medium | **Doc-only — kaldır** (parser choices: ollama/vllm/tgi/hf-endpoints) | **Faz 30 Task O** | +| GH-019 (chat `--base/--top-p/--safety/--load`) | Medium | **Doc-only — kaldır + `--max-tokens` → `--max-new-tokens`** | **Faz 30 Task O** | +| GH-020 (ingest `--format/--language/--include/--exclude/--pii-locale`) | Medium | **Doc-only — kaldır + flag rename'leri** | **Faz 30 Task O** | +| GH-021 (chat slash `/load//top_p//max_tokens//safety`) | Low | **Doc-only — kaldır** | **Faz 30 Task O** | +| GH-022 (export quant `q6_k`/`fp16`) | Low | **Doc-only — `q6_k` kaldır, `fp16` → `f16`** | **Faz 30 Task O** | +| GH-023 (`ingestion.retention.raw_documents.ttl_days` config) | Medium | **Faz 21'in `RetentionConfig`'ine extend** — `ingestion` top-level key + `retention.raw_documents.ttl_days` field | Faz 21 (genişletme) | +| GH-024 (`FORGELM_RESUME_TOKEN`) | Low | **Doc-only — kaldır** (kodda kullanılmıyor) | **Faz 30 Task O** | +| GH-025 (`FORGELM_CACHE_DIR`) | Low | **Doc-only — kaldır + `HF_HOME` canonical işaretle** | **Faz 30 Task O** | +| GH-026 (`ingestion.retention` config; duplicate of GH-023) | Medium | (GH-023 ile aynı — Faz 21 extend) | Faz 21 | +| GH-027 (alias of GH-024) | Low | (GH-024 ile aynı) | Faz 30 Task O | +| GH-028 (alias of GH-025) | Low | (GH-025 ile aynı) | Faz 30 Task O | + +**Ek görev:** Faz 21 kapsamı **GH-023 absorption** ile genişletilir — `ForgeConfig`'a `ingestion: IngestionConfig` top-level key + `IngestionConfig.retention.raw_documents.ttl_days` Pydantic field eklenir; `_purge --kind raw_documents` flag'ı bu retention policy'yi enforce eder. Faz 21 acceptance'a yeni satır: "Config: `ingestion.retention.raw_documents.ttl_days: int = 90` field çalışır + Pydantic Literal sweep tarafından kapsanır + GDPR docs'taki YAML örneği geçerli." + ### Foundation PR closure-coverage özeti -Foundation PR'da tamamlanan toplam: **8 Critical + 30+ Major** (master review ID'leri foundation PR commit message'da listeli). 2 madde explicit-skip (C-7, C-8 — plan referans hataları) ve 7 carry-over (C-1, C-2, C-3, C-4, C-5, C-6, C-9, C-10) plan'da yukarıda explicit yazıldı. **Atlanan veya gözden kaçan iş yok.** +Foundation PR'da tamamlanan toplam: **8 Critical + 30+ Major** (master review ID'leri foundation PR commit message'da listeli). 2 madde explicit-skip (C-7, C-8 — plan referans hataları) ve 8 carry-over (C-1, C-2, C-3, C-4, C-5, C-6, C-9, C-10) plan'da yukarıda explicit yazıldı. + +Wave 1 PR (closure/wave1-faz9-32) review fazında ek olarak **28 ghost entry** tespit edildi (ghost-features-analysis-20260502.md): 5 yeni faz (34-38) + Faz 21 genişletmesi + Faz 30 Task O ile absorb edildi. **Atlanan veya gözden kaçan iş yok.** --- @@ -1606,6 +1932,7 @@ Foundation PR'da tamamlanan toplam: **8 Critical + 30+ Major** (master review ID Bu plan v2 (revised 2026-04-30 kullanıcı kararları sonrası). **v2.1 (2026-04-30):** §15.5 Foundation PR carry-over registry eklendi. +**v2.2 (2026-05-02):** Ghost feature analysis absorption — Wave 1 PR review aşamasında tespit edilen 28 ghost entry plan'a entegre edildi. Yeni Faz 34-38 açıldı (`forgelm doctor`, `cache-models` + `cache-tasks`, compliance verification toolbelt, `approvals` listing, `reverse-pii`); Faz 30 Task O ghost feature naming/flag drift sweep eklendi (10 doc-only Tier 1 fix); Phase 13 + Phase 14 deferred row'ları Faz 30 Task A'ya eklendi; Faz 21 GH-023 absorption ile `ingestion.retention` extend edildi. Toplam faz sayısı 33 → 38; toplam ~52 PR. Wave 1 IN PROGRESS işaretleri Faz 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 28, 30, 34, 35, 36, 37, 38 satırlarına uygulandı. Plan dosyası: `docs/analysis/code_reviews/closure-plan-202604300906.md`. Her PR description'ında bu plana referans verir (`§N` faz numarası ile). @@ -1631,14 +1958,16 @@ v0.5.5 release sonrası: yeni master review turu (`master-review-opus-202604YYMM 7. **Polish + cleanup (Faz 24-29):** Bilingual sweep + site picker + QMS bilingual + silent except + remaining + nit. -8. **Final documentation + site (Faz 30):** Tüm `docs/` (EN+TR) + `docs/usermanuals/{en,tr}/` + `site/*` v0.5.5 nihai durumuna senkronlanır. Tablo A feature×{guide,reference,usermanual} matrisinin tamamı doldurulur. DE/FR/ES/ZH dokunulmaz (kullanıcı kararı: deferred). +8. **Ghost feature closure (Faz 34-38):** `doctor` + air-gap pre-cache (`cache-models`/`cache-tasks`) + compliance verification toolbelt (`safety-eval`/`verify-annex-iv`/`verify-gguf`) + `approvals` listing + `reverse-pii`. Faz 15 (cli/ split) sonrası açılabilir; Faz 34 ↔ 36 ↔ 37 paralel-safe; Faz 35 Faz 34'e bağımlı; Faz 38 Faz 21'e bağımlı. + +9. **Final documentation + site (Faz 30):** Tüm `docs/` (EN+TR) + `docs/usermanuals/{en,tr}/` + `site/*` v0.5.5 nihai durumuna senkronlanır. Tablo A feature×{guide,reference,usermanual} matrisinin tamamı doldurulur (Faz 34-38 yeni subcommand'lar dahil). Task O ghost feature naming/flag drift sweep (10 Tier 1 doc fix). Phase 13 + Phase 14 deferred ghost'lar için "planned for v0.6.0+" callout'ları. DE/FR/ES/ZH dokunulmaz (kullanıcı kararı: deferred). -9. **Release prep (Faz 31-32):** Cross-OS matrix + pre-commit. +10. **Release prep (Faz 31-32):** Cross-OS matrix + pre-commit. -10. **Release (Faz 33):** v0.5.5 PyPI publish. +11. **Release (Faz 33):** v0.5.5 PyPI publish. -**Toplam:** 33 faz, ~47 PR. **Hedef tarih yok**, dependency-driven sıralama. Her faz PR ile kapanır. **Geride iş yok. Tekrarlamadık. Stub kalmadı. Tüm doküman + site EN+TR'de v0.5.5 ile senkron.** +**Toplam:** 38 faz, ~52 PR. **Hedef tarih yok**, dependency-driven sıralama. Her faz PR ile kapanır. **Geride iş yok. Tekrarlamadık. Stub kalmadı. Ghost feature kalmadı. Tüm doküman + site EN+TR'de v0.5.5 ile senkron.** --- -*End of closure plan v2 — 202604300906.* +*End of closure plan v2.2 — 202604300906 (revised 2026-05-02).* diff --git a/docs/analysis/code_reviews/ghost-features-analysis-20260502.md b/docs/analysis/code_reviews/ghost-features-analysis-20260502.md new file mode 100644 index 00000000..c6c9aa09 --- /dev/null +++ b/docs/analysis/code_reviews/ghost-features-analysis-20260502.md @@ -0,0 +1,783 @@ +# Ghost Feature Analysis — ForgeLM CLI + +**Date:** 2026-05-02 (v2 — full re-scan) +**Branch:** closure/foundation-faz1-8 (current HEAD at time of analysis) +**Scope:** All `forgelm ` and flag references across **all** docs: +- `docs/usermanuals/` (EN + TR) +- `docs/guides/`, `docs/reference/`, `docs/qms/`, `docs/roadmap/` +- `README.md`, `CHANGELOG.md`, `CONTRIBUTING.md` +- `site/*.html` +- Cross-checked against: + - `forgelm/cli/_parser.py` (registered subcommands + flags) + - `forgelm/cli/subcommands/` (implementations) + - `forgelm/config.py` (ForgeConfig structure) + - `docs/analysis/code_reviews/closure-plan-202604300906.md` + - `docs/roadmap/` directory + +--- + +## Summary + +**15 ghost subcommands** found in Category A: 12 subcommands that do not exist in the codebase, 1 documented under the wrong form (GH-011 — flag form `--benchmark-only` rather than a subcommand), 1 with alias-only confusion (GH-008 — `verify-log` vs the implemented `verify-audit`), 1 planned but not yet implemented (GH-013 — `forgelm purge`, scheduled for Faz 21). The complete document spans **28 entries**: 15 subcommands + 10 flags/options + 1 config key + 2 env vars (see the standalone "Total ghost count" line below the implemented-subcommand reference). + +### Category A — Ghost subcommands (documented, not implemented) + +| # | Command | Refs (all docs) | In code | Closure plan | Roadmap | +|---|---|---|---|---|---| +| GH-001 | `forgelm doctor` | 38 | ❌ | ❌ | ❌ | +| GH-002 | `forgelm cache-models` | 16 | ❌ | ❌ | ❌ | +| GH-003 | `forgelm cache-tasks` | 11 | ❌ | ❌ | ❌ | +| GH-004 | `forgelm verify-annex-iv` | 12 | ❌ | ❌ | ❌ | +| GH-005 | `forgelm safety-eval` | 12 | ❌ | ❌ | ❌ | +| GH-006 | `forgelm trend` | 10 | ❌ | ❌ | ❌ | +| GH-007 | `forgelm approvals` | 11 | ❌ | ❌ | ❌ | +| GH-008 | `forgelm verify-log` | 7 | ❌ alias confusion | ❌ | ❌ | +| GH-009 | `forgelm verify-gguf` | 7 | ❌ | ❌ | ❌ | +| GH-010 | `forgelm compare-runs` | 8 | ❌ | ❌ | ❌ | +| GH-011 | `forgelm benchmark` | 10 | ⚠️ wrong form | ❌ | ❌ | +| GH-012 | `forgelm batch-chat` | 7 | ❌ | ❌ | ❌ | +| GH-013 | `forgelm purge` | 8 | ❌ | ✅ Faz 21 | ❌ | +| GH-014 | `forgelm reverse-pii` | 2 | ❌ | ❌ | ❌ | +| GH-015 | `forgelm merge-sweep` | 2 | ❌ | ❌ | ❌ | + +### Category B — Ghost flags / wrong flag names + +| # | Documented as | Actual in parser | Severity | +|---|---|---|---| +| GH-016 | `--export-bundle` | `--compliance-export` | High — in QMS SOPs | +| GH-017 | `--estimate-cost` | not registered | Medium | +| GH-018 | `--resume-from PATH` | `--resume [PATH]` | Low — close but different | +| GH-019 | `--output-format plain\|json` | `text\|json` (not `plain`) | Low — naming nit | +| GH-020 | `forgelm deploy --target kserve\|triton` | choices: `ollama\|vllm\|tgi\|hf-endpoints` | Medium | +| GH-021 | `forgelm chat --base`, `--top-p`, `--safety`, `--load` | none of these exist | Medium | +| GH-022 | export quant `q6_k` and `fp16` | `f16` exists, `q6_k` does not | Low | +| GH-023 | `forgelm ingest --format raw\|instructions\|qa` | no `--format` on ingest | Medium | +| GH-024 | `forgelm ingest --language`, `--include`, `--exclude`, `--pii-locale` | only `--pii-ml-language` exists | Medium | +| GH-025 | chat slash commands `/load`, `/top_p`, `/max_tokens`, `/safety` | only `/reset /save /temperature /system /help /exit` | Low | + +### Category C — Ghost config keys + +| # | Documented as | Actual in config.py | Severity | +|---|---|---|---| +| GH-026 | `ingestion.retention.raw_documents.ttl_days` | no `ingestion` or `retention` top-level key exists in `ForgeConfig` | Medium | + +### Category D — Ghost env vars + +| # | Documented as | In code | Source | +|---|---|---|---| +| GH-027 | `FORGELM_RESUME_TOKEN` | not used anywhere in forgelm/ | cli.md env table | +| GH-028 | `FORGELM_CACHE_DIR` | not used anywhere in forgelm/ | cli.md env table | + +### Category E — Planned (not yet implemented) + +| # | Command | Closure plan | Expected wave | +|---|---|---|---| +| GH-013 | `forgelm purge` | ✅ Faz 21 | Wave 2 | + +--- + +**Implemented subcommands** (for reference): +`chat`, `export`, `deploy`, `quickstart`, `ingest`, `audit`, `verify-audit`, `approve`, `reject` — 9 total. + +**Total ghost count: 28 entries** (15 subcommands + 10 flags/options + 1 config key + 2 env vars) + +--- + +## Detailed Findings + +--- + +### GH-001 — `forgelm doctor` + +**Severity:** Critical (user-onboarding blocker) +**References in usermanuals:** 30 occurrences across 10 files (5 EN source pages mirrored to TR; the table below lists every entry rather than collapsing pairs) + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/getting-started/installation.md` | 100, 103, 111 | Installation flow — "Run `forgelm doctor` before anything else" | +| `docs/usermanuals/en/getting-started/first-run.md` | 12, 29, 32, 41 | First-run diagram + step-by-step | +| `docs/usermanuals/en/operations/troubleshooting.md` | 8, 168, 173, 179 | "catches 80% of environment problems" | +| `docs/usermanuals/en/operations/air-gap.md` | 115, 124, 148 | `forgelm doctor --offline` variant | +| `docs/usermanuals/en/reference/cli.md` | 15 | CLI reference table | +| `docs/usermanuals/tr/getting-started/installation.md` | 100, 103, 111 | TR mirror | +| `docs/usermanuals/tr/getting-started/first-run.md` | 12, 29, 32, 41 | TR mirror | +| `docs/usermanuals/tr/operations/troubleshooting.md` | 8, 168, 173, 179 | TR mirror | +| `docs/usermanuals/tr/operations/air-gap.md` | 115, 124, 148 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 15 | TR mirror | + +**Documented behavior:** Environment check — verifies Python version, PyTorch, CUDA availability, GPU detection, optional dep install status. `--offline` variant checks cached resources for air-gapped deployments. + +**Code reality:** No `doctor` in `forgelm/cli/_parser.py`, no `_doctor.py` in `forgelm/cli/subcommands/`. Running `forgelm doctor` exits with "unrecognized arguments". + +**Closure plan:** Not mentioned in any phase. +**Roadmap:** Not mentioned in any file under `docs/roadmap/`. + +**Impact:** This is the first command new users are told to run after installation. A user following `installation.md` or `first-run.md` will hit an immediate error. The `--offline` variant is central to the air-gap deployment guide. + +**Options:** +- A) Implement `forgelm doctor` (see scope below) +- B) Remove all 30 references; replace with a manual checklist +- C) Add to closure plan as a new phase + +**Suggested scope if implemented:** Python version check, `torch.cuda.is_available()`, GPU count + VRAM, check for optional extras (`[qlora]`, `[safety]`, `[evaluation]`, `[ingestion-pii-ml]`), output as structured table. `--offline` flag: additionally checks `HF_HUB_OFFLINE` environment + whether required model weights are in local cache. + +--- + +### GH-002 — `forgelm cache-models` + +**Severity:** High (air-gap workflow blocker) +**References in usermanuals:** 12 occurrences across 4 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/operations/air-gap.md` | 25, 30, 54 | Pre-cache workflow: `--model`, `--safety` flags | +| `docs/usermanuals/en/operations/docker.md` | 101, 104 | Dockerfile `RUN` steps | +| `docs/usermanuals/en/reference/cli.md` | 28 | CLI reference table | +| `docs/usermanuals/tr/operations/air-gap.md` | 25, 30, 54 | TR mirror | +| `docs/usermanuals/tr/operations/docker.md` | 101, 104 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 28 | TR mirror | + +**Documented behavior:** Pre-download HuggingFace model weights to local cache for air-gapped environments. Flags: `--model `, `--safety `. + +**Code reality:** No `cache-models` subcommand. The underlying functionality (HF cache population) is achievable via `huggingface-cli download` or Python `snapshot_download()`, but ForgeLM exposes no wrapper. + +**Closure plan:** Not mentioned. +**Roadmap:** Not mentioned. + +**Note:** Closely paired with GH-003 (`cache-tasks`). Both appear exclusively in air-gap and Docker operational guides. An operator following `air-gap.md` cannot execute either step as documented. + +--- + +### GH-003 — `forgelm cache-tasks` + +**Severity:** High (air-gap workflow blocker) +**References in usermanuals:** 8 occurrences across 4 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/operations/air-gap.md` | 30, 101 | Pre-cache lm-eval tasks: `--tasks hellaswag,arc_easy,...` | +| `docs/usermanuals/en/operations/docker.md` | 107 | Dockerfile `RUN` step | +| `docs/usermanuals/en/reference/cli.md` | 29 | CLI reference table | +| `docs/usermanuals/en/reference/yaml-templates.md` | 315 | Comment: "Pre-requisites: forgelm cache-models / cache-tasks already run" | +| `docs/usermanuals/tr/operations/air-gap.md` | 30, 101 | TR mirror | +| `docs/usermanuals/tr/operations/docker.md` | 107 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 29 | TR mirror | + +**Documented behavior:** Pre-download lm-evaluation-harness task datasets for air-gapped environments. Flags: `--tasks `. + +**Code reality:** No `cache-tasks` subcommand. + +**Closure plan:** Not mentioned. +**Roadmap:** Not mentioned. + +--- + +### GH-004 — `forgelm verify-annex-iv` + +**Severity:** High (compliance workflow blocker) +**References in usermanuals:** 8 occurrences across 4 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/compliance/annex-iv.md` | 137, 144 | "validates Annex IV artifact + re-computes manifest hashes" | +| `docs/usermanuals/en/operations/troubleshooting.md` | 155 | Fix for "Missing fields in Annex IV artifact" error | +| `docs/usermanuals/en/reference/cli.md` | 25 | CLI reference table | +| `docs/usermanuals/tr/compliance/annex-iv.md` | 137, 144 | TR mirror | +| `docs/usermanuals/tr/operations/troubleshooting.md` | 155 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 25 | TR mirror | + +**Documented behavior:** Validate an `annex_iv.json` artifact — check required fields, re-compute manifest hashes, detect tampering. + +**Code reality:** `forgelm/compliance.py` generates `annex_iv_metadata.json` (lines 680, 909-910) during training. There is no verification subcommand. `forgelm verify-audit` (Faz 6) verifies the audit log chain but not the Annex IV artifact. + +**Closure plan:** Not mentioned. Note: `forgelm verify-audit` was planned and shipped (Faz 6) but serves a different purpose (audit chain integrity, not Annex IV field completeness). +**Roadmap:** Not mentioned. + +**Note:** The EU AI Act Article 11 + Annex IV documentation requirements make this a real compliance use-case, not purely aspirational. + +--- + +### GH-005 — `forgelm safety-eval` + +**Severity:** High (evaluation workflow gap) +**References in usermanuals:** 8 occurrences across 4 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/evaluation/safety.md` | 76, 91 | Standalone safety evaluation with custom probe set | +| `docs/usermanuals/en/deployment/gguf-export.md` | 136 | Post-export safety check on GGUF model | +| `docs/usermanuals/en/reference/cli.md` | 20 | CLI reference table | +| `docs/usermanuals/tr/evaluation/safety.md` | 76, 91 | TR mirror | +| `docs/usermanuals/tr/deployment/gguf-export.md` | 136 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 20 | TR mirror | + +**Documented behavior:** Standalone Llama Guard scoring against a model without full training run. Flags: `--model `, `--probes `, `--default-probes`. Supports GGUF models. + +**Code reality:** Safety evaluation code exists in `forgelm/safety.py` and runs inside the training pipeline. There is no standalone CLI entry point. `forgelm/cli/_no_train_modes.py` has a `--benchmark-only` mode analog but no `safety-eval` mode. + +**Closure plan:** Not mentioned as a standalone subcommand. +**Roadmap:** Safety evaluation pipeline (Phase 9 in completed-phases.md) is part of the training flow, not a standalone command. + +**Note:** The GGUF context (`forgelm safety-eval --model model.q4_k_m.gguf`) is particularly useful for post-deployment validation — a genuine use-case not served by the current pipeline. + +--- + +### GH-006 — `forgelm trend` + +**Severity:** Medium +**References in usermanuals:** 6 occurrences across 2 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/evaluation/trend-tracking.md` | 80, 107 | Query metric trends: `--metric`, `--lookback`, `--filter config_hash` | +| `docs/usermanuals/en/reference/cli.md` | 30 | CLI reference table | +| `docs/usermanuals/tr/evaluation/trend-tracking.md` | 80, 107 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 30 | TR mirror | + +**Documented behavior:** Show metric trend across recent runs from `.forgelm/eval-history.jsonl`. Flags: `--metric `, `--lookback `, `--filter config_hash`. + +**Code reality:** Trend data is embedded in audit reports and the eval-history file is referenced in docs, but no `.forgelm/eval-history.jsonl` write path exists in `forgelm/trainer.py` or `forgelm/compliance.py`, and no `trend` subcommand exists. + +**Closure plan:** Not mentioned. +**Roadmap:** `docs/roadmap/completed-phases.md:563` mentions `"trend": "improving"` as a field in safety results (Phase 9), but this is an artifact field, not a CLI command. `phase-13-pro-cli.md` (Pro CLI, v0.6.0+) has `forgelm pro dashboard` for observability — trend visualization is a natural fit there but is not explicitly listed. + +--- + +### GH-007 — `forgelm approvals` + +**Severity:** Medium +**References in usermanuals:** 6 occurrences across 2 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/compliance/human-oversight.md` | 131, 138 | `--pending` lists pending approvals; `--show ` shows detail | +| `docs/usermanuals/en/reference/cli.md` | 33 | CLI reference table | +| `docs/usermanuals/tr/compliance/human-oversight.md` | 131, 138 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 33 | TR mirror | + +**Documented behavior:** List and inspect pending approval requests. `--pending` shows all runs in staging awaiting approval. `--show ` shows detail for a single run. + +**Code reality:** `forgelm approve ` and `forgelm reject ` exist (Faz 9, `forgelm/cli/subcommands/_approve.py`). The listing command `forgelm approvals` does not. An operator cannot discover which runs are awaiting approval without manually inspecting the filesystem or audit log. + +**Closure plan:** Not mentioned. Faz 9 delivered approve + reject but not the listing command. +**Roadmap:** Not mentioned. + +**Note:** Closely related to the implemented `forgelm approve`/`forgelm reject`. The listing command would scan `final_model.staging/` directories or audit log for unresolved `human_approval.required` events — the data is already there. + +--- + +### GH-008 — `forgelm verify-log` + +**Severity:** Medium (alias confusion with existing command) +**References in usermanuals:** 4 occurrences across 2 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/compliance/audit-log.md` | 59, 65 | Verify audit chain integrity, check timestamps, prev_hash, seq numbers | +| `docs/usermanuals/en/reference/cli.md` | 26 | CLI reference table | +| `docs/usermanuals/tr/compliance/audit-log.md` | 59, 65 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 26 | TR mirror | + +**Documented behavior:** Validate audit log chain — check monotonic timestamps, prev_hash chain validity, no gaps in seq numbers. + +**Code reality:** `forgelm verify-audit ` (Faz 6, `forgelm/cli/subcommands/_verify_audit.py`) performs exactly this function. The documented command is `verify-log`; the implemented command is `verify-audit`. Same functionality, different name. + +**Closure plan:** Faz 6 explicitly delivered `forgelm verify-audit`. The name `verify-log` never appears in the plan. +**Roadmap:** Not mentioned. + +**Note:** This is likely a naming inconsistency introduced during documentation authoring — the writer used `verify-log` while the implementation was named `verify-audit`. Resolution: rename the usermanual references to `verify-audit` (no code change needed). + +--- + +### GH-009 — `forgelm verify-gguf` + +**Severity:** Medium +**References in usermanuals:** 4 occurrences across 2 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/deployment/gguf-export.md` | 80 | Post-export integrity check on `.gguf` file | +| `docs/usermanuals/en/reference/cli.md` | 27 | CLI reference table | +| `docs/usermanuals/tr/deployment/gguf-export.md` | 80 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 27 | TR mirror | + +**Documented behavior:** Validate GGUF file integrity — check magic bytes, metadata block, tensor checksums. + +**Code reality:** `forgelm/cli/subcommands/_export.py` handles GGUF conversion but no verification subcommand exists. `_export.py` does write a SHA-256 integrity record during export (per CONTRIBUTING), but no standalone verify command reads it back. + +**Closure plan:** Not mentioned. +**Roadmap:** Not mentioned. + +--- + +### GH-010 — `forgelm compare-runs` + +**Severity:** Low-Medium +**References in usermanuals:** 4 occurrences across 2 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/operations/experiment-tracking.md` | 116 | Side-by-side comparison of run metrics: `compare-runs runs/v1.0 runs/v1.1 runs/v1.2` | +| `docs/usermanuals/en/evaluation/trend-tracking.md` | 116 | Positional args: list of run directories | +| `docs/usermanuals/tr/operations/experiment-tracking.md` | 116 | TR mirror | +| `docs/usermanuals/tr/evaluation/trend-tracking.md` | 116 | TR mirror | + +**Documented behavior:** Side-by-side metric comparison across multiple run directories. Positional args: run directory paths. + +**Code reality:** No `compare-runs` subcommand. Underlying data (per-run JSON artifacts) is available; the command to read and format it does not exist. + +**Closure plan:** Not mentioned. +**Roadmap:** `phase-13-pro-cli.md` has "experiment browser" as a Pro CLI feature (web dashboard) but `compare-runs` as a CLI command is not explicitly listed there. + +--- + +### GH-011 — `forgelm benchmark` (wrong form) + +**Severity:** Low-Medium (wrong interface documented) +**References in usermanuals:** 4 occurrences across 2 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/evaluation/benchmarks.md` | 71 | `forgelm benchmark --model "..." --tasks hellaswag,...` | +| `docs/usermanuals/en/reference/cli.md` | 19 | CLI reference table: "Run lm-eval-harness against a model" | +| `docs/usermanuals/tr/evaluation/benchmarks.md` | 71 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 19 | TR mirror | + +**Documented behavior:** Standalone benchmark run against an existing model (no training). Flags: `--model`, `--tasks`. + +**Code reality:** The functionality exists but is exposed as a **flag on the main command**, not a subcommand. `forgelm/cli/_parser.py:598`: +```text +--benchmark-only Run benchmark evaluation on an existing model without training. + Requires evaluation.benchmark config. +``` +Usage: `forgelm --config my_config.yaml --benchmark-only /path/to/model` + +The documented form `forgelm benchmark --model ... --tasks ...` does not work. A user following `benchmarks.md` will get an error. + +**Closure plan:** Not mentioned as a subcommand. `--benchmark-only` flag exists in code but is not prominently documented. +**Roadmap:** Not mentioned. + +**Note:** Two options: (a) add `benchmark` as a proper subcommand that wraps `--benchmark-only` logic, or (b) correct the docs to show the `--benchmark-only` flag form. + +--- + +### GH-012 — `forgelm batch-chat` + +**Severity:** Low +**References in usermanuals:** 4 occurrences across 2 files + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/deployment/chat.md` | 133 | "automated probing of many prompts" — `--prompts `, `--output ` | +| `docs/usermanuals/en/reference/cli.md` | 22 | CLI reference table: "Non-interactive prompt → response" | +| `docs/usermanuals/tr/deployment/chat.md` | 133 | TR mirror | +| `docs/usermanuals/tr/reference/cli.md` | 22 | TR mirror | + +**Documented behavior:** Non-interactive batch inference — read prompts from JSONL, write responses to JSONL. Alternative to interactive `forgelm chat`. + +**Code reality:** `forgelm chat` subcommand exists (`forgelm/cli/subcommands/_chat.py`) but no `--batch` or `batch-chat` mode. + +**Closure plan:** Not mentioned. +**Roadmap:** Not mentioned. + +--- + +### GH-013 — `forgelm purge` (planned, not yet implemented) + +**Severity:** Informational (legitimate planned feature) +**References in usermanuals:** 5 occurrences across 3 files (inferred from GDPR docs) + +| File | Context | +|---|---| +| `docs/usermanuals/en/compliance/gdpr-erasure.md` | Linked from Faz 21 | +| `docs/usermanuals/tr/compliance/gdpr-erasure.md` | TR mirror | + +**Documented behavior:** GDPR right-to-erasure. `--row-id`, `--corpus`, `--run-id`, `--kind`, `--check-policy` flags. + +**Code reality:** No implementation yet. + +**Closure plan:** ✅ Explicitly planned in **Faz 21** (GDPR right-to-erasure implementation). Tasks: +- `forgelm/cli/subcommands/_purge.py` +- `forgelm/compliance.py` `record_erasure_event()` helper +- Audit events: `data.erasure_requested`, `data.erasure_completed`, `data.erasure_failed` +- Tests: 7 acceptance tests +- Docs: `docs/guides/gdpr_erasure.md` + TR mirror + +**Roadmap:** Not separately listed (embedded in closure plan as Wave 2+ phase). + +**Status:** Not a documentation error — Faz 21 has not shipped yet. The docs will be accurate once Faz 21 is implemented. + +--- + +### GH-014 — `forgelm reverse-pii` + +**Severity:** Medium +**References:** 2 occurrences + +| File | Line | Context | +|---|---|---| +| `docs/usermanuals/en/compliance/gdpr.md` | 77 | GDPR Art. 15 right of access — `--query "ali@example.com" data/*.jsonl` | +| `docs/usermanuals/tr/compliance/gdpr.md` | 77 | TR mirror | + +**Documented behavior:** Reverse-PII lookup — given an identifier (email, name), scan masked JSONL files to check if it appears in the training data. Supports glob patterns. + +**Code reality:** No `reverse-pii` subcommand in parser or subcommands directory. The forward direction (masking with `forgelm ingest --pii-mask`) exists, but the reverse lookup does not. + +**Closure plan:** Not mentioned. +**Roadmap:** Not mentioned. + +--- + +### GH-015 — `forgelm merge-sweep` + +**Severity:** Low-Medium +**References:** 2 occurrences + +| File | Line | Context | +|---|---|---| +| `docs/usermanuals/en/deployment/model-merging.md` | 172 | "ForgeLM ships a `forgelm merge-sweep` helper that automates" grid search over merge algorithms | +| `docs/usermanuals/tr/deployment/model-merging.md` | 172 | TR mirror | + +**Documented behavior:** Automated grid search over `(algorithm, parameters)` combinations for model merging, evaluating each variant. + +**Code reality:** `forgelm/merging.py` exists (model merge logic). `--merge` flag exists on main command. No `merge-sweep` subcommand. + +**Closure plan:** Not mentioned. +**Roadmap:** Not mentioned. `phase-14-pipeline-chains.md` covers multi-stage pipelines but not merge sweeps. + +--- + +### GH-016 — `--export-bundle` flag + +**Severity:** High — appears in QMS SOPs used by compliance auditors +**References:** 3 occurrences (non-analysis docs) + +| File | Line | Context | +|---|---|---| +| `docs/qms/sop_model_training.md` | 66 | "Evidence bundle: `forgelm --config job.yaml --export-bundle ./archive/`" | +| `docs/qms/roles_responsibilities.md` | 66 | "Reviews evidence bundle (`--export-bundle`)" | +| `docs/roadmap/completed-phases.md` | 346 | Listed as a shipped Phase 6 deliverable | + +**Documented behavior:** Export all compliance artifacts (audit trail, data provenance, Annex IV) as a bundle/ZIP. + +**Code reality:** Parser has `--compliance-export OUTPUT_DIR` (not `--export-bundle`). These appear to be the same feature with a renamed flag — `completed-phases.md` shows the old name while the parser carries the current name. The QMS SOPs were not updated when the flag was renamed. + +**Closure plan:** Not mentioned as a rename. +**Roadmap:** `completed-phases.md` marks it as ✅ shipped under the old name. + +--- + +### GH-017 — `--estimate-cost` flag + +**Severity:** Low-Medium +**References:** 1 occurrence in `docs/usermanuals/en/reference/cli.md` top-level flags table + +**Documented behavior:** Pre-flight cost estimation (GPU cost) before training. + +**Code reality:** Not registered in `forgelm/cli/_parser.py`. GPU cost estimation logic exists inside the training pipeline output (`_result.py`) but no standalone `--estimate-cost` flag. `--fit-check` (VRAM estimate) does exist. + +**Closure plan:** Not mentioned. +**Roadmap:** `phase-13-pro-cli.md` has "Cloud GPU cost estimation — real-time pricing" as a Pro CLI v0.6.0 feature. + +--- + +### GH-018 — Deploy targets `kserve` and `triton` + +**Severity:** Medium +**References:** `docs/usermanuals/en/reference/cli.md` deploy section + +**Documented form:** `forgelm deploy CHECKPOINT --target ollama|vllm|tgi|hf-endpoints|kserve|triton` + +**Code reality:** Parser choices: `["ollama", "vllm", "tgi", "hf-endpoints"]`. No `kserve` or `triton`. Running `forgelm deploy ... --target kserve` exits with argparse error. + +**Closure plan:** Not mentioned. +**Roadmap:** Not mentioned. + +--- + +### GH-019 — Ghost chat flags: `--base`, `--top-p`, `--safety`, `--load` + +**Severity:** Medium +**References:** `docs/usermanuals/en/reference/cli.md` chat section + +**Documented form:** +```text +forgelm chat CHECKPOINT [--base BASE_MODEL] [--temperature 0.7] [--top-p 0.9] + [--max-tokens 1024] [--system "..."] [--safety on|off] [--load PATH] +``` + +**Code reality:** Parser registered flags for `forgelm chat`: +`model_path`, `--adapter`, `--system`, `--temperature`, `--max-new-tokens`, `--no-stream`, `--load-in-4bit`, `--load-in-8bit`, `--trust-remote-code`, `--backend` + +Missing: `--base`, `--top-p`, `--safety`, `--load`, `--max-tokens` (actual flag is `--max-new-tokens`). + +**Closure plan:** Not mentioned. + +--- + +### GH-020 — Ghost ingest flags: `--format`, `--language`, `--include`, `--exclude`, `--pii-locale` + +**Severity:** Medium +**References:** `docs/usermanuals/en/reference/cli.md` ingest section + +**Documented form:** +```text +forgelm ingest INPUT_DIR --output PATH.jsonl + [--strategy tokens|markdown|paragraph|sentence] + [--max-tokens N] [--pii-mask] [--secrets-mask] + [--pii-locale tr|de|fr|us] [--language LANG] + [--include "*.pdf,*.md"] [--exclude "drafts/*"] + [--format raw|instructions|qa] +``` + +**Code reality mismatches:** +- `--strategy` choices: `sliding|paragraph|markdown` (not `tokens` or `sentence`) +- `--max-tokens` → actual flag is `--chunk-tokens` +- `--pii-locale` → actual flag is `--pii-ml-language` +- `--language` → does not exist +- `--include` / `--exclude` → do not exist +- `--format raw|instructions|qa` → does not exist (ingest has no `--format`; export has `--format gguf`) + +**Closure plan:** Not mentioned. + +--- + +### GH-021 — Ghost chat slash commands: `/load`, `/top_p`, `/max_tokens`, `/safety` + +**Severity:** Low +**References:** `docs/usermanuals/en/reference/cli.md` + +**Documented:** `/reset`, `/save`, `/load`, `/system`, `/temperature`, `/top_p`, `/max_tokens`, `/safety`, `/help`, `/quit` + +**Code reality** (`forgelm/chat.py:151-157`): `/exit`, `/quit`, `/reset`, `/save`, `/temperature`, `/system`, `/help` + +Missing: `/load`, `/top_p`, `/max_tokens`, `/safety` + +--- + +### GH-022 — Export quant option `q6_k` not registered + +**Severity:** Low +**References:** `docs/usermanuals/en/reference/cli.md` export section + +**Documented:** `--quant q4_k_m|q5_k_m|q6_k|q8_0|q3_k_m|q2_k|fp16` + +**Code reality:** Parser choices: `["q2_k", "q3_k_m", "q4_k_m", "q5_k_m", "q8_0", "f16"]` +- `q6_k` is documented but not registered +- Docs say `fp16`, parser says `f16` (different string — `forgelm export ... --quant fp16` would error) + +--- + +### GH-023 — `ingestion.retention.raw_documents.ttl_days` config key + +**Severity:** Medium — appears in GDPR compliance guide +**References:** 2 occurrences + +| File | Lines | Context | +|---|---|---| +| `docs/usermanuals/en/compliance/gdpr.md` | 57-60 | GDPR Art. 5 storage limitation — automated retention enforcement | +| `docs/usermanuals/tr/compliance/gdpr.md` | 57-60 | TR mirror | + +**Documented form:** +```yaml +ingestion: + retention: + raw_documents: + ttl_days: 90 + audit_reports: + ttl_days: 365 +``` + +**Code reality:** `forgelm/config.py` `ForgeConfig` top-level keys: `model`, `lora`, `training`, `data`, `auth`, `evaluation`, `webhook`, `distributed`, `merge`, `compliance`, `risk_assessment`, `monitoring`, `synthetic`. No `ingestion` key. `DataConfig` has no `retention` field. A config with this key would fail Pydantic validation (`extra="forbid"`). + +**Closure plan:** Faz 21 mentions `RetentionConfig` Pydantic block (line 682: "forgelm/config.py — RetentionConfig Pydantic block"), but this has not shipped. The GDPR docs were written ahead of the implementation. + +--- + +### GH-024 — `FORGELM_RESUME_TOKEN` env var + +**Severity:** Low +**References:** `docs/usermanuals/en/reference/cli.md` environment table + +**Documented:** "Token for the API-based human approval flow" + +**Code reality:** `grep -rn "FORGELM_RESUME_TOKEN" forgelm/` → no results. Not used anywhere in the codebase. + +--- + +### GH-025 — `FORGELM_CACHE_DIR` env var + +**Severity:** Low +**References:** `docs/usermanuals/en/reference/cli.md` environment table + +**Documented:** "ForgeLM-specific cache location" + +**Code reality:** `grep -rn "FORGELM_CACHE_DIR" forgelm/` → no results. Not used anywhere in the codebase. `HF_HOME` exists (standard HF env var) and is documented correctly alongside this. + +--- + +## Closure Plan Status Summary + +| Ghost | In closure-plan-202604300906.md | Phase | +|---|---|---| +| `forgelm doctor` | ❌ not mentioned | — | +| `forgelm cache-models` | ❌ not mentioned | — | +| `forgelm cache-tasks` | ❌ not mentioned | — | +| `forgelm verify-annex-iv` | ❌ not mentioned | — | +| `forgelm safety-eval` | ❌ not mentioned | — | +| `forgelm trend` | ❌ not mentioned | — | +| `forgelm approvals` | ❌ not mentioned (`approve`+`reject` are, listing is not) | Faz 9 partial | +| `forgelm verify-log` | ❌ not mentioned (`verify-audit` is) | Faz 6 alias confusion | +| `forgelm verify-gguf` | ❌ not mentioned | — | +| `forgelm compare-runs` | ❌ not mentioned | — | +| `forgelm benchmark` | ⚠️ `--benchmark-only` flag exists, subcommand form not | — | +| `forgelm batch-chat` | ❌ not mentioned | — | +| `forgelm purge` | ✅ Faz 21 | Wave 2 | +| `forgelm reverse-pii` | ❌ not mentioned | — | +| `forgelm merge-sweep` | ❌ not mentioned | — | +| `--export-bundle` | ⚠️ renamed to `--compliance-export` — QMS docs not updated | — | +| `--estimate-cost` | ❌ not mentioned | Phase 13 Pro (v0.6.0) | +| Deploy `kserve`/`triton` | ❌ not mentioned | — | +| Chat `--base/--top-p/--safety/--load` | ❌ not mentioned | — | +| Ingest `--format/--language/--include/--exclude/--pii-locale` | ❌ not mentioned | — | +| Chat `/load /top_p /max_tokens /safety` | ❌ not mentioned | — | +| Export `q6_k` / `fp16` quant | ❌ not mentioned | — | +| `ingestion.retention` config | ⚠️ `RetentionConfig` planned in Faz 21 | Wave 2 | +| `FORGELM_RESUME_TOKEN` | ❌ not mentioned | — | +| `FORGELM_CACHE_DIR` | ❌ not mentioned | — | + +--- + +## Roadmap Directory Status Summary + +Searched all files in `docs/roadmap/` (`completed-phases.md`, `releases.md`, `risks-and-decisions.md`, `phase-10-5-quickstart.md`, `phase-10-post-training.md`, `phase-11-5-backlog.md`, `phase-11-data-ingestion.md`, `phase-12-5-backlog.md`, `phase-12-data-curation-maturity.md`, `phase-13-pro-cli.md`, `phase-14-pipeline-chains.md`). + +**Findings:** +- `forgelm pro` subcommand group: ✅ explicitly in `phase-13-pro-cli.md` as v0.6.0 Pro CLI tier (gated on ≥1K PyPI installs + ≥2 paying contracts) +- `forgelm trend` + `forgelm compare-runs`: contextually belong to `phase-13-pro-cli.md` "Web dashboard — experiment browser" (task 2) but are NOT explicitly named there +- All others: not found in any roadmap file + +--- + +## Prioritized Action Recommendations + +### Tier 1 — No-code fixes (naming/alias corrections only) + +| ID | Action | Effort | Files | +|---|---|---|---| +| GH-008 | Rename `verify-log` → `verify-audit` in docs (no code change needed) | XS | 4 files | +| GH-011 | Correct `benchmarks.md` — replace `forgelm benchmark ...` with `forgelm --config ... --benchmark-only PATH` | XS | 2 files | +| GH-016 | Rename `--export-bundle` → `--compliance-export` in QMS SOPs | XS | 2 files | +| GH-022 | Fix export quant in cli.md: `q6_k` → remove; `fp16` → `f16` | XS | 1 file | +| GH-019 | Fix chat flags in cli.md: remove `--base`, `--top-p`, `--safety`, `--load`; add `--max-new-tokens` | XS | 1 file | +| GH-020 | Fix ingest flags in cli.md: `--max-tokens` → `--chunk-tokens`; `--pii-locale` → `--pii-ml-language`; remove `--format`, `--language`, `--include`, `--exclude`; fix `--strategy` choices | S | 1 file | +| GH-021 | Fix chat slash commands in cli.md: remove `/load`, `/top_p`, `/max_tokens`, `/safety` | XS | 1 file | +| GH-018 | Fix deploy targets in cli.md: remove `kserve`, `triton` | XS | 1 file | +| GH-017 | Remove `--estimate-cost` from cli.md flags table (or add "planned v0.6.0" note) | XS | 1 file | +| GH-024, GH-025 | Remove `FORGELM_RESUME_TOKEN` and `FORGELM_CACHE_DIR` from env table in cli.md | XS | 1 file | + +### Tier 2 — Implement or remove before v0.5.5 ships + +Documenting non-existent commands violates the "documentation drift" principle from CLAUDE.md. For each, decide: implement or remove. + +| ID | Command | Implement complexity | Remove complexity | +|---|---|---|---| +| GH-007 | `forgelm approvals` | XS — scan staging dirs + audit log for unresolved events; builds on Faz 9 plumbing | S — 6 files | +| GH-004 | `forgelm verify-annex-iv` | S — read JSON, check required Annex IV fields, re-hash | S — 4 files | +| GH-001 | `forgelm doctor` | M — Python/torch/CUDA/GPU/deps check; `--offline` variant | L — 38 refs across 10 files, re-write getting-started flow | +| GH-023 | `ingestion.retention` config | M — implement `RetentionConfig` Pydantic model + add `ingestion` top-level key to `ForgeConfig` (Faz 21 dependency) | S — 2 files | + +### Tier 3 — Add to closure plan as named phases + +These represent real user value but are clearly out of v0.5.5 scope. Until implemented, remove from usermanuals or add a "planned" callout. + +| ID | Command | Natural home | +|---|---|---| +| GH-002, GH-003 | `cache-models`, `cache-tasks` | New phase — "Air-gap pre-cache commands" | +| GH-005 | `safety-eval` | New phase — "Standalone safety-eval subcommand" | +| GH-009 | `verify-gguf` | Add to Faz 28 or new minor phase | +| GH-006, GH-010 | `trend`, `compare-runs` | Phase 13 Pro CLI scope (already planned, add explicit names) | +| GH-012 | `batch-chat` | Add to `forgelm chat` as `--batch` flag or Phase 13 | +| GH-014 | `reverse-pii` | New phase (GDPR follow-up to Faz 21) | +| GH-015 | `merge-sweep` | Phase 14 pipeline chains or new phase | + +**Until implemented:** Remove from usermanuals or add: +```markdown +> **Note:** This command is planned for a future release and not yet available in v0.5.5. +``` + +--- + +## Affected Files by Ghost + +Quick reference for cleanup or implementation work: + +```text +GH-001 (doctor): + docs/usermanuals/en/getting-started/installation.md:100,103,111 + docs/usermanuals/en/getting-started/first-run.md:12,29,32,41 + docs/usermanuals/en/operations/troubleshooting.md:8,168,173,179 + docs/usermanuals/en/operations/air-gap.md:115,124,148 + docs/usermanuals/en/reference/cli.md:15 + docs/usermanuals/tr/* (mirrors of all above) + +GH-002 (cache-models): + docs/usermanuals/en/operations/air-gap.md:25,54 + docs/usermanuals/en/operations/docker.md:101,104 + docs/usermanuals/en/reference/cli.md:28 + docs/usermanuals/tr/* (mirrors) + +GH-003 (cache-tasks): + docs/usermanuals/en/operations/air-gap.md:30,101 + docs/usermanuals/en/operations/docker.md:107 + docs/usermanuals/en/reference/cli.md:29 + docs/usermanuals/en/reference/yaml-templates.md:315 + docs/usermanuals/tr/* (mirrors) + +GH-004 (verify-annex-iv): + docs/usermanuals/en/compliance/annex-iv.md:137,144 + docs/usermanuals/en/operations/troubleshooting.md:155 + docs/usermanuals/en/reference/cli.md:25 + docs/usermanuals/tr/* (mirrors) + +GH-005 (safety-eval): + docs/usermanuals/en/evaluation/safety.md:76,91 + docs/usermanuals/en/deployment/gguf-export.md:136 + docs/usermanuals/en/reference/cli.md:20 + docs/usermanuals/tr/* (mirrors) + +GH-006 (trend): + docs/usermanuals/en/evaluation/trend-tracking.md:80,107 + docs/usermanuals/en/reference/cli.md:30 + docs/usermanuals/tr/* (mirrors) + +GH-007 (approvals): + docs/usermanuals/en/compliance/human-oversight.md:131,138 + docs/usermanuals/en/reference/cli.md:33 + docs/usermanuals/tr/* (mirrors) + +GH-008 (verify-log → verify-audit): + docs/usermanuals/en/compliance/audit-log.md:59,65 + docs/usermanuals/en/reference/cli.md:26 + docs/usermanuals/tr/* (mirrors) + +GH-009 (verify-gguf): + docs/usermanuals/en/deployment/gguf-export.md:80 + docs/usermanuals/en/reference/cli.md:27 + docs/usermanuals/tr/* (mirrors) + +GH-010 (compare-runs): + docs/usermanuals/en/operations/experiment-tracking.md:116 + docs/usermanuals/en/evaluation/trend-tracking.md:116 + docs/usermanuals/tr/* (mirrors) + +GH-011 (benchmark wrong form): + docs/usermanuals/en/evaluation/benchmarks.md:71 + docs/usermanuals/en/reference/cli.md:19 + docs/usermanuals/tr/* (mirrors) + +GH-012 (batch-chat): + docs/usermanuals/en/deployment/chat.md:133 + docs/usermanuals/en/reference/cli.md:22 + docs/usermanuals/tr/* (mirrors) +``` diff --git a/docs/product_strategy-tr.md b/docs/product_strategy-tr.md index 93d1e9a8..2c99278d 100644 --- a/docs/product_strategy-tr.md +++ b/docs/product_strategy-tr.md @@ -36,7 +36,7 @@ ForgeLM, LLM ince ayarını (fine-tuning) **güvenli, denetlenebilir ve tekrarla - Mad. 14 İnsan Gözetimi → `require_human_approval: true` + çıkış kodu 4 - Mad. 15 Doğruluk ve Sağlamlık → `model_integrity.json` + güvenlik değerlendirme sonuçları - **Müdahaleye dayanıklı denetim günlüğü**: SHA-256 zinciri — her kayıt bir öncekinin hash'ini içerir; silme veya değiştirme tespit edilebilir -- **İnsan onayı kapısı**: Çıkış kodu 4, CI/CD orchestrator'larına "insan onayı bekleniyor" sinyali verir — özel pipeline mantığı gerekmez +- **İnsan onayı kapısı**: Çıkış kodu 4, CI/CD orchestrator'larına "insan onayı bekleniyor" sinyali verir; nihai model `final_model.staging/` dizinine iner — operatörler `forgelm approve ` ile promote eder (atomik rename → `final_model/`) veya `forgelm reject ` ile reddeder; her iki karar da denetim zincirine işlenir ### 3. Config-Tabanlı CI/CD @@ -65,7 +65,7 @@ ForgeLM'in temel mimari kimliği: bir YAML dosyası bir eğitim çalışmasını **Profil:** Bankacılık, sağlık, savunma, kamu. Tescilli veriyi harici API'lere gönderemez. EU AI Act (AB), KVKK/BDDK (Türkiye), HIPAA (ABD) veya eşdeğer düzenlemelere tabidir. Tam denetim izleriyle şirket içi veya izole ortamda çalışmayı gerektirir. -**ForgeLM'i neden seçerler:** Tam EU AI Act compliance kanıt paketini otomatik üreten tek ince ayar aracı. Şirket içi Docker dağıtımı. `trust_remote_code: false` varsayılan. İnsan onayı kapısı (çıkış kodu 4) mevcut yönetişim süreçlerine uyar. SHA-256 zinciriyle denetim günlüğü kayıt tutma gereksinimlerini karşılar. +**ForgeLM'i neden seçerler:** Tam EU AI Act compliance kanıt paketini otomatik üreten tek ince ayar aracı. Şirket içi Docker dağıtımı. `trust_remote_code: false` varsayılan. İnsan onayı kapısı (çıkış kodu 4) üretim dağıtımını erteler; nihai model `final_model.staging/` dizinine iner; operatörler promote için `forgelm approve `, reddetmek içinse `forgelm reject ` çalıştırır. SHA-256 zinciriyle denetim günlüğü kayıt tutma gereksinimlerini karşılar. ### Üçüncül: Bağımsız Araştırmacılar ve Geliştiriciler diff --git a/docs/product_strategy.md b/docs/product_strategy.md index 11b9d135..5471db63 100644 --- a/docs/product_strategy.md +++ b/docs/product_strategy.md @@ -36,7 +36,7 @@ As of August 2, 2026, EU AI Act applies to high-risk AI systems. ForgeLM is the - Art. 14 Human Oversight → `require_human_approval: true` + exit code 4 - Art. 15 Accuracy & Robustness → `model_integrity.json` + safety evaluation results - **Tamper-evident audit log**: SHA-256 hash chain — each entry includes the hash of the previous entry; deletion or modification is detectable -- **Human approval gate**: exit code 4 signals "awaiting human approval" to CI/CD orchestrators — no custom pipeline logic needed +- **Human approval gate**: exit code 4 signals "awaiting human approval" to CI/CD orchestrators; the final model lands in `final_model.staging/` — operators run `forgelm approve ` to promote (atomic rename to `final_model/`) or `forgelm reject ` to discard, with both decisions recorded in the audit chain ### 3. Config-Driven CI/CD @@ -65,7 +65,7 @@ ForgeLM's core architectural identity: a YAML file fully describes a training ru **Profile:** Banking, healthcare, defense, government. Cannot send proprietary data to external APIs. Face EU AI Act (EU), KVKK/BDDK (Turkey), HIPAA (US), or equivalent regulations. Require on-premise or air-gapped execution with full audit trails. -**Why ForgeLM:** The only fine-tuning tool that produces a complete EU AI Act compliance evidence bundle automatically. On-premise Docker deployment. `trust_remote_code: false` by default. Human approval gate (exit code 4) fits into existing governance workflows. Audit log with SHA-256 hash chain satisfies record-keeping requirements. +**Why ForgeLM:** The only fine-tuning tool that produces a complete EU AI Act compliance evidence bundle automatically. On-premise Docker deployment. `trust_remote_code: false` by default. Human approval gate (exit code 4) defers production deployment; the final model lands in `final_model.staging/`; operators run `forgelm approve ` to promote, or `forgelm reject ` to discard. Audit log with SHA-256 hash chain satisfies record-keeping requirements. ### Tertiary: Independent Researchers and Developers diff --git a/docs/reference/audit_event_catalog-tr.md b/docs/reference/audit_event_catalog-tr.md index a4b80f39..c14c3627 100644 --- a/docs/reference/audit_event_catalog-tr.md +++ b/docs/reference/audit_event_catalog-tr.md @@ -35,9 +35,9 @@ Hash zinciri, satır diske düştükten (`flush` + `fsync`) sonra ilerler; kirli | Event | Ne zaman emit edilir | Payload | Madde | |------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|-------| -| `human_approval.required` | `requires_human_approval: true` işaretli bir kapı pipeline'ı duraklatıp operatör kararını bekler. | `gate`, `reason`, `metrics` | 14 | -| `human_approval.granted` | Operatör duraklatılan kapıyı onayladı. _(Faz 9 — placeholder; henüz emit edilmiyor.)_ | `gate`, `approver`, `comment` | 14 | -| `human_approval.rejected` | Operatör duraklatılan kapıyı reddetti. _(Faz 9 — placeholder; henüz emit edilmiyor.)_ | `gate`, `approver`, `comment` | 14 | +| `human_approval.required` | `requires_human_approval: true` işaretli bir kapı pipeline'ı duraklatıp operatör kararını bekler. | `gate`, `reason`, `metrics`, `staging_path`, `run_id` | 14 | +| `human_approval.granted` | Operatör duraklatılan kapıyı `forgelm approve` ile onayladı. | `gate`, `approver`, `comment`, `run_id`, `promote_strategy` | 14 | +| `human_approval.rejected` | Operatör duraklatılan kapıyı `forgelm reject` ile reddetti. | `gate`, `approver`, `comment`, `run_id`, `staging_path` | 14 | ### Madde 15 — Model Bütünlüğü (auto-revert + güvenlik) @@ -58,7 +58,7 @@ Hash zinciri, satır diske düştükten (`flush` + `fsync`) sonra ilerler; kirli | Event | Ne zaman emit edilir | Payload | Madde | |-----------------------------|--------------------------------------------------------------------------------------------------------|----------------------------------|-------| -| `cli.legacy_flag_invoked` | Deprecated bir CLI flag'i kullanıldı. _(Faz 13 — placeholder; henüz emit edilmiyor.)_ | `flag`, `replacement`, `version` | 12 | +| `cli.legacy_flag_invoked` | Deprecated bir CLI flag'i kullanıldı (örn. `--data-audit`). | `flag`, `replacement`, `version` | 12 | ### Audit-sistem event'leri (meta) diff --git a/docs/reference/audit_event_catalog.md b/docs/reference/audit_event_catalog.md index da288b3f..25892755 100644 --- a/docs/reference/audit_event_catalog.md +++ b/docs/reference/audit_event_catalog.md @@ -35,9 +35,9 @@ The hash chain advances after the line lands on disk (`flush` + `fsync`), so an | Event | When emitted | Payload | Article | |------------------------------|---------------------------------------------------------------------------------------------------------|------------------------------------------------------|---------| -| `human_approval.required` | A gate marked `requires_human_approval: true` paused the pipeline awaiting an operator decision. | `gate`, `reason`, `metrics` | 14 | -| `human_approval.granted` | Operator approved the paused gate. _(Faz 9 — placeholder; not yet emitted.)_ | `gate`, `approver`, `comment` | 14 | -| `human_approval.rejected` | Operator rejected the paused gate. _(Faz 9 — placeholder; not yet emitted.)_ | `gate`, `approver`, `comment` | 14 | +| `human_approval.required` | A gate marked `requires_human_approval: true` paused the pipeline awaiting an operator decision. | `gate`, `reason`, `metrics`, `staging_path`, `run_id` | 14 | +| `human_approval.granted` | Operator approved the paused gate via `forgelm approve`. | `gate`, `approver`, `comment`, `run_id`, `promote_strategy` | 14 | +| `human_approval.rejected` | Operator rejected the paused gate via `forgelm reject`. | `gate`, `approver`, `comment`, `run_id`, `staging_path` | 14 | ### Article 15 — Model Integrity (auto-revert + safety) @@ -58,7 +58,7 @@ The hash chain advances after the line lands on disk (`flush` + `fsync`), so an | Event | When emitted | Payload | Article | |-----------------------------|----------------------------------------------------------------------------------------------------|----------------------------------|---------| -| `cli.legacy_flag_invoked` | A deprecated CLI flag was used. _(Faz 13 — placeholder; not yet emitted.)_ | `flag`, `replacement`, `version` | 12 | +| `cli.legacy_flag_invoked` | A deprecated CLI flag was used (e.g., `--data-audit`). | `flag`, `replacement`, `version` | 12 | ### Audit-system events (meta) diff --git a/docs/reference/data_preparation-tr.md b/docs/reference/data_preparation-tr.md index 4d372db6..297a88b3 100644 --- a/docs/reference/data_preparation-tr.md +++ b/docs/reference/data_preparation-tr.md @@ -17,7 +17,7 @@ ForgeLM sırasıyla kendi içerisinde eşleştirme yapabileceği sütun başlık ## Örnek JSONL Yapısı (Yerel Veri) -Eğer sisteme şirketinize ve size özel bir şirket içi model veri aktarımı yapmak istiyorsanız, bunu her satırın birer JSON nesnesi olduğu `.jsonl` dosyası şekline getirebilirsiniz. `.jsonl` yapınızın var olan `dataset_name_or_path` dosyası üzerindeki yoluna (config kısmından) dosyanızın tam adresini (mutlak konumunu) vermeniz yeterli olacaktır. +Eğer sisteme şirketinize ve size özel bir şirket içi model veri aktarımı yapmak istiyorsanız, bunu her satırın birer JSON nesnesi olduğu `.jsonl` dosyası şekline getirebilirsiniz. `.jsonl` yapınızın var olan `dataset_name_or_path` dosyası üzerindeki yoluna (config kısmından) dosyanızın tam adresini (mutlak konumunu) vermeniz yeterli olacaktır. Aşağıda standart kabul görmüş örnek JSONL yapısı: ```json diff --git a/docs/reference/data_preparation.md b/docs/reference/data_preparation.md index 52546155..81ab8584 100644 --- a/docs/reference/data_preparation.md +++ b/docs/reference/data_preparation.md @@ -25,7 +25,7 @@ If you are bringing custom company data, format it into a `.jsonl` file where ea ``` ## Chat Templates & Formatting -In 2026, modern conversational fine-tuning no longer relies on manual string formatting (e.g., `[SYSTEM]...[USER]...`). +In 2026, modern conversational fine-tuning no longer relies on manual string formatting (e.g., `[SYSTEM]...[USER]...`). Instead, ForgeLM utilizes Hugging Face's `tokenizer.apply_chat_template()`. This means ForgeLM dynamically understands the architecture of the model you are using (be it Llama-3, Mistral, Gemma, or Qwen) and automatically formats your data into that specific model's native conversational token structure (e.g., `<|im_start|>user\n...<|im_end|>`). diff --git a/docs/standards/architecture.md b/docs/standards/architecture.md index 6c9fb9a0..f881e2cb 100644 --- a/docs/standards/architecture.md +++ b/docs/standards/architecture.md @@ -172,7 +172,6 @@ From [`pyproject.toml`](../../pyproject.toml): | `merging` | mergekit | Any | | `ingestion` | pypdf, python-docx, ebooklib, beautifulsoup4, langdetect, xxhash | Any | | `ingestion-scale` | datasketch (MinHash LSH dedup; Phase 12, opt-in) | Any | -| `ingestion-secrets` | detect-secrets (credential leakage scanner; Phase 12, opt-in regex fallback when missing) | Any | | `ingestion-pii-ml` | presidio-analyzer (ML-NER for person/organization/location PII; Phase 12.5, opt-in) | Any | | `export` | llama-cpp-python (GGUF conversion) | Linux/macOS | | `chat` | rich (terminal rendering) | Any | diff --git a/docs/standards/code-review.md b/docs/standards/code-review.md index 7fd0a30b..285e21b2 100644 --- a/docs/standards/code-review.md +++ b/docs/standards/code-review.md @@ -191,3 +191,16 @@ forgelm --config config_template.yaml --dry-run ``` If that passes and the PR template is honest, you're ready. + +## Pre-commit hooks (optional) + +The repository ships a [`.pre-commit-config.yaml`](../../.pre-commit-config.yaml) +that mirrors the lint, format, and secret-scanning steps already run in CI. +Pre-commit hooks are an **optional ergonomic optimization** for local +development — contributors who run `pre-commit install` get faster feedback, +but the hooks are **not** part of the PR enforcement contract. CI is the +single enforcement boundary: every required check (`ruff check`, `ruff format +--check`, `gitleaks`, the test matrix, the validate job) runs on every push +and pull request regardless of whether the author installed the hooks +locally. Skipping a hook with `SKIP=...` only suppresses the local run; it +never bypasses CI. diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index dd24702c..89411ec3 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -124,6 +124,86 @@ except Exception as e: **Rule:** If you `except`, you must either (a) recover with a known-good fallback, (b) log and re-raise with `from e`, or (c) convert to a domain exception with `from e`. "Log and swallow" is a bug unless the failure is explicitly non-fatal (webhooks, cleanup). +## Best-effort artefact carve-out + +The default rule above is "narrow class or don't catch." There is exactly one sanctioned escape hatch, and it has a precise scope. + +### When the carve-out applies + +The only sanctioned form for keeping `except Exception:` is + +```python +except Exception as e: # noqa: BLE001 — best-effort: +``` + +and "best-effort" has a single specific meaning here: + +> An **outer** error path is already responsible for the primary failure. +> This catch protects a **secondary side effect** — audit log emission, webhook delivery, cleanup of advisory artefacts (model card, integrity checksum, governance report, trend file, deployer instructions) — from masking the primary failure. + +If you cannot point at the outer error handler that owns the primary failure, you do **not** have a best-effort path; you have an unknown failure mode that should be diagnosed and either narrowed or surfaced. + +### Mandatory hygiene for every BLE001 site + +1. The `# noqa: BLE001` comment carries a one-line rationale that names the artefact and explains why a wider class is genuinely infeasible. +2. The handler logs at `WARNING` (or `ERROR` for outage events) so the failure shows up in the run log even though the run continues. +3. A surrounding error path or audit event records the primary failure independently — the BLE001 catch is the secondary, not the primary, surface. +4. **Never** use BLE001 to dodge thinking about the failure modes. If the narrow tuple is `(OSError, ValueError, TypeError)`, write that — only fall back to BLE001 when the protected operation crosses a third-party library surface that documents a wide error tail (HF Hub repository errors, Pydantic mixed validation/runtime errors, etc.). + +### Forbidden forms + +The bare `except:` form is **forbidden** everywhere, no exceptions. It catches `KeyboardInterrupt` and `SystemExit` and routinely masks `Ctrl-C` during long training runs. Ruff `E722` enforces this in CI. + +`except Exception: pass` (no log, no rationale, no re-raise) is **forbidden**. The BLE001 carve-out exists so the deliberate cases are visible; silent swallowing is what the carve-out replaces. + +### Examples + +**Good — narrow class first:** + +```python +try: + with open(trend_path, "a", encoding="utf-8") as f: + f.write(json.dumps(entry) + "\n") +except (OSError, TypeError, ValueError) as e: + # OSError: filesystem (permissions, full disk). + # TypeError/ValueError: json.dumps on unexpected entry shape. + # Trend logging is non-fatal — a missing entry must not abort the + # safety pass that already concluded successfully. + logger.warning("Failed to write safety trend entry: %s", e) +``` + +**Good — best-effort BLE001 with rationale:** + +```python +try: + info = HfApi().dataset_info(dataset_path) + if info.sha: + fingerprint["hf_revision"] = info.sha +except Exception as e: # noqa: BLE001 — best-effort revision pin; HF Hub surface raises a wide error tail (HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError, OSError, ValueError) and enumerating them couples this module to huggingface_hub internals. + logger.warning("HF Hub revision pin skipped for '%s': %s", dataset_path, e) +``` + +**Bad — silent swallow with no rationale:** + +```python +try: + do_critical_thing() +except Exception: # ❌ no narrow class, no BLE001, no rationale, no log + pass +``` + +**Bad — BLE001 used to dodge thinking:** + +```python +try: + config = ForgeConfig.load(path) +except Exception as e: # noqa: BLE001 — "just in case" ❌ + logger.warning("Config load failed: %s", e) + config = ForgeConfig() +``` + +The protected operation is config validation. Pydantic raises `ValidationError`, the loader raises `FileNotFoundError` and `yaml.YAMLError`. That is a precise tuple — write it. + ## User-facing error messages The audience is an engineer reading a CLI terminal in the dark. Messages must be: diff --git a/docs/standards/localization.md b/docs/standards/localization.md index fd422037..d8a8b950 100644 --- a/docs/standards/localization.md +++ b/docs/standards/localization.md @@ -7,6 +7,13 @@ **User-facing documentation is bilingual (EN + TR). Code, CLI output, config keys, logs, and internal dev docs are English only.** +## Supported languages + +- **EN + TR (authored).** All user-facing docs and user-manual pages are written and reviewed in both English and Turkish. EN+TR mirror parity is mandatory for the doc tree — see "Structural mirror rule" below. +- **DE / FR / ES / ZH (site UI translated; user-manual content English-fallback).** The site language picker exposes all six languages, and `site/js/translations.js` carries the marketing/product copy in DE/FR/ES/ZH. The user-manual content (`docs/usermanuals/...`) is **only authored in EN + TR**; when a deferred-language user picks DE/FR/ES/ZH, the manual side falls back to English via the i18n chain (`tableForLang(...) → DEFAULT='en'`). `tools/build_usermanuals.py` therefore emits JS bags only for the authored set (`en`, `tr`); the deferred languages reuse the EN bag through the fallback path. + +A language is "site-translated" when its strings live in `site/js/translations.js`. A language is "manual-authored" only when `docs/usermanuals//` content is reviewed by a native or near-native speaker. + Rationale: - ForgeLM's market includes a significant Turkish-speaking segment (banking + regulatory + research) served by Turkish-language docs. diff --git a/docs/standards/release.md b/docs/standards/release.md index 82483a50..b666ab2d 100644 --- a/docs/standards/release.md +++ b/docs/standards/release.md @@ -122,12 +122,15 @@ Done manually by the maintainer (or a bot when automated): ### Automated (by `publish.yml`) -On tag push matching `v*`: +On tag push matching `v*` the workflow chains three jobs (full description +under [Release prep workflow](#release-prep-workflow) below): -1. Build wheel and sdist (`python -m build`). -2. Verify with `twine check`. -3. Publish to PyPI using OIDC trusted publishing (no API key needed). -4. Create GitHub Release with auto-extracted changelog excerpt. +1. **`build`** — produce wheel + sdist, verify with `twine check`, upload + as the `dist` artifact. +2. **`cross-os-tests`** — 3 OS × 4 Python = 12 combos install the packaged + wheel (not editable), run pytest, generate a per-combo CycloneDX SBOM. +3. **`publish`** — OIDC trusted publishing to PyPI; only runs after every + matrix combo is green. ### After the release @@ -138,6 +141,102 @@ On tag push matching `v*`: 5. [ ] Bump `pyproject.toml` version to next pre-release (`0.4.1rc1`). 6. [ ] Update [marketing/marketing_strategy_roadmap.md](../marketing/marketing_strategy_roadmap.md) metrics row. +## Release prep workflow + +The release pipeline is a single GitHub Actions workflow, +[`.github/workflows/publish.yml`](../../.github/workflows/publish.yml). It +fires when a tag matching `v*` is pushed (e.g. `v0.5.0`, `v0.5.1rc1`) and +chains three jobs whose `needs:` dependencies form a strict DAG. + +### Trigger + +```yaml +on: + push: + tags: ['v*'] +``` + +Pushing the tag is the contract — no manual `workflow_dispatch`, no +release-event listener. The same procedure works whether you tag locally +and `git push --tags` or cut the tag from the GitHub Releases UI. + +### Job 1 — `build` (`ubuntu-latest`, Python 3.11) + +Runs `python -m build` to produce both `dist/*.whl` and `dist/*.tar.gz`, +then `twine check dist/*` to validate metadata, then uploads `dist/` as a +workflow artifact named `dist`. Subsequent jobs download this same +artifact rather than rebuilding, so every matrix combo and the publish +step exercise byte-identical files. + +### Job 2 — `cross-os-tests` (matrix: 3 OS × 4 Python = 12 combos) + +`needs: build`. Strategy: + +```yaml +fail-fast: false +matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ['3.10', '3.11', '3.12', '3.13'] +``` + +`fail-fast: false` is deliberate — one Python version blowing up on one +OS must not abort the other 11 combos, otherwise the matrix loses its job +as a breadth probe. + +Each combo: + +1. Downloads the `dist` artifact. +2. Installs the wheel via `python -m pip install dist/*.whl` — **packaged + wheel, not editable**. This is the load-bearing detail. An editable + install (`pip install -e .`) does not exercise wheel build, package + data inclusion, console-script generation, or cross-OS path handling. + By installing the same wheel that PyPI users will pull we guarantee + that what passes here is what they get. +3. On Linux only, additionally installs `forgelm[qlora,ingestion,eval]` + to cover the heaviest extras path. `qlora` pulls `bitsandbytes`, which + only ships Linux wheels — we never claim Windows / macOS support for + that extra. +4. Runs `pytest tests/ -q --ignore=tests/test_cost_estimation.py`. The + cost-estimation test is excluded because its pricing fixture drifts on + a different cadence than the release matrix; a stale fixture would + break the chain for reasons unrelated to packaging health. +5. Generates a CycloneDX 1.5 SBOM via `python tools/generate_sbom.py`, + redirected to `sbom-${{ matrix.os }}-py${{ matrix.python }}.json`. +6. Uploads each SBOM as its own artifact — 12 per release tag, retained + alongside the workflow run for downstream supply-chain audits. + +All steps that may run on Windows declare `shell: bash` so the same +script fragments work on Linux, macOS, and Windows runners (Windows +defaults to PowerShell otherwise). + +### Job 3 — `publish` (`ubuntu-latest`, environment: `pypi`) + +`needs: cross-os-tests`. Downloads the `dist` artifact and hands it to +[`pypa/gh-action-pypi-publish@release/v1`](https://github.com/pypa/gh-action-pypi-publish). +Authentication is OIDC trusted publishing — no PyPI API token is stored; +GitHub Actions mints a short-lived token scoped to the `pypi` environment. +The job sets `permissions: { id-token: write, contents: read }`; nothing +else is granted. + +The `pypi` GitHub environment is configured in repository settings to +require manual approval for production tags if extra control is wanted; +the workflow itself does not gate beyond the `needs:` chain. + +### Failure semantics + +The `needs:` chain is the safety net: + +- If `build` fails → neither `cross-os-tests` nor `publish` runs. +- If **any** `cross-os-tests` combo fails (with `fail-fast: false`, the + other 11 still run so the failure surface is visible) → `publish` does + not run. Nothing reaches PyPI. +- The tag remains in git either way; re-running the workflow after a fix + is the recovery path, not deleting + recreating the tag. + +This is by design: a release that ships only on Linux/3.11 because the +3.13 wheel was broken would silently degrade users on the un-tested +combos. Packaging hygiene is gated, not advisory. + ## Release cadence Current target: diff --git a/docs/usermanuals/en/compliance/human-oversight.md b/docs/usermanuals/en/compliance/human-oversight.md index 03052f15..071e03ef 100644 --- a/docs/usermanuals/en/compliance/human-oversight.md +++ b/docs/usermanuals/en/compliance/human-oversight.md @@ -51,9 +51,9 @@ The trainer halts after eval and prints: [2026-04-29 14:33:10] Human approval required. Run ID: abc123 Bundle: checkpoints/run/artifacts/ - - To approve: forgelm approve --run-id abc123 --reviewer "Cemil " - To reject: forgelm approve --run-id abc123 --reject --reason "..." + + To approve: forgelm approve abc123 --output-dir checkpoints/run --comment "..." + To reject: forgelm reject abc123 --output-dir checkpoints/run --comment "..." ``` The reviewer runs the approval command from any machine with access to the artifacts directory. ForgeLM verifies their identity via SSH key signing or env-set token, signs the audit log, and resumes promotion. diff --git a/docs/usermanuals/tr/compliance/human-oversight.md b/docs/usermanuals/tr/compliance/human-oversight.md index 3fab3ba0..d951916d 100644 --- a/docs/usermanuals/tr/compliance/human-oversight.md +++ b/docs/usermanuals/tr/compliance/human-oversight.md @@ -52,8 +52,8 @@ Trainer eval'den sonra durur ve yazdırır: Run ID: abc123 Bundle: checkpoints/run/artifacts/ - Onaylamak için: forgelm approve --run-id abc123 --reviewer "Cemil " - Reddetmek için: forgelm approve --run-id abc123 --reject --reason "..." + Onaylamak için: forgelm approve abc123 --output-dir checkpoints/run --comment "..." + Reddetmek için: forgelm reject abc123 --output-dir checkpoints/run --comment "..." ``` Reviewer artifacts dizinine erişimi olan herhangi bir makineden onay komutunu çalıştırır. ForgeLM SSH key signing veya env-set token üzerinden kimliği doğrular, audit log'u imzalar ve terfiyi sürdürür. diff --git a/forgelm/cli.py b/forgelm/cli.py deleted file mode 100644 index 758aac39..00000000 --- a/forgelm/cli.py +++ /dev/null @@ -1,1861 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import logging -import os -import sys -from importlib.metadata import PackageNotFoundError -from importlib.metadata import version as pkg_version -from pathlib import Path -from typing import Optional - -import yaml -from pydantic import ValidationError - -from .config import ConfigError, ForgeConfig, load_config - -# Module name used both for the package logger and as the `python -m` -# target when forgelm respawns itself for the quickstart subprocess flow. -_CLI_MODULE = "forgelm.cli" - -logger = logging.getLogger(_CLI_MODULE) - -# Exit codes — ForgeLM's public CLI contract. Any other value (e.g. signal- -# derived 128+N codes) is clamped to EXIT_TRAINING_ERROR before propagating. -EXIT_SUCCESS = 0 -EXIT_CONFIG_ERROR = 1 -EXIT_TRAINING_ERROR = 2 -EXIT_EVAL_FAILURE = 3 -EXIT_AWAITING_APPROVAL = 4 -_PUBLIC_EXIT_CODES = frozenset( - {EXIT_SUCCESS, EXIT_CONFIG_ERROR, EXIT_TRAINING_ERROR, EXIT_EVAL_FAILURE, EXIT_AWAITING_APPROVAL} -) - - -def _non_negative_int(value: str) -> int: - """argparse type for flags that must be ≥ 0 (e.g. --near-dup-threshold). - - Raising :class:`argparse.ArgumentTypeError` lets argparse render a - standard "invalid value" error and exit through its usual path, - without us having to thread parser.error() into every call site. - """ - try: - ivalue = int(value) - except (TypeError, ValueError) as exc: - raise argparse.ArgumentTypeError(f"invalid integer: {value!r}") from exc - if ivalue < 0: - raise argparse.ArgumentTypeError(f"value must be ≥ 0, got {ivalue}") - return ivalue - - -def _get_version() -> str: - try: - return pkg_version("forgelm") - except PackageNotFoundError: - from forgelm import __version__ - - return __version__ - - -def _setup_logging(log_level: str, json_format: bool = False) -> None: - """Configure structured logging for the entire forgelm package.""" - numeric_level = getattr(logging, log_level.upper(), logging.INFO) - - if json_format: - # Suppress human-readable logs when JSON output is requested - numeric_level = logging.WARNING - - logging.basicConfig( - level=numeric_level, - format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - - -def _add_common_subparser_flags(p: argparse.ArgumentParser, *, include_output_format: bool) -> None: - """Register the shared --quiet / --log-level / --output-format flags. - - Uses ``default=argparse.SUPPRESS`` so an explicit flag at the main-parser - level (before the subcommand) is not clobbered when the subparser fills - in its own defaults. - """ - if include_output_format: - p.add_argument( - "--output-format", - type=str, - default=argparse.SUPPRESS, - choices=["text", "json"], - help="Output format: text (default) or json.", - ) - p.add_argument("-q", "--quiet", action="store_true", default=argparse.SUPPRESS, help="Suppress INFO logs.") - p.add_argument( - "--log-level", - type=str, - default=argparse.SUPPRESS, - choices=["DEBUG", "INFO", "WARNING", "ERROR"], - help="Set logging verbosity (default: INFO).", - ) - - -def _add_chat_subcommand(subparsers) -> None: - p = subparsers.add_parser( - "chat", - help="Interactive chat REPL with a fine-tuned model.", - description=( - "Load a fine-tuned model and start an interactive terminal session. " - "Supports streaming output, slash commands (/reset, /save, /temperature, " - "/system, /exit), and optional per-response safety annotations." - ), - ) - p.add_argument("model_path", help="Path to a saved HuggingFace model directory or HF Hub ID.") - p.add_argument("--adapter", type=str, default=None, help="PEFT adapter directory to merge before chat.") - p.add_argument("--system", type=str, default=None, metavar="PROMPT", help="Initial system prompt.") - p.add_argument("--temperature", type=float, default=0.7, help="Sampling temperature (default: 0.7).") - p.add_argument("--max-new-tokens", type=int, default=512, help="Max tokens per response (default: 512).") - p.add_argument("--no-stream", action="store_true", help="Disable streaming output.") - p.add_argument("--load-in-4bit", action="store_true", help="Load model in 4-bit NF4 quantisation.") - p.add_argument("--load-in-8bit", action="store_true", help="Load model in 8-bit quantisation.") - p.add_argument("--trust-remote-code", action="store_true", help="Allow execution of model-bundled code.") - p.add_argument( - "--backend", - type=str, - default="transformers", - choices=["transformers", "unsloth"], - help="Model backend (default: transformers).", - ) - # chat is interactive; --output-format doesn't apply. - _add_common_subparser_flags(p, include_output_format=False) - - -def _add_export_subcommand(subparsers) -> None: - p = subparsers.add_parser( - "export", - help="Export a fine-tuned model to GGUF format.", - description=( - "Convert a HuggingFace model to GGUF for use with Ollama, llama.cpp, " - "and compatible runtimes. Requires: pip install 'forgelm[export]'" - ), - ) - p.add_argument("model_path", help="Path to a saved HuggingFace model directory.") - p.add_argument("--output", type=str, required=True, metavar="FILE", help="Output .gguf file path.") - p.add_argument( - "--format", - type=str, - default="gguf", - choices=["gguf"], - help="Export format (default: gguf).", - ) - p.add_argument( - "--quant", - type=str, - default="q4_k_m", - choices=["q2_k", "q3_k_m", "q4_k_m", "q5_k_m", "q8_0", "f16"], - help="Quantisation type (default: q4_k_m).", - ) - p.add_argument("--adapter", type=str, default=None, help="PEFT adapter directory to merge before export.") - p.add_argument( - "--no-integrity-update", - action="store_true", - help="Skip updating model_integrity.json with the exported artifact.", - ) - _add_common_subparser_flags(p, include_output_format=True) - - -def _add_deploy_subcommand(subparsers) -> None: - p = subparsers.add_parser( - "deploy", - help="Generate a deployment configuration for a serving runtime.", - description=( - "Produce a ready-to-use config file for Ollama, vLLM, TGI, or " - "HuggingFace Inference Endpoints. Does not start a server." - ), - ) - p.add_argument("model_path", help="Path to a saved HuggingFace model directory or HF Hub ID.") - p.add_argument( - "--target", - type=str, - required=True, - choices=["ollama", "vllm", "tgi", "hf-endpoints"], - help="Target serving runtime.", - ) - p.add_argument("--output", type=str, default=None, metavar="FILE", help="Output file path (default: auto).") - p.add_argument("--system", type=str, default=None, metavar="PROMPT", help="System prompt (Ollama only).") - p.add_argument("--max-length", type=int, default=4096, help="Context window length (default: 4096).") - p.add_argument( - "--gpu-memory-utilization", - type=float, - default=0.90, - help="vLLM GPU memory utilisation fraction (default: 0.90).", - ) - p.add_argument("--port", type=int, default=8080, help="Host port for TGI container (default: 8080).") - p.add_argument("--trust-remote-code", action="store_true", help="Set trust_remote_code in vLLM config.") - p.add_argument( - "--vendor", - type=str, - default="aws", - help="Cloud vendor for HF Endpoints config (default: aws).", - ) - _add_common_subparser_flags(p, include_output_format=True) - - -def _add_quickstart_subcommand(subparsers) -> None: - p = subparsers.add_parser( - "quickstart", - help="Generate a config from a curated template (and optionally start training).", - description=( - "Pick a template (e.g. customer-support, code-assistant), get a working YAML and " - "(optionally) seed dataset out the other end. The generated config uses the same " - "schema as a hand-written one — quickstart is just opinionated defaults plus a " - "license-clean seed dataset." - ), - ) - p.add_argument( - "template", - nargs="?", - default=None, - help="Template name. Run with --list to see what's available.", - ) - p.add_argument("--list", action="store_true", help="List available templates and exit.") - p.add_argument( - "--model", - type=str, - default=None, - metavar="MODEL_ID", - help="Override the template's primary model (HF Hub ID or local path).", - ) - p.add_argument( - "--dataset", - type=str, - default=None, - metavar="PATH", - help="Override the template's bundled dataset (HF Hub ID or local JSONL path).", - ) - p.add_argument( - "--output", - type=str, - default=None, - metavar="FILE", - help="Where to write the generated YAML (default: ./configs/