diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d44dd02..dfb89c9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,11 @@ All notable changes to ForgeLM are documented here. ### Fixed +- The `forgelm` CLI no longer sends Hugging Face's anonymous usage ping. TRL's trainers call `send_telemetry` when they are + built, and `huggingface_hub` sends it unless `HF_HUB_DISABLE_TELEMETRY` is true or offline mode is on, which contradicted the + privacy page ("does not send usage data anywhere"). `main()` now sets `HF_HUB_DISABLE_TELEMETRY=1` before anything imports + `huggingface_hub`, and an operator's own value wins (`HF_HUB_DISABLE_TELEMETRY=0` opts back in). Using ForgeLM as a library + does not set it (#95). - Fix the KTO bullet on the Alignment overview user-manual page (English and Turkish): it named the fields `(prompt, response, good?)`, but the KTO loader reads `prompt`, `completion` and a boolean `label` (#965). diff --git a/docs/usermanuals/en/operations/air-gap.md b/docs/usermanuals/en/operations/air-gap.md index aacf64e5..40a2807c 100644 --- a/docs/usermanuals/en/operations/air-gap.md +++ b/docs/usermanuals/en/operations/air-gap.md @@ -63,7 +63,7 @@ ERROR: model "meta-llama/Llama-Guard-3-8B" not in local cache. | OpenAI / Anthropic judges | Disable API-backed evaluation or use a local judge; `--offline` does not disable API clients. | | Webhooks | `--offline` does not disable configured delivery. Omit the `webhook:` block, or leave `url` / `url_env` unset, to disable it; `allow_private_destinations` permits private destinations in addition to public ones. | | pip install (during runtime) | Install required dependencies before disconnecting; `--offline` does not control pip. | -| Telemetry | ForgeLM has no own telemetry service; configure external integrations separately. | +| Telemetry | ForgeLM has no own telemetry service. The `forgelm` CLI sets `HF_HUB_DISABLE_TELEMETRY=1` (unless you set it yourself) so TRL and `huggingface_hub` send no anonymous usage ping; using ForgeLM as a library does not set it. Configure external integrations separately. | ## Local synthetic data diff --git a/docs/usermanuals/tr/operations/air-gap.md b/docs/usermanuals/tr/operations/air-gap.md index 58dc233e..c5744481 100644 --- a/docs/usermanuals/tr/operations/air-gap.md +++ b/docs/usermanuals/tr/operations/air-gap.md @@ -63,7 +63,7 @@ ERROR: model "meta-llama/Llama-Guard-3-8B" not in local cache. | OpenAI / Anthropic judge'lar | API tabanlı değerlendirmeyi kapatın veya yerel judge kullanın; `--offline` API istemcilerini kapatmaz. | | Webhook'lar | `--offline`, yapılandırılmış teslimatları kapatmaz. Kapatmak için `webhook:` bloğunu çıkarın veya `url` / `url_env` değerlerini boş bırakın; `allow_private_destinations`, genel hedeflerin yanında özel hedeflere de izin verir. | | Runtime'da pip install | Gerekli bağımlılıkları bağlantıyı kesmeden kurun; `--offline`, pip'i yönetmez. | -| Telemetry | ForgeLM'in kendi telemetri servisi yoktur; harici entegrasyonları ayrıca yapılandırın. | +| Telemetry | ForgeLM'in kendi telemetri servisi yoktur. `forgelm` CLI'ı `HF_HUB_DISABLE_TELEMETRY=1`'i (siz ayarlamadıysanız) ayarlar; böylece TRL ve `huggingface_hub` anonim kullanım ping'i göndermez. ForgeLM'i kütüphane olarak kullanmak bunu ayarlamaz. Harici entegrasyonları ayrıca yapılandırın. | ## Yerel sentetik veri diff --git a/forgelm/cli/_config_load.py b/forgelm/cli/_config_load.py index 4e5ac477..74c1ad78 100644 --- a/forgelm/cli/_config_load.py +++ b/forgelm/cli/_config_load.py @@ -47,6 +47,20 @@ def _load_config_or_exit(config_path: str, json_output: bool): sys.exit(EXIT_CONFIG_ERROR) +def _disable_hub_telemetry() -> None: + """Keep the ``forgelm`` CLI from sending Hugging Face's anonymous usage ping. + + A training run makes ``huggingface_hub`` send a small usage ping to huggingface.co: TRL's trainers call + ``send_telemetry`` when they are built, and ``huggingface_hub`` skips it only when ``HF_HUB_DISABLE_TELEMETRY`` is + true or offline mode is on. ForgeLM's privacy page says the tool sends no usage data anywhere, so the CLI sets + the variable by default. ``setdefault`` leaves an operator's own value alone (``HF_HUB_DISABLE_TELEMETRY=0`` + opts back in). It must run before ``huggingface_hub`` is imported, which reads it once at import; importing + ``forgelm.cli`` does not import it, and ``main()`` calls this first. Using ForgeLM as a library does not go + through ``main()``, so a library user sets the variable themselves. + """ + os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1") + + def _apply_offline_flag(config, offline_arg: bool) -> None: if offline_arg: config.model.offline = True diff --git a/forgelm/cli/_dispatch.py b/forgelm/cli/_dispatch.py index e4b64387..6339f13a 100644 --- a/forgelm/cli/_dispatch.py +++ b/forgelm/cli/_dispatch.py @@ -10,7 +10,7 @@ import sys from .._strict_json import dumps_strict -from ._config_load import _apply_offline_flag, _load_config_or_exit +from ._config_load import _apply_offline_flag, _disable_hub_telemetry, _load_config_or_exit from ._exit_codes import EXIT_CONFIG_ERROR, EXIT_SUCCESS, EXIT_TRAINING_ERROR, _clamp_exit_code from ._logging import _setup_logging, logger from ._no_train_modes import _maybe_run_no_train_mode @@ -163,6 +163,7 @@ def main(): # (= 128+SIGINT) — outside the documented public 0/1/2/3/4 surface # and surprising to CI/CD scripts that branch on exit code 2. _force_utf8_console_streams() + _disable_hub_telemetry() try: return _main_inner() except KeyboardInterrupt: diff --git a/tests/test_cli.py b/tests/test_cli.py index a9765895..7f5ed278 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -319,3 +319,76 @@ def test_facade_reexports_every_subcommand_registrar(self): ] for name in registrars: assert callable(getattr(cli, name, None)), f"{name} is not re-exported from forgelm.cli" + + +class TestHubTelemetryDefault: + """The CLI keeps TRL / huggingface_hub from sending the anonymous usage ping (#95).""" + + @staticmethod + def _unset(monkeypatch): + """Make the variable absent *and* make monkeypatch restore that, so a later set does not leak.""" + monkeypatch.setenv("HF_HUB_DISABLE_TELEMETRY", "placeholder") + monkeypatch.delenv("HF_HUB_DISABLE_TELEMETRY") + + @staticmethod + def _clean_env(): + env = {k: v for k, v in os.environ.items() if k != "HF_HUB_DISABLE_TELEMETRY"} + env["PYTHONPATH"] = os.getcwd() + os.pathsep + env.get("PYTHONPATH", "") + return env + + def test_it_is_set_when_the_operator_did_not_choose(self, monkeypatch): + from forgelm.cli._config_load import _disable_hub_telemetry + + self._unset(monkeypatch) + _disable_hub_telemetry() + assert os.environ["HF_HUB_DISABLE_TELEMETRY"] == "1" + + @pytest.mark.parametrize("chosen", ["0", "1", "false"]) + def test_an_operators_own_value_wins(self, monkeypatch, chosen): + from forgelm.cli._config_load import _disable_hub_telemetry + + monkeypatch.setenv("HF_HUB_DISABLE_TELEMETRY", chosen) + _disable_hub_telemetry() + assert os.environ["HF_HUB_DISABLE_TELEMETRY"] == chosen + + def test_main_sets_it_before_anything_else_runs(self, monkeypatch): + from forgelm.cli import _dispatch + + self._unset(monkeypatch) + seen = {} + + def fake_inner(): + seen["at_dispatch"] = os.environ.get("HF_HUB_DISABLE_TELEMETRY") + return 0 + + with patch.object(_dispatch, "_main_inner", fake_inner): + _dispatch.main() + assert seen["at_dispatch"] == "1" + + def test_importing_the_cli_does_not_import_huggingface_hub(self): + # huggingface_hub reads HF_HUB_DISABLE_TELEMETRY once, at import. main() sets the variable, so nothing the CLI + # module imports at load time may import it first (a top-level `import transformers` in forgelm.cli would). + import subprocess + import sys + + code = "import sys, forgelm.cli; print(any(m == 'huggingface_hub' or m.startswith('huggingface_hub.') for m in sys.modules))" + out = subprocess.run( + [sys.executable, "-c", code], env=self._clean_env(), capture_output=True, text=True, timeout=120 + ) + assert out.returncode == 0, out.stderr + assert out.stdout.strip() == "False" + + def test_a_fresh_process_that_calls_it_reads_the_variable_as_true(self): + pytest.importorskip("huggingface_hub") + import subprocess + import sys + + code = ( + "from forgelm.cli._config_load import _disable_hub_telemetry; _disable_hub_telemetry(); " + "import huggingface_hub.constants as c; print(c.HF_HUB_DISABLE_TELEMETRY)" + ) + out = subprocess.run( + [sys.executable, "-c", code], env=self._clean_env(), capture_output=True, text=True, timeout=120 + ) + assert out.returncode == 0, out.stderr + assert out.stdout.strip() == "True"