diff --git a/.env.example b/.env.example index db4f184..67f70ba 100644 --- a/.env.example +++ b/.env.example @@ -333,6 +333,12 @@ ## password does not re-auth against a generated one; ## see docs/operating.md for the migration. Skipped ## when CLICKHOUSE_HOST points at an external instance. +## DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD - the hunt runner's own ClickHouse user, +## dfe_hunt_runner. The engine creates the user on +## this password and the runner connects with it, so +## it reaches the data database and nothing else. +## Needed with an external CLICKHOUSE_HOST too: the +## engine creates the user there. ## ## Most are covered again in their own sections below. diff --git a/docker-compose.yml b/docker-compose.yml index b6f6b46..46063e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1159,6 +1159,9 @@ services: DFE_CLICKHOUSE_USERNAME: ${CLICKHOUSE_USERNAME:-default} DFE_CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-} DFE_CLICKHOUSE_SECURE: ${CLICKHOUSE_SECURE:-false} + # The password the engine gives dfe_hunt_runner: the same value the runner + # below connects with, minted by `make init`. + DFE_CLICKHOUSE_HUNT_RUNNER_PASSWORD: ${DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD:-} # Default TTL for every time-series table; 0 = none; a source or a # dfe-schemas TTL overrides it. DFE_CLICKHOUSE_DEFAULT_TTL_DAYS: ${DFE_CLICKHOUSE_DEFAULT_TTL_DAYS:-90} @@ -1301,8 +1304,11 @@ services: DFE_CLICKHOUSE_PORT: ${CLICKHOUSE_EXTERNAL_HTTP_PORT:-8123} DFE_CLICKHOUSE_NATIVE_PORT: ${CLICKHOUSE_EXTERNAL_NATIVE_PORT:-9000} DFE_CLICKHOUSE_DATABASE: ${CLICKHOUSE_DB:-default} - DFE_CLICKHOUSE_USERNAME: ${CLICKHOUSE_USERNAME:-default} - DFE_CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-} + # Its own user, not the admin account: the worker runs INSERT ... SELECT + # built from rule text, and dfe_hunt_runner can reach nothing but the data + # database. The engine names the user, so it is not a dial. + DFE_CLICKHOUSE_USERNAME: dfe_hunt_runner + DFE_CLICKHOUSE_PASSWORD: ${DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD:-} DFE_CLICKHOUSE_SECURE: ${CLICKHOUSE_SECURE:-false} DFE_CONFIG_DIR: ${DFE_ENGINE_CONFIG_DIR:-/app/config} # Same volume as the API's, so both read the one signing key. diff --git a/docs/configuration.md b/docs/configuration.md index 1474ed6..20ce35f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -193,6 +193,7 @@ it is a second deployment of the one component, not a component of its own. | `CLICKHOUSE_DB` | ClickHouse initialisation database | `default` | | `CLICKHOUSE_USERNAME` | ClickHouse username to connect with | `default` | | `CLICKHOUSE_PASSWORD` | ClickHouse password associated to user | - | +| `DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD` | Password of `dfe_hunt_runner`, the hunt runner's own ClickHouse user; the engine creates the user on it and the runner connects with it | generated | | `DFE_CLICKHOUSE_DEFAULT_TTL_DAYS` | Days every time-series table keeps rows, the OTel tables included; 0 disables the default TTL; a source or dfe-schemas TTL overrides it | `90` | ### Kafka - General diff --git a/docs/operating.md b/docs/operating.md index 1671751..b5e9851 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -348,10 +348,9 @@ position requires zero reference to the artefact, that is the remaining edge. ## Secrets: generated by make init -`make init` mints a random value for `DFE_UI_NEXTAUTH_SECRET`, -`HYPERDX_POSTGRES_PASSWORD`, `HYPERDX_EXPRESS_SESSION_SECRET`, -`HYPERDX_TOKEN_ENCRYPTION_KEY` and `CLICKHOUSE_PASSWORD`, including topping up an -existing `.env` that predates any of them (`scripts/init.py`). Compose carries a +`make init` mints a random value for every key in `GENERATED_SECRETS` +(`scripts/init.py`), including topping up an existing `.env` that predates any +of them. Compose carries a sentinel (or empty) default rather than a `${VAR:?}` hard-fail -- interpolation is not profile-gated, so a hard-fail would abort `make down` too, for a service the operator may not even run. The check lives in the power-on self test instead: @@ -368,6 +367,11 @@ skips it when `CLICKHOUSE_HOST` points at an external instance, because then the credential is the operator's, not the stack's. See the upgrade note below -- a generated password against an existing warehouse volume is a breaking change. +`DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD` is the hunt runner's own ClickHouse user, +`dfe_hunt_runner`. The engine creates that user on this password, with `SELECT` +and `INSERT` on the data database and nothing else, and the runner connects with +the same value instead of the admin account. + Never commit `.env`. ## Persistence: what survives, and what `make clean` destroys diff --git a/scripts/init.py b/scripts/init.py index 2bdd5ff..74dfa3b 100755 --- a/scripts/init.py +++ b/scripts/init.py @@ -47,10 +47,14 @@ # Secrets that must not be left at their weak/empty default. scripts/post.py # enforces them: DFE_UI_NEXTAUTH_SECRET, HYPERDX_POSTGRES_PASSWORD, -# HYPERDX_EXPRESS_SESSION_SECRET and HYPERDX_TOKEN_ENCRYPTION_KEY via its -# WEAK_SECRET_DEFAULTS service-map, CLICKHOUSE_PASSWORD via its own external-CH -# check (its default is empty, not a sentinel string), and -# DFE_AUTH_LOCAL_ADMIN_PASSWORD by logging in with it. Keep them in step. +# HYPERDX_EXPRESS_SESSION_SECRET, HYPERDX_TOKEN_ENCRYPTION_KEY and +# DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD via its WEAK_SECRET_DEFAULTS service-map, +# CLICKHOUSE_PASSWORD via its own external-CH check (its default is empty, not a +# sentinel string), and DFE_AUTH_LOCAL_ADMIN_PASSWORD by logging in with it. Keep +# them in step. +# +# DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD is dfe_hunt_runner's: the engine creates +# that ClickHouse user on this password and the hunt runner connects with it. # # The deploy mints two logins: DFE_AUTH_LOCAL_ADMIN_PASSWORD is `admin`, issued with # a forced change at first login, which `make post` makes and records back in .env. @@ -89,6 +93,7 @@ "CLICKHOUSE_PASSWORD": _LEN_CREDENTIAL, "DFE_AUTH_BREAKGLASS_PASSWORD": _LEN_CREDENTIAL, "DFE_AUTH_LOCAL_ADMIN_PASSWORD": _LEN_CREDENTIAL, + "DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD": _LEN_CREDENTIAL, "HYPERDX_POSTGRES_PASSWORD": _LEN_CREDENTIAL, "DFE_API_JWT_SECRET": _LEN_KEY, "DFE_UI_NEXTAUTH_SECRET": _LEN_KEY, diff --git a/scripts/post.py b/scripts/post.py index 0296e32..d730271 100644 --- a/scripts/post.py +++ b/scripts/post.py @@ -313,6 +313,8 @@ ), # Empty, not a sentinel: HyperDX refuses to start on a malformed key. "HYPERDX_TOKEN_ENCRYPTION_KEY": ("", "hyperdx"), + # Empty leaves dfe_hunt_runner on a password the engine mints, which the runner never sees. + "DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD": ("", "dfe-hunt-runner"), } diff --git a/scripts/tests/test_post.py b/scripts/tests/test_post.py index d09ed25..6fd7e12 100644 --- a/scripts/tests/test_post.py +++ b/scripts/tests/test_post.py @@ -900,3 +900,41 @@ def test_the_hunt_events_carry_a_match_and_near_misses() -> None: """Without both sets the verdict cannot tell an exact rule from an over-broad one.""" assert post.HUNT_EVENTS[post._detection.MATCHING] assert post.HUNT_EVENTS[post._detection.OTHER] + + +_RUNNER_PASSWORD_KEY = "DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD" + + +def _only_the_runner_password_in_play( + monkeypatch: pytest.MonkeyPatch, services: list[str] +) -> None: + """The bundled ClickHouse with its admin password set, and these services running.""" + monkeypatch.setattr(post, "_resolved_services", lambda: services) + monkeypatch.delenv("CLICKHOUSE_HOST", raising=False) + monkeypatch.setenv("CLICKHOUSE_PASSWORD", "aMintedAdminValue123") + + +def test_an_unset_hunt_runner_password_fails_the_self_test( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Empty leaves dfe_hunt_runner on a password the engine mints and the runner never sees.""" + _only_the_runner_password_in_play(monkeypatch, ["dfe-engine", "dfe-hunt-runner"]) + monkeypatch.delenv(_RUNNER_PASSWORD_KEY, raising=False) + + assert [name for name, _value in post._weak_secrets()] == [_RUNNER_PASSWORD_KEY] + + +def test_a_minted_hunt_runner_password_passes(monkeypatch: pytest.MonkeyPatch) -> None: + _only_the_runner_password_in_play(monkeypatch, ["dfe-engine", "dfe-hunt-runner"]) + monkeypatch.setenv(_RUNNER_PASSWORD_KEY, "aMintedRunnerValue123") + + assert post._weak_secrets() == [] + + +def test_a_stack_without_the_runner_needs_no_runner_password( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _only_the_runner_password_in_play(monkeypatch, ["dfe-engine"]) + monkeypatch.delenv(_RUNNER_PASSWORD_KEY, raising=False) + + assert post._weak_secrets() == []